Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix:builds the AMD64 architecture image explicitly and pushes to Airbyte's repo for suing it in Airbyte Cloud #13

Merged
merged 1 commit into from
Aug 7, 2024
Merged

fix:builds the AMD64 architecture image explicitly and pushes to Airbyte's repo for suing it in Airbyte Cloud #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions airbyte-integrations/connectors/destination-glide/scripts/build-docker-image.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@ this_dir=$(cd $(dirname "$0"); pwd) # this script's directory
this_script=$(basename $0)

# segment sometimes fails, so AIRBYTE_CI_DISABLE_TELEMETRY=true to disable it per airbyte-ci/connectors/pipelines/README.md
AIRBYTE_CI_DISABLE_TELEMETRY=true airbyte-ci connectors --name=destination-glide build

# see --architecture flag and available values in airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py
AIRBYTE_CI_DISABLE_TELEMETRY=true airbyte-ci connectors --name=destination-glide build \
--architecture linux/amd64 --architecture linux/arm64
43 changes: 33 additions & 10 deletions airbyte-integrations/connectors/destination-glide/scripts/push-docker-image.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,33 @@
this_dir=$(cd $(dirname "$0"); pwd) # this script's directory
this_script=$(basename $0)

#!/usr/bin/env bash
LOCAL_BUILT_IMAGE_NAME=airbyte/destination-glide:dev
GLIDE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide
# So... it looks like when the airbyte-ci builds the multi-architecture images, it puts each one on a different tag locally.
# We should probably push amd64 only or at least make it the "default" and make arm tagged sepraately.
# if we built and pushed all at once we could have a single tag for both architectures. However, we're using the airbyte-ci to build and that makes pushing a separate step.
# It looks like there is experimental support for using docker manfiest to combine tags into a single multi-arch image at https://docs.docker.com/reference/cli/docker/manifest/
LOCAL_BUILT_IMAGE_TAG_AMD64=dev-linux-amd64
LOCAL_BUILT_IMAGE_TAG_ARM64=dev-linux-arm64

LOCAL_BUILT_IMAGE_NAME=airbyte/destination-glide
# GLIDE_DOCKER_IMAGE_NAME is really only useful if we want to use it entirely in Airbyte OSS. To use in Airbyte cloud we *must* push to the airbyte-managed repo.
#GLIDE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/glide-connectors/airbyte-glide-destination/destination-glide

# AIRBYTE_DOCKER_IMAGE_NAME is the repo we use for Airbyte's Cloud instance.
GLIDE_AT_AIRBYTE_COMPANY_NAME=heyglide
AIRBYTE_DOCKER_IMAGE_NAME=us-central1-docker.pkg.dev/airbyte-custom-connectors/$GLIDE_AT_AIRBYTE_COMPANY_NAME/destination-glide

echo "Pushing to remote image repository: $GLIDE_DOCKER_IMAGE_NAME"
docker inspect --format='Pushing the local image "{{index .RepoTags 0}}" created at "{{.Created}}"' $LOCAL_BUILT_IMAGE_NAME
echo "Will push to remote image repository: $AIRBYTE_DOCKER_IMAGE_NAME"

docker inspect --format='Pushing the local image "{{index .RepoTags 0}}" created at "{{.Created}}"' $LOCAL_BUILT_IMAGE_NAME:$LOCAL_BUILT_IMAGE_TAG_AMD64
if [ $? -ne 0 ]; then
echo "Error: The local image $LOCAL_BUILT_IMAGE_NAME does not exist. Please build the image first."
exit 1
fi

# Fetch the list of tags from Docker Hub
#tags=$(wget -q -O - "https://hub.docker.com/v2/namespaces/activescott/repositories/destination-glide/tags?page_size=10" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E '([0-9]+\.)+[0-9]+' )
# Get the list of tags using gcloud with the appropriate format
tags=$(gcloud artifacts docker tags list $GLIDE_DOCKER_IMAGE_NAME --format="get(tag)")
tags=$(gcloud artifacts docker tags list $AIRBYTE_DOCKER_IMAGE_NAME --format="get(tag)")

# Sort the tags and get the highest one
highest_tag=$(echo "$tags" | awk -F'/' '{print $NF}' | sed 's/latest//g' | sort -V | tail -n 1)
Expand All @@ -38,18 +53,26 @@ fi
# Tag the local image with the new version
# NOTE: airbyte/destination-glide is the local docker image that airbyte's CI builds and names locally.

docker image tag $LOCAL_BUILT_IMAGE_NAME $GLIDE_DOCKER_IMAGE_NAME:$new_version
docker image tag $LOCAL_BUILT_IMAGE_NAME:$LOCAL_BUILT_IMAGE_TAG_AMD64 $AIRBYTE_DOCKER_IMAGE_NAME:$new_version
if [ $? -ne 0 ]; then
echo "Error: The local image $LOCAL_BUILT_IMAGE_NAME does not exist. Please build the image first."
exit 1
fi

# Push the image with the new tag
docker push $GLIDE_DOCKER_IMAGE_NAME:$new_version
docker push $AIRBYTE_DOCKER_IMAGE_NAME:$new_version
if [ $? -ne 0 ]; then
echo "Error: The local image $LOCAL_BUILT_IMAGE_NAME does not exist. Please build the image first."
exit 1
fi

echo "Do you want to make this new version the 'latest' tag in Docker too (so that new pulls get this version by default)? (y/n)"
read answer

if [ "$answer" != "${answer#[Yy]}" ] ; then
echo "Taging latest..."
docker image tag $LOCAL_BUILT_IMAGE_NAME $GLIDE_DOCKER_IMAGE_NAME:latest
docker push $GLIDE_DOCKER_IMAGE_NAME:latest
docker image tag $AIRBYTE_DOCKER_IMAGE_NAME:$new_version $AIRBYTE_DOCKER_IMAGE_NAME:latest
docker push $AIRBYTE_DOCKER_IMAGE_NAME:latest
else
echo "Exiting..."
exit 0
Expand Down
Loading