From 7902a41c87240d607179d07c28cce462ea135c5f Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Mon, 26 Feb 2024 10:39:49 -0500 Subject: [PATCH] chore: add docker image for hermetic build scripts (#2493) Similar changes to those of https://github.com/googleapis/sdk-platform-java/pull/2298 by @mpeddada1 This setup uses two triggers: - The first one is [library-generation-presubmit-sdk-platform-java](https://pantheon.corp.google.com/cloud-build/triggers;region=global/edit/5783744f-0820-419f-bc5e-abbbece4be0b?e=13803378&mods=monitoring_api_prod&project=cloud-devrel-kokoro-resources), which runs on each pull request. It builds a docker image with the contents of `library_generation` at HEAD and runs `library_generation/integration_tests.py` with such image - The second one is [library-generation-sdk-platform-java](https://pantheon.corp.google.com/cloud-build/triggers;region=global/edit/e3373892-82a2-4eac-a1f2-95523966df70?e=13803378&mods=monitoring_api_prod&project=cloud-devrel-kokoro-resources), which: - is triggered upon a commit pushed to the `main` branch, then - builds a docker image with two tags - `latest`, which will be constantly updated to match the latest build - a tag based on the branch `${COMMIT_SHA}` - then the image is pushed with both tags ### tasks - [x] create dockerfile - [x] create cloudbuild.yaml for testing - [x] create cloudbuild test infra - [x] create cloudbuild.yaml for releasing the image - [x] create cloudbuild release infra --------- Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> --- .../cloudbuild-library-generation.yaml | 33 +++++++++++++++ .../cloudbuild-test-library-generation.yaml | 31 ++++++++++++++ .cloudbuild/library_generation.Dockerfile | 40 +++++++++++++++++++ library_generation/dockerignore | 6 +++ library_generation/generate_repo.py | 0 library_generation/postprocess_library.sh | 37 ++++++++++++++--- library_generation/setup.py | 0 .../test/container_integration_tests.sh | 39 ++++++++++++++++++ library_generation/test/integration_tests.py | 40 +++++++++++++++---- 9 files changed, 213 insertions(+), 13 deletions(-) create mode 100644 .cloudbuild/cloudbuild-library-generation.yaml create mode 100644 .cloudbuild/cloudbuild-test-library-generation.yaml create mode 100644 .cloudbuild/library_generation.Dockerfile create mode 100644 library_generation/dockerignore mode change 100644 => 100755 library_generation/generate_repo.py mode change 100644 => 100755 library_generation/setup.py create mode 100644 library_generation/test/container_integration_tests.sh mode change 100644 => 100755 library_generation/test/integration_tests.py diff --git a/.cloudbuild/cloudbuild-library-generation.yaml b/.cloudbuild/cloudbuild-library-generation.yaml new file mode 100644 index 0000000000..7bdf371bfb --- /dev/null +++ b/.cloudbuild/cloudbuild-library-generation.yaml @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +timeout: 7200s # 2 hours +substitutions: + _IMAGE_NAME: "gcr.io/cloud-devrel-public-resources/java-library-generation" + _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}" + _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest" +steps: + # Library generation build + - name: gcr.io/cloud-builders/docker + args: [ + "build", + "-t", "${_SHA_IMAGE_ID}", + "-t", "${_LATEST_IMAGE_ID}", + "--file", ".cloudbuild/library_generation.Dockerfile", "."] + id: library-generation-build + waitFor: ["-"] + +images: + - ${_SHA_IMAGE_ID} + - ${_LATEST_IMAGE_ID} diff --git a/.cloudbuild/cloudbuild-test-library-generation.yaml b/.cloudbuild/cloudbuild-test-library-generation.yaml new file mode 100644 index 0000000000..909a6439cf --- /dev/null +++ b/.cloudbuild/cloudbuild-test-library-generation.yaml @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +timeout: 7200s # 2 hours +substitutions: + _TEST_IMAGE_ID: 'gcr.io/cloud-devrel-public-resources/java-library-generation:${COMMIT_SHA}' + +steps: + # Library generation build + - name: gcr.io/cloud-builders/docker + args: ["build", "-t", "${_TEST_IMAGE_ID}", "--file", ".cloudbuild/library_generation.Dockerfile", "."] + id: library-generation-build + waitFor: ["-"] + - name: ${_TEST_IMAGE_ID} + entrypoint: bash + args: [ './library_generation/test/container_integration_tests.sh' ] + waitFor: [ "library-generation-build" ] + env: + - 'TEST_IMAGE_ID=${_TEST_IMAGE_ID}' + diff --git a/.cloudbuild/library_generation.Dockerfile b/.cloudbuild/library_generation.Dockerfile new file mode 100644 index 0000000000..62a8c516de --- /dev/null +++ b/.cloudbuild/library_generation.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build from the root of this repo: +FROM gcr.io/cloud-devrel-public-resources/python + +# install tools +RUN apt-get update && apt-get install -y \ + unzip openjdk-17-jdk rsync maven \ + && apt-get clean + +COPY library_generation /src + +RUN rm $(which python3) +RUN ln -s $(which python3.11) /usr/local/bin/python +RUN ln -s $(which python3.11) /usr/local/bin/python3 +RUN python -m pip install --upgrade pip +RUN cd /src && python -m pip install -r requirements.in +RUN cd /src && python -m pip install . + +# set dummy git credentials for empty commit used in postprocessing +RUN git config --global user.email "cloud-java-bot@google.com" +RUN git config --global user.name "Cloud Java Bot" + +WORKDIR /workspace +RUN chmod 750 /workspace +RUN chmod 750 /src/generate_repo.py + +CMD [ "/src/generate_repo.py" ] diff --git a/library_generation/dockerignore b/library_generation/dockerignore new file mode 100644 index 0000000000..7e978caea9 --- /dev/null +++ b/library_generation/dockerignore @@ -0,0 +1,6 @@ +README.md +**/__pycache__/ +**/*.egg-info/ +**/output/ +**/build/ +**/google-cloud-java/ diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py old mode 100644 new mode 100755 diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 2951a8e9a4..9f9b5a3786 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -61,22 +61,48 @@ fi # we determine the location of the .OwlBot.yaml file by checking if the target # folder is a monorepo folder or not -if [[ "${postprocessing_target}" == *google-cloud-java* ]]; then +if [[ "${is_monorepo}" == "true" ]]; then owlbot_yaml_relative_path=".OwlBot.yaml" else owlbot_yaml_relative_path=".github/.OwlBot.yaml" fi +# Default values for running copy-code directly from host +repo_binding="${postprocessing_target}" +repo_workspace="/repo" +preprocessed_libraries_binding="${owlbot_cli_source_folder}" + +# When running docker inside docker, we run into the issue of volume bindings +# being mapped from the host machine to the child container (instead of the +# parent container to child container) because we bind the `docker.sock` socket +# to the parent container (i.e. docker calls use the host's filesystem context) +# see https://serverfault.com/a/819371 +# We solve this by referencing environment variables that will be +# set to produce the correct volume mapping. +# +# The workflow is: to check if we are in a docker container (via passed env var) +# and use managed volumes (docker volume create) instead of bindings +# (-v /path:/other-path). The volume names are also received as env vars. + +if [[ -n "${RUNNING_IN_DOCKER}" ]]; then + set -u # temporarily fail on unset variables + repo_binding="${REPO_BINDING_VOLUME}" + set +u + if [[ "${is_monorepo}" == "true" ]]; then + repo_workspace="/repo/$(echo "${postprocessing_target}" | rev | cut -d'/' -f1 | rev)" + fi +fi + docker run --rm \ --user "$(id -u)":"$(id -g)" \ - -v "${postprocessing_target}:/repo" \ - -v "${owlbot_cli_source_folder}:/pre-processed-libraries" \ - -w /repo \ + -v "${repo_binding}:/repo" \ + -v "/tmp:/tmp" \ + -w "${repo_workspace}" \ --env HOME=/tmp \ gcr.io/cloud-devrel-public-resources/owlbot-cli@"${owlbot_cli_image_sha}" \ copy-code \ --source-repo-commit-hash=none \ - --source-repo=/pre-processed-libraries \ + --source-repo="${preprocessed_libraries_binding}" \ --config-file="${owlbot_yaml_relative_path}" # we clone the synthtool library and manually build it @@ -86,6 +112,7 @@ pushd /tmp/synthtool if [ ! -d "synthtool" ]; then git clone https://github.com/googleapis/synthtool.git fi +git config --global --add safe.directory /tmp/synthtool/synthtool pushd "synthtool" git reset --hard "${synthtool_commitish}" diff --git a/library_generation/setup.py b/library_generation/setup.py old mode 100644 new mode 100755 diff --git a/library_generation/test/container_integration_tests.sh b/library_generation/test/container_integration_tests.sh new file mode 100644 index 0000000000..9a17e585a6 --- /dev/null +++ b/library_generation/test/container_integration_tests.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# This is a wrapper of integration_tests.py that sets up the environment to run +# the script in a docker container + +set -xe +if [[ -z "${TEST_IMAGE_ID}" ]]; then + echo "required environemnt variable TEST_IMAGE_ID is not set" + exit 1 +fi + +if [[ ! -d google-cloud-java ]]; then + git clone https://github.com/googleapis/google-cloud-java +fi +pushd google-cloud-java +git reset --hard main +popd + +# We use a volume to hold the google-cloud-java repository used in the +# integration tests. This is because the test container creates a child +# container using the host machine's docker socket, meaning that we can only +# reference volumes created from within the host machine (i.e. the machine +# running this script) +# +# To summarize, we create a special volume that can be referenced both in the +# main container and in any child containers created by this one. +if [[ $(docker volume inspect repo) != '[]' ]]; then + docker volume rm repo +fi +docker volume create --name "repo" --opt "type=none" --opt "device=$(pwd)/google-cloud-java" --opt "o=bind" + +docker run --rm \ + -v repo:/workspace \ + -v /tmp:/tmp \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e "RUNNING_IN_DOCKER=true" \ + -e "REPO_BINDING_VOLUME=repo" \ + -w "/src" \ + "${TEST_IMAGE_ID}" \ + python -m unittest /src/test/integration_tests.py diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py old mode 100644 new mode 100755 index 4f806cf17f..c552a52e4a --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -26,8 +26,11 @@ from library_generation.generate_repo import generate_from_yaml from library_generation.model.generation_config import from_yaml from library_generation.test.compare_poms import compare_xml -from library_generation.utilities import get_library_name -from library_generation.utilities import sh_util as shell_call +from library_generation.utilities import ( + get_library_name, + sh_util as shell_call, + run_process_and_print_output, +) config_name = "generation_config.yaml" script_dir = os.path.dirname(os.path.realpath(__file__)) @@ -48,8 +51,9 @@ def test_generate_repo(self): config_files = self.__get_config_files(config_dir) i = 0 for repo, config_file in config_files.items(): - repo_dest = f"{output_folder}/{repo}" - self.__pull_repo_to(Path(repo_dest), repo, committish_list[i]) + repo_dest = self.__pull_repo_to( + Path(f"{golden_dir}/{repo}"), repo, committish_list[i] + ) library_names = self.__get_library_names_from_config(config_file) # prepare golden files for library_name in library_names: @@ -109,11 +113,31 @@ def test_generate_repo(self): i += 1 @classmethod - def __pull_repo_to(cls, dest: Path, repo: str, committish: str): - repo_url = f"{repo_prefix}/{repo}" - print(f"Cloning repository {repo_url}") - repo = Repo.clone_from(repo_url, dest) + def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str: + if "RUNNING_IN_DOCKER" in os.environ: + # the docker image expects the repo to be in /workspace + dest_in_docker = "/workspace" + run_process_and_print_output( + [ + "git", + "config", + "--global", + "--add", + "safe.directory", + dest_in_docker, + ], + "Add /workspace to safe directories", + ) + dest = Path(dest_in_docker) + repo = Repo(dest) + else: + dest = default_dest + repo_dest = f"{golden_dir}/{repo}" + repo_url = f"{repo_prefix}/{repo}" + print(f"Cloning repository {repo_url}") + repo = Repo.clone_from(repo_url, dest) repo.git.checkout(committish) + return str(dest) @classmethod def __get_library_names_from_config(cls, config_path: str) -> List[str]: