From cf44b0754258b18fe3ae29725e9713b548beff80 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Thu, 23 May 2024 10:23:01 -0400 Subject: [PATCH] chore: bake protoc into the hermetic build docker image (#2707) ### Does the config work with renovate bot? Yes. After a local run using `LOG_LEVEL=debug npx --yes --package renovate -- renovate --platform=local &> renovate.out`, we obtained a dependency update in the logs: ``` { "deps": [ { "depName": "protocolbuffers/protobuf", "currentValue": "25.2", "datasource": "github-releases", "extractVersion": "^v(?.*)$", "replaceString": "ARG PROTOC_VERSION=25.2\n", "updates": [ { "bucket": "non-major", "newVersion": "25.3", "newValue": "25.3", "releaseTimestamp": "2024-02-15T23:20:43.000Z", "newMajor": 25, "newMinor": 3, "updateType": "minor", "branchName": "renovate/protocolbuffers-protobuf-25.x" }, { "bucket": "major", "newVersion": "26.1", "newValue": "26.1", "releaseTimestamp": "2024-03-27T20:28:47.000Z", "newMajor": 26, "newMinor": 1, "updateType": "major", "branchName": "renovate/protocolbuffers-protobuf-26.x" } ], "packageName": "protocolbuffers/protobuf", "versioning": "semver-coerced", "warnings": [], "sourceUrl": "https://github.com/protocolbuffers/protobuf", "registryUrl": "https://github.com", "currentVersion": "25.2", "currentVersionTimestamp": "2024-01-09T23:52:55.000Z", "isSingleVersion": true, "fixedVersion": "25.2" } ], ``` However, we disabled the setting for now. This log entry confirms it. `DEBUG: Dependency: protocolbuffers/protobuf, is disabled (repository=local)` --------- Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> --- .../library_generation.Dockerfile | 19 +++++++-- .../test/generate_library_unit_tests.sh | 39 +++++++++++++++++++ library_generation/test/integration_tests.py | 8 ---- .../model/generation_config_unit_tests.py | 2 +- ...generation_config_comparator_unit_tests.py | 6 +-- library_generation/utils/utilities.sh | 24 ++++++++++-- renovate.json | 15 ++++++- 7 files changed, 93 insertions(+), 20 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index e962dc5e51..9de0bc1c2c 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -15,8 +15,11 @@ # build from the root of this repo: FROM gcr.io/cloud-devrel-public-resources/python -ARG SYNTHTOOL_COMMITTISH=a2c9b4a5da2d7f583c8a1869fd2843c206145834 +SHELL [ "/bin/bash", "-c" ] + +ARG SYNTHTOOL_COMMITTISH=e36d2f164ca698f0264fb6f79ddc4b0fa024a940 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 +ARG PROTOC_VERSION=25.3 ENV HOME=/home # install OS tools @@ -24,15 +27,23 @@ RUN apt-get update && apt-get install -y \ unzip openjdk-17-jdk rsync maven jq \ && apt-get clean +# copy source code +COPY library_generation /src + +# install protoc +WORKDIR /protoc +RUN source /src/utils/utilities.sh \ + && download_protoc "${PROTOC_VERSION}" "linux-x86_64" +# we indicate protoc is available in the container via env vars +ENV DOCKER_PROTOC_LOCATION=/protoc +ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}" + # use python 3.11 (the base image has several python versions; here we define the default one) RUN rm $(which python3) RUN ln -s $(which python3.11) /usr/local/bin/python RUN ln -s $(which python3.11) /usr/local/bin/python3 RUN python -m pip install --upgrade pip -# copy source code -COPY library_generation /src - # install scripts as a python package WORKDIR /src RUN python -m pip install -r requirements.txt diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index 9b5f8bc456..5384d6982c 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -28,6 +28,15 @@ get_grpc_version_failed_with_invalid_generator_version_test() { assertEquals 1 $((res)) } +get_protoc_version_succeed_docker_env_var_test() { + local version_with_docker + local version_without_docker + export DOCKER_PROTOC_VERSION="9.9.9" + version_with_docker=$(get_protoc_version "2.24.0") + assertEquals "${DOCKER_PROTOC_VERSION}" "${version_with_docker}" + unset DOCKER_PROTOC_VERSION +} + get_protoc_version_succeed_with_valid_generator_version_test() { local actual_version actual_version=$(get_protoc_version "2.24.0") @@ -134,6 +143,34 @@ download_protoc_failed_with_invalid_arch_test() { assertEquals 1 $((res)) } +download_tools_succeed_with_baked_protoc() { + # This mimics a docker container scenario. + # This test consists of creating an empty /tmp/.../protoc-99.99/bin folder and map + # it to the DOCKER_PROTOC_LOCATION env var (which is treated specially in the + # `download_tools` function). If `DOCKER_PROTOC_VERSION` matches exactly as + # the version passed to `download_protoc`, then we will not download protoc + # but simply have the variable `protoc_path` pointing to DOCKER_PROTOC_LOCATION + # (which we manually created in this test) + local test_dir=$(mktemp -d) + pushd "${test_dir}" + export DOCKER_PROTOC_LOCATION=$(mktemp -d) + export DOCKER_PROTOC_VERSION="99.99" + export output_folder=$(get_output_folder) + mkdir "${output_folder}" + local protoc_bin_folder="${DOCKER_PROTOC_LOCATION}/protoc-99.99/bin" + mkdir -p "${protoc_bin_folder}" + + local test_ggj_version="2.40.0" + local test_grpc_version="1.64.0" + download_tools "${test_ggj_version}" "99.99" "${test_grpc_version}" "linux-x86_64" + assertEquals "${protoc_bin_folder}" "${protoc_path}" + + rm -rdf "${output_folder}" + unset DOCKER_PROTOC_LOCATION + unset DOCKER_PROTOC_VERSION + unset output_folder +} + download_grpc_plugin_succeed_with_valid_version_linux_test() { download_grpc_plugin "1.55.1" "linux-x86_64" assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe" @@ -256,6 +293,7 @@ test_list=( extract_folder_name_test get_grpc_version_succeed_with_valid_generator_version_test get_grpc_version_failed_with_invalid_generator_version_test + get_protoc_version_succeed_docker_env_var_test get_protoc_version_succeed_with_valid_generator_version_test get_protoc_version_failed_with_invalid_generator_version_test get_gapic_opts_with_rest_test @@ -268,6 +306,7 @@ test_list=( download_protoc_succeed_with_valid_version_macos_test download_protoc_failed_with_invalid_version_linux_test download_protoc_failed_with_invalid_arch_test + download_tools_succeed_with_baked_protoc download_grpc_plugin_succeed_with_valid_version_linux_test download_grpc_plugin_succeed_with_valid_version_macos_test download_grpc_plugin_failed_with_invalid_version_linux_test diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index e4d298cd9c..5e5d219c61 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -52,10 +52,6 @@ class IntegrationTest(unittest.TestCase): def setUpClass(cls) -> None: IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir) - @classmethod - def tearDownClass(cls) -> None: - cls.__remove_docker_image() - @classmethod def setUp(cls) -> None: cls.__remove_generated_files() @@ -307,7 +303,3 @@ def __recursive_diff_files( cls.__recursive_diff_files( sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" ) - - @classmethod - def __remove_docker_image(cls): - subprocess.check_call(["docker", "image", "rmi", image_tag]) diff --git a/library_generation/test/model/generation_config_unit_tests.py b/library_generation/test/model/generation_config_unit_tests.py index 8f94ecae1e..dafbe238da 100644 --- a/library_generation/test/model/generation_config_unit_tests.py +++ b/library_generation/test/model/generation_config_unit_tests.py @@ -52,7 +52,7 @@ def test_from_yaml_succeeds(self): self.assertEqual( "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish ) - self.assertEqual("26.37.0", config.libraris_bom_version) + self.assertEqual("26.37.0", config.libraries_bom_version) self.assertEqual( [ ".github/*", diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py index 5792ead347..be0568c1f4 100644 --- a/library_generation/test/utils/generation_config_comparator_unit_tests.py +++ b/library_generation/test/utils/generation_config_comparator_unit_tests.py @@ -90,8 +90,8 @@ def test_compare_config_generator_update(self): self.assertEqual("1.2.4", config_change.current_value) def test_compare_config_libraries_bom_update(self): - self.baseline_config.libraris_bom_version = "26.36.0" - self.current_config.libraris_bom_version = "26.37.0" + self.baseline_config.libraries_bom_version = "26.36.0" + self.current_config.libraries_bom_version = "26.37.0" result = compare_config( baseline_config=self.baseline_config, current_config=self.current_config, @@ -100,7 +100,7 @@ def test_compare_config_libraries_bom_update(self): len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("libraris_bom_version", config_change.changed_param) + self.assertEqual("libraries_bom_version", config_change.changed_param) self.assertEqual("26.37.0", config_change.current_value) def test_compare_protobuf_update(self): diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh index 237d3c23a9..2e26ed6027 100755 --- a/library_generation/utils/utilities.sh +++ b/library_generation/utils/utilities.sh @@ -116,6 +116,11 @@ get_grpc_version() { get_protoc_version() { local gapic_generator_version=$1 local protoc_version + if [[ -n "${DOCKER_PROTOC_VERSION}" ]]; then + >&2 echo "Using protoc version baked into the container: ${DOCKER_PROTOC_VERSION}" + echo "${DOCKER_PROTOC_VERSION}" + return + fi pushd "${output_folder}" > /dev/null # get protobuf version from gapic-generator-java-pom-parent/pom.xml download_gapic_generator_pom_parent "${gapic_generator_version}" @@ -131,7 +136,17 @@ download_tools() { local os_architecture=$4 pushd "${output_folder}" download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-${gapic_generator_version}.jar" - download_protoc "${protoc_version}" "${os_architecture}" + + # the variable protoc_path is used in generate_library.sh. It is explicitly + # exported to make clear that it is used outside this utilities file. + if [[ "${DOCKER_PROTOC_VERSION}" == "${protoc_version}" ]]; then + # if the specified protoc_version matches the one baked in the docker + # container, we just point protoc_path to its location. + export protoc_path="${DOCKER_PROTOC_LOCATION}/protoc-${protoc_version}/bin" + else + export protoc_path=$(download_protoc "${protoc_version}" "${os_architecture}") + fi + download_grpc_plugin "${grpc_version}" "${os_architecture}" popd } @@ -162,7 +177,10 @@ download_generator_artifact() { download_protoc() { local protoc_version=$1 local os_architecture=$2 - if [ ! -d "protoc-${protoc_version}" ]; then + + local protoc_path="${output_folder}/protoc-${protoc_version}/bin" + + if [ ! -d "${protoc_path}" ]; then # pull proto files and protoc from protobuf repository as maven central # doesn't have proto files download_from \ @@ -173,8 +191,8 @@ download_protoc() { cp -r "protoc-${protoc_version}/include/google" . rm "protoc-${protoc_version}.zip" fi + echo "${protoc_path}" - protoc_path="${output_folder}/protoc-${protoc_version}/bin" } download_grpc_plugin() { diff --git a/renovate.json b/renovate.json index 55c61cc419..27d778c0c8 100644 --- a/renovate.json +++ b/renovate.json @@ -28,6 +28,18 @@ "depNameTemplate": "com.google.protobuf:protobuf-java", "datasourceTemplate": "maven" }, + { + "customType": "regex", + "fileMatch": [ + "^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$" + ], + "matchStrings": [ + "ARG PROTOC_VERSION=[\"']?(?.+?)[\"']?\\s+" + ], + "datasourceTemplate": "github-releases", + "depNameTemplate": "protocolbuffers/protobuf", + "extractVersionTemplate": "^v(?.*)$" + }, { "customType": "regex", "fileMatch": [ @@ -141,7 +153,8 @@ }, { "matchPackagePatterns": [ - "^com.google.protobuf" + "^com.google.protobuf", + "^protocolbuffers/protobuf" ], "groupName": "Protobuf dependencies", "enabled": false