chore: bake protoc into the hermetic build docker image

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -15,24 +15,35 @@
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
-ARG SYNTHTOOL_COMMITTISH=a2c9b4a5da2d7f583c8a1869fd2843c206145834
+SHELL [ "/bin/bash", "-c" ]
+
+ARG SYNTHTOOL_COMMITTISH=e36d2f164ca698f0264fb6f79ddc4b0fa024a940
 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
+ARG PROTOC_VERSION=25.3
 ENV HOME=/home
 
 # install OS tools
 RUN apt-get update && apt-get install -y \
 	unzip openjdk-17-jdk rsync maven jq \
 	&& apt-get clean
 
+# copy source code
+COPY library_generation /src
+
+# install protoc
+WORKDIR /protoc
+RUN source /src/utils/utilities.sh \
+	&& download_protoc "${PROTOC_VERSION}" "linux-x86_64"
+# we indicate protoc is available in the container via env vars
+ENV DOCKER_PROTOC_LOCATION=/protoc
+ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"
+
 # use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3
 RUN python -m pip install --upgrade pip
 
-# copy source code
-COPY library_generation /src
-
 # install scripts as a python package
 WORKDIR /src
 RUN python -m pip install -r requirements.txt

library_generation/test/generate_library_unit_tests.sh

@@ -28,6 +28,15 @@ get_grpc_version_failed_with_invalid_generator_version_test() {
   assertEquals 1 $((res))
 }
 
+get_protoc_version_succeed_docker_env_var_test() {
+  local version_with_docker
+  local version_without_docker
+  export DOCKER_PROTOC_VERSION="9.9.9"
+  version_with_docker=$(get_protoc_version "2.24.0")
+  assertEquals "${DOCKER_PROTOC_VERSION}" "${version_with_docker}"
+  unset DOCKER_PROTOC_VERSION
+}
+
 get_protoc_version_succeed_with_valid_generator_version_test() {
   local actual_version
   actual_version=$(get_protoc_version "2.24.0")
@@ -134,6 +143,34 @@ download_protoc_failed_with_invalid_arch_test() {
   assertEquals 1 $((res))
 }
 
+download_tools_succeed_with_baked_protoc() {
+  # This mimics a docker container scenario.
+  # This test consists of creating an empty /tmp/.../protoc-99.99/bin folder and map
+  # it to the DOCKER_PROTOC_LOCATION env var (which is treated specially in the
+  # `download_tools` function). If `DOCKER_PROTOC_VERSION` matches exactly as
+  # the version passed to `download_protoc`, then we will not download protoc
+  # but simply have the variable `protoc_path` pointing to DOCKER_PROTOC_LOCATION 
+  # (which we manually created in this test)
+  local test_dir=$(mktemp -d)
+  pushd "${test_dir}"
+  export DOCKER_PROTOC_LOCATION=$(mktemp -d)
+  export DOCKER_PROTOC_VERSION="99.99"
+  export output_folder=$(get_output_folder)
+  mkdir "${output_folder}"
+  local protoc_bin_folder="${DOCKER_PROTOC_LOCATION}/protoc-99.99/bin"
+  mkdir -p "${protoc_bin_folder}"
+
+  local test_ggj_version="2.40.0"
+  local test_grpc_version="1.64.0"
+  download_tools "${test_ggj_version}" "99.99" "${test_grpc_version}" "linux-x86_64"
+  assertEquals "${protoc_bin_folder}" "${protoc_path}"
+
+  rm -rdf "${output_folder}"
+  unset DOCKER_PROTOC_LOCATION
+  unset DOCKER_PROTOC_VERSION
+  unset output_folder
+}
+
 download_grpc_plugin_succeed_with_valid_version_linux_test() {
   download_grpc_plugin "1.55.1" "linux-x86_64"
   assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe"
@@ -256,6 +293,7 @@ test_list=(
   extract_folder_name_test
   get_grpc_version_succeed_with_valid_generator_version_test
   get_grpc_version_failed_with_invalid_generator_version_test
+  get_protoc_version_succeed_docker_env_var_test
   get_protoc_version_succeed_with_valid_generator_version_test
   get_protoc_version_failed_with_invalid_generator_version_test
   get_gapic_opts_with_rest_test
@@ -268,6 +306,7 @@ test_list=(
   download_protoc_succeed_with_valid_version_macos_test
   download_protoc_failed_with_invalid_version_linux_test
   download_protoc_failed_with_invalid_arch_test
+  download_tools_succeed_with_baked_protoc
   download_grpc_plugin_succeed_with_valid_version_linux_test
   download_grpc_plugin_succeed_with_valid_version_macos_test
   download_grpc_plugin_failed_with_invalid_version_linux_test

library_generation/test/integration_tests.py

@@ -52,10 +52,6 @@ class IntegrationTest(unittest.TestCase):
     def setUpClass(cls) -> None:
         IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir)
 
-    @classmethod
-    def tearDownClass(cls) -> None:
-        cls.__remove_docker_image()
-
     @classmethod
     def setUp(cls) -> None:
         cls.__remove_generated_files()
@@ -308,6 +304,3 @@ def __recursive_diff_files(
                 sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/"
             )
 
-    @classmethod
-    def __remove_docker_image(cls):
-        subprocess.check_call(["docker", "image", "rmi", image_tag])

library_generation/test/model/generation_config_unit_tests.py

@@ -52,7 +52,7 @@ def test_from_yaml_succeeds(self):
         self.assertEqual(
             "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish
         )
-        self.assertEqual("26.37.0", config.libraris_bom_version)
+        self.assertEqual("26.37.0", config.libraries_bom_version)
         self.assertEqual(
             [
                 ".github/*",

library_generation/test/utils/generation_config_comparator_unit_tests.py

@@ -90,8 +90,8 @@ def test_compare_config_generator_update(self):
         self.assertEqual("1.2.4", config_change.current_value)
 
     def test_compare_config_libraries_bom_update(self):
-        self.baseline_config.libraris_bom_version = "26.36.0"
-        self.current_config.libraris_bom_version = "26.37.0"
+        self.baseline_config.libraries_bom_version = "26.36.0"
+        self.current_config.libraries_bom_version = "26.37.0"
         result = compare_config(
             baseline_config=self.baseline_config,
             current_config=self.current_config,
@@ -100,7 +100,7 @@ def test_compare_config_libraries_bom_update(self):
             len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1
         )
         config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0]
-        self.assertEqual("libraris_bom_version", config_change.changed_param)
+        self.assertEqual("libraries_bom_version", config_change.changed_param)
         self.assertEqual("26.37.0", config_change.current_value)
 
     def test_compare_protobuf_update(self):

library_generation/utils/utilities.sh

@@ -116,6 +116,11 @@ get_grpc_version() {
 get_protoc_version() {
   local gapic_generator_version=$1
   local protoc_version
+  if [[ -n "${DOCKER_PROTOC_VERSION}" ]]; then
+    >&2 echo "Using protoc version baked into the container: ${DOCKER_PROTOC_VERSION}"
+    echo "${DOCKER_PROTOC_VERSION}"
+    return
+  fi
   pushd "${output_folder}" > /dev/null
   # get protobuf version from gapic-generator-java-pom-parent/pom.xml
   download_gapic_generator_pom_parent "${gapic_generator_version}"
@@ -131,7 +136,17 @@ download_tools() {
   local os_architecture=$4
   pushd "${output_folder}"
   download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-${gapic_generator_version}.jar"
-  download_protoc "${protoc_version}" "${os_architecture}"
+
+  # the variable protoc_path is used in generate_library.sh. It is explicitly
+  # exported to make clear that it is used outside this utilities file.
+  if [[ "${DOCKER_PROTOC_VERSION}" == "${protoc_version}" ]]; then
+    # if the specified protoc_version matches the one baked in the docker
+    # container, we just point protoc_path to its location.
+    export protoc_path="${DOCKER_PROTOC_LOCATION}/protoc-${protoc_version}/bin"
+  else
+    export protoc_path=$(download_protoc "${protoc_version}" "${os_architecture}")
+  fi
+
   download_grpc_plugin "${grpc_version}" "${os_architecture}"
   popd
 }
@@ -162,7 +177,10 @@ download_generator_artifact() {
 download_protoc() {
   local protoc_version=$1
   local os_architecture=$2
-  if [ ! -d "protoc-${protoc_version}" ]; then
+
+  local protoc_path="${output_folder}/protoc-${protoc_version}/bin"
+
+  if [ ! -d "${protoc_path}" ]; then
     # pull proto files and protoc from protobuf repository as maven central
     # doesn't have proto files
     download_from \
@@ -173,8 +191,8 @@ download_protoc() {
     cp -r "protoc-${protoc_version}/include/google" .
     rm "protoc-${protoc_version}.zip"
   fi
+  echo "${protoc_path}"
 
-  protoc_path="${output_folder}/protoc-${protoc_version}/bin"
 }
 
 download_grpc_plugin() {

renovate.json

@@ -28,6 +28,18 @@
       "depNameTemplate": "com.google.protobuf:protobuf-java",
       "datasourceTemplate": "maven"
     },
+    {
+      "customType": "regex",
+      "fileMatch": [
+		"^\\.cloudbuild/library_generation/library_generation\\.Dockerfile$"
+      ],
+      "matchStrings": [
+        "ARG PROTOC_VERSION=[\"']?(?<currentValue>.+?)[\"']?\\s+"
+      ],
+      "datasourceTemplate": "github-releases",
+      "depNameTemplate": "protocolbuffers/protobuf",
+      "extractVersionTemplate": "^v(?<version>.*)$"
+    },
     {
       "customType": "regex",
       "fileMatch": [
@@ -141,7 +153,8 @@
     },
     {
       "matchPackagePatterns": [
-        "^com.google.protobuf"
+        "^com.google.protobuf",
+		"^protocolbuffers/protobuf"
       ],
       "groupName": "Protobuf dependencies",
       "enabled": false