chore: secure hermetic-build docker image

.cloudbuild/library_generation/cloudbuild-library-generation-integration-tests.yaml

@@ -0,0 +1,41 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a tentative Cloud Build workflow to replace the existing integration
+# tests setup in GitHub Actions
+timeout: 7200s # 2 hours
+substitutions:
+  _IMAGE_NAME: "hermetic-build"
+steps:
+  # Library generation build
+  - name: gcr.io/cloud-builders/docker
+    args: [
+      "build",
+      "-t", "${_IMAGE_NAME}",
+      "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
+    id: library-generation-image-build
+    env:
+    - 'DOCKER_BUILDKIT=1'
+    waitFor: ["-"]
+  # Python scripts compilation
+  - name: python
+    args: [ "python", "-m", "pip", "install", "library_generation" ]
+    id: library-generation-python-compile
+    waitFor: ["library-generation-image-build"]
+  # Python integration tests execution
+  - name: python
+    args: [ "python", "-m", "unittest",
+            "library_generation/test/integration_tests.py" ]
+    id: library-generation-python-compile
+    waitFor: ["library-generation-python-compile"]

.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml

@@ -29,6 +29,8 @@ steps:
       "-t", "${_VERSIONED_IMAGE_ID}",
       "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
     id: library-generation-build
+    env:
+    - 'DOCKER_BUILDKIT=1'
     waitFor: ["-"]
 
 images:

.cloudbuild/library_generation/image-configuration/airlock-pip.conf

@@ -0,0 +1,5 @@
+[global]
+index-url = https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/
+# TODO: use the following index URL when `lxml` and `versions` are available in the `trusted` airlock registry
+# We can confirm their availability in https://airlock.corp.goog/search?query=&type=Python
+# index-url = https://us-python.pkg.dev/artifact-foundry-prod/python-3p-trusted/simple/

.cloudbuild/library_generation/image-configuration/airlock-pypirc

@@ -0,0 +1,11 @@
+[distutils]
+index-servers = ah-3p-staging-python
+# TODO: use this index instead when `lxml` and `versions` are available in the `trusted` airlock registry
+# We can confirm their availability in https://airlock.corp.goog/search?query=&type=Python
+# index-servers = python-3p-trusted
+
+[ah-3p-staging-python]
+repository: https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/
+# TODO: use this repository instead when `lxml` and `versions` are available in the `trusted` airlock registry
+# We can confirm their availability in https://airlock.corp.goog/search?query=&type=Python
+# repository: https://us-python.pkg.dev/artifact-foundry-prod/python-3p-trusted/

.cloudbuild/library_generation/image-configuration/owlbot-cli-build-config.js

@@ -0,0 +1,20 @@
+/**
+ * @fileoverview This file contains the esbuild configuration to compile the
+ * owlbot-cli source code into a single bundled javascript file
+ * @author diegomarquezp
+ */
+const { build } = require("esbuild");
+
+
+const sharedConfig = {
+  entryPoints: ["src/bin/owl-bot.ts"],
+  bundle: true,
+  minify: false,
+};
+
+build({
+  ...sharedConfig,
+  platform: 'node',
+  format: 'cjs',
+  outfile: "build/bundle.js",
+});

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -11,107 +11,197 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Creates the owl-bot binary (no node runtime needed)
 
-# install gapic-generator-java in a separate layer so we don't overload the image
-# with the transferred source code and jars
-FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build
+# node:22.1-alpine
+FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/node@sha256:487dc5d5122d578e13f2231aa4ac0f63068becd921099c4c677c850df93bede8 as owlbot-cli-build
+ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
+
+# install tools
+RUN apk add git
+
+# Clone the owlbot-cli source code
+WORKDIR /tools
+RUN git clone https://github.com/googleapis/repo-automation-bots
+WORKDIR /tools/repo-automation-bots/packages/owl-bot
+RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
+
+# Part of the code path (that we don't use) ends up touching a dependency called
+# @google-cloud/datastore that tries a fs.readFileSync that is not handled by
+# default by esbundle (esbundle is good a figuring out imports but doesn't
+# actively scan filesystem interactions such as fs.readFileSync). This makes the
+# app to fetch a file at runtime that is not available in the bundle context.
+# This is why we remove this import and its usage from the entrypoint.
+RUN sed -i '/testWebhook/d' src/bin/owl-bot.ts
+
+# Bundle the source code and its dependencies into a single javascript file
+# with all its dependencies embedded.
+# This is because SEA (see below) cannot
+# resolve external modules in a multi-file project.
+# We use the esbuild tool. See https://esbuild.github.io/
+COPY ./.cloudbuild/library_generation/image-configuration/owlbot-cli-build-config.js .
+RUN npm i esbuild
+RUN node owlbot-cli-build-config.js
+
+# Compile the bundled javascript file into a Linux executable
+# Create a Standalone Executable Application (SEA) configuration file.
+# See https://nodejs.org/api/single-executable-applications.html
+RUN echo '{ "main": "bundle.js", "output": "sea-prep.blob" }' > build/sea-config.json
+WORKDIR /tools/repo-automation-bots/packages/owl-bot/build
+RUN node --experimental-sea-config sea-config.json
+RUN cp $(command -v node) owl-bot-bin
+RUN npx postject owl-bot-bin NODE_SEA_BLOB sea-prep.blob \
+    --sentinel-fuse NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2
+
+# move to a simple path for convenience
+RUN cp ./owl-bot-bin /owl-bot-bin
+
+# Creates the generator jar
+# maven:3.8.6-openjdk-11-slim
+FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/maven@sha256:2cb7c73ba2fd0f7ae64cfabd99180030ec85841a1197b4ae821d21836cb0aa3b  as ggj-build
 
 WORKDIR /sdk-platform-java
 COPY . .
 # {x-version-update-start:gapic-generator-java:current}
 ENV DOCKER_GAPIC_GENERATOR_VERSION="2.45.1-SNAPSHOT" 
 # {x-version-update-end}
 
-RUN mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
-RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \
-  "./gapic-generator-java.jar"
+# use Docker Buildkit caching for faster local builds
+RUN --mount=type=cache,target=/root/.m2 mvn install -B -ntp -T 1.5C \
+    -Dclirr.skip -Dcheckstyle.skip -Djacoco.skip -Dmaven.test.skip \
+    -Dmaven.site.skikip -Dmaven.javadoc.skip -pl gapic-generator-java -am
 
-# build from the root of this repo:
-FROM gcr.io/cloud-devrel-public-resources/python
+RUN --mount=type=cache,target=/root/.m2 cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \
+  "/gapic-generator-java.jar"
 
-SHELL [ "/bin/bash", "-c" ]
+# Builds the python scripts in library_generation
+# python:3.11-alpine
+FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/python@sha256:0b5ed25d3cc27cd35c7b0352bac8ef2ebc8dd3da72a0c03caaf4eb15d9ec827a as python-scripts-build
 
+# This will use GOOGLE_APPLICATION_CREDENTIALS if passed in docker build command.
+# If not passed will leave it unset to support GCE Metadata in CI builds
+ARG GOOGLE_APPLICATION_CREDENTIALS
+
+RUN apk add bash curl
+
+# Install gcloud to obtain the credentials to use the Airlock repostiory
+RUN curl -sSL https://sdk.cloud.google.com | bash -e
+ENV PATH $PATH:/root/google-cloud-sdk/bin
+
+
+# Configure the Airlock pip package repository
+RUN pip install keyrings.google-artifactregistry-auth -i https://pypi.org/simple/
+COPY .cloudbuild/library_generation/image-configuration/airlock-pypirc /root/.pypirc
+COPY .cloudbuild/library_generation/image-configuration/airlock-pip.conf /etc/pip.conf
+RUN chmod 600 /root/.pypirc /etc/pip.conf
+
+COPY library_generation /src
+
+# install main scripts as a python package
+WORKDIR /src
+
+RUN --mount=type=secret,id=credentials python -m pip install --target /usr/local/lib/python3.11 -r requirements.txt
+RUN python -m pip install --target /usr/local/lib/python3.11 .
+
+# Final image. Installs the rest of the dependencies and gets the binaries
+# from the previous stages. We use the node base image for it to be compatible
+# with the standalone binary owl-bot compiled in the previous stage
+# node:22.1-alpine
+FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/node@sha256:487dc5d5122d578e13f2231aa4ac0f63068becd921099c4c677c850df93bede8 as final
 
-ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ARG PROTOC_VERSION=25.4
 ARG GRPC_VERSION=1.66.0
+# This SHA is the latest known-to-work version of this binary compatibility tool
+ARG GLIB_MUS_SHA=7717dd4dc26377dd9cedcc92b72ebf35f9e68a2d
 ENV HOME=/home
-ENV OS_ARCHITECTURE="linux-x86_64"
+ENV OS_ARCH="linux-x86_64"
+
+# Install shell script tools. Keep them in sorted order.
+RUN apk update && apk add \
+    bash \
+    curl \
+    git \
+    jq \
+    maven \
+    py-pip \
+    python3 \
+    rsync \
+    sudo \
+    unzip
+SHELL [ "/bin/bash", "-c" ]
+
+# Install compatibility layer to run glibc-based programs (such as the
+# grpc plugin).
+# Alpine, by default, only supports musl-based binaries, and there is no public
+# downloadable distrubution of the grpc that is Alpine (musl) compatible.
+# This is one of the recommended approaches to ensure glibc-compatibility
+# as per https://wiki.alpinelinux.org/wiki/Running_glibc_programs
+WORKDIR /home
+RUN git clone https://gitlab.com/manoel-linux1/GlibMus-HQ.git
+WORKDIR /home/GlibMus-HQ
+# We lock the tool to the latest known-to-work version
+RUN git checkout "${GLIB_MUS_SHA}"
+RUN chmod a+x compile-x86_64-alpine-linux.sh
+RUN ./compile-x86_64-alpine-linux.sh
+WORKDIR /home
+RUN rm -rf /home/GlibMus-HQ
+# We remove some unnecessary compatibility SOs and archive files
+WORKDIR /usr/lib
+RUN rm -rf LibLLVM-17* libatomic.a gcc llvm17 libexec
+# We also remove unnecessary programs installed by this tool
+WORKDIR /usr/bin
+RUN rm -rf lto-dump
 
-# install OS tools
-RUN apt-get update && apt-get install -y \
-	unzip openjdk-17-jdk rsync maven jq \
-	&& apt-get clean
 
-# copy source code
-COPY library_generation /src
+
+# Use utilites script to download dependencies
+COPY library_generation/utils/utilities.sh /utilities.sh
 
 # install protoc
 WORKDIR /protoc
-RUN source /src/utils/utilities.sh \
-	&& download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}"
+RUN source /utilities.sh && download_protoc "${PROTOC_VERSION}" "${OS_ARCH}"
 # we indicate protoc is available in the container via env vars
 ENV DOCKER_PROTOC_LOCATION=/protoc
 ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"
 
 # install grpc
 WORKDIR /grpc
-RUN source /src/utils/utilities.sh \
-	&& download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}"
+RUN source /utilities.sh && download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCH}"
 # similar to protoc, we indicate grpc is available in the container via env vars
-ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
+ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCH}.exe"
 ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 
+# Remove utilities script now that we downloaded the generation tools
+RUN rm /utilities.sh
 
-# Here we transfer gapic-generator-java from the previous stage.
-# Note that the destination is a well-known location that will be assumed at runtime
-# We hard-code the location string to avoid making it configurable (via ARG) as
-# well as to avoid it making it overridable at runtime (via ENV).
-COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
-RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
-
-#  use python 3.11 (the base image has several python versions; here we define the default one)
-RUN rm $(which python3)
-RUN ln -s $(which python3.11) /usr/local/bin/python
-RUN ln -s $(which python3.11) /usr/local/bin/python3
-RUN python -m pip install --upgrade pip
+# Make home folder accessible for all users since the container is usually
+# launched using the -u $(user -i) argument.
+# Execution is needed for gapic-generator-java.jar, whereas write permission is
+# needed for writing .gitconfig and creating .gitconfig.lock (postprocessing).
+# Note that this is NOT a recursive permission setting.
+RUN chmod 777 "${HOME}"
+RUN touch "${HOME}/.bashrc" && chmod 755 "${HOME}/.bashrc"
 
-# install main scripts as a python package
-WORKDIR /src
-RUN python -m pip install -r requirements.txt
-RUN python -m pip install .
+# Here we transfer gapic-generator-java from the previous stage.
+# Note that the destination is a well-known location that will be assumed at runtime.
+# We hard-code the location string so it cannot be overriden.
+COPY --from=ggj-build "/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
+RUN chmod 755 "${HOME}/.library_generation"
+RUN chmod 555 "${HOME}/.library_generation/gapic-generator-java.jar"
 
-# Install nvm with node and npm
-ENV NODE_VERSION 20.12.0
-WORKDIR /home
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
-RUN chmod o+rx /home/.nvm
-ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin
-ENV PATH=${PATH}:${NODE_PATH}
-RUN node --version
-RUN npm --version
-
-# install the owl-bot CLI
-WORKDIR /tools
-RUN git clone https://github.com/googleapis/repo-automation-bots
-WORKDIR /tools/repo-automation-bots/packages/owl-bot
-RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
-RUN npm i && npm run compile && npm link
-RUN owl-bot copy-code --version
-RUN chmod -R o+rx ${NODE_PATH}
-RUN ln -sf ${NODE_PATH}/* /usr/local/bin
+# Copy the owlbot-cli binary
+COPY --from=owlbot-cli-build "/owl-bot-bin" "/usr/bin/owl-bot"
+RUN chmod 555 "/usr/bin/owl-bot"
 
-# allow users to access the script folders
-RUN chmod -R o+rx /src
+# Copy the library_generation python packages
+COPY --from=python-scripts-build "/usr/local/lib/python3.11/" "/usr/lib/python3.11/"
 
 # set dummy git credentials for the empty commit used in postprocessing
 # we use system so all users using the container will use this configuration
 RUN git config --system user.email "[email protected]"
 RUN git config --system user.name "Cloud Java Bot"
-
-# allow read-write for /home and execution for binaries in /home/.nvm
-RUN chmod -R a+rw /home
-RUN chmod -R a+rx /home/.nvm
+RUN touch "${HOME}/.gitconfig"
+RUN chmod 666 "${HOME}/.gitconfig"
 
 WORKDIR /workspace
-ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ]
+ENTRYPOINT [ "python", "-m", "library_generation.cli.entry_point", "generate" ]

.github/workflows/verify_library_generation.yaml

@@ -38,40 +38,6 @@ jobs:
         head_repo_url: ${{ github.event.pull_request.head.repo.html_url }}
         head_repo_name: ${{ github.event.pull_request.head.repo.full_name  }}
         base_repo: ${{ github.repository }}
-  library-generation-integration-tests:
-    runs-on: ubuntu-22.04
-    needs: should-run-library-generation-tests
-    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: 3.11
-    - name: install pyenv
-      shell: bash
-      run: |
-        set -ex
-        curl https://pyenv.run | bash
-        # setup environment
-        export PYENV_ROOT="$HOME/.pyenv"
-        export PATH="$PYENV_ROOT/bin:$PATH"
-        echo "PYENV_ROOT=${PYENV_ROOT}" >> $GITHUB_ENV
-        echo "PATH=${PATH}" >> $GITHUB_ENV
-
-        set +ex
-    - name: install python dependencies
-      shell: bash
-      run: |
-        set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        pip install .
-        popd
-    - name: Run integration tests
-      shell: bash
-      run: |
-        set -x
-        python -m unittest library_generation/test/integration_tests.py
   library-generation-unit-tests:
     runs-on: ubuntu-22.04
     needs: should-run-library-generation-tests