chore: secure hermetic-build docker image

.cloudbuild/library_generation/image-configuration/airlock-pip.conf

@@ -0,0 +1,2 @@
+[global]
+index-url = https://us-python.pkg.dev/artifact-foundry-prod/python-3p-trusted/simple/

.cloudbuild/library_generation/image-configuration/airlock-pypirc

@@ -0,0 +1,5 @@
+[distutils]
+index-servers = python-3p-trusted
+
+[python-3p-trusted]
+repository: https://us-python.pkg.dev/artifact-foundry-prod/python-3p-trusted/

.cloudbuild/library_generation/image-configuration/owlbot-cli-build-config.js

@@ -0,0 +1,20 @@
+/**
+ * @fileoverview This file contains the esbuild configuration to compile the
+ * owlbot-cli source code into a single bundled javascript file
+ * @author diegomarquezp
+ */
+const { build } = require("esbuild");
+
+
+const sharedConfig = {
+  entryPoints: ["src/bin/owl-bot.ts"],
+  bundle: true,
+  minify: false,
+};
+
+build({
+  ...sharedConfig,
+  platform: 'node',
+  format: 'cjs',
+  outfile: "build/bundle.js",
+});

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -11,107 +11,149 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Creates the owl-bot binary (no node runtime needed)
+FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/node:22.1-alpine as owlbot-cli-build
+ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
+
+# install tools
+RUN apk add git
+
+# Clone the owlbot-cli source code
+WORKDIR /tools
+RUN git clone https://github.com/googleapis/repo-automation-bots
+WORKDIR /tools/repo-automation-bots/packages/owl-bot
+RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
 
-# install gapic-generator-java in a separate layer so we don't overload the image
-# with the transferred source code and jars
-FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build
+# Part of the code path (that we don't use) ends up touching a dependency called
+# @google-cloud/datastore that tries a fs.readFileSync that is not handled by
+# default by esbundle (esbundle is good a figuring out imports but doesn't
+# actively scan filesystem interactions such as fs.readFileSync). This makes the
+# app to fetch a file at runtime that is not available in the bundle context.
+# This is why we remove this import and its usage from the entrypoint.
+RUN sed -i '/testWebhook/d' src/bin/owl-bot.ts
+
+# Bundle the source code and its dependencies into a single javascript file
+# with all its dependencies embedded.
+# This is because SEA (see below) cannot
+# resolve external modules in a multi-file project.
+# We use the esbuild tool see https://esbuild.github.io/
+COPY ./.cloudbuild/library_generation/image-configuration/owlbot-cli-build-config.js .
+RUN npm i esbuild
+RUN node owlbot-cli-build-config.js
+
+# Compile the bundled javascript file into a Linux executable
+# Create a Standalone Executable Application (SEA) configuration file.
+# See https://nodejs.org/api/single-executable-applications.html
+RUN echo '{ "main": "bundle.js", "output": "sea-prep.blob" }' > build/sea-config.json
+WORKDIR /tools/repo-automation-bots/packages/owl-bot/build
+RUN node --experimental-sea-config sea-config.json
+RUN cp $(command -v node) owl-bot-bin
+RUN npx postject owl-bot-bin NODE_SEA_BLOB sea-prep.blob \
+    --sentinel-fuse NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2
+
+# move to a simple path for convenience
+RUN cp ./owl-bot-bin /owl-bot-bin
+
+# Creates the generator jar
+FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/maven:3.8.6-openjdk-11-slim as ggj-build
 
 WORKDIR /sdk-platform-java
 COPY . .
 # {x-version-update-start:gapic-generator-java:current}
 ENV DOCKER_GAPIC_GENERATOR_VERSION="2.45.1-SNAPSHOT" 
 # {x-version-update-end}
 
-RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip
+RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip \
+    -pl gapic-generator-java -am
 RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \
-  "./gapic-generator-java.jar"
+  "/gapic-generator-java.jar"
 
-# build from the root of this repo:
-FROM gcr.io/cloud-devrel-public-resources/python
+# Builds the python scripts in library_generation
+FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/python:3.12.3-alpine3.18 as python-scripts-build
 
-SHELL [ "/bin/bash", "-c" ]
+# This will use GOOGLE_APPLICATION_CREDENTIALS if passed in docker build command.
+# If not passed will leave it unset to support GCE Metadata in CI builds
+ARG GOOGLE_APPLICATION_CREDENTIALS
+
+# Use bash to allow multiline echoes
+RUN apk add bash curl
+
+# Install gcloud to obtain the credentials to use the Airlock repostiory
+RUN curl -sSL https://sdk.cloud.google.com | bash -e
+ENV PATH $PATH:/root/google-cloud-sdk/bin
+#RUN --mount=type=secret,id=credentials \
+#      gcloud auth application-default print-access-token && exit 1
+
+
+# Configure the Airlock pip package repository
+RUN pip install keyrings.google-artifactregistry-auth -i https://pypi.org/simple/
+COPY .cloudbuild/library_generation/image-configuration/airlock-pypirc /root/.pypirc
+COPY .cloudbuild/library_generation/image-configuration/airlock-pip.conf /etc/pip.conf
+RUN chmod 600 /root/.pypirc /etc/pip.conf
+
+COPY library_generation /src
+
+# install main scripts as a python package
+WORKDIR /src
+RUN --mount=type=secret,id=credentials python -m pip install --user -r requirements.txt
+RUN python -m pip install --user .
+RUN ls -lah  /root/.local
+
+# Final image. Installs the rest of the dependencies and gets the binaries
+# from the previous stages
+FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/python:3.12.3-alpine3.18 as final
 
 
-ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ARG PROTOC_VERSION=25.4
 ARG GRPC_VERSION=1.66.0
 ENV HOME=/home
 ENV OS_ARCHITECTURE="linux-x86_64"
 
-# install OS tools
-RUN apt-get update && apt-get install -y \
-	unzip openjdk-17-jdk rsync maven jq \
-	&& apt-get clean
+# Install shell script tools
+RUN apk update && apk add unzip rsync jq git maven bash curl
 
-# copy source code
-COPY library_generation /src
+SHELL [ "/bin/bash", "-c" ]
+
+# Use utilites script to download dependencies
+COPY library_generation/utils/utilities.sh /utilities.sh
 
 # install protoc
 WORKDIR /protoc
-RUN source /src/utils/utilities.sh \
+RUN source /utilities.sh \
 	&& download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}"
 # we indicate protoc is available in the container via env vars
 ENV DOCKER_PROTOC_LOCATION=/protoc
 ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"
 
 # install grpc
 WORKDIR /grpc
-RUN source /src/utils/utilities.sh \
+RUN source /utilities.sh \
 	&& download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}"
 # similar to protoc, we indicate grpc is available in the container via env vars
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
 ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 
+# Remove utilities script now that we downloaded the generation tools
+RUN rm /utilities.sh
 
 # Here we transfer gapic-generator-java from the previous stage.
 # Note that the destination is a well-known location that will be assumed at runtime
-# We hard-code the location string to avoid making it configurable (via ARG) as
-# well as to avoid it making it overridable at runtime (via ENV).
-COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
-RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
+# We hard-code the location string so it cannot be overriden
+COPY --from=ggj-build "/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
+RUN chmod 555 "${HOME}/.library_generation/gapic-generator-java.jar"
 
-#  use python 3.11 (the base image has several python versions; here we define the default one)
-RUN rm $(which python3)
-RUN ln -s $(which python3.11) /usr/local/bin/python
-RUN ln -s $(which python3.11) /usr/local/bin/python3
-RUN python -m pip install --upgrade pip
+# Copy the owlbot-cli binary
+COPY --from=owlbot-cli-build "/owl-bot-bin" "/usr/bin/owl-bot"
+RUN chmod 555 "/usr/bin/owl-bot"
 
-# install main scripts as a python package
-WORKDIR /src
-RUN python -m pip install -r requirements.txt
-RUN python -m pip install .
-
-# Install nvm with node and npm
-ENV NODE_VERSION 20.12.0
-WORKDIR /home
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
-RUN chmod o+rx /home/.nvm
-ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin
-ENV PATH=${PATH}:${NODE_PATH}
-RUN node --version
-RUN npm --version
-
-# install the owl-bot CLI
-WORKDIR /tools
-RUN git clone https://github.com/googleapis/repo-automation-bots
-WORKDIR /tools/repo-automation-bots/packages/owl-bot
-RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
-RUN npm i && npm run compile && npm link
-RUN owl-bot copy-code --version
-RUN chmod -R o+rx ${NODE_PATH}
-RUN ln -sf ${NODE_PATH}/* /usr/local/bin
+# Copy the library_generation python packages
+COPY --from=python-scripts-build "/root/.local" "${HOME}/.local"
 
-# allow users to access the script folders
-RUN chmod -R o+rx /src
 
 # set dummy git credentials for the empty commit used in postprocessing
 # we use system so all users using the container will use this configuration
 RUN git config --system user.email "[email protected]"
 RUN git config --system user.name "Cloud Java Bot"
 
-# allow read-write for /home and execution for binaries in /home/.nvm
-RUN chmod -R a+rw /home
-RUN chmod -R a+rx /home/.nvm
-
 WORKDIR /workspace
-ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ]
+ENTRYPOINT [ "python", "-m", "library_generation.cli.entry_point", "generate" ]

library_generation/requirements.in

@@ -3,7 +3,7 @@
 absl-py==2.1.0
 attr==0.3.2
 attrs==23.2.0
-black==24.4.2
+black==24.8.0
 click==8.1.7
 gitdb==4.0.11
 GitPython==3.1.43
@@ -26,3 +26,7 @@ jinja2==3.1.4
 # a list where typing extensions is pinned to >=4.0.1. This will produce an error saying "all requirements
 # must have their versions pinned with ==". The following line pins the dependency to a specific version via ==
 typing-extensions==4.0.1
+# we also pin the following dependencies to the ones available in the
+# Airlock Artifact registry
+argcomplete==3.4.0
+filelock==3.15.4