diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml new file mode 100644 index 0000000000..c054cdf39d --- /dev/null +++ b/.github/workflows/hermetic_library_generation.yaml @@ -0,0 +1,54 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Hermetic library generation upon generation config change through pull requests +on: + pull_request: + +jobs: + library_generation: + # skip pull requests come from a forked repository + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Build image + shell: bash + run: | + docker build \ + -f .cloudbuild/library_generation/library_generation.Dockerfile \ + -t gcr.io/cloud-devrel-public-resources/java-library-generation:latest \ + . + - name: Install gapic-generator-java-pom-parent + shell: bash + run: | + mvn clean install -pl gapic-generator-java-pom-parent + - name: Generate changed libraries + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash library_generation/scripts/hermetic_library_generation.sh \ + --target_branch "${base_ref}" \ + --current_branch "${head_ref}" \ + --image_tag latest + env: + base_ref: ${{ github.base_ref }} + head_ref: ${{ github.head_ref }} + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.github/workflows/update_googleapis_commit.yaml b/.github/workflows/update_googleapis_commit.yaml new file mode 100644 index 0000000000..3308d6978b --- /dev/null +++ b/.github/workflows/update_googleapis_commit.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update googleapis commit +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + update-googleapis-commit: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update googleapis commit to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash library_generation/scripts/update_googleapis_commit.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/library_generation/scripts/hermetic_library_generation.sh b/library_generation/scripts/hermetic_library_generation.sh new file mode 100755 index 0000000000..5bd4c63970 --- /dev/null +++ b/library_generation/scripts/hermetic_library_generation.sh @@ -0,0 +1,122 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to, when a pull request changes the generation +# configuration (generation_config.yaml by default): +# 1. Find whether the last commit in this pull request contains changes to +# the generation configuration and exit early if it doesn't have such a change +# since the generation result would be the same. +# 2. Compare generation configurations in the current branch (with which the +# pull request associated) and target branch (into which the pull request is +# merged); +# 3. Generate changed libraries using library_generation image; +# 4. Commit the changes to the pull request, if any. +# 5. Edit the PR body with generated pull request description, if applicable. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. docker + +# The parameters of this script is: +# 1. target_branch, the branch into which the pull request is merged. +# 2. current_branch, the branch with which the pull request is associated. +# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation. +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --target_branch) + target_branch="$2" + shift + ;; + --current_branch) + current_branch="$2" + shift + ;; + --image_tag) + image_tag="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${target_branch}" ]; then + echo "missing required argument --target_branch" + exit 1 +fi + +if [ -z "${current_branch}" ]; then + echo "missing required argument --current_branch" + exit 1 +fi + +if [ -z "${image_tag}" ]; then + echo "missing required argument --image_tag" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config=generation_config.yaml + echo "Use default generation config: ${generation_config}" +fi + +workspace_name="/workspace" +baseline_generation_config="baseline_generation_config.yaml" +message="chore: generate libraries at $(date)" + +git checkout "${target_branch}" +git checkout "${current_branch}" +# if the last commit doesn't contain changes to generation configuration, +# do not generate again as the result will be the same. +change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)" +if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then + echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true + exit 0 +fi +# copy generation configuration from target branch to current branch. +git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" +config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) + +# run hermetic code generation docker image. +docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${workspace_name}" \ + -v "$HOME"/.m2:/home/.m2 \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" +# gapic-libraries-bom/pom.xml is specific to google-cloud-java, only commit +# this file if it exists. +if [ -f gapic-libraries-bom/pom.xml ]; then + git add gapic-libraries-bom/pom.xml +fi + +git add java-* pom.xml versions.txt +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "There is no generated code change with the generation config change ${config_diff}." + echo "Skip committing to the pull request." + exit 0 +fi + +echo "Configuration diff:" +echo "${config_diff}" +git commit -m "${message}" +git push +# set pr body if pr_description.txt is generated. +if [[ -f "pr_description.txt" ]]; then + pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") + gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" +fi diff --git a/library_generation/scripts/update_googleapis_commit.sh b/library_generation/scripts/update_googleapis_commit.sh new file mode 100644 index 0000000000..b950be9af4 --- /dev/null +++ b/library_generation/scripts/update_googleapis_commit.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to update googleapis commit to latest in generation +# configuration at the time of running and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: update googleapis commit at $(date)" + +# try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +mkdir tmp-googleapis +# use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}" + +git add "${generation_config}" +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "The latest googleapis commit is not changed." + echo "Skip committing to the pull request." + exit 0 +fi +git commit -m "${title}" +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q --unshallow remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push +fi