From ff37c6c64cb0f100e440c1dde187c25def4d5666 Mon Sep 17 00:00:00 2001 From: Filipe Constantinov Menezes Date: Mon, 23 Sep 2024 10:45:31 +0100 Subject: [PATCH] CLOUDP-268278 Automate documentation update with Copybara (#3265) --- build/ci/copy.bara.sky.template | 54 ++++++++++++++++++++++ build/ci/evergreen.yml | 11 +++-- build/ci/release.yml | 74 ++++++++++++++++++++++++++++- build/ci/run-copybara.sh | 82 +++++++++++++++++++++++++++++++++ build/package/package.sh | 10 ---- 5 files changed, 216 insertions(+), 15 deletions(-) create mode 100644 build/ci/copy.bara.sky.template create mode 100755 build/ci/run-copybara.sh diff --git a/build/ci/copy.bara.sky.template b/build/ci/copy.bara.sky.template new file mode 100644 index 0000000000..7a3760627f --- /dev/null +++ b/build/ci/copy.bara.sky.template @@ -0,0 +1,54 @@ +""" +Copy definitions +""" + +release_tag = "${TAG}" +source_url = "https://github.com/mongodb/mongodb-atlas-cli.git" +destination_url = "https://github.com/10gen/cloud-docs.git" +destination_url_cli_docs = "https://github.com/mongodb/docs-atlas-cli.git" +author = "apix-bot[bot] <168195273+apix-bot[bot]@users.noreply.github.com>" + +core.workflow( + name = "cloud-docs", + origin = git.origin( + url = source_url, + ref = release_tag, + ), + destination = git.github_pr_destination( + url = destination_url, + destination_ref = "master", + pr_branch = "apix_bot_copybara_pull_request_"+release_tag, + title = "Copies Atlas CLI commands for release tag: " + release_tag, + body = "Copies Atlas CLI commands from the source repo. See the attached Netlify comment for build log and staging.", + integrates = [], + ), + origin_files = glob(["docs/command/**"], exclude = ["docs/command/atlas-completion**"]), + destination_files = glob(["source/includes/command/**"]), + authoring = authoring.pass_thru(author), + transformations = [ + core.move("docs/command", "source/includes/command"), + core.rename(".txt", ".rst", suffix = True) + ], +) + +core.workflow( + name = "docs-atlas-cli", + origin = git.origin( + url = source_url, + ref = release_tag, + ), + destination = git.github_pr_destination( + url = destination_url_cli_docs, + destination_ref = "master", + pr_branch = "apix_bot_copybara_pull_request_"+release_tag, + title = "Copies Atlas CLI commands for release tag: " + release_tag, + body = "Copies Atlas CLI commands from the source repo. See the attached Netlify comment for build log and staging.", + integrates = [], + ), + origin_files = glob(["docs/command/**"]), + destination_files = glob(["source/command/**"]), + authoring = authoring.pass_thru(author), + transformations = [ + core.move("docs/command", "source/command"), + ], +) diff --git a/build/ci/evergreen.yml b/build/ci/evergreen.yml index 9cdc5c2ca1..4d13eaca99 100644 --- a/build/ci/evergreen.yml +++ b/build/ci/evergreen.yml @@ -45,6 +45,7 @@ variables: <<: *go_env pre: - func: "clone" + - func: "set-expansions" post: - command: attach.xunit_results params: @@ -401,14 +402,18 @@ functions: sleep 10 done shell: bash - "set-home": + "set-expansions": - command: shell.exec params: <<: *go_options shell: bash script: | set -Eeou pipefail - echo "home: $HOME" > expansions.yaml + cat < expansions.yaml + home: $HOME + tag: $(git tag --list 'atlascli/v*' --sort=-taggerdate | head -1) + version: $(git tag --list 'atlascli/v*' --sort=-taggerdate | head -1 | cut -d 'v' -f 2) + EOF - command: expansions.update params: ignore_missing_file: true @@ -1575,8 +1580,8 @@ tasks: task_groups: - name: atlas_deployments_windows_group setup_task: - - func: "set-home" - func: clone + - func: "set-expansions" - func: "vendor deps" - func: "install terraform" - func: "install azure cli" diff --git a/build/ci/release.yml b/build/ci/release.yml index 07a0f9e0f9..38f8e84692 100644 --- a/build/ci/release.yml +++ b/build/ci/release.yml @@ -71,6 +71,9 @@ functions: <<: *go_options binary: build/package/generate-notices.sh "package": + - command: github.generate_token + params: + expansion_name: github_token - command: subprocess.exec type: test params: @@ -81,6 +84,7 @@ functions: ARTIFACTORY_PASSWORD: ${artifactory_password} GRS_USERNAME: ${garasign_username} GRS_PASSWORD: ${garasign_password} + GITHUB_TOKEN: ${github_token} include_expansions_in_env: - go_base_path - workdir @@ -88,8 +92,6 @@ functions: - notary_service_secret - notary_service_url - goreleaser_key - - github_app_id - - github_app_pem - unstable binary: build/package/package.sh "install go-msi": @@ -256,6 +258,41 @@ functions: - mongo_repo working_dir: src/github.com/mongodb/mongodb-atlas-cli/build/package/docker/repo binary: ./docker-build.sh + "build-copybara": + - command: subprocess.exec + params: + <<: *go_options + binary: git + args: + - clone + - --depth=1 + - https://github.com/google/copybara + - ${workdir}/copybara + - command: subprocess.exec + params: + <<: *go_options + binary: docker + args: + - build + - -t + - google/copybara + - ${workdir}/copybara + "run-copybara": + - command: shell.exec + params: + <<: *go_options + env: + <<: *go_env + TAG: ${tag} + GH_TOKEN: ${gh_token} + WORKFLOW: ${workflow} + EVERGREEN_USER: ${evergreen_user} + EVERGREEN_API_KEY: ${evergreen_api_key} + DOCS_SLACK_CHANNEL: ${docs_slack_channel} + working_dir: src/github.com/mongodb/mongodb-atlas-cli/build/ci + shell: bash + script: | + ./run-copybara.sh tasks: - name: package_msi tags: ["packaging"] @@ -419,6 +456,30 @@ tasks: - func: "powershell build" vars: file: windows_post_package_test + - name: copybara + depends_on: + - name: package_goreleaser + variant: release_atlascli_github + commands: + - func: "build-copybara" + - command: github.generate_token + params: + expansion_name: docs_atlas_cli_token + owner: mongodb + repo: docs-atlas-cli + - func: "run-copybara" + vars: + gh_token: ${docs_atlas_cli_token} + workflow: docs-atlas-cli + - command: github.generate_token + params: + expansion_name: cloud_docs_token + owner: 10gen + repo: cloud-docs + - func: "run-copybara" + vars: + gh_token: ${cloud_docs_token} + workflow: cloud-docs buildvariants: - name: go_atlascli_msi_snapshot display_name: "Packaging AtlasCLI (go-msi)" @@ -469,6 +530,15 @@ buildvariants: depends_on: - name: package_msi variant: release_atlascli_msi + - name: copybara + display_name: "Copybara" + git_tag_only: true + run_on: + - ubuntu2404-small + expansions: + <<: *go_linux_version + tasks: + - name: copybara - name: release_atlascli_msi display_name: "Release AtlasCLI (go-msi)" run_on: diff --git a/build/ci/run-copybara.sh b/build/ci/run-copybara.sh new file mode 100755 index 0000000000..d858a0553e --- /dev/null +++ b/build/ci/run-copybara.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +# Copyright 2024 MongoDB Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeou pipefail + +if [[ "${TAG:?}" == "" ]]; then + echo "missing \$TAG" + exit 1 +fi + +if [[ "${GH_TOKEN:?}" == "" ]]; then + echo "missing \$GH_TOKEN" + exit 1 +fi + +if [[ "${WORKFLOW:?}" == "" ]]; then + echo "missing \$WORKFLOW" + exit 1 +fi + +if [[ "${EVERGREEN_USER:?}" == "" ]]; then + echo "missing \$EVERGREEN_USER" + exit 1 +fi + +if [[ "${EVERGREEN_API_KEY:?}" == "" ]]; then + echo "missing \$EVERGREEN_API_KEY" + exit 1 +fi + +if [[ "${DOCS_SLACK_CHANNEL:?}" == "" ]]; then + echo "missing \$DOCS_SLACK_CHANNEL" + exit 1 +fi + +cat < .gitconfig +[user] + name = apix-bot[bot] + email = 168195273+apix-bot[bot]@users.noreply.github.com +[credential] + helper = store +EOF + +envsubst < copy.bara.sky.template > copy.bara.sky + +echo "https://x-access-token:${GH_TOKEN:?}@github.com" > .git-credentials +echo "https://x-access-token:${GH_TOKEN:?}@api.github.com" >> .git-credentials + +docker run \ + --name copybara-container \ + -v "${PWD}:/usr/src/app" \ + -v "${PWD}/.gitconfig:/root/.gitconfig" \ + -v "${PWD}/.git-credentials:/root/.git-credentials" \ + -e "COPYBARA_WORKFLOW=$WORKFLOW" \ + -e "COPYBARA_OPTIONS=--github-api-bearer-auth true" \ + google/copybara + +PR_URL=$(docker logs copybara-container 2>&1 | grep "/pull/" | sed -E 's/^.*(https\:[^\ ]+).*$/\1/') + +rm -rf .git-credentials .gitconfig copy.bara.sky +docker rm -f copybara-container + +TARGET="$DOCS_SLACK_CHANNEL" +MSG="Hey team :wave: ${PR_URL} is ready for review :thankyou:" +echo "{\"target\":\"$TARGET\",\"msg\":\"$MSG\"}" +curl --header "Api-User:${EVERGREEN_USER:?}" \ + --header "Api-Key:${EVERGREEN_API_KEY:?}" \ + --request POST "https://evergreen.mongodb.com/rest/v2/notifications/slack" \ + --data "{\"target\":\"$TARGET\",\"msg\":\"$MSG\"}" diff --git a/build/package/package.sh b/build/package/package.sh index 20e27ecb34..70f0ec8c12 100755 --- a/build/package/package.sh +++ b/build/package/package.sh @@ -17,20 +17,12 @@ set -Eeou pipefail export GOROOT="${GOROOT:?}" -export GITHUB_APP_ID=${github_app_id:?} -export GITHUB_APP_PEM=${github_app_pem:?} export NOTARY_SERVICE_URL=${notary_service_url:?} export MACOS_NOTARY_KEY=${notary_service_key_id:?} export MACOS_NOTARY_SECRET=${notary_service_secret:?} export GORELEASER_KEY=${goreleaser_key:?} export VERSION_GIT -echo "$GITHUB_APP_PEM" > app.pem -GITHUB_INSTALLATION_ID=$(gh-token installations --app-id "$GITHUB_APP_ID" --key ./app.pem | jq '.[] | select(.account.login == "mongodb") | .id' | head -1) -GITHUB_TOKEN=$(gh-token generate --app-id "$GITHUB_APP_ID" --key ./app.pem --installation-id "$GITHUB_INSTALLATION_ID" -t) -rm -rf app.pem -export GITHUB_TOKEN - VERSION_GIT="$(git tag --list "atlascli/v*" --sort=taggerdate | tail -1 | cut -d "v" -f 2)" if [[ "${unstable-}" == "-unstable" ]]; then @@ -41,8 +33,6 @@ else ./bin/goreleaser --config "build/package/.goreleaser.yml" --clean --release-notes "CHANGELOG.md" -p 1 fi -gh-token revoke -t "$GITHUB_TOKEN" - # check that the notarization service signed the mac binaries SIGNED_FILE_NAME=mongodb-atlas-cli_macos_signed.zip if [[ -f "dist/$SIGNED_FILE_NAME" ]]; then