From 18ce7a678dc95e3749770a837b533a15f2dc10f6 Mon Sep 17 00:00:00 2001 From: Pawel Leszczynski Date: Mon, 22 Jul 2024 14:48:49 +0200 Subject: [PATCH] CI and deploy steps for docs Signed-off-by: Pawel Leszczynski --- .circleci/continue_config.yml | 23 ---- .../openlineage-integration-publish.yml | 11 -- .circleci/workflows/openlineage-java.yml | 10 +- .github/workflows/javadoc-deploy.yml | 51 ++++++++ .github/workflows/site-deploy.yml | 37 ++++++ .github/workflows/spec-deploy.yml | 118 ++++++++++++++++++ client/java/release-javadoc.sh | 46 ------- .../openlineage/client/OpenLineageClient.java | 2 +- spec/OpenLineage.json | 2 +- spec/release.sh | 80 ------------ website/CNAME | 2 +- 11 files changed, 210 insertions(+), 172 deletions(-) delete mode 100644 .circleci/workflows/openlineage-integration-publish.yml create mode 100644 .github/workflows/javadoc-deploy.yml create mode 100644 .github/workflows/site-deploy.yml create mode 100644 .github/workflows/spec-deploy.yml delete mode 100755 client/java/release-javadoc.sh delete mode 100755 spec/release.sh diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 5c0685c270..cf5fa44060 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -993,29 +993,6 @@ jobs: - run: mkdir -p target/wheels && cp target/wheels/* dist/ - run: python -m twine upload --non-interactive --verbose --repository pypi dist/* - publish-javadoc: - working_directory: ~/openlineage/client/java - docker: - - image: cimg/openjdk:11.0 - steps: - - *checkout_project_root - - add_ssh_keys: - fingerprints: - - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91" - - run: ./gradlew --console=plain javadoc - - run: ./release-javadoc.sh - - publish-spec: - working_directory: ~/openlineage - docker: - - image: cimg/base:2021.07 - steps: - - *checkout_project_root - - add_ssh_keys: - fingerprints: - - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91" - - run: spec/release.sh - build-proxy-fluentd: working_directory: ~/openlineage/proxy/fluentd docker: diff --git a/.circleci/workflows/openlineage-integration-publish.yml b/.circleci/workflows/openlineage-integration-publish.yml deleted file mode 100644 index 10ac69c238..0000000000 --- a/.circleci/workflows/openlineage-integration-publish.yml +++ /dev/null @@ -1,11 +0,0 @@ -workflows: - openlineage-integration-publish: - jobs: - - publish-spec: - filters: - branches: - only: main - context: release - - workflow_complete: - requires: - - publish-spec diff --git a/.circleci/workflows/openlineage-java.yml b/.circleci/workflows/openlineage-java.yml index 4ff1484262..750d346c8d 100644 --- a/.circleci/workflows/openlineage-java.yml +++ b/.circleci/workflows/openlineage-java.yml @@ -23,12 +23,4 @@ workflows: requires: - compile-integration-sql-java-linux-arm - compile-integration-sql-java-linux-x86 - - compile-integration-sql-java-macos - - publish-javadoc: - filters: - branches: - only: main - context: release - - workflow_complete: - requires: - - publish-javadoc \ No newline at end of file + - compile-integration-sql-java-macos \ No newline at end of file diff --git a/.github/workflows/javadoc-deploy.yml b/.github/workflows/javadoc-deploy.yml new file mode 100644 index 0000000000..973ed41601 --- /dev/null +++ b/.github/workflows/javadoc-deploy.yml @@ -0,0 +1,51 @@ +name: Build & Deploy javadoc + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + branches: + # testing only, not allowed to run on OpenLineage/OpenLineage repo + - test-docs-deploy + +jobs: + generate_javadoc: + if: | + (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || + (github.ref != 'refs/heads/test-docs-deploy' && github.repository == 'OpenLineage/OpenLineage') + name: "Generate Java docs" + env: + branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main' }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: '11' + - uses: actions/checkout@v4 + - name: Generate new java doc + run: | + cd client/java + ./gradlew --console=plain javadoc + - name: Get git branch + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git fetch + git checkout ${{ env.branch }} + - name: Clear existing javadoc + run: rm -rf website/static/apidocs/javadoc + - name: Copy docs to website directory + run: cp -r client/java/build/docs/javadoc website/static/apidocs + - name: Commit javadoc + run: | + git pull + git add website/static/apidocs/javadoc/ + git commit -m "[generated] reloading javadoc" + git push + + site-deploy-changes: + uses: ./.github/workflows/site-deploy.yml + needs: generate_javadoc + with: + branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main' }} \ No newline at end of file diff --git a/.github/workflows/site-deploy.yml b/.github/workflows/site-deploy.yml new file mode 100644 index 0000000000..1ef7e71449 --- /dev/null +++ b/.github/workflows/site-deploy.yml @@ -0,0 +1,37 @@ +name: 'Reusable netlify deploy workflow' + +on: + workflow_call: + inputs: + branch: + required: true + type: string + +jobs: + site-deploy: + name: Deploy to GitHub Pages + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: "npm" + cache-dependency-path: website/package-lock.json + - name: Install dependencies + run: yarn install --frozen-lockfile + - name: Build openapi docs + run: yarn run build:docs + - name: Build website + run: yarn build + - name: Setup Pages + uses: actions/configure-pages@v5 + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: './website/build' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.github/workflows/spec-deploy.yml b/.github/workflows/spec-deploy.yml new file mode 100644 index 0000000000..1d7a143798 --- /dev/null +++ b/.github/workflows/spec-deploy.yml @@ -0,0 +1,118 @@ +name: Build & Deploy spec to Netlify + +on: + pull_request: + branches: + - main + push: + branches: + # testing only, not allowed to run on OpenLineage/OpenLineage repo + - test-docs-deploy + +jobs: + generate_spec: + if: | + (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || + (github.event.pull_request.merged == true && github.repository == 'OpenLineage/OpenLineage') + name: 'Commit Spec definition' + runs-on: ubuntu-latest + env: + branch: ${{ github.head_ref || github.ref_name }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: shogo82148/actions-setup-perl@v1 + with: + perl-version: "5.38" + - name: Setup NodeJS + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: "npm" + cache-dependency-path: website/package-lock.json + - name: Check changes + id: check-changes + run: | + if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then + echo "no changes in spec detected, skipping publishing spec" + exit 1 + fi + echo "spec changes detected" + exit 0 + continue-on-error: true + - name: Check spec changes + if: steps.check-changes.outcome == 'success' + run: | + # Copy changed spec JSON files to target location + git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do + # ignore registry files + if [[ $LINE =~ "registry.json" ]]; then + continue + fi + + # extract target file name from $id field in spec files + URL=$(cat $LINE | jq -r '.["$id"]') + + # extract target location in website repo + LOC="website/static/${URL#*//*/}" + LOC_DIR="${LOC%/*}" + + # create dir if necessary, and copy files + echo "change detected in $LINE" + mkdir -p $LOC_DIR + cp $LINE $LOC + done + - name: Generate openapi + if: steps.check-changes.outcome == 'success' + run: | + SPEC_DIR="$(pwd)/website/static/spec" + APIDOC_DIR="$(pwd)/website/static/apidocs" + + pushd $SPEC_DIR + LATEST_VERSION=$(find . -maxdepth 1 | grep -v 'facets' | grep '[0-9]*-[0-9]-[0-9]' | sort -Vr | head -1) + echo latest version is $LATEST_VERSION + rm -f ./OpenLineage.json 2>/dev/null + perl -i -pe"s/version: [[:alnum:]\.-]*/version: ${LATEST_VERSION:2}/g" ./OpenLineage.yml + echo Versions updated + + mkdir "${LATEST_VERSION}/facets" + for i in $(ls -d ./facets/* | sort); do cp $i/*.json ${LATEST_VERSION}/facets; done; + + echo "LATEST_VERSION=${LATEST_VERSION}" >> "$GITHUB_ENV" + + pushd $LATEST_VERSION + ln -sf ../OpenLineage.yml . + popd + - name: Generate redoc + if: steps.check-changes.outcome == 'success' + run : | + npm install redoc-cli + yarn run redoc-cli build -o website/static/apidocs/openapi/index.html website/static/spec/$LATEST_VERSION/OpenLineage.yml --title "OpenLineage API Docs" + - name: Clean temp files + run: | + pushd website/static/spec/$LATEST_VERSION + rm -rf facets + rm OpenLineage.yml + popd + - name: Commit changes + if: steps.check-changes.outcome == 'success' + run: | + # verify if there are any changes + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git config user.name github-actions + git config user.email github-actions@github.com + git fetch + git checkout ${{ env.branch }} + git add website/static/* + git commit -m "[generated] adding spec changes" + git push + else + echo "nothing to commit" + fi + + site-deploy-changes: + uses: ./.github/workflows/site-deploy.yml + needs: generate_spec + with: + branch: ${{ github.head_ref || github.ref_name }} # shall be `main` or `test-docs-deploy` for testing \ No newline at end of file diff --git a/client/java/release-javadoc.sh b/client/java/release-javadoc.sh deleted file mode 100755 index 9bd9f70aca..0000000000 --- a/client/java/release-javadoc.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018-2024 contributors to the OpenLineage project -# SPDX-License-Identifier: Apache-2.0 -# - -set -e - -function git-website() { - command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@ -} - -git config --global user.email "openlineage-bot-key@openlineage.io" -git config --global user.name "OpenLineage deploy bot" - -WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website} -REPO="git@github.com:OpenLineage/docs" - -if [[ -d $WEBSITE_DIR ]]; then - # Check if we're in git repository and the repository points at website - if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then - # Make sure we're at the head of the main branch - git checkout main - git reset --hard origin/master - else - echo "$WEBSITE_DIR is not empty - failing" - exit 1 - fi -else - git clone --depth 1 $REPO $WEBSITE_DIR -fi - -# check if there are any changes in javadoc in the latest commit -if [[ $(diff -qr $WEBSITE_DIR/static/apidocs/javadoc './build/docs/javadoc' | wc -l) -eq 0 ]]; then - echo "no changes in javadoc detected, skipping publishing javadoc" - exit 0 -fi - -echo "Changes detected, updating javadoc..." -rm -rf $WEBSITE_DIR/static/apidocs/javadoc -mv ./build/docs/javadoc $WEBSITE_DIR/static/apidocs - -# commit new spec and push -git-website add -A static/apidocs/javadoc -git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage javadoc update" -git-website push diff --git a/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java b/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java index 6610ebb012..0d61398391 100644 --- a/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java +++ b/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java @@ -20,7 +20,7 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. */ +/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. DUMMY TEST CHANGE */ @Slf4j public final class OpenLineageClient { final Transport transport; diff --git a/spec/OpenLineage.json b/spec/OpenLineage.json index 58167f7140..dbbeaf1bfe 100644 --- a/spec/OpenLineage.json +++ b/spec/OpenLineage.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json", + "$id": "https://openlineage.io/spec/2-0-10/OpenLineage.json", "$defs": { "BaseEvent": { "type": "object", diff --git a/spec/release.sh b/spec/release.sh deleted file mode 100755 index 138f0a6ba6..0000000000 --- a/spec/release.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018-2024 contributors to the OpenLineage project -# SPDX-License-Identifier: Apache-2.0 - -set -e - -function git-website() { - command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@ -} - -git config --global user.email "openlineage-bot-key@openlineage.io" -git config --global user.name "OpenLineage deploy bot" - -WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website} -REPO="git@github.com:OpenLineage/docs" - -if [[ -d $WEBSITE_DIR ]]; then - # Check if we're in git repository and the repository points at website - if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then - # Make sure we're at the head of the main branch - git checkout main - git reset --hard origin/master - else - echo "$WEBSITE_DIR is not empty - failing" - exit 1 - fi -else - git clone --depth 1 $REPO $WEBSITE_DIR -fi - -WEBSITE_COMMIT_FILE="$WEBSITE_DIR/.last_spec_commit_id" - -# Check on which commit we deployed spec last -if [[ -f $WEBSITE_COMMIT_FILE ]]; then - PREV_SPEC_COMMIT=$(cat "$WEBSITE_COMMIT_FILE") -else - # Before lifecycle state facet - PREV_SPEC_COMMIT="d66c41872f3cc7f7cd5c99664d401e070e60ff48" -fi - -# check if there are any changes in spec in the latest commit -if git diff --name-only --exit-code $PREV_SPEC_COMMIT HEAD 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then - echo "no changes in spec detected, skipping publishing spec" - exit 0 -fi - -echo "Copying spec files from commit $PREV_SPEC_COMMIT" - -# Mark last commit on which we finished copying spec -echo "$CIRCLE_SHA1" > "$WEBSITE_COMMIT_FILE" - -# Copy changed spec YML file to target location -cp spec/OpenLineage.yml ${WEBSITE_DIR}/static/spec/OpenLineage.yml - -# Copy changed spec JSON files to target location -git diff --name-only $PREV_SPEC_COMMIT HEAD 'spec/*.json' | while read LINE; do - - #ignore registry files - if [[ $LINE =~ "registry.json" ]]; then - continue - fi - - # extract target file name from $id field in spec files - URL=$(cat $LINE | jq -r '.["$id"]') - - # extract target location in website repo - LOC="${WEBSITE_DIR}/static/${URL#*//*/}" - LOC_DIR="${LOC%/*}" - - # create dir if necessary, and copy files - mkdir -p $LOC_DIR - cp $LINE $LOC -done - -# commit new spec and push -git-website add -A .last_spec_commit_id -git-website add -A static/spec/ -git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage specification update" -git-website push diff --git a/website/CNAME b/website/CNAME index 622635b5d7..347a3963ff 100644 --- a/website/CNAME +++ b/website/CNAME @@ -1 +1 @@ -openlineage.io +pawel-big-lebowski.github.io \ No newline at end of file