diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 5c0685c270..cf5fa44060 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -993,29 +993,6 @@ jobs: - run: mkdir -p target/wheels && cp target/wheels/* dist/ - run: python -m twine upload --non-interactive --verbose --repository pypi dist/* - publish-javadoc: - working_directory: ~/openlineage/client/java - docker: - - image: cimg/openjdk:11.0 - steps: - - *checkout_project_root - - add_ssh_keys: - fingerprints: - - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91" - - run: ./gradlew --console=plain javadoc - - run: ./release-javadoc.sh - - publish-spec: - working_directory: ~/openlineage - docker: - - image: cimg/base:2021.07 - steps: - - *checkout_project_root - - add_ssh_keys: - fingerprints: - - "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91" - - run: spec/release.sh - build-proxy-fluentd: working_directory: ~/openlineage/proxy/fluentd docker: diff --git a/.circleci/workflows/openlineage-integration-publish.yml b/.circleci/workflows/openlineage-integration-publish.yml deleted file mode 100644 index 10ac69c238..0000000000 --- a/.circleci/workflows/openlineage-integration-publish.yml +++ /dev/null @@ -1,11 +0,0 @@ -workflows: - openlineage-integration-publish: - jobs: - - publish-spec: - filters: - branches: - only: main - context: release - - workflow_complete: - requires: - - publish-spec diff --git a/.circleci/workflows/openlineage-java.yml b/.circleci/workflows/openlineage-java.yml index 4ff1484262..750d346c8d 100644 --- a/.circleci/workflows/openlineage-java.yml +++ b/.circleci/workflows/openlineage-java.yml @@ -23,12 +23,4 @@ workflows: requires: - compile-integration-sql-java-linux-arm - compile-integration-sql-java-linux-x86 - - compile-integration-sql-java-macos - - publish-javadoc: - filters: - branches: - only: main - context: release - - workflow_complete: - requires: - - publish-javadoc \ No newline at end of file + - compile-integration-sql-java-macos \ No newline at end of file diff --git a/.github/workflows/javadoc-deploy.yml b/.github/workflows/javadoc-deploy.yml new file mode 100644 index 0000000000..15eb3aea0d --- /dev/null +++ b/.github/workflows/javadoc-deploy.yml @@ -0,0 +1,53 @@ +name: Build & Deploy javadoc + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + branches: + # testing only, not allowed to run on OpenLineage/OpenLineage repo + - test-docs-deploy + +jobs: + generate_javadoc: + if: | + (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || + (github.ref != 'refs/heads/test-docs-deploy' && github.repository == 'OpenLineage/OpenLineage') + name: "Generate Java docs" + env: + branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main' }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: '11' + - uses: actions/checkout@v4 + - name: Generate new java doc + run: | + cd client/java + ./gradlew --console=plain javadoc + - name: Get git branch + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git fetch + git checkout ${{ env.branch }} + - name: Clear existing javadoc + run: rm -rf website/static/apidocs/javadoc + - name: Copy docs to website directory + run: cp -r client/java/build/docs/javadoc website/static/apidocs + - name: Commit javadoc + run: | + git add website/static/apidocs/javadoc/ + git commit -m "[generated] reloading javadoc" + git push + + netlify-deploy-changes: + uses: ./.github/workflows/netlify-deploy.yml + needs: generate_javadoc + with: + branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main' }} + secrets: + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + NETLIFY_TOKEN_SECRET: ${{ secrets.NETLIFY_TOKEN_SECRET }} \ No newline at end of file diff --git a/.github/workflows/netlify-deploy.yml b/.github/workflows/netlify-deploy.yml new file mode 100644 index 0000000000..5e02904609 --- /dev/null +++ b/.github/workflows/netlify-deploy.yml @@ -0,0 +1,50 @@ +name: 'Reusable netlify deploy workflow' + +on: + workflow_call: + inputs: + branch: + required: true + type: string + secrets: + NETLIFY_SITE_ID: + required: true + NETLIFY_TOKEN_SECRET: + required: true + +jobs: + netlify-deploy: + runs-on: ubuntu-latest + steps: + - name: Repository Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - name: Setup NodeJS + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: "npm" + cache-dependency-path: website/package-lock.json + - name: Install Netlify + run: | + cd website + npm install netlify-cli@17.10.1 -g + - name: Install Dependencies + run: | + cd website + yarn + - name: Build project + run: | + cd website + yarn build + - name: Deploy to Netlify + id: netlify_deploy + run: | + cd website/build + netlify deploy \ + --cwd . \ + --dir . \ + --site ${{ secrets.NETLIFY_SITE_ID }} \ + --auth ${{ secrets.NETLIFY_TOKEN_SECRET }} \ + --prod \ No newline at end of file diff --git a/.github/workflows/spec-deploy.yml b/.github/workflows/spec-deploy.yml new file mode 100644 index 0000000000..36d54a487e --- /dev/null +++ b/.github/workflows/spec-deploy.yml @@ -0,0 +1,113 @@ +name: Build & Deploy spec to Netlify + +on: + pull_request: + branches: + - main + push: + branches: + # testing only, not allowed to run on OpenLineage/OpenLineage repo + - test-docs-deploy + +jobs: + generate_spec: + if: | + (github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) || + (github.event.pull_request.merged == true && github.repository == 'OpenLineage/OpenLineage') + name: 'Commit Spec definition' + runs-on: ubuntu-latest + env: + branch: ${{ github.head_ref || github.ref_name }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: shogo82148/actions-setup-perl@v1 + with: + perl-version: "5.38" + - name: Check changes + id: check-changes + run: | + if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then + echo "no changes in spec detected, skipping publishing spec" + exit 1 + fi + echo "spec changes detected" + exit 0 + continue-on-error: true + - name: Check spec changes + if: steps.check-changes.outcome == 'success' + run: | + # Copy changed spec JSON files to target location + git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do + # ignore registry files + if [[ $LINE =~ "registry.json" ]]; then + continue + fi + + # extract target file name from $id field in spec files + URL=$(cat $LINE | jq -r '.["$id"]') + + # extract target location in website repo + LOC="website/static/${URL#*//*/}" + LOC_DIR="${LOC%/*}" + + # create dir if necessary, and copy files + echo "change detected in $LINE" + mkdir -p $LOC_DIR + cp $LINE $LOC + done + - name: Generate openapi + if: steps.check-changes.outcome == 'success' + run: | + SPEC_DIR="$(pwd)/website/static/spec" + APIDOC_DIR="$(pwd)/website/static/apidocs" + + pushd $SPEC_DIR + LATEST_VERSION=$(find . -maxdepth 1 | grep -v 'facets' | grep '[0-9]*-[0-9]-[0-9]' | sort -Vr | head -1) + echo latest version is $LATEST_VERSION + rm -f ./OpenLineage.json 2>/dev/null + perl -i -pe"s/version: [[:alnum:]\.-]*/version: ${LATEST_VERSION:2}/g" ./OpenLineage.yml + echo Versions updated + + mkdir "${LATEST_VERSION}/facets" + for i in $(ls -d ./facets/* | sort); do cp $i/*.json ${LATEST_VERSION}/facets; done; + + echo "LATEST_VERSION=${LATEST_VERSION}" >> "$GITHUB_ENV" + + pushd $LATEST_VERSION + ln -sf ../OpenLineage.yml . + popd + - name: Generate redoc + uses: seeebiii/redoc-cli-github-action@v10 + with: + args: 'build --output "${APIDOC_DIR}/openapi/index.html" "${SPEC_DIR}/${LATEST_VERSION}/OpenLineage.yml" --title 'OpenLineage API Docs'' + - name: Clean temp files + pushd $LATEST_VERSION + rm -rf facets + rm OpenLineage.yml + popd + - name: Commit changes + if: steps.check-changes.outcome == 'success' + run: | + # verify if there are any changes + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git config user.name github-actions + git config user.email github-actions@github.com + git fetch + git checkout ${{ env.branch }} + git add website/static/* + git commit -m "[generated] adding spec changes" + git push + else + echo "nothing to commit" + fi + + netlify-deploy-changes: + uses: ./.github/workflows/netlify-deploy.yml + needs: generate_spec + with: + branch: ${{ github.head_ref || github.ref_name }} # shall be `main` or `test-docs-deploy` for testing + secrets: + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + NETLIFY_TOKEN_SECRET: ${{ secrets.NETLIFY_TOKEN_SECRET }} diff --git a/client/java/release-javadoc.sh b/client/java/release-javadoc.sh deleted file mode 100755 index 9bd9f70aca..0000000000 --- a/client/java/release-javadoc.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018-2024 contributors to the OpenLineage project -# SPDX-License-Identifier: Apache-2.0 -# - -set -e - -function git-website() { - command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@ -} - -git config --global user.email "openlineage-bot-key@openlineage.io" -git config --global user.name "OpenLineage deploy bot" - -WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website} -REPO="git@github.com:OpenLineage/docs" - -if [[ -d $WEBSITE_DIR ]]; then - # Check if we're in git repository and the repository points at website - if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then - # Make sure we're at the head of the main branch - git checkout main - git reset --hard origin/master - else - echo "$WEBSITE_DIR is not empty - failing" - exit 1 - fi -else - git clone --depth 1 $REPO $WEBSITE_DIR -fi - -# check if there are any changes in javadoc in the latest commit -if [[ $(diff -qr $WEBSITE_DIR/static/apidocs/javadoc './build/docs/javadoc' | wc -l) -eq 0 ]]; then - echo "no changes in javadoc detected, skipping publishing javadoc" - exit 0 -fi - -echo "Changes detected, updating javadoc..." -rm -rf $WEBSITE_DIR/static/apidocs/javadoc -mv ./build/docs/javadoc $WEBSITE_DIR/static/apidocs - -# commit new spec and push -git-website add -A static/apidocs/javadoc -git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage javadoc update" -git-website push diff --git a/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java b/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java index 6610ebb012..0d61398391 100644 --- a/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java +++ b/client/java/src/main/java/io/openlineage/client/OpenLineageClient.java @@ -20,7 +20,7 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. */ +/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. DUMMY TEST CHANGE */ @Slf4j public final class OpenLineageClient { final Transport transport; diff --git a/spec/OpenLineage.json b/spec/OpenLineage.json index 58167f7140..dbbeaf1bfe 100644 --- a/spec/OpenLineage.json +++ b/spec/OpenLineage.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json", + "$id": "https://openlineage.io/spec/2-0-10/OpenLineage.json", "$defs": { "BaseEvent": { "type": "object", diff --git a/spec/release.sh b/spec/release.sh deleted file mode 100755 index 138f0a6ba6..0000000000 --- a/spec/release.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2018-2024 contributors to the OpenLineage project -# SPDX-License-Identifier: Apache-2.0 - -set -e - -function git-website() { - command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@ -} - -git config --global user.email "openlineage-bot-key@openlineage.io" -git config --global user.name "OpenLineage deploy bot" - -WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website} -REPO="git@github.com:OpenLineage/docs" - -if [[ -d $WEBSITE_DIR ]]; then - # Check if we're in git repository and the repository points at website - if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then - # Make sure we're at the head of the main branch - git checkout main - git reset --hard origin/master - else - echo "$WEBSITE_DIR is not empty - failing" - exit 1 - fi -else - git clone --depth 1 $REPO $WEBSITE_DIR -fi - -WEBSITE_COMMIT_FILE="$WEBSITE_DIR/.last_spec_commit_id" - -# Check on which commit we deployed spec last -if [[ -f $WEBSITE_COMMIT_FILE ]]; then - PREV_SPEC_COMMIT=$(cat "$WEBSITE_COMMIT_FILE") -else - # Before lifecycle state facet - PREV_SPEC_COMMIT="d66c41872f3cc7f7cd5c99664d401e070e60ff48" -fi - -# check if there are any changes in spec in the latest commit -if git diff --name-only --exit-code $PREV_SPEC_COMMIT HEAD 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then - echo "no changes in spec detected, skipping publishing spec" - exit 0 -fi - -echo "Copying spec files from commit $PREV_SPEC_COMMIT" - -# Mark last commit on which we finished copying spec -echo "$CIRCLE_SHA1" > "$WEBSITE_COMMIT_FILE" - -# Copy changed spec YML file to target location -cp spec/OpenLineage.yml ${WEBSITE_DIR}/static/spec/OpenLineage.yml - -# Copy changed spec JSON files to target location -git diff --name-only $PREV_SPEC_COMMIT HEAD 'spec/*.json' | while read LINE; do - - #ignore registry files - if [[ $LINE =~ "registry.json" ]]; then - continue - fi - - # extract target file name from $id field in spec files - URL=$(cat $LINE | jq -r '.["$id"]') - - # extract target location in website repo - LOC="${WEBSITE_DIR}/static/${URL#*//*/}" - LOC_DIR="${LOC%/*}" - - # create dir if necessary, and copy files - mkdir -p $LOC_DIR - cp $LINE $LOC -done - -# commit new spec and push -git-website add -A .last_spec_commit_id -git-website add -A static/spec/ -git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage specification update" -git-website push