Skip to content

Commit

Permalink
CI and deploy steps for docs
Browse files Browse the repository at this point in the history
Signed-off-by: Pawel Leszczynski <[email protected]>
  • Loading branch information
pawel-big-lebowski committed Aug 1, 2024
1 parent c0f6c01 commit 18ce7a6
Show file tree
Hide file tree
Showing 11 changed files with 210 additions and 172 deletions.
23 changes: 0 additions & 23 deletions .circleci/continue_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -993,29 +993,6 @@ jobs:
- run: mkdir -p target/wheels && cp target/wheels/* dist/
- run: python -m twine upload --non-interactive --verbose --repository pypi dist/*

publish-javadoc:
working_directory: ~/openlineage/client/java
docker:
- image: cimg/openjdk:11.0
steps:
- *checkout_project_root
- add_ssh_keys:
fingerprints:
- "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
- run: ./gradlew --console=plain javadoc
- run: ./release-javadoc.sh

publish-spec:
working_directory: ~/openlineage
docker:
- image: cimg/base:2021.07
steps:
- *checkout_project_root
- add_ssh_keys:
fingerprints:
- "7d:bc:78:35:09:c9:25:04:63:f9:eb:4b:f1:f4:d1:91"
- run: spec/release.sh

build-proxy-fluentd:
working_directory: ~/openlineage/proxy/fluentd
docker:
Expand Down
11 changes: 0 additions & 11 deletions .circleci/workflows/openlineage-integration-publish.yml

This file was deleted.

10 changes: 1 addition & 9 deletions .circleci/workflows/openlineage-java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,4 @@ workflows:
requires:
- compile-integration-sql-java-linux-arm
- compile-integration-sql-java-linux-x86
- compile-integration-sql-java-macos
- publish-javadoc:
filters:
branches:
only: main
context: release
- workflow_complete:
requires:
- publish-javadoc
- compile-integration-sql-java-macos
51 changes: 51 additions & 0 deletions .github/workflows/javadoc-deploy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: Build & Deploy javadoc

on:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
branches:
# testing only, not allowed to run on OpenLineage/OpenLineage repo
- test-docs-deploy

jobs:
generate_javadoc:
if: |
(github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) ||
(github.ref != 'refs/heads/test-docs-deploy' && github.repository == 'OpenLineage/OpenLineage')
name: "Generate Java docs"
env:
branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main' }}
runs-on: ubuntu-latest
steps:
- uses: actions/setup-java@v4
with:
distribution: 'corretto'
java-version: '11'
- uses: actions/checkout@v4
- name: Generate new java doc
run: |
cd client/java
./gradlew --console=plain javadoc
- name: Get git branch
run: |
git config user.name github-actions
git config user.email [email protected]
git fetch
git checkout ${{ env.branch }}
- name: Clear existing javadoc
run: rm -rf website/static/apidocs/javadoc
- name: Copy docs to website directory
run: cp -r client/java/build/docs/javadoc website/static/apidocs
- name: Commit javadoc
run: |
git pull
git add website/static/apidocs/javadoc/
git commit -m "[generated] reloading javadoc"
git push
site-deploy-changes:
uses: ./.github/workflows/site-deploy.yml
needs: generate_javadoc
with:
branch: ${{ (github.head_ref || github.ref_name) == 'test-docs-deploy' && 'test-docs-deploy' || 'main' }}
37 changes: 37 additions & 0 deletions .github/workflows/site-deploy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: 'Reusable netlify deploy workflow'

on:
workflow_call:
inputs:
branch:
required: true
type: string

jobs:
site-deploy:
name: Deploy to GitHub Pages
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.branch }}
- uses: actions/setup-node@v4
with:
node-version: 22
cache: "npm"
cache-dependency-path: website/package-lock.json
- name: Install dependencies
run: yarn install --frozen-lockfile
- name: Build openapi docs
run: yarn run build:docs
- name: Build website
run: yarn build
- name: Setup Pages
uses: actions/configure-pages@v5
- name: Upload artifact
uses: actions/upload-pages-artifact@v3
with:
path: './website/build'
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4
118 changes: 118 additions & 0 deletions .github/workflows/spec-deploy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
name: Build & Deploy spec to Netlify

on:
pull_request:
branches:
- main
push:
branches:
# testing only, not allowed to run on OpenLineage/OpenLineage repo
- test-docs-deploy

jobs:
generate_spec:
if: |
(github.ref == 'refs/heads/test-docs-deploy' && github.repository != 'OpenLineage/OpenLineage' ) ||
(github.event.pull_request.merged == true && github.repository == 'OpenLineage/OpenLineage')
name: 'Commit Spec definition'
runs-on: ubuntu-latest
env:
branch: ${{ github.head_ref || github.ref_name }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- uses: shogo82148/actions-setup-perl@v1
with:
perl-version: "5.38"
- name: Setup NodeJS
uses: actions/setup-node@v4
with:
node-version: 22
cache: "npm"
cache-dependency-path: website/package-lock.json
- name: Check changes
id: check-changes
run: |
if git diff --name-only --exit-code HEAD^ HEAD -- 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
echo "no changes in spec detected, skipping publishing spec"
exit 1
fi
echo "spec changes detected"
exit 0
continue-on-error: true
- name: Check spec changes
if: steps.check-changes.outcome == 'success'
run: |
# Copy changed spec JSON files to target location
git diff --name-only HEAD^ HEAD -- 'spec/*.json' | while read LINE; do
# ignore registry files
if [[ $LINE =~ "registry.json" ]]; then
continue
fi
# extract target file name from $id field in spec files
URL=$(cat $LINE | jq -r '.["$id"]')
# extract target location in website repo
LOC="website/static/${URL#*//*/}"
LOC_DIR="${LOC%/*}"
# create dir if necessary, and copy files
echo "change detected in $LINE"
mkdir -p $LOC_DIR
cp $LINE $LOC
done
- name: Generate openapi
if: steps.check-changes.outcome == 'success'
run: |
SPEC_DIR="$(pwd)/website/static/spec"
APIDOC_DIR="$(pwd)/website/static/apidocs"
pushd $SPEC_DIR
LATEST_VERSION=$(find . -maxdepth 1 | grep -v 'facets' | grep '[0-9]*-[0-9]-[0-9]' | sort -Vr | head -1)
echo latest version is $LATEST_VERSION
rm -f ./OpenLineage.json 2>/dev/null
perl -i -pe"s/version: [[:alnum:]\.-]*/version: ${LATEST_VERSION:2}/g" ./OpenLineage.yml
echo Versions updated
mkdir "${LATEST_VERSION}/facets"
for i in $(ls -d ./facets/* | sort); do cp $i/*.json ${LATEST_VERSION}/facets; done;
echo "LATEST_VERSION=${LATEST_VERSION}" >> "$GITHUB_ENV"
pushd $LATEST_VERSION
ln -sf ../OpenLineage.yml .
popd
- name: Generate redoc
if: steps.check-changes.outcome == 'success'
run : |
npm install redoc-cli
yarn run redoc-cli build -o website/static/apidocs/openapi/index.html website/static/spec/$LATEST_VERSION/OpenLineage.yml --title "OpenLineage API Docs"
- name: Clean temp files
run: |
pushd website/static/spec/$LATEST_VERSION
rm -rf facets
rm OpenLineage.yml
popd
- name: Commit changes
if: steps.check-changes.outcome == 'success'
run: |
# verify if there are any changes
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
git config user.name github-actions
git config user.email [email protected]
git fetch
git checkout ${{ env.branch }}
git add website/static/*
git commit -m "[generated] adding spec changes"
git push
else
echo "nothing to commit"
fi
site-deploy-changes:
uses: ./.github/workflows/site-deploy.yml
needs: generate_spec
with:
branch: ${{ github.head_ref || github.ref_name }} # shall be `main` or `test-docs-deploy` for testing
46 changes: 0 additions & 46 deletions client/java/release-javadoc.sh

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;

/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. */
/** HTTP client used to emit {@link OpenLineage.RunEvent}s to HTTP backend. DUMMY TEST CHANGE */
@Slf4j
public final class OpenLineageClient {
final Transport transport;
Expand Down
2 changes: 1 addition & 1 deletion spec/OpenLineage.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://openlineage.io/spec/2-0-2/OpenLineage.json",
"$id": "https://openlineage.io/spec/2-0-10/OpenLineage.json",
"$defs": {
"BaseEvent": {
"type": "object",
Expand Down
Loading

0 comments on commit 18ce7a6

Please sign in to comment.