Add workflow to determine API changes and comment on PRs (#297)

* Determine API Changes and comment on PR Signed-off-by: Thomas Farr <[email protected]> * Add tool to calculate coverage Signed-off-by: Thomas Farr <[email protected]> * Implement tool to dump the cluster's spec Signed-off-by: Thomas Farr <[email protected]> * Reimplement commenting Signed-off-by: Thomas Farr <[email protected]> * Ensure directory exists when writing to file Signed-off-by: Thomas Farr <[email protected]> * Align tool spec location defaults Signed-off-by: Thomas Farr <[email protected]> * Tidy up workflows Signed-off-by: Thomas Farr <[email protected]> * Documentation Signed-off-by: Thomas Farr <[email protected]> * Fix links Signed-off-by: Thomas Farr <[email protected]> --------- Signed-off-by: Thomas Farr <[email protected]>
opensearch-project · May 22, 2024 · c58908d · c58908d
1 parent 9013205
commit c58908d
Show file tree

Hide file tree

Showing 26 changed files with 912 additions and 269 deletions.
diff --git a/.github/pr-comment-templates/pr-change-analysis.template.md b/.github/pr-comment-templates/pr-change-analysis.template.md
@@ -0,0 +1,23 @@
+# Changes Analysis
+
+**Commit SHA:** {{.after_sha}}
+**Comparing To SHA:** {{.before_sha}}
+
+## API Changes
+
+### Summary
+{{.api_changes_summary}}
+
+### Report
+The full API changes report is available at: {{.api_changes_report_url}}
+
+## API Coverage
+{{with .api_coverage}}
+
+|               | Before                                              | After                                             | Δ                                                 |
+|--------------:|-----------------------------------------------------|---------------------------------------------------|---------------------------------------------------|
+|   Covered (%) | {{.before.covered}} ({{.before.covered_pct}} %)     | {{.after.covered}} ({{.after.covered_pct}} %)     | {{.covered_delta}} ({{.covered_pct_delta}} %)     |
+| Uncovered (%) | {{.before.uncovered}} ({{.before.uncovered_pct}} %) | {{.after.uncovered}} ({{.after.uncovered_pct}} %) | {{.uncovered_delta}} ({{.uncovered_pct_delta}} %) |
+|       Unknown | {{.before.specified_but_not_provided}}              | {{.after.specified_but_not_provided}}             | {{.specified_but_not_provided_delta}}             |
+
+{{end}}
diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml
@@ -1,19 +1,20 @@
-name: Apply 'untriaged' label during issue lifecycle
-
-on:
-  issues:
-    types: [opened, reopened, transferred]
-
-jobs:
-  apply-label:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/github-script@v6
-        with:
-          script: |
-            github.rest.issues.addLabels({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              labels: ['untriaged']
-            })
+name: Apply 'untriaged' label during issue lifecycle
+
+on:
+  issues:
+    types: [opened, reopened, transferred]
+
+jobs:
+  apply-label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add `untriaged` Label
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ['untriaged']
+            })
diff --git a/.github/workflows/analyze-pr-changes.yml b/.github/workflows/analyze-pr-changes.yml
@@ -0,0 +1,219 @@
+name: Analyze PR Changes
+
+on: [pull_request]
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Environment Variables
+        shell: bash -eo pipefail {0}
+        run: |
+          BEFORE_SHA=$(git merge-base "$BASE_SHA" "$HEAD_SHA")
+          AFTER_SHA=${HEAD_SHA}
+          CLUSTER_SPEC=/tmp/opensearch-openapi-CLUSTER.yaml
+          BEFORE_SPEC=/tmp/opensearch-openapi-${BEFORE_SHA}.yaml
+          AFTER_SPEC=/tmp/opensearch-openapi-${AFTER_SHA}.yaml
+          BEFORE_COVERAGE=/tmp/coverage-api-${BEFORE_SHA}.json
+          AFTER_COVERAGE=/tmp/coverage-api-${AFTER_SHA}.json
+          COVERAGE_DIFF=/tmp/coverage-api-${BEFORE_SHA}-${AFTER_SHA}-DIFF.json
+          
+          vars=(
+            BEFORE_SHA
+            AFTER_SHA
+            CLUSTER_SPEC
+            BEFORE_SPEC
+            AFTER_SPEC
+            BEFORE_COVERAGE
+            AFTER_COVERAGE
+            COVERAGE_DIFF
+          )
+          
+          { 
+            for var in "${vars[@]}"
+            do
+              echo "${var}=${!var}"
+            done
+          } | tee "$GITHUB_ENV"
+        env:
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+
+      - name: Dump OpenSearch Cluster's API
+        shell: bash -eo pipefail {0}
+        run: |
+          docker build coverage --tag opensearch-with-api-plugin
+          
+          docker run \
+            --name opensearch \
+            --rm -d \
+            -p 9200:9200 -p 9600:9600 \
+            -e "discovery.type=single-node" \
+            -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_PASSWORD" \
+            opensearch-with-api-plugin
+          
+          npm install
+          
+          npm run dump-cluster-spec -- --insecure --output $CLUSTER_SPEC
+          
+          docker stop opensearch
+        env:
+          OPENSEARCH_PASSWORD: BobgG7YrtsdKf9M
+
+      - name: Checkout BEFORE Spec
+        shell: bash -eo pipefail {0}
+        run: git checkout $BEFORE_SHA
+
+      - name: Build BEFORE Spec
+        shell: bash -eo pipefail {0}
+        run: |
+          npm install
+          npm run merge -- --source ./spec --output $BEFORE_SPEC
+
+      - name: Checkout AFTER Spec
+        shell: bash -eo pipefail {0}
+        run: git checkout $AFTER_SHA
+
+      - name: Build AFTER Spec
+        shell: bash -eo pipefail {0}
+        run: |
+          npm install
+          npm run merge -- --source ./spec --output $AFTER_SPEC
+
+      - name: Calculate Coverage
+        shell: bash -eo pipefail {0}
+        run: |
+          npm run coverage:spec -- \
+              --cluster $CLUSTER_SPEC \
+              --specification $BEFORE_SPEC \
+              --output $BEFORE_COVERAGE
+          
+          npm run coverage:spec -- \
+              --cluster $CLUSTER_SPEC \
+              --specification $AFTER_SPEC \
+              --output $AFTER_COVERAGE
+          
+          jq . $AFTER_COVERAGE
+          
+          jq --slurp '
+            [ .[].counts ]
+            | {
+              "before": (.[0]),
+              "after": (.[1]),
+              "covered_delta": (.[1].covered - .[0].covered),
+              "covered_pct_delta": ((.[1].covered_pct - .[0].covered_pct) * 100 | round / 100),
+              "uncovered_delta": (.[1].uncovered - .[0].uncovered),
+              "uncovered_pct_delta": ((.[1].uncovered_pct - .[0].uncovered_pct) * 100 | round / 100),
+              "specified_but_not_provided_delta": (.[1].specified_but_not_provided - .[0].specified_but_not_provided)
+            }
+            ' \
+            $BEFORE_COVERAGE \
+            $AFTER_COVERAGE \
+            | tee $COVERAGE_DIFF
+
+      - name: Upload Coverage Data
+        id: upload-coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-api
+          path: |
+            /tmp/coverage-api-*.json
+
+      - name: Install openapi-changes
+        shell: bash -eo pipefail {0}
+        run: npm install --global @pb33f/openapi-changes
+
+      - name: Generate API Changes HTML Report
+        shell: bash -eo pipefail {0}
+        run: openapi-changes html-report --no-logo --no-color $BEFORE_SPEC $AFTER_SPEC
+
+      - name: Upload API Changes HTML Report
+        id: upload-api-changes-report
+        uses: actions/upload-artifact@v4
+        with:
+          name: api-changes-report
+          path: |
+            report.html
+            /tmp/opensearch-openapi-*.yaml
+
+      - name: Generate API Changes Summary
+        shell: bash -eo pipefail {0}
+        run: |
+          if ! openapi-changes summary --no-logo --no-color --markdown $BEFORE_SPEC $AFTER_SPEC >output.md ; then
+            if ! grep -q 'breaking changes discovered' output.md ; then
+              cat output.md >/dev/stderr
+              exit 1
+            fi
+          fi
+          
+          gawk '
+            BEGIN {
+              RS = "(\r|\n|\r\n)"
+              WAS_BLANK = 0
+              HAD_CHANGES = 0
+            }
+            
+            /^starting work/ || /^Building original model/ || /^Date:/ || /^SPEC:/ || /^ERROR:/ || /^DONE:/ {
+              next
+            }
+              
+            /^[[:space:]]*$/ {
+              WAS_BLANK = 1
+              next
+            }
+              
+            WAS_BLANK {
+              WAS_BLANK = 0
+              print ""
+            }
+              
+            {
+              HAD_CHANGES = 1
+              print
+            }
+          
+            END {
+              if (!HAD_CHANGES) {
+                print "**NO CHANGES**\n"
+              }
+            }
+          ' output.md | tee changes-summary.md
+
+      - name: Construct Comment Data Payload
+        shell: bash -eo pipefail {0}
+        run: |
+          jq \
+            --arg pr_number ${PR_NUMBER} \
+            --arg before_sha ${BEFORE_SHA} \
+            --arg after_sha ${AFTER_SHA} \
+            --arg api_changes_report_url "${API_CHANGES_REPORT_URL}" \
+            --rawfile api_changes_summary ./changes-summary.md \
+            --slurpfile api_coverage $COVERAGE_DIFF \
+            --null-input '
+            {
+              "pr_number": ($pr_number),
+              "comment_identifier": "# Changes Analysis",
+              "template_name": "pr-change-analysis",
+              "template_data": {
+                "before_sha": ($before_sha),
+                "after_sha": ($after_sha),
+                "api_changes_report_url": ($api_changes_report_url),
+                "api_changes_summary": ($api_changes_summary),
+                "api_coverage": ($api_coverage[0])
+              }
+            }
+            ' | tee pr-comment.json
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          API_CHANGES_REPORT_URL: ${{ steps.upload-api-changes-report.outputs.artifact-url }}
+
+      - name: Upload PR Comment Payload
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-comment
+          path: pr-comment.json
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -13,25 +13,23 @@ jobs:
 
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Node.js
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: '20'
 
       - name: Build
-        run: |-
-          mkdir -p ./build
-          npm install
-          npm run merge -- --source ./spec --output ./build/opensearch-openapi.yaml
+        run: npm install && npm run merge
 
       - name: Extract Branch Name
         id: branch
         shell: bash
         run: echo "name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
 
-      - uses: marvinpinto/[email protected]
+      - name: Release Specification to GitHub
+        uses: marvinpinto/[email protected]
         with:
           repo_token: "${{ secrets.GITHUB_TOKEN }}"
           automatic_release_tag: ${{ steps.branch.outputs.name }}

diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml
@@ -8,7 +8,8 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout Repo
+        uses: actions/checkout@v4
 
       - name: Link Checker
         uses: lycheeverse/lychee-action@v1

diff --git a/.github/workflows/coverage-api.yml b/.github/workflows/coverage-api.yml