feat: test production gateways

.github/workflows/release-fixtures.yml

@@ -0,0 +1,23 @@
+name: Release Fixtures
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: [Release]
+    types:
+      - completed
+
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Ganerate Fixtures
+        env:
+          W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
+        run: |
+          make fixtures.car
+      - name: Upload fixtures
+        uses: web3-storage/add-to-web3@v2
+        with:
+          web3_token: ${{ secrets.W3STORAGE_TOKEN }}
+          path_to_add: 'fixtures.car'

.github/workflows/test-prod-e2e.yml

@@ -0,0 +1,115 @@
+name: Test Production (e2e)
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  upload-fixtures:
+    runs-on: "ubuntu-latest"
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.4
+      - uses: actions/checkout@v3
+        with:
+          path: "gateway-conformance"
+      - name: Extract fixtures
+        uses: ./gateway-conformance/.github/actions/extract-fixtures
+        with:
+          output: ./
+          merged: true
+      # https://web3.storage/docs/how-tos/store/#storing-ipfs-content-archives
+      - name: Upload fixtures
+        id: upload
+        uses: web3-storage/add-to-web3@v2
+        with:
+          web3_token: ${{ secrets.W3STORAGE_TOKEN }}
+          path_to_add: 'fixtures.car'
+      - name: Wait for pinning
+        run: |
+          sleep 180 # 3 minutes
+          # see rational in https://github.com/ipfs/gateway-conformance/pull/108#discussion_r1274628865
+  test:
+    runs-on: "ubuntu-latest"
+    strategy:
+      matrix:
+        target: ["ipfs.runfission.com", "w3s.link"]
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash
+    needs: upload-fixtures
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.4
+      - uses: actions/checkout@v3
+        with:
+          path: "gateway-conformance"
+      - name: Run the tests
+        uses: ./gateway-conformance/.github/actions/test
+        with:
+          gateway-url: https://${{ matrix.target }}
+          subdomain-url: https://${{ matrix.target }}
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+      - name: Upload one-page HTML report
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance-${{ matrix.target }}.html
+          path: ./output.html
+      - name: Upload JSON output
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance-${{ matrix.target }}.json
+          path: ./output.json
+  aggregate:
+    runs-on: "ubuntu-latest"
+    needs: [test]
+    # the tests might have failed
+    if: always()
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: "gateway-conformance"
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Aggregate results
+        working-directory: ./artifacts
+        run: |
+          mkdir ./aggregates
+
+          # download-artifact downloads artifacts in a directory named after the artifact
+          # details: https://github.com/actions/download-artifact#download-all-artifacts
+          for folder in ./conformance-*.json; do
+            file="${folder}/output.json"
+            new_file="aggregates/${folder#conformance-}"
+            jq -ns 'inputs' "$file" | node ../gateway-conformance/aggregate.js 1 > "${new_file}"
+          done
+
+          node ../gateway-conformance/aggregate-into-table.js ./aggregates/*.json > ./table.md
+      - name: Set summary
+        if: (failure() || success())
+        run: cat ./artifacts/table.md >> $GITHUB_STEP_SUMMARY

.github/workflows/update-badge.yml

@@ -0,0 +1,63 @@
+# Note: this workflow requires the repository to give Write access to Github Workflows.
+# in Settings > Actions > General > Workflow permissions
+
+permissions:
+  contents: write
+name: Update Badge
+
+on:
+  workflow_run:
+    workflows:
+      - Test Production (e2e)
+    types:
+      - completed
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  update-badge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: pl-strflt/job-summary-url-action@v1
+        id: metadata
+        with:
+          workflow: test-prod-e2e.yml # ${{ github.event.workflow.path }}
+          run_id: ${{ github.event.workflow_run.id }}
+          run_attempt: ${{ github.event.workflow_run.run_attempt }}
+          job: aggregate
+      - uses: actions/checkout@v3
+      # https://github.com/orgs/community/discussions/26560
+      - run: |
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config user.name "github-actions[bot]"
+      - run: |
+          echo GITHUB_JOB_SUMMARY_URL=${GITHUB_JOB_SUMMARY_URL}
+          IN='[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](.*)'
+          ESCAPED_IN=$(printf '%s\n' "$IN" | sed -e 's/[][\/!&]/\\&/g')
+
+          OUT="[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](${GITHUB_JOB_SUMMARY_URL})"
+
+          sed -i "s;${ESCAPED_IN};${OUT};" README.md
+        env:
+          GITHUB_JOB_SUMMARY_URL: ${{ steps.metadata.outputs.job_summary_url }}
+          REPOSITORY: ${{ github.repository }}
+      - id: git
+         run: |
+          if [[ -n $(git diff --shortstat 2> /dev/null | tail -n1) ]]; then
+            echo "dirty=1" >> $GITHUB_OUTPUT
+          else
+            echo "dirty=0" >> $GITHUB_OUTPUT
+          fi
+      - if: steps.git.outputs.dirty == '1'
+        run: |
+          git add README.md
+          git commit -m 'chore: update the link to the dashboard [skip ci]'
+          git push

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - finalized port of Kubo's sharness tests. [PR](https://github.com/ipfs/gateway-conformance/pull/92)
+- `extract-fixtures --merged` generates a car version 1 with a single root now
 
 ## [0.2.0] - 2023-06-26
 ### Added

README.md

@@ -2,6 +2,8 @@
 
 `gateway-conformance` is a tool designed to test if an IPFS Gateway implementation complies with the IPFS Gateway Specification correctly. The tool is distributed as a Docker image, as well as a GitHub Action(s).
 
+[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)]()
+
 ## Table of Contents
 
 - [Commands](#commands)

aggregate-into-table.js

@@ -0,0 +1,86 @@
+const fs = require("fs");
+
+// retrieve the list of input files from the command line
+const files = process.argv.slice(2);
+
+// read all input files (json)
+const inputs = files.map((file) => {
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+);
+
+// merge all the unique keys from all the inputs
+let keys = new Set();
+inputs.forEach((input) => {
+    Object.keys(input).forEach((key) => {
+        keys.add(key);
+    });
+});
+keys = Array.from(keys).sort();
+
+// generate a table
+const columns = [];
+
+// add the leading column ("gateway", "key1", "key2", ... "keyN")
+const leading = ["gateway"];
+keys.forEach((key) => {
+    // Skip the "Test" prefix
+    const niceKey = key.replace(/^Test/, '');
+    leading.push(niceKey);
+});
+columns.push(leading);
+
+// add the data for every input
+const cellRender = (cell) => {
+    if (cell === null) {
+        return '';
+    }
+
+    if (cell['fail'] > 0) {
+        return `:red_circle: (${cell['pass']} / ${cell['total']})`;
+    }
+    if (cell['skip'] > 0) {
+        return `:yellow_circle: (skipped)`;
+    }
+    if (cell['pass'] > 0) {
+        return `:green_circle: (${cell['pass']} / ${cell['total']})`;
+    }
+
+    throw new Error(`Unhandled cell value: ${JSON.stringify(cell)}`);
+}
+
+inputs.forEach((input, index) => {
+    // clean name (remove path and extension)
+    let name = files[index].replace(/\.json$/, '').replace(/^.*\//, '');
+
+    const col = [name];
+    keys.forEach((key) => {
+        col.push(cellRender(input[key] || null));
+    });
+    columns.push(col);
+});
+
+// # Rotate the table
+// it's easier to create the table by column, but we want to render it by row
+let rows = columns[0].map((_, i) => columns.map(col => col[i]));
+
+// # Render the table into a markdown table
+
+// add the hyphen header row after the first row
+const hyphenated = rows[0].map((x, i) => {
+    if (i === 0) {
+        return '-'.repeat(Math.max(0, x.length - 2)) + '-:'
+    }
+    return ':-' + '-'.repeat(Math.max(0, x.length - 2));
+})
+
+rows = [
+    rows[0],
+    hyphenated,
+    ...rows.slice(1),
+]
+
+let markdown = rows.map(row => '| ' + row.join(' | ') + ' |').join('\n');
+
+// output the table to stdout
+fs.writeFileSync(1, markdown);