feat: add version flag

.github/actions/extract-fixtures/action.yml

@@ -25,3 +25,5 @@ runs:
         ref: ${{ steps.github.outputs.action_sha || steps.github.outputs.action_ref }}
         dockerfile: Dockerfile
         args: extract-fixtures --directory="$OUTPUT" --merged="$MERGED"
+        build-args: |
+          VERSION:${{ steps.github.outputs.action_ref }}

.github/actions/test/action.yml

@@ -46,6 +46,8 @@ runs:
         dockerfile: Dockerfile
         opts: --network=host
         args: test --url="$URL" --json="$JSON" --specs="$SPECS" --subdomain-url="$SUBDOMAIN" -- ${{ inputs.args }}
+        build-args: |
+          VERSION:${{ steps.github.outputs.action_ref }}
     - name: Create the XML
       if: (inputs.xml || inputs.html || inputs.markdown) && (failure() || success())
       uses: pl-strflt/gotest-json-to-junit-xml@v1

.github/workflows/release-docker.yml

@@ -2,6 +2,9 @@ name: Release Docker
 on:
   workflow_dispatch:
     inputs:
+      tag:
+        description: 'The tag that is being released.'
+        required: false
       tags:
         description: 'Comma separated list of tags to apply to the image.'
         required: false
@@ -27,6 +30,7 @@ jobs:
     outputs:
       draft: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].draft || false }}
       tags: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].tags || github.event.inputs.tags }}
+      tag: ${{ fromJSON(steps.workflow-run.outputs.artifacts)['release'].files['release.json'].tag || github.event.inputs.tag }}
     steps:
       # This step will download the release artifact either from the workflow
       # run that triggered this workflow or from the artifacts-url input. It
@@ -65,6 +69,8 @@ jobs:
       - uses: docker/build-push-action@v4
         with:
           context: .
+          build-args: |
+            VERSION:${{ needs.release.outputs.tag }}
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.tags.outputs.tags }}

CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## Unreleased
+### Added
+- `--version` flag shows the current version
+- Metadata logging used to associate tests with custom data like versions, specs identifiers, etc.
 
 ## [0.3.0] - 2023-07-31
 ### Added

Dockerfile

@@ -6,6 +6,7 @@ COPY ./go.mod ./go.sum ./
 RUN go mod download
 
 COPY . .
-RUN go build -o ./gateway-conformance ./cmd/gateway-conformance
+ARG VERSION=dev
+RUN go build -ldflags="-X github.com/ipfs/gateway-conformance/tooling.Version=${VERSION}" -o ./gateway-conformance ./cmd/gateway-conformance
 
 ENTRYPOINT ["/app/gateway-conformance"]

Makefile

@@ -1,3 +1,7 @@
+GIT_COMMIT := $(shell git rev-parse --short HEAD)
+DIRTY_SUFFIX := $(shell test -n "`git status --porcelain`" && echo "-dirty" || true)
+CLI_VERSION := dev-$(GIT_COMMIT)$(DIRTY_SUFFIX)
+
 all: gateway-conformance
 
 clean: clean-docker
@@ -30,7 +34,7 @@ fixtures.car: gateway-conformance
 	./gateway-conformance extract-fixtures --merged=true --dir=.
 
 gateway-conformance:
-	go build -o ./gateway-conformance ./cmd/gateway-conformance
+	go build -ldflags="-X github.com/ipfs/gateway-conformance/tooling.Version=$(CLI_VERSION)" -o ./gateway-conformance ./cmd/gateway-conformance
 
 test-docker: docker fixtures.car gateway-conformance
 	./gc test
@@ -44,7 +48,7 @@ test-docker: docker fixtures.car gateway-conformance
 	open ./reports/output.html
 
 docker:
-	docker build -t gateway-conformance .
+	docker build --build-arg VERSION="$(CLI_VERSION)" -t gateway-conformance .
 
 clean-docker:
 	@if command -v docker >/dev/null 2>&1 && docker image inspect gateway-conformance >/dev/null 2>&1; then \

aggregate-into-table.js

@@ -1,13 +1,14 @@
 const fs = require("fs");
 
+const TestMetadata = "TestMetadata";
+
 // retrieve the list of input files from the command line
 const files = process.argv.slice(2);
 
 // read all input files (json)
 const inputs = files.map((file) => {
     return JSON.parse(fs.readFileSync(file, 'utf8'));
-}
-);
+});
 
 // merge all the unique keys from all the inputs
 let keys = new Set();
@@ -16,13 +17,14 @@ inputs.forEach((input) => {
         keys.add(key);
     });
 });
+keys.delete(TestMetadata); // Extract TestMetadata which is a special case
 keys = Array.from(keys).sort();
 
 // generate a table
 const columns = [];
 
-// add the leading column ("gateway", "key1", "key2", ... "keyN")
-const leading = ["gateway"];
+// add the leading column ("gateway", "version", "key1", "key2", ... "keyN")
+const leading = ["gateway", "version"];
 keys.forEach((key) => {
     // Skip the "Test" prefix
     const niceKey = key.replace(/^Test/, '');
@@ -53,7 +55,13 @@ inputs.forEach((input, index) => {
     // clean name (remove path and extension)
     let name = files[index].replace(/\.json$/, '').replace(/^.*\//, '');
 
-    const col = [name];
+    // extract TestMetadata & version
+    const metadata = input[TestMetadata]["meta"];
+    const version = metadata['version'];
+
+    const col = [name, version];
+
+    // extract results
     keys.forEach((key) => {
         col.push(cellRender(input[key] || null));
     });

aggregate.js

@@ -1,5 +1,8 @@
 const fs = require("fs");
 
+// # we group test results by Path, depth is the number of levels to group by
+const depth = process.argv[2] && parseInt(process.argv[2], 10) || 1;
+
 // # read json from stdin:
 let lines = fs.readFileSync(0, "utf-8");
 lines = JSON.parse(lines);
@@ -10,9 +13,44 @@ lines = lines.filter((line) => {
   return Test !== undefined;
 });
 
+// # extract test metadata
+//   action is output, and starts with ".* --- META: (.*)"
+//   see details in https://github.com/ipfs/gateway-conformance/pull/125
+const getMetadata = (line) => {
+  const { Action, Output } = line;
+
+  if (Action !== "output") {
+    return null;
+  }
+
+  const match = Output.match(/.* --- META: (.*)/);
+
+  if (!match) {
+    return null;
+  }
+
+  const metadata = match[1];
+  return JSON.parse(metadata);
+}
+
+lines = lines.map((line) => {
+  const metadata = getMetadata(line);
+
+  if (!metadata) {
+    return line;
+  }
+
+  return {
+    ...line,
+    Action: "meta",
+    Metadata: metadata,
+  }
+});
+
+// # keep the test result lines and metadata only
 lines = lines.filter((line) => {
   const { Action } = line;
-  return ["pass", "fail", "skip"].includes(Action);
+  return ["pass", "fail", "skip", "meta"].includes(Action);
 });
 
 // # add "Path" field by parsing "Name" and split by "/"
@@ -69,8 +107,6 @@ lines = lines.filter((line) => {
 
 // # Aggregate by Path and count actions
 
-const depth = process.argv[2] && parseInt(process.argv[2], 10) || 1;
-
 // test result is a map { [path_str]: { [path], [action]: count } }
 const testResults = {};
 
@@ -82,12 +118,18 @@ lines.forEach((line) => {
   const key = path.join(" > ");
 
   if (!current[key]) {
-    current[key] = {Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0};
+    current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": {} };
   }
   current = current[key];
 
-  current[Action] += 1;
-  current["total"] += 1;
+  if (Action === "meta") {
+    const { Metadata } = line;
+    current["meta"] = { ...current["meta"], ...Metadata };
+    return;
+  } else {
+    current[Action] += 1;
+    current["total"] += 1;
+  }
 });
 
 // output result to stdout

cmd/gateway-conformance/main.go

@@ -78,8 +78,9 @@ func main() {
 	var verbose bool
 
 	app := &cli.App{
-		Name:  "gateway-conformance",
-		Usage: "Tooling for the gateway test suite",
+		Name:    "gateway-conformance",
+		Usage:   "Tooling for the gateway test suite",
+		Version: tooling.Version,
 		Commands: []*cli.Command{
 			{
 				Name:    "test",
@@ -126,6 +127,9 @@ func main() {
 						args = append(args, fmt.Sprintf("-specs=%s", specs))
 					}
 
+					ldFlag := fmt.Sprintf("-ldflags=-X github.com/ipfs/gateway-conformance/tooling.Version=%s", tooling.Version)
+					args = append(args, ldFlag)
+
 					args = append(args, cCtx.Args().Slice()...)
 
 					fmt.Println("go " + strings.Join(args, " "))

tests/metadata_test.go

@@ -0,0 +1,11 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/ipfs/gateway-conformance/tooling"
+)
+
+func TestMetadata(t *testing.T) {
+	tooling.LogVersion(t)
+}

tooling/env.go

@@ -6,6 +6,10 @@ import (
 	"runtime"
 )
 
+var (
+	Version = "dev"
+)
+
 func Home() string {
 	home := os.Getenv("GATEWAY_CONFORMANCE_HOME")
 	if home == "" {

tooling/metadata.go

@@ -0,0 +1,23 @@
+package tooling
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func LogMetadata(t *testing.T, value interface{}) {
+	jsonValue, err := json.Marshal(value)
+	if err != nil {
+		t.Errorf("Failed to encode value: %v", err)
+		return
+	}
+	t.Logf("--- META: %s", string(jsonValue))
+}
+
+func LogVersion(t *testing.T) {
+	LogMetadata(t, struct {
+		Version string `json:"version"`
+	}{
+		Version: Version,
+	})
+}