From 499ab0e4e2c236c2d1a076d697f5d45f56b57b03 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 11 Sep 2023 11:41:00 +0200 Subject: [PATCH 1/9] feat: accept errors & log comments --- .github/workflows/test-kubo-e2e.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-kubo-e2e.yml b/.github/workflows/test-kubo-e2e.yml index 815b80171..cbc1a6791 100644 --- a/.github/workflows/test-kubo-e2e.yml +++ b/.github/workflows/test-kubo-e2e.yml @@ -68,9 +68,36 @@ jobs: xml: output.xml html: output.html markdown: output.md + accept-test-failure: true - name: Set summary - if: (failure() || success()) + if: (failure() || success()) && github.event.pull_request run: cat ./output.md >> $GITHUB_STEP_SUMMARY + - name: Prepare Comment + if: (failure() || success()) && github.event.pull_request + env: + TARGET: ${{ matrix.target }} + run: | + echo "Results against Kubo ${{ matrix.target }}:" > comment.md + + # Strip everything after the results table + cat output.md | sed '/Failures\/Errors/,$d' >> comment.md + + cat >> comment.md < Date: Mon, 11 Sep 2023 12:25:24 +0200 Subject: [PATCH 2/9] simplif error handling --- .github/workflows/test-kubo-e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-kubo-e2e.yml b/.github/workflows/test-kubo-e2e.yml index cbc1a6791..d8f1c3cc0 100644 --- a/.github/workflows/test-kubo-e2e.yml +++ b/.github/workflows/test-kubo-e2e.yml @@ -70,10 +70,9 @@ jobs: markdown: output.md accept-test-failure: true - name: Set summary - if: (failure() || success()) && github.event.pull_request run: cat ./output.md >> $GITHUB_STEP_SUMMARY - name: Prepare Comment - if: (failure() || success()) && github.event.pull_request + if: github.event.pull_request env: TARGET: ${{ matrix.target }} run: | @@ -86,12 +85,14 @@ jobs: Check the action's summary for the full results. EOF - name: Find latest comment - uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 id: find-comment + if: github.event.pull_request + uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 with: issue-number: ${{ github.event.pull_request.number }} body-includes: "Results against Kubo ${{ matrix.target }}" - name: Create comment + if: github.event.pull_request uses: peter-evans/create-or-update-comment@c6c9a1a66007646a28c153e2a8580a5bad27bcfa # v3.0.2 with: issue-number: ${{ github.event.pull_request.number }} @@ -99,7 +100,6 @@ jobs: edit-mode: replace body-path: comment.md - name: Upload one-page HTML report - if: (failure() || success()) uses: actions/upload-artifact@v3 with: name: conformance-${{ matrix.target }}.html From 874a766253703a84141ce63376c1baa6f19ab98a Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 11 Sep 2023 12:32:09 +0200 Subject: [PATCH 3/9] cleaner output --- .github/workflows/test-kubo-e2e.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test-kubo-e2e.yml b/.github/workflows/test-kubo-e2e.yml index d8f1c3cc0..815cf4517 100644 --- a/.github/workflows/test-kubo-e2e.yml +++ b/.github/workflows/test-kubo-e2e.yml @@ -77,13 +77,10 @@ jobs: TARGET: ${{ matrix.target }} run: | echo "Results against Kubo ${{ matrix.target }}:" > comment.md + echo "(check the action's summary for the full results)" >> comment.md # Strip everything after the results table cat output.md | sed '/Failures\/Errors/,$d' >> comment.md - - cat >> comment.md < Date: Fri, 4 Aug 2023 10:08:41 +0200 Subject: [PATCH 4/9] feat: introduce test group --- CHANGELOG.md | 1 + aggregate-into-table.js | 74 +++++++++++++++++++++++++--- aggregate.js | 44 +++++++++++++++-- tests/metadata_test.go | 8 +++ tests/path_gateway_cors_test.go | 3 ++ tests/path_gateway_dag_test.go | 13 +++++ tests/path_gateway_ipns_test.go | 3 ++ tests/path_gateway_raw_test.go | 3 ++ tests/path_gateway_tar_test.go | 3 ++ tests/path_gateway_unixfs_test.go | 11 +++++ tests/subdomain_gateway_ipfs_test.go | 5 ++ tests/subdomain_gateway_ipns_test.go | 5 ++ tests/trustless_gateway_car_test.go | 13 +++++ tests/trustless_gateway_ipns_test.go | 3 ++ tests/trustless_gateway_raw_test.go | 5 ++ tooling/metadata.go | 10 ++++ 16 files changed, 193 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f9f26c7a..eb1a1975d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Metadata logging used to associate tests with custom data like versions, specs identifiers, etc. - Output Github's workflow URL with metadata. [PR](https://github.com/ipfs/gateway-conformance/pull/145) - Basic Dashboard Output with content generation. [PR](https://github.com/ipfs/gateway-conformance/pull/152) +- Test Group Metadata on Tests. [PR](https://github.com/ipfs/gateway-conformance/pull/156) ## [0.3.0] - 2023-07-31 ### Added diff --git a/aggregate-into-table.js b/aggregate-into-table.js index a48981430..0d6032025 100644 --- a/aggregate-into-table.js +++ b/aggregate-into-table.js @@ -1,6 +1,7 @@ const fs = require("fs"); const TestMetadata = "TestMetadata"; +const METADATA_TEST_GROUP = "group"; // retrieve the list of input files from the command line const files = process.argv.slice(2); @@ -10,24 +11,77 @@ const inputs = files.map((file) => { return JSON.parse(fs.readFileSync(file, 'utf8')); }); -// merge all the unique keys from all the inputs -let keys = new Set(); +// merge all the unique keys & metadata from all the inputs +const metadata = {} inputs.forEach((input) => { Object.keys(input).forEach((key) => { - keys.add(key); + metadata[key] = { ...metadata[key], ...input[key]["meta"] || {} }; }); }); -keys.delete(TestMetadata); // Extract TestMetadata which is a special case -keys = Array.from(keys).sort(); +delete metadata[TestMetadata]; // Extract TestMetadata which is a special case + +// generate groups: an array of {group, key} objects +// where group is the group name (or undefined), and key is the test key name (or undefined) +// It represents the table leftmost column. +// +// Group1 +// Group1 - Test1 +// Group1 - Test2 +// Group2 +// ... +const groups = [] +const groupsAdded = new Set(); +Object.entries(metadata).forEach(([key, value]) => { + const group = value[METADATA_TEST_GROUP] || undefined; + + if (!groupsAdded.has(group)) { + groups.push({ group, key: undefined }); + groupsAdded.add(group); + } + + groups.push({ group, key }); +}); + +// sort the groups so that the tests are ordered by group, then by key. +// undefined groups are always at the end. +groups.sort((a, b) => { + if (a.group === b.group) { + if (a.key === undefined) { + return -1; + } + if (b.key === undefined) { + return 1; + } + return a.key.localeCompare(b.key); + } + + if (a.group === undefined) { + return 1; + } + + if (b.group === undefined) { + return -1; + } + + return a.group.localeCompare(b.group); +}); // generate a table const columns = []; // add the leading column ("gateway", "version", "key1", "key2", ... "keyN") const leading = ["gateway", "version"]; -keys.forEach((key) => { +groups.forEach(({ group, key }) => { + if (key === undefined) { + leading.push(`**${group || 'Other'}**`); + return; + } + + const m = metadata[key]; + // Skip the "Test" prefix - const niceKey = key.replace(/^Test/, ''); + let niceKey = key.replace(/^Test/, ''); + leading.push(niceKey); }); columns.push(leading); @@ -73,7 +127,11 @@ inputs.forEach((input, index) => { const col = [name, versionCell]; // extract results - keys.forEach((key) => { + groups.forEach(({ group, key }) => { + if (key === undefined) { + col.push(null); + return; + } col.push(cellRender(input[key] || null)); }); columns.push(col); diff --git a/aggregate.js b/aggregate.js index f19462e34..783932cf5 100644 --- a/aggregate.js +++ b/aggregate.js @@ -16,7 +16,7 @@ lines = lines.filter((line) => { // # extract test metadata // action is output, and starts with ".* --- META: (.*)" // see details in https://github.com/ipfs/gateway-conformance/pull/125 -const getMetadata = (line) => { +const extractMetadata = (line) => { const { Action, Output } = line; if (Action !== "output") { @@ -34,7 +34,7 @@ const getMetadata = (line) => { } lines = lines.map((line) => { - const metadata = getMetadata(line); + const metadata = extractMetadata(line); if (!metadata) { return line; @@ -87,6 +87,44 @@ lines.forEach((line) => { }); }) +// prepare metadata up the tree +const metadataTree = {}; + +// sort lines so that the one with the longest path is processed first +const sortedLines = lines.sort((a, b) => { + return b.Path.length - a.Path.length; +}); + +sortedLines.forEach((line) => { + const { Path, Action, Metadata } = line; + let current = metadataTree; + + if (Action !== "meta") { + return; + } + + Path.forEach((path) => { + if (!current[path]) { + current[path] = {}; + } + current = current[path]; + current["meta"] = { ...current["meta"], ...Metadata }; + }); +}); + +const getMetadata = (path) => { + let current = metadataTree; + + path.forEach((path) => { + if (!current[path]) { + return null; + } + current = current[path]; + }); + + return current["meta"]; +} + // # Drop all lines where the Test "Path" does not point to a leaf // if the test has children then we don't really care about it's pass / fail / skip status, // we'll aggregate its children results' @@ -118,7 +156,7 @@ lines.forEach((line) => { const key = path.join(" > "); if (!current[key]) { - current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": {} }; + current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": getMetadata(path) || {} }; } current = current[key]; diff --git a/tests/metadata_test.go b/tests/metadata_test.go index a8b060eb0..f9f70de2d 100644 --- a/tests/metadata_test.go +++ b/tests/metadata_test.go @@ -11,3 +11,11 @@ func TestMetadata(t *testing.T) { tooling.LogJobURL(t) tooling.LogGatewayURL(t) } + +const ( + GroupTrustlessGateway = "Trustless Gateway" + GroupPathGateway = "Path Gateway" + GroupSubdomainGateway = "Subdomain Gateway" + GroupCORS = "CORS" + GroupIPNS = "IPNS" +) diff --git a/tests/path_gateway_cors_test.go b/tests/path_gateway_cors_test.go index 563a2a71a..7bab7878c 100644 --- a/tests/path_gateway_cors_test.go +++ b/tests/path_gateway_cors_test.go @@ -3,11 +3,14 @@ package tests import ( "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) func TestCors(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + cidHello := "bafkqabtimvwgy3yk" // hello tests := SugarTests{ diff --git a/tests/path_gateway_dag_test.go b/tests/path_gateway_dag_test.go index 31fa64b12..2cb90f564 100644 --- a/tests/path_gateway_dag_test.go +++ b/tests/path_gateway_dag_test.go @@ -3,6 +3,7 @@ package tests import ( "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/ipns" @@ -12,6 +13,8 @@ import ( ) func TestGatewayJsonCbor(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car") fileJSON := fixture.MustGetNode("ą", "ę", "t.json") @@ -66,6 +69,8 @@ func TestGatewayJsonCbor(t *testing.T) { // ## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON // ## (returns representation defined in https://ipld.io/specs/codecs/dag-pb/spec/#logical-format) func TestDagPbConversion(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car") dir := fixture.MustGetRoot() @@ -205,6 +210,8 @@ func TestDagPbConversion(t *testing.T) { // # Requesting CID with plain json (0x0200) and cbor (0x51) codecs // # (note these are not UnixFS, not DAG-* variants, just raw block identified by a CID with a special codec) func TestPlainCodec(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + table := []struct { Name string Format string @@ -308,6 +315,8 @@ func TestPlainCodec(t *testing.T) { // ## Pathing, traversal over DAG-JSON and DAG-CBOR func TestPathing(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + dagJSONTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-json-traversal.car").MustGetRoot() dagCBORTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-cbor-traversal.car").MustGetRoot() @@ -381,6 +390,8 @@ func TestPathing(t *testing.T) { // ## NATIVE TESTS for DAG-JSON (0x0129) and DAG-CBOR (0x71): // ## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested func TestNativeDag(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + missingCID := car.RandomCID() table := []struct { @@ -570,6 +581,8 @@ func TestNativeDag(t *testing.T) { } func TestGatewayJSONCborAndIPNS(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + ipnsIdDagJSON := "k51qzi5uqu5dhjghbwdvbo6mi40htrq6e2z4pwgp15pgv3ho1azvidttzh8yy2" ipnsIdDagCBOR := "k51qzi5uqu5dghjous0agrwavl8vzl64xckoqzwqeqwudfr74kfd11zcyk3b7l" diff --git a/tests/path_gateway_ipns_test.go b/tests/path_gateway_ipns_test.go index f705dec94..f3563d766 100644 --- a/tests/path_gateway_ipns_test.go +++ b/tests/path_gateway_ipns_test.go @@ -3,11 +3,14 @@ package tests import ( "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) func TestRedirectCanonicalIPNS(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + tests := SugarTests{ { Name: "GET for /ipns/{b58-multihash-of-ed25519-key} redirects to /ipns/{cidv1-libp2p-key-base36}", diff --git a/tests/path_gateway_raw_test.go b/tests/path_gateway_raw_test.go index ca0a81c15..b4be6bff0 100644 --- a/tests/path_gateway_raw_test.go +++ b/tests/path_gateway_raw_test.go @@ -5,12 +5,15 @@ import ( "strings" "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) func TestGatewayBlock(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + fixture := car.MustOpenUnixfsCar("gateway-raw-block.car") tests := SugarTests{ diff --git a/tests/path_gateway_tar_test.go b/tests/path_gateway_tar_test.go index df7145bb0..1bd8e81c4 100644 --- a/tests/path_gateway_tar_test.go +++ b/tests/path_gateway_tar_test.go @@ -3,6 +3,7 @@ package tests import ( "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/specs" @@ -11,6 +12,8 @@ import ( ) func TestTar(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + fixtureOutside := car.MustOpenUnixfsCar("path_gateway_tar/outside-root.car") fixtureInside := car.MustOpenUnixfsCar("path_gateway_tar/inside-root.car") diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go index 04f355062..329fca88a 100644 --- a/tests/path_gateway_unixfs_test.go +++ b/tests/path_gateway_unixfs_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/ipns" @@ -13,6 +14,8 @@ import ( ) func TestUnixFSDirectoryListing(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + fixture := car.MustOpenUnixfsCar("dir_listing/fixtures.car") root := fixture.MustGetNode() file := fixture.MustGetNode("ą", "ę", "file-źł.txt") @@ -76,6 +79,8 @@ func TestUnixFSDirectoryListing(t *testing.T) { } func TestGatewayCache(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + fixture := car.MustOpenUnixfsCar("gateway-cache/fixtures.car") tests := SugarTests{ @@ -308,6 +313,8 @@ func TestGatewayCache(t *testing.T) { } func TestGatewayCacheWithIPNS(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + fixture := car.MustOpenUnixfsCar("gateway-cache/fixtures.car") ipns := ipns.MustOpenIPNSRecordWithKey("gateway-cache/k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe.ipns-record") ipnsKey := ipns.Key() @@ -404,6 +411,8 @@ func TestGatewayCacheWithIPNS(t *testing.T) { } func TestGatewaySymlink(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + fixture := car.MustOpenUnixfsCar("path_gateway_unixfs/symlink.car") rootDirCID := fixture.MustGetCid() @@ -443,6 +452,8 @@ func TestGatewaySymlink(t *testing.T) { } func TestGatewayUnixFSFileRanges(t *testing.T) { + tooling.LogTestGroup(t, GroupPathGateway) + // Multi-range requests MUST conform to the HTTP semantics. The server does not // need to be able to support returning multiple ranges. However, it must respond // correctly. diff --git a/tests/subdomain_gateway_ipfs_test.go b/tests/subdomain_gateway_ipfs_test.go index 397cfce9a..5a6bf13cc 100644 --- a/tests/subdomain_gateway_ipfs_test.go +++ b/tests/subdomain_gateway_ipfs_test.go @@ -4,6 +4,7 @@ import ( "net/url" "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/helpers" @@ -12,6 +13,8 @@ import ( ) func TestUnixFSDirectoryListingOnSubdomainGateway(t *testing.T) { + tooling.LogTestGroup(t, GroupSubdomainGateway) + fixture := car.MustOpenUnixfsCar("dir_listing/fixtures.car") root := fixture.MustGetNode() file := fixture.MustGetNode("ą", "ę", "file-źł.txt") @@ -105,6 +108,8 @@ func TestUnixFSDirectoryListingOnSubdomainGateway(t *testing.T) { } func TestGatewaySubdomains(t *testing.T) { + tooling.LogTestGroup(t, GroupSubdomainGateway) + fixture := car.MustOpenUnixfsCar("subdomain_gateway/fixtures.car") CIDVal := string(fixture.MustGetRawData("hello-CIDv1")) // hello diff --git a/tests/subdomain_gateway_ipns_test.go b/tests/subdomain_gateway_ipns_test.go index d99d8b5e0..4a4293f05 100644 --- a/tests/subdomain_gateway_ipns_test.go +++ b/tests/subdomain_gateway_ipns_test.go @@ -4,6 +4,7 @@ import ( "net/url" "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" "github.com/ipfs/gateway-conformance/tooling/dnslink" "github.com/ipfs/gateway-conformance/tooling/helpers" @@ -15,6 +16,8 @@ import ( ) func TestGatewaySubdomainAndIPNS(t *testing.T) { + tooling.LogTestGroup(t, GroupSubdomainGateway) + tests := SugarTests{} rsaFixture := ipns.MustOpenIPNSRecordWithKey("subdomain_gateway/QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3.ipns-record") @@ -159,6 +162,8 @@ func TestGatewaySubdomainAndIPNS(t *testing.T) { } func TestSubdomainGatewayDNSLinkInlining(t *testing.T) { + tooling.LogTestGroup(t, GroupSubdomainGateway) + tests := SugarTests{} // We're going to run the same test against multiple gateways (localhost, and a subdomain gateway) diff --git a/tests/trustless_gateway_car_test.go b/tests/trustless_gateway_car_test.go index 6dbb6166e..c0f5ca8d6 100644 --- a/tests/trustless_gateway_car_test.go +++ b/tests/trustless_gateway_car_test.go @@ -3,6 +3,7 @@ package tests import ( "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/helpers" @@ -11,6 +12,8 @@ import ( ) func TestTrustlessCarPathing(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + subdirTwoSingleBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-two-single-block-files.car") singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") dirWithDagCborWithLinksFixture := car.MustOpenUnixfsCar("trustless_gateway_car/dir-with-dag-cbor-with-links.car") @@ -118,6 +121,8 @@ func TestTrustlessCarPathing(t *testing.T) { } func TestTrustlessCarDagScopeBlock(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + subdirTwoSingleBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-two-single-block-files.car") singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") @@ -201,6 +206,8 @@ func TestTrustlessCarDagScopeBlock(t *testing.T) { } func TestTrustlessCarDagScopeEntity(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + subdirTwoSingleBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-two-single-block-files.car") singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") subdirWithMixedBlockFiles := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-mixed-block-files.car") @@ -335,6 +342,8 @@ func TestTrustlessCarDagScopeEntity(t *testing.T) { } func TestTrustlessCarDagScopeAll(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + subdirWithMixedBlockFiles := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-mixed-block-files.car") tests := SugarTests{ @@ -398,6 +407,8 @@ func TestTrustlessCarDagScopeAll(t *testing.T) { } func TestTrustlessCarEntityBytes(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") subdirWithMixedBlockFiles := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-mixed-block-files.car") missingBlockFixture := car.MustOpenUnixfsCar("trustless_gateway_car/file-3k-and-3-blocks-missing-block.car") @@ -658,6 +669,8 @@ func TestTrustlessCarEntityBytes(t *testing.T) { } func TestTrustlessCarOrderAndDuplicates(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + dirWithDuplicateFiles := car.MustOpenUnixfsCar("trustless_gateway_car/dir-with-duplicate-files.car") // This array is defined at the SPEC level and should not depend on library behavior // See the recipe for `dir-with-duplicate-files.car` diff --git a/tests/trustless_gateway_ipns_test.go b/tests/trustless_gateway_ipns_test.go index e99449fbf..138ae07f5 100644 --- a/tests/trustless_gateway_ipns_test.go +++ b/tests/trustless_gateway_ipns_test.go @@ -3,6 +3,7 @@ package tests import ( "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/ipns" "github.com/ipfs/gateway-conformance/tooling/specs" @@ -10,6 +11,8 @@ import ( ) func TestGatewayIPNSRecord(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + fixture := car.MustOpenUnixfsCar("ipns_records/fixtures.car") file := fixture.MustGetRoot() fileCID := file.Cid() diff --git a/tests/trustless_gateway_raw_test.go b/tests/trustless_gateway_raw_test.go index 8094f92d5..a0cd27568 100644 --- a/tests/trustless_gateway_raw_test.go +++ b/tests/trustless_gateway_raw_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/specs" @@ -12,6 +13,8 @@ import ( ) func TestTrustlessRaw(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + fixture := car.MustOpenUnixfsCar("gateway-raw-block.car") tests := SugarTests{ @@ -128,6 +131,8 @@ func TestTrustlessRaw(t *testing.T) { } func TestTrustlessRawRanges(t *testing.T) { + tooling.LogTestGroup(t, GroupTrustlessGateway) + // Multi-range requests MUST conform to the HTTP semantics. The server does not // need to be able to support returning multiple ranges. However, it must respond // correctly. diff --git a/tooling/metadata.go b/tooling/metadata.go index 547b431b8..cd6a1c503 100644 --- a/tooling/metadata.go +++ b/tooling/metadata.go @@ -18,6 +18,16 @@ func LogMetadata(t *testing.T, value interface{}) { t.Logf("--- META: %s", string(jsonValue)) } +func LogTestGroup(t *testing.T, name string) { + t.Helper() + + LogMetadata(t, struct { + Group string `json:"group"` + }{ + Group: name, + }) +} + func LogVersion(t *testing.T) { LogMetadata(t, struct { Version string `json:"version"` From 5106287edf36898b77fa5fac23f555638a16d778 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 6 Sep 2023 16:58:38 +0200 Subject: [PATCH 5/9] feat: complete grouping --- tests/dnslink_gateway_test.go | 3 +++ tests/metadata_test.go | 13 ++++++++----- tests/path_gateway_cors_test.go | 2 +- tests/path_gateway_dag_test.go | 12 ++++++------ tests/path_gateway_ipns_test.go | 2 +- tests/path_gateway_raw_test.go | 2 +- tests/path_gateway_tar_test.go | 2 +- tests/path_gateway_unixfs_test.go | 10 +++------- tests/redirects_file_test.go | 2 ++ tests/subdomain_gateway_ipfs_test.go | 4 ++-- tests/subdomain_gateway_ipns_test.go | 4 ++-- tests/trustless_gateway_car_test.go | 12 ++++++------ tests/trustless_gateway_ipns_test.go | 2 +- tests/trustless_gateway_raw_test.go | 4 ++-- 14 files changed, 39 insertions(+), 35 deletions(-) diff --git a/tests/dnslink_gateway_test.go b/tests/dnslink_gateway_test.go index 0a18a23b2..110ec21c3 100644 --- a/tests/dnslink_gateway_test.go +++ b/tests/dnslink_gateway_test.go @@ -4,6 +4,7 @@ import ( "net/url" "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/dnslink" @@ -14,6 +15,8 @@ import ( ) func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) { + tooling.LogTestGroup(t, GroupDNSLink) + fixture := car.MustOpenUnixfsCar("dir_listing/fixtures.car") file := fixture.MustGetNode("ą", "ę", "file-źł.txt") diff --git a/tests/metadata_test.go b/tests/metadata_test.go index f9f70de2d..022d9dda3 100644 --- a/tests/metadata_test.go +++ b/tests/metadata_test.go @@ -13,9 +13,12 @@ func TestMetadata(t *testing.T) { } const ( - GroupTrustlessGateway = "Trustless Gateway" - GroupPathGateway = "Path Gateway" - GroupSubdomainGateway = "Subdomain Gateway" - GroupCORS = "CORS" - GroupIPNS = "IPNS" + GroupSubdomains = "Subdomains" + GroupCORS = "CORS" + GroupIPNS = "IPNS" + GroupDNSLink = "DNSLink" + GroupJSONCbor = "JSON-CBOR" + GroupBlockCar = "Block-CAR" + GroupTar = "Tar" + GroupUnixFS = "UnixFS" ) diff --git a/tests/path_gateway_cors_test.go b/tests/path_gateway_cors_test.go index 7bab7878c..84bf79906 100644 --- a/tests/path_gateway_cors_test.go +++ b/tests/path_gateway_cors_test.go @@ -9,7 +9,7 @@ import ( ) func TestCors(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupCORS) cidHello := "bafkqabtimvwgy3yk" // hello diff --git a/tests/path_gateway_dag_test.go b/tests/path_gateway_dag_test.go index 2cb90f564..1ea706373 100644 --- a/tests/path_gateway_dag_test.go +++ b/tests/path_gateway_dag_test.go @@ -13,7 +13,7 @@ import ( ) func TestGatewayJsonCbor(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupJSONCbor) fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car") @@ -69,7 +69,7 @@ func TestGatewayJsonCbor(t *testing.T) { // ## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON // ## (returns representation defined in https://ipld.io/specs/codecs/dag-pb/spec/#logical-format) func TestDagPbConversion(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupJSONCbor) fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car") @@ -210,7 +210,7 @@ func TestDagPbConversion(t *testing.T) { // # Requesting CID with plain json (0x0200) and cbor (0x51) codecs // # (note these are not UnixFS, not DAG-* variants, just raw block identified by a CID with a special codec) func TestPlainCodec(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupJSONCbor) table := []struct { Name string @@ -315,7 +315,7 @@ func TestPlainCodec(t *testing.T) { // ## Pathing, traversal over DAG-JSON and DAG-CBOR func TestPathing(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupJSONCbor) dagJSONTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-json-traversal.car").MustGetRoot() dagCBORTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-cbor-traversal.car").MustGetRoot() @@ -390,7 +390,7 @@ func TestPathing(t *testing.T) { // ## NATIVE TESTS for DAG-JSON (0x0129) and DAG-CBOR (0x71): // ## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested func TestNativeDag(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupJSONCbor) missingCID := car.RandomCID() @@ -581,7 +581,7 @@ func TestNativeDag(t *testing.T) { } func TestGatewayJSONCborAndIPNS(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupIPNS) ipnsIdDagJSON := "k51qzi5uqu5dhjghbwdvbo6mi40htrq6e2z4pwgp15pgv3ho1azvidttzh8yy2" ipnsIdDagCBOR := "k51qzi5uqu5dghjous0agrwavl8vzl64xckoqzwqeqwudfr74kfd11zcyk3b7l" diff --git a/tests/path_gateway_ipns_test.go b/tests/path_gateway_ipns_test.go index f3563d766..010517ea8 100644 --- a/tests/path_gateway_ipns_test.go +++ b/tests/path_gateway_ipns_test.go @@ -9,7 +9,7 @@ import ( ) func TestRedirectCanonicalIPNS(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupIPNS) tests := SugarTests{ { diff --git a/tests/path_gateway_raw_test.go b/tests/path_gateway_raw_test.go index b4be6bff0..d5861e3dc 100644 --- a/tests/path_gateway_raw_test.go +++ b/tests/path_gateway_raw_test.go @@ -12,7 +12,7 @@ import ( ) func TestGatewayBlock(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupBlockCar) fixture := car.MustOpenUnixfsCar("gateway-raw-block.car") diff --git a/tests/path_gateway_tar_test.go b/tests/path_gateway_tar_test.go index 1bd8e81c4..7bf715495 100644 --- a/tests/path_gateway_tar_test.go +++ b/tests/path_gateway_tar_test.go @@ -12,7 +12,7 @@ import ( ) func TestTar(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupTar) fixtureOutside := car.MustOpenUnixfsCar("path_gateway_tar/outside-root.car") fixtureInside := car.MustOpenUnixfsCar("path_gateway_tar/inside-root.car") diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go index 329fca88a..990c2de05 100644 --- a/tests/path_gateway_unixfs_test.go +++ b/tests/path_gateway_unixfs_test.go @@ -14,7 +14,7 @@ import ( ) func TestUnixFSDirectoryListing(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupUnixFS) fixture := car.MustOpenUnixfsCar("dir_listing/fixtures.car") root := fixture.MustGetNode() @@ -79,8 +79,6 @@ func TestUnixFSDirectoryListing(t *testing.T) { } func TestGatewayCache(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) - fixture := car.MustOpenUnixfsCar("gateway-cache/fixtures.car") tests := SugarTests{ @@ -313,7 +311,7 @@ func TestGatewayCache(t *testing.T) { } func TestGatewayCacheWithIPNS(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupIPNS) fixture := car.MustOpenUnixfsCar("gateway-cache/fixtures.car") ipns := ipns.MustOpenIPNSRecordWithKey("gateway-cache/k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe.ipns-record") @@ -411,8 +409,6 @@ func TestGatewayCacheWithIPNS(t *testing.T) { } func TestGatewaySymlink(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) - fixture := car.MustOpenUnixfsCar("path_gateway_unixfs/symlink.car") rootDirCID := fixture.MustGetCid() @@ -452,7 +448,7 @@ func TestGatewaySymlink(t *testing.T) { } func TestGatewayUnixFSFileRanges(t *testing.T) { - tooling.LogTestGroup(t, GroupPathGateway) + tooling.LogTestGroup(t, GroupUnixFS) // Multi-range requests MUST conform to the HTTP semantics. The server does not // need to be able to support returning multiple ranges. However, it must respond diff --git a/tests/redirects_file_test.go b/tests/redirects_file_test.go index af1c85c9a..3fb7481f6 100644 --- a/tests/redirects_file_test.go +++ b/tests/redirects_file_test.go @@ -4,6 +4,7 @@ import ( "net/url" "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/dnslink" @@ -235,6 +236,7 @@ func TestRedirectsFileSupport(t *testing.T) { } func TestRedirectsFileSupportWithDNSLink(t *testing.T) { + tooling.LogTestGroup(t, GroupDNSLink) dnsLinks := dnslink.MustOpenDNSLink("redirects_file/dnslink.yml") dnsLink := dnsLinks.MustGet("custom-dnslink") diff --git a/tests/subdomain_gateway_ipfs_test.go b/tests/subdomain_gateway_ipfs_test.go index 5a6bf13cc..8fd11a682 100644 --- a/tests/subdomain_gateway_ipfs_test.go +++ b/tests/subdomain_gateway_ipfs_test.go @@ -13,7 +13,7 @@ import ( ) func TestUnixFSDirectoryListingOnSubdomainGateway(t *testing.T) { - tooling.LogTestGroup(t, GroupSubdomainGateway) + tooling.LogTestGroup(t, GroupUnixFS) fixture := car.MustOpenUnixfsCar("dir_listing/fixtures.car") root := fixture.MustGetNode() @@ -108,7 +108,7 @@ func TestUnixFSDirectoryListingOnSubdomainGateway(t *testing.T) { } func TestGatewaySubdomains(t *testing.T) { - tooling.LogTestGroup(t, GroupSubdomainGateway) + tooling.LogTestGroup(t, GroupSubdomains) fixture := car.MustOpenUnixfsCar("subdomain_gateway/fixtures.car") diff --git a/tests/subdomain_gateway_ipns_test.go b/tests/subdomain_gateway_ipns_test.go index 4a4293f05..58211eacc 100644 --- a/tests/subdomain_gateway_ipns_test.go +++ b/tests/subdomain_gateway_ipns_test.go @@ -16,7 +16,7 @@ import ( ) func TestGatewaySubdomainAndIPNS(t *testing.T) { - tooling.LogTestGroup(t, GroupSubdomainGateway) + tooling.LogTestGroup(t, GroupSubdomains) tests := SugarTests{} @@ -162,7 +162,7 @@ func TestGatewaySubdomainAndIPNS(t *testing.T) { } func TestSubdomainGatewayDNSLinkInlining(t *testing.T) { - tooling.LogTestGroup(t, GroupSubdomainGateway) + tooling.LogTestGroup(t, GroupSubdomains) tests := SugarTests{} diff --git a/tests/trustless_gateway_car_test.go b/tests/trustless_gateway_car_test.go index c0f5ca8d6..50f09c2f3 100644 --- a/tests/trustless_gateway_car_test.go +++ b/tests/trustless_gateway_car_test.go @@ -12,7 +12,7 @@ import ( ) func TestTrustlessCarPathing(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupBlockCar) subdirTwoSingleBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-two-single-block-files.car") singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") @@ -121,7 +121,7 @@ func TestTrustlessCarPathing(t *testing.T) { } func TestTrustlessCarDagScopeBlock(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupBlockCar) subdirTwoSingleBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-two-single-block-files.car") singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") @@ -206,7 +206,7 @@ func TestTrustlessCarDagScopeBlock(t *testing.T) { } func TestTrustlessCarDagScopeEntity(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupBlockCar) subdirTwoSingleBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-two-single-block-files.car") singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") @@ -342,7 +342,7 @@ func TestTrustlessCarDagScopeEntity(t *testing.T) { } func TestTrustlessCarDagScopeAll(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupBlockCar) subdirWithMixedBlockFiles := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-mixed-block-files.car") @@ -407,7 +407,7 @@ func TestTrustlessCarDagScopeAll(t *testing.T) { } func TestTrustlessCarEntityBytes(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupBlockCar) singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") subdirWithMixedBlockFiles := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-mixed-block-files.car") @@ -669,7 +669,7 @@ func TestTrustlessCarEntityBytes(t *testing.T) { } func TestTrustlessCarOrderAndDuplicates(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupBlockCar) dirWithDuplicateFiles := car.MustOpenUnixfsCar("trustless_gateway_car/dir-with-duplicate-files.car") // This array is defined at the SPEC level and should not depend on library behavior diff --git a/tests/trustless_gateway_ipns_test.go b/tests/trustless_gateway_ipns_test.go index 138ae07f5..5f0867b58 100644 --- a/tests/trustless_gateway_ipns_test.go +++ b/tests/trustless_gateway_ipns_test.go @@ -11,7 +11,7 @@ import ( ) func TestGatewayIPNSRecord(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupIPNS) fixture := car.MustOpenUnixfsCar("ipns_records/fixtures.car") file := fixture.MustGetRoot() diff --git a/tests/trustless_gateway_raw_test.go b/tests/trustless_gateway_raw_test.go index a0cd27568..03c31d974 100644 --- a/tests/trustless_gateway_raw_test.go +++ b/tests/trustless_gateway_raw_test.go @@ -13,7 +13,7 @@ import ( ) func TestTrustlessRaw(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupBlockCar) fixture := car.MustOpenUnixfsCar("gateway-raw-block.car") @@ -131,7 +131,7 @@ func TestTrustlessRaw(t *testing.T) { } func TestTrustlessRawRanges(t *testing.T) { - tooling.LogTestGroup(t, GroupTrustlessGateway) + tooling.LogTestGroup(t, GroupBlockCar) // Multi-range requests MUST conform to the HTTP semantics. The server does not // need to be able to support returning multiple ranges. However, it must respond From 0481a94a47a71ba2c19db1e3682dc639030d986f Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Thu, 7 Sep 2023 17:40:56 +0200 Subject: [PATCH 6/9] feat: add Spec(s) operations --- tests/metadata_test.go | 13 ++++++++++++- tests/path_gateway_dag_test.go | 4 +++- tests/path_gateway_tar_test.go | 1 + tests/path_gateway_unixfs_test.go | 6 ++++++ tests/redirects_file_test.go | 5 ++++- tests/trustless_gateway_car_test.go | 1 + tests/trustless_gateway_raw_test.go | 6 ++++-- tooling/metadata.go | 14 +++++++------- tooling/test/sugar.go | 22 ++++++++++++++++++++++ tooling/test/test.go | 21 +++++++++++++++++++++ tooling/test/validate.go | 11 ++++++++--- 11 files changed, 89 insertions(+), 15 deletions(-) diff --git a/tests/metadata_test.go b/tests/metadata_test.go index 022d9dda3..738d44aef 100644 --- a/tests/metadata_test.go +++ b/tests/metadata_test.go @@ -4,12 +4,23 @@ import ( "testing" "github.com/ipfs/gateway-conformance/tooling" + "github.com/ipfs/gateway-conformance/tooling/test" ) +func logGatewayURL(t *testing.T) { + tooling.LogMetadata(t, struct { + GatewayURL string `json:"gateway_url"` + SubdomainGatewayURL string `json:"subdomain_gateway_url"` + }{ + GatewayURL: test.GatewayURL, + SubdomainGatewayURL: test.SubdomainGatewayURL, + }) +} + func TestMetadata(t *testing.T) { tooling.LogVersion(t) tooling.LogJobURL(t) - tooling.LogGatewayURL(t) + logGatewayURL(t) } const ( diff --git a/tests/path_gateway_dag_test.go b/tests/path_gateway_dag_test.go index 1ea706373..9c4d5e42a 100644 --- a/tests/path_gateway_dag_test.go +++ b/tests/path_gateway_dag_test.go @@ -42,6 +42,7 @@ func TestGatewayJsonCbor(t *testing.T) { }, { Name: "GET UnixFS file with JSON bytes is returned with application/json Content-Type - with headers", + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#accept-request-header", Hint: ` ## Quick regression check for JSON stored on UnixFS: ## it has nothing to do with DAG-JSON and JSON codecs, @@ -478,7 +479,7 @@ func TestNativeDag(t *testing.T) { Response: Expect(). Headers( Header("Content-Type").Hint("expected Content-Type").Equals("application/vnd.ipld.dag-{{format}}", row.Format), - Header("Content-Length").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())), + Header("Content-Length").Spec("specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())), Header("Content-Disposition").Hint("includes Content-Disposition").Contains(`{{disposition}}; filename="{{cid}}.{{format}}"`, row.Disposition, dagTraversalCID, row.Format), Header("X-Content-Type-Options").Hint("includes nosniff hint").Contains("nosniff"), ), @@ -552,6 +553,7 @@ func TestNativeDag(t *testing.T) { }, { Name: Fmt("HEAD {{name}} with only-if-cached for missing block returns HTTP 412 Precondition Failed", row.Name), + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#only-if-cached", Request: Request(). Path("/ipfs/{{cid}}", missingCID). Header("Cache-Control", "only-if-cached"). diff --git a/tests/path_gateway_tar_test.go b/tests/path_gateway_tar_test.go index 7bf715495..d6131d0c7 100644 --- a/tests/path_gateway_tar_test.go +++ b/tests/path_gateway_tar_test.go @@ -76,6 +76,7 @@ func TestTar(t *testing.T) { }, { Name: "GET TAR with explicit ?filename= succeeds with modified Content-Disposition header", + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header", Request: Request(). Path("/ipfs/{{cid}}", dirCID). Query("filename", "testтест.tar"). diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go index 990c2de05..c0e6fc500 100644 --- a/tests/path_gateway_unixfs_test.go +++ b/tests/path_gateway_unixfs_test.go @@ -202,6 +202,7 @@ func TestGatewayCache(t *testing.T) { // ========== { Name: "GET for /ipfs/ file with matching Etag in If-None-Match returns 304 Not Modified", + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header", Request: Request(). Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()). Headers( @@ -212,6 +213,7 @@ func TestGatewayCache(t *testing.T) { }, { Name: "GET for /ipfs/ dir with index.html file with matching Etag in If-None-Match returns 304 Not Modified", + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header", Request: Request(). Path("/ipfs/{{cid}}/root2/root3/root4/", fixture.MustGetCid()). Headers( @@ -222,6 +224,7 @@ func TestGatewayCache(t *testing.T) { }, { Name: "GET for /ipfs/ file with matching third Etag in If-None-Match returns 304 Not Modified", + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header", Request: Request(). Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()). Headers( @@ -232,6 +235,7 @@ func TestGatewayCache(t *testing.T) { }, { Name: "GET for /ipfs/ file with matching weak Etag in If-None-Match returns 304 Not Modified", + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header", Request: Request(). Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()). Headers( @@ -242,6 +246,7 @@ func TestGatewayCache(t *testing.T) { }, { Name: "GET for /ipfs/ file with wildcard Etag in If-None-Match returns 304 Not Modified", + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header", Request: Request(). Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()). Headers( @@ -252,6 +257,7 @@ func TestGatewayCache(t *testing.T) { }, { Name: "GET for /ipfs/ dir listing with matching weak Etag in If-None-Match returns 304 Not Modified", + Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header", Request: Request(). Path("/ipfs/{{cid}}/root2/root3/", fixture.MustGetCid()). Headers( diff --git a/tests/redirects_file_test.go b/tests/redirects_file_test.go index 3fb7481f6..9322c74ff 100644 --- a/tests/redirects_file_test.go +++ b/tests/redirects_file_test.go @@ -15,6 +15,7 @@ import ( ) func TestRedirectsFileSupport(t *testing.T) { + tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/web-redirects-file/") fixture := car.MustOpenUnixfsCar("redirects_file/redirects.car") redirectDir := fixture.MustGetNode("examples") redirectDirCID := redirectDir.Base32Cid() @@ -165,7 +166,8 @@ func TestRedirectsFileSupport(t *testing.T) { Contains("could not parse _redirects:"), Contains(`forced redirects (or "shadowing") are not supported`), ), - ), + ).Spec("specs.ipfs.tech/http-gateways/web-redirects-file/#no-forced-redirects"), + Spec: "specs.ipfs.tech/http-gateways/web-redirects-file/#error-handling", }, { Name: "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file", @@ -180,6 +182,7 @@ func TestRedirectsFileSupport(t *testing.T) { Contains("redirects file size cannot exceed"), ), ), + Spec: "specs.ipfs.tech/http-gateways/web-redirects-file/#max-file-size", }, }...) diff --git a/tests/trustless_gateway_car_test.go b/tests/trustless_gateway_car_test.go index 50f09c2f3..5a5114c63 100644 --- a/tests/trustless_gateway_car_test.go +++ b/tests/trustless_gateway_car_test.go @@ -408,6 +408,7 @@ func TestTrustlessCarDagScopeAll(t *testing.T) { func TestTrustlessCarEntityBytes(t *testing.T) { tooling.LogTestGroup(t, GroupBlockCar) + tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/trustless-gateway/#entity-bytes-request-query-parameter") singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car") subdirWithMixedBlockFiles := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-mixed-block-files.car") diff --git a/tests/trustless_gateway_raw_test.go b/tests/trustless_gateway_raw_test.go index 03c31d974..608d0ffa2 100644 --- a/tests/trustless_gateway_raw_test.go +++ b/tests/trustless_gateway_raw_test.go @@ -14,7 +14,8 @@ import ( func TestTrustlessRaw(t *testing.T) { tooling.LogTestGroup(t, GroupBlockCar) - + tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw") + fixture := car.MustOpenUnixfsCar("gateway-raw-block.car") tests := SugarTests{ @@ -132,6 +133,7 @@ func TestTrustlessRaw(t *testing.T) { func TestTrustlessRawRanges(t *testing.T) { tooling.LogTestGroup(t, GroupBlockCar) + // @lidel: "The optional entity-bytes=from:to parameter is available only for CAR requests." // Multi-range requests MUST conform to the HTTP semantics. The server does not // need to be able to support returning multiple ranges. However, it must respond @@ -145,7 +147,7 @@ func TestTrustlessRawRanges(t *testing.T) { RunWithSpecs(t, SugarTests{ { - Name: "GETaa with application/vnd.ipld.raw with single range request includes correct bytes", + Name: "GET with application/vnd.ipld.raw with single range request includes correct bytes", Request: Request(). Path("/ipfs/{{cid}}", fixture.MustGetCid("dir", "ascii.txt")). Headers( diff --git a/tooling/metadata.go b/tooling/metadata.go index cd6a1c503..25636d5d1 100644 --- a/tooling/metadata.go +++ b/tooling/metadata.go @@ -3,8 +3,6 @@ package tooling import ( "encoding/json" "testing" - - "github.com/ipfs/gateway-conformance/tooling/test" ) func LogMetadata(t *testing.T, value interface{}) { @@ -44,12 +42,14 @@ func LogJobURL(t *testing.T) { }) } -func LogGatewayURL(t *testing.T) { +func LogSpecs(t *testing.T, specs ...string) { + if len(specs) == 0 { + return + } + LogMetadata(t, struct { - GatewayURL string `json:"gateway_url"` - SubdomainGatewayURL string `json:"subdomain_gateway_url"` + Specs []string `json:"specs"` }{ - GatewayURL: test.GatewayURL, - SubdomainGatewayURL: test.SubdomainGatewayURL, + Specs: specs, }) } diff --git a/tooling/test/sugar.go b/tooling/test/sugar.go index 966c96859..3527738d5 100644 --- a/tooling/test/sugar.go +++ b/tooling/test/sugar.go @@ -137,6 +137,7 @@ type ExpectBuilder struct { StatusCode_ int `json:"statusCode,omitempty"` Headers_ []HeaderBuilder `json:"headers,omitempty"` Body_ interface{} `json:"body,omitempty"` + Specs_ []string `json:"specs,omitempty"` } func Expect() ExpectBuilder { @@ -157,6 +158,16 @@ func (e ExpectBuilder) Header(h HeaderBuilder) ExpectBuilder { return e } +func (e ExpectBuilder) Spec(spec string) ExpectBuilder { + e.Specs_ = []string{spec} + return e +} + +func (e ExpectBuilder) Specs(specs ...string) ExpectBuilder { + e.Specs_ = specs + return e +} + func (e ExpectBuilder) Bytes(body string) ExpectBuilder { e.Body_ = []byte(body) return e @@ -213,6 +224,7 @@ type HeaderBuilder struct { Value_ string `json:"value,omitempty"` Check_ check.Check[[]string] `json:"check,omitempty"` Hint_ string `json:"hint,omitempty"` + Specs_ []string `json:"specs,omitempty"` Not_ bool `json:"not,omitempty"` } @@ -245,6 +257,16 @@ func (h HeaderBuilder) Hint(hint string) HeaderBuilder { return h } +func (h HeaderBuilder) Specs(specs ...string) HeaderBuilder { + h.Specs_ = specs + return h +} + +func (h HeaderBuilder) Spec(spec string) HeaderBuilder { + h.Specs_ = []string{spec} + return h +} + func (h HeaderBuilder) Equals(value string, args ...any) HeaderBuilder { h.Check_ = check.IsUniqAnd(check.IsEqual(value, args...)) return h diff --git a/tooling/test/test.go b/tooling/test/test.go index 33245cc07..f9a470843 100644 --- a/tooling/test/test.go +++ b/tooling/test/test.go @@ -6,12 +6,15 @@ import ( "testing" "time" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/specs" ) type SugarTest struct { Name string Hint string + Spec string + Specs []string Request RequestBuilder Requests []RequestBuilder Response ExpectBuilder @@ -20,6 +23,22 @@ type SugarTest struct { type SugarTests []SugarTest +func (s *SugarTest) AllSpecs() []string { + if len(s.Specs) > 0 && s.Spec != "" { + panic("cannot have both Spec and Specs") + } + + if len(s.Specs) > 0 { + return s.Specs + } + + if s.Spec != "" { + return []string{s.Spec} + } + + return []string{} +} + func RunWithSpecs( t *testing.T, tests SugarTests, @@ -51,6 +70,7 @@ func run(t *testing.T, tests SugarTests) { if len(test.Requests) > 0 { t.Run(test.Name, func(t *testing.T) { + tooling.LogSpecs(t, test.AllSpecs()...) responses := make([]*http.Response, 0, len(test.Requests)) for _, req := range test.Requests { @@ -63,6 +83,7 @@ func run(t *testing.T, tests SugarTests) { }) } else { t.Run(test.Name, func(t *testing.T) { + tooling.LogSpecs(t, test.AllSpecs()...) _, res, localReport := runRequest(timeout, t, test, test.Request) validateResponse(t, test.Response, res, localReport) }) diff --git a/tooling/test/validate.go b/tooling/test/validate.go index 2187d06d4..15bc77f57 100644 --- a/tooling/test/validate.go +++ b/tooling/test/validate.go @@ -6,6 +6,7 @@ import ( "net/http" "testing" + "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/check" ) @@ -16,15 +17,19 @@ func validateResponse( localReport Reporter, ) { t.Helper() + tooling.LogSpecs(t, expected.Specs_...) if expected.StatusCode_ != 0 { - if res.StatusCode != expected.StatusCode_ { - localReport(t, "Status code is not %d. It is %d", expected.StatusCode_, res.StatusCode) - } + t.Run("Status code", func(t *testing.T) { + if res.StatusCode != expected.StatusCode_ { + localReport(t, "Status code is not %d. It is %d", expected.StatusCode_, res.StatusCode) + } + }) } for _, header := range expected.Headers_ { t.Run(fmt.Sprintf("Header %s", header.Key_), func(t *testing.T) { + tooling.LogSpecs(t, header.Specs_...) actual := res.Header.Values(header.Key_) c := header.Check_ From b3c9aa350e5b6bb7525c3f15dd448fabf4081bff Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 8 Sep 2023 10:50:11 +0200 Subject: [PATCH 7/9] feat: output specs --- aggregate-into-table.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/aggregate-into-table.js b/aggregate-into-table.js index 0d6032025..061808713 100644 --- a/aggregate-into-table.js +++ b/aggregate-into-table.js @@ -23,7 +23,7 @@ delete metadata[TestMetadata]; // Extract TestMetadata which is a special case // generate groups: an array of {group, key} objects // where group is the group name (or undefined), and key is the test key name (or undefined) // It represents the table leftmost column. -// +// // Group1 // Group1 - Test1 // Group1 - Test2 @@ -69,7 +69,7 @@ groups.sort((a, b) => { // generate a table const columns = []; -// add the leading column ("gateway", "version", "key1", "key2", ... "keyN") +// add the leading column ("gateway", "version", "group1", "test11", ... "test42") const leading = ["gateway", "version"]; groups.forEach(({ group, key }) => { if (key === undefined) { @@ -77,11 +77,19 @@ groups.forEach(({ group, key }) => { return; } - const m = metadata[key]; - // Skip the "Test" prefix let niceKey = key.replace(/^Test/, ''); + const m = metadata[key]; + if (m.specs && m.specs.length > 0) { + if (m.specs.length === 1) { + niceKey = `[${niceKey}](https://${m.specs[0]})` + } else { + const urls = m.specs.map((url, index) => `[${index}](https://${url})`); + niceKey = `${niceKey} (${urls.join(', ')})`; + } + } + leading.push(niceKey); }); columns.push(leading); From 43f6668a2cf33abc8563c1d6aaf6a2f3f0330f72 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 8 Sep 2023 10:56:07 +0200 Subject: [PATCH 8/9] fix: gofmt --- tests/trustless_gateway_raw_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/trustless_gateway_raw_test.go b/tests/trustless_gateway_raw_test.go index 608d0ffa2..8bef78ea4 100644 --- a/tests/trustless_gateway_raw_test.go +++ b/tests/trustless_gateway_raw_test.go @@ -15,7 +15,7 @@ import ( func TestTrustlessRaw(t *testing.T) { tooling.LogTestGroup(t, GroupBlockCar) tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw") - + fixture := car.MustOpenUnixfsCar("gateway-raw-block.car") tests := SugarTests{ From 07a5a329bcc04ef2239b3135dab35f38045f0e4f Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 8 Sep 2023 10:57:11 +0200 Subject: [PATCH 9/9] CHANGELOG: update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb1a1975d..c033938c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Output Github's workflow URL with metadata. [PR](https://github.com/ipfs/gateway-conformance/pull/145) - Basic Dashboard Output with content generation. [PR](https://github.com/ipfs/gateway-conformance/pull/152) - Test Group Metadata on Tests. [PR](https://github.com/ipfs/gateway-conformance/pull/156) +- Specs Metadata on Tests. [PR](https://github.com/ipfs/gateway-conformance/pull/159) ## [0.3.0] - 2023-07-31 ### Added