From 3b5e34ebc1338e4c12b0dae1f1dc96afd7b62eee Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 4 Jul 2023 09:22:23 +0200
Subject: [PATCH 01/37] ci: add test production e2e

---
 .github/workflows/test-prod-e2e.yml | 70 +++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 .github/workflows/test-prod-e2e.yml

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
new file mode 100644
index 000000000..1597a489d
--- /dev/null
+++ b/.github/workflows/test-prod-e2e.yml
@@ -0,0 +1,70 @@
+name: Test Kubo (e2e)
+
+on:
+  workflow_dispatch:
+  push:
+    branches: 
+      - main
+  pull_request:
+
+jobs:
+  upload-fixtures:
+    runs-on: 'ubuntu-latest'
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.4
+      - uses: actions/checkout@v3
+        with:
+          path: 'gateway-conformance'
+      - name: Extract fixtures
+        uses: ./gateway-conformance/.github/actions/extract-fixtures
+        with:
+          output: ./
+          merged: true
+      # https://web3.storage/docs/how-tos/store/#storing-ipfs-content-archives
+      - name: Upload fixtures
+        env: 
+          W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
+        run: |
+          curl --headers "Authorization: Bearer $W3STORAGE_TOKEN" --data-binary @fixtures.car
+  test:
+    runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ["ipfs.runfission.com"]
+    defaults:
+      run:
+        shell: bash
+    needs: upload-fixtures
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.4
+      - uses: actions/checkout@v3
+        with:
+          path: 'gateway-conformance'
+      - name: Run the tests
+        uses: ./gateway-conformance/.github/actions/test
+        with:
+          gateway-url: https://${{ matrix.target }}
+          subdomain-url: https://${{ matrix.target }}
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+      - name: Set summary
+        if: (failure() || success())
+        run: cat ./output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload one-page HTML report
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance.html
+          path: ./output.html

From f2ad9a3d962191c5a621706df7d686a3b7fdd5ae Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 4 Jul 2023 09:29:53 +0200
Subject: [PATCH 02/37] fix: missing curl option

---
 .github/workflows/test-prod-e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 1597a489d..5d8e2ea61 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -31,7 +31,7 @@ jobs:
         env: 
           W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
         run: |
-          curl --headers "Authorization: Bearer $W3STORAGE_TOKEN" --data-binary @fixtures.car
+          curl --oauth2-bearer "$W3STORAGE_TOKEN" --data-binary @fixtures.car
   test:
     runs-on: 'ubuntu-latest'
     strategy:

From e20ec2b3facdec65a1af2da454a763f743e51394 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 4 Jul 2023 10:28:42 +0200
Subject: [PATCH 03/37] feat: pin & wait for status

---
 .github/workflows/test-prod-e2e.yml | 40 ++++++++++++++++++++++++-----
 tooling/car/merge.go                |  4 ++-
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 5d8e2ea61..9fde485e6 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -3,13 +3,13 @@ name: Test Kubo (e2e)
 on:
   workflow_dispatch:
   push:
-    branches: 
+    branches:
       - main
   pull_request:
 
 jobs:
   upload-fixtures:
-    runs-on: 'ubuntu-latest'
+    runs-on: "ubuntu-latest"
     defaults:
       run:
         shell: bash
@@ -20,7 +20,7 @@ jobs:
           go-version: 1.20.4
       - uses: actions/checkout@v3
         with:
-          path: 'gateway-conformance'
+          path: "gateway-conformance"
       - name: Extract fixtures
         uses: ./gateway-conformance/.github/actions/extract-fixtures
         with:
@@ -28,12 +28,38 @@ jobs:
           merged: true
       # https://web3.storage/docs/how-tos/store/#storing-ipfs-content-archives
       - name: Upload fixtures
-        env: 
+        id: upload
+        env:
           W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
         run: |
-          curl --oauth2-bearer "$W3STORAGE_TOKEN" --data-binary @fixtures.car
+          response=$(curl --oauth2-bearer "$W3STORAGE_TOKEN" --data-binary @fixtures.car "https://api.web3.storage/car")
+          status=$?
+
+          if [ $status -eq 0 ]; then
+            cid=$(echo $response | jq -r .cid)
+            echo "Pinned fixtures to w3.storage with CID: $cid"
+            echo "cid=$cid" >> "$GITHUB_OUTPUT"
+          else
+            exit $status
+          fi
+      - name: Wait for pinning
+        env:
+          W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
+          CID: ${{ steps.upload.outputs.cid }}
+        run: |
+          pin_status=""
+
+          while [[ "$pin_status" != "Pinned" ]]; do
+            response=$(curl --oauth2-bearer "$W3STORAGE_TOKEN" "https://api.web3.storage/status/$cid")
+            pin_status=$(echo $response | jq -r '.pins[0].status')
+            echo "Current pin status: $pin_status"
+
+            if [[ "$pin_status" != "Pinned" ]]; then
+              sleep 10 # Sleep for 10 seconds before trying again
+            fi
+          done
   test:
-    runs-on: 'ubuntu-latest'
+    runs-on: "ubuntu-latest"
     strategy:
       fail-fast: false
       matrix:
@@ -49,7 +75,7 @@ jobs:
           go-version: 1.20.4
       - uses: actions/checkout@v3
         with:
-          path: 'gateway-conformance'
+          path: "gateway-conformance"
       - name: Run the tests
         uses: ./gateway-conformance/.github/actions/test
         with:
diff --git a/tooling/car/merge.go b/tooling/car/merge.go
index 6c00f4595..7c7a270c1 100644
--- a/tooling/car/merge.go
+++ b/tooling/car/merge.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/ipfs/go-cid"
+	carv2 "github.com/ipld/go-car/v2"
 	"github.com/ipld/go-car/v2/blockstore"
 )
 
@@ -31,7 +32,8 @@ func Merge(inputPaths []string, outputPath string) error {
 
 	// Now prepare our new CAR file
 	fmt.Printf("Opening the %s file, with roots: %v\n", outputPath, roots)
-	rout, err := blockstore.OpenReadWrite(outputPath, roots)
+	options := []carv2.Option{blockstore.WriteAsCarV1(true)}
+	rout, err := blockstore.OpenReadWrite(outputPath, roots[0:1], options...)
 	if err != nil {
 		return err
 	}

From 7f92d35c4cdfc937f26320fa138530072e564eee Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 4 Jul 2023 10:43:19 +0200
Subject: [PATCH 04/37] fix cid

---
 .github/workflows/test-prod-e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 9fde485e6..53454b74f 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -50,7 +50,7 @@ jobs:
           pin_status=""
 
           while [[ "$pin_status" != "Pinned" ]]; do
-            response=$(curl --oauth2-bearer "$W3STORAGE_TOKEN" "https://api.web3.storage/status/$cid")
+            response=$(curl --oauth2-bearer "$W3STORAGE_TOKEN" "https://api.web3.storage/status/$CID")
             pin_status=$(echo $response | jq -r '.pins[0].status')
             echo "Current pin status: $pin_status"
 

From f0322039d56b4a37e8f006f687fdd8697a83b80b Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 4 Jul 2023 10:45:46 +0200
Subject: [PATCH 05/37] ci: error catching

---
 .github/workflows/test-prod-e2e.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 53454b74f..4d5c556d2 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -32,25 +32,25 @@ jobs:
         env:
           W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
         run: |
-          response=$(curl --oauth2-bearer "$W3STORAGE_TOKEN" --data-binary @fixtures.car "https://api.web3.storage/car")
-          status=$?
+          set -e
+          set -o pipefail
 
-          if [ $status -eq 0 ]; then
-            cid=$(echo $response | jq -r .cid)
-            echo "Pinned fixtures to w3.storage with CID: $cid"
-            echo "cid=$cid" >> "$GITHUB_OUTPUT"
-          else
-            exit $status
-          fi
+          response=$(curl --fail --oauth2-bearer "$W3STORAGE_TOKEN" --data-binary @fixtures.car "https://api.web3.storage/car")
+          cid=$(echo $response | jq -r .cid)
+          echo "Pinned fixtures to w3.storage with CID: $cid"
+          echo "cid=$cid" >> "$GITHUB_OUTPUT"
       - name: Wait for pinning
         env:
           W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
           CID: ${{ steps.upload.outputs.cid }}
         run: |
+          set -e
+          set -o pipefail
+
           pin_status=""
 
           while [[ "$pin_status" != "Pinned" ]]; do
-            response=$(curl --oauth2-bearer "$W3STORAGE_TOKEN" "https://api.web3.storage/status/$CID")
+            response=$(curl --fail --oauth2-bearer "$W3STORAGE_TOKEN" "https://api.web3.storage/status/$CID")
             pin_status=$(echo $response | jq -r '.pins[0].status')
             echo "Current pin status: $pin_status"
 

From c26577f59efd7ceac7adc942ee56db4f2358f2a9 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 4 Jul 2023 10:49:16 +0200
Subject: [PATCH 06/37] ci: naming

---
 .github/workflows/test-prod-e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 4d5c556d2..1404078cb 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -1,4 +1,4 @@
-name: Test Kubo (e2e)
+name: Test Production (e2e)
 
 on:
   workflow_dispatch:

From 4adbae2b0c62ed20ddbaceda6cbda2e989a982ff Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 4 Jul 2023 10:58:05 +0200
Subject: [PATCH 07/37] test: public gateways

---
 .github/workflows/test-prod-e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 1404078cb..227d8a3a8 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -63,7 +63,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target: ["ipfs.runfission.com"]
+        target: ["ipfs.runfission.com", "w3s.link"]
     defaults:
       run:
         shell: bash

From 1aa64637a816874c9a2dd998a81d4721c7019869 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Wed, 5 Jul 2023 10:16:18 +0200
Subject: [PATCH 08/37] wip: merge car

---
 tooling/car/merge.go | 46 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/tooling/car/merge.go b/tooling/car/merge.go
index 7c7a270c1..5fdbe0271 100644
--- a/tooling/car/merge.go
+++ b/tooling/car/merge.go
@@ -7,6 +7,11 @@ import (
 	"github.com/ipfs/go-cid"
 	carv2 "github.com/ipld/go-car/v2"
 	"github.com/ipld/go-car/v2/blockstore"
+	"github.com/ipld/go-ipld-prime/fluent"
+	"github.com/ipld/go-ipld-prime/linking"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	basicnode "github.com/ipld/go-ipld-prime/node/basic"
+	"github.com/ipld/go-ipld-prime/storage/memstore"
 )
 
 func Merge(inputPaths []string, outputPath string) error {
@@ -30,18 +35,55 @@ func Merge(inputPaths []string, outputPath string) error {
 		roots = append(roots, r...)
 	}
 
+	// Then aggregate all roots under a single one
+	lsys := cidlink.DefaultLinkSystem()
+	store := memstore.Store{Bag: make(map[string][]byte)}
+	lsys.SetWriteStorage(&store)
+	lsys.SetReadStorage(&store)
+
+	node := fluent.MustBuildList(basicnode.Prototype.List, int64(len(roots)), func(na fluent.ListAssembler) {
+		for _, root := range roots {
+			na.AssembleValue().AssignLink(cidlink.Link{Cid: root})
+		}
+	})
+
+	lp := cidlink.LinkPrototype{Prefix: cid.Prefix{
+		Version:  1,    // Usually '1'.
+		Codec:    0x71, // 0x71 means "dag-cbor" -- See the multicodecs table: https://github.com/multiformats/multicodec/
+		MhType:   0x13, // 0x20 means "sha2-512" -- See the multicodecs table: https://github.com/multiformats/multicodec/
+		MhLength: 64,   // sha2-512 hash has a 64-byte sum.
+	}}
+
+	lnk, err := lsys.Store(
+		linking.LinkContext{},
+		lp,
+		node)
+	if err != nil {
+		return err
+	}
+
+	rootCid := lnk.(cidlink.Link).Cid
+
+	fmt.Printf("Root CID: %s\n", rootCid.String())
+
 	// Now prepare our new CAR file
 	fmt.Printf("Opening the %s file, with roots: %v\n", outputPath, roots)
 	options := []carv2.Option{blockstore.WriteAsCarV1(true)}
-	rout, err := blockstore.OpenReadWrite(outputPath, roots[0:1], options...)
+	rout, err := blockstore.OpenReadWrite(outputPath, []cid.Cid{rootCid}, options...)
 	if err != nil {
 		return err
 	}
 
+	// Add blocks from our store (root block)
+	// TODO: how to?
+	// for every block in our store, add it to `rout`
+	// for ever k, v in store.Bag ????
+
 	// Then aggregate all our blocks.
 	for _, path := range inputPaths {
 		fmt.Printf("processing %s\n", path)
-		robs, err := blockstore.OpenReadOnly(path,
+		robs, err := blockstore.OpenReadOnly(
+			path,
 			blockstore.UseWholeCIDs(true),
 		)
 		if err != nil {

From cda83ce72fe0417a65b6ba7e92ae188c846a1361 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Wed, 5 Jul 2023 11:02:57 +0200
Subject: [PATCH 09/37] attempt to more tweaks

---
 tooling/car/merge.go | 71 +++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 63 insertions(+), 8 deletions(-)

diff --git a/tooling/car/merge.go b/tooling/car/merge.go
index 5fdbe0271..89bc446b7 100644
--- a/tooling/car/merge.go
+++ b/tooling/car/merge.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	carv2 "github.com/ipld/go-car/v2"
 	"github.com/ipld/go-car/v2/blockstore"
@@ -47,23 +48,52 @@ func Merge(inputPaths []string, outputPath string) error {
 		}
 	})
 
+	// NOTE: by default we generate super large CIDs (from the doc)
+	// which are not compatible with dns.
+	// lp := cidlink.LinkPrototype{Prefix: cid.Prefix{
+	// 	Version:  1,    // Usually '1'.
+	// 	Codec:    0x71, // 0x71 means "dag-cbor" -- See the multicodecs table: https://github.com/multiformats/multicodec/
+	// 	MhType:   0x13, // 0x20 means "sha2-512" -- See the multicodecs table: https://github.com/multiformats/multicodec/
+	// 	MhLength: 64,   // sha2-512 hash has a 64-byte sum.
+	// }}
+	// cid: bafyrgqhpkthtuhnrvrnzobebylknmj4ayxac2f3kfm7pxm5ywmhu65ztzuyz4mmrhwf4sjliwntozivctgwk6qxiquospjybg37o4aiyvzt64
+	// So I switch to sah2-256:
+
+	// lp := cidlink.LinkPrototype{Prefix: cid.Prefix{
+	// 	Version:  1,
+	// 	Codec:    0x71,
+	// 	MhType:   0x12,  // sha2-256
+	// 	MhLength: 32,
+	// }}
+	// Trying to publish that configuratino:
+	/*
+		curl --oauth2-bearer "$W3STORAGE_TOKEN" --data-binary @fixtures.car "https://api.web3.storage/car"
+		{"code":"HTTP_ERROR","message":"protobuf: (PBNode) invalid wireType, expected 2, got 0"}%
+
+		What is that wireType? Maybe we can only publish dag-pb, so let's try.
+	*/
+
 	lp := cidlink.LinkPrototype{Prefix: cid.Prefix{
-		Version:  1,    // Usually '1'.
-		Codec:    0x71, // 0x71 means "dag-cbor" -- See the multicodecs table: https://github.com/multiformats/multicodec/
-		MhType:   0x13, // 0x20 means "sha2-512" -- See the multicodecs table: https://github.com/multiformats/multicodec/
-		MhLength: 64,   // sha2-512 hash has a 64-byte sum.
+		Version:  1,
+		Codec:    0x70, // dag-pb
+		MhType:   0x12,
+		MhLength: 32, // sha2-256
 	}}
+	/**
+	Trying to run with that code, no the code doesn't complete:
+	2023/07/05 10:57:36 func called on wrong kind: "AssignNode" called on a dagpb.PBNode node (kind: list), but only makes sense on map
+	*/
 
 	lnk, err := lsys.Store(
 		linking.LinkContext{},
 		lp,
 		node)
+
 	if err != nil {
 		return err
 	}
 
 	rootCid := lnk.(cidlink.Link).Cid
-
 	fmt.Printf("Root CID: %s\n", rootCid.String())
 
 	// Now prepare our new CAR file
@@ -71,13 +101,34 @@ func Merge(inputPaths []string, outputPath string) error {
 	options := []carv2.Option{blockstore.WriteAsCarV1(true)}
 	rout, err := blockstore.OpenReadWrite(outputPath, []cid.Cid{rootCid}, options...)
 	if err != nil {
+		fmt.Println("Error here0:", err)
 		return err
 	}
 
 	// Add blocks from our store (root block)
-	// TODO: how to?
-	// for every block in our store, add it to `rout`
-	// for ever k, v in store.Bag ????
+	for k, v := range store.Bag {
+		fmt.Println("Adding block", k)
+		c, err := cid.Parse(k)
+		if err != nil {
+			fmt.Println("Error here:", c, err)
+		}
+
+		c, err = cid.Decode(k)
+		if err != nil {
+			fmt.Println("Error here:", c, err)
+			// Adding block q@T:['-j+>Ow311=%hf좢B'6g
+			// 2023/07/05 10:24:43 invalid cid: selected encoding not supported
+			// return err
+		}
+
+		// blk, err := blocks.NewBlockWithCid(v, c)
+		// if err != nil {
+		// 	return err
+		// }
+
+		blk := blocks.NewBlock(v)
+		rout.Put(context.Background(), blk)
+	}
 
 	// Then aggregate all our blocks.
 	for _, path := range inputPaths {
@@ -87,11 +138,13 @@ func Merge(inputPaths []string, outputPath string) error {
 			blockstore.UseWholeCIDs(true),
 		)
 		if err != nil {
+			fmt.Println("Error here:1", err)
 			return err
 		}
 
 		cids, err := robs.AllKeysChan(context.Background())
 		if err != nil {
+			fmt.Println("Error here:2", err)
 			return err
 		}
 
@@ -99,6 +152,7 @@ func Merge(inputPaths []string, outputPath string) error {
 			fmt.Printf("Adding %s\n", c.String())
 			block, err := robs.Get(context.Background(), c)
 			if err != nil {
+				fmt.Println("Error here:3", err)
 				return err
 			}
 
@@ -108,5 +162,6 @@ func Merge(inputPaths []string, outputPath string) error {
 
 	fmt.Printf("Finalizing...\n")
 	err = rout.Finalize()
+
 	return err
 }

From 2e8f64863b544f1cd491e14214001364c7803df8 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Wed, 5 Jul 2023 11:14:37 +0200
Subject: [PATCH 10/37] trying out more tweaks: using maps and dag pb, no
 success

---
 tooling/car/merge.go | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/tooling/car/merge.go b/tooling/car/merge.go
index 89bc446b7..5f422d781 100644
--- a/tooling/car/merge.go
+++ b/tooling/car/merge.go
@@ -42,9 +42,30 @@ func Merge(inputPaths []string, outputPath string) error {
 	lsys.SetWriteStorage(&store)
 	lsys.SetReadStorage(&store)
 
-	node := fluent.MustBuildList(basicnode.Prototype.List, int64(len(roots)), func(na fluent.ListAssembler) {
+	// THIS was not compatible with dag pb I think,
+	// node := fluent.MustBuildList(basicnode.Prototype.List, int64(len(roots)), func(na fluent.ListAssembler) {
+	// 	for _, root := range roots {
+	// 		na.AssembleValue().AssignLink(cidlink.Link{Cid: root})
+	// 	}
+	// })
+
+	// creating list of uniq roots
+	uniqRoots := make(map[string]cid.Cid)
+	for _, root := range roots {
+		uniqRoots[root.String()] = root
+	}
+	roots = make([]cid.Cid, 0)
+	for _, root := range uniqRoots {
+		roots = append(roots, root)
+	}
+
+	// Adding to a map, they won't accept duplicate, hence the code above
+	node := fluent.MustBuildMap(basicnode.Prototype.Map, int64(len(roots)), func(ma fluent.MapAssembler) {
 		for _, root := range roots {
-			na.AssembleValue().AssignLink(cidlink.Link{Cid: root})
+			fmt.Println("adding root", root)
+			ma.AssembleEntry(root.String()).AssignLink(cidlink.Link{Cid: root})
+			// getting error:
+			// 2023/07/05 11:13:54 invalid key for map dagpb.PBNode: "bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu": no such field
 		}
 	})
 

From 90ea81249d9478a45c223f7fbe4daae5b1c9a47c Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Wed, 5 Jul 2023 17:15:41 +0200
Subject: [PATCH 11/37] more tweaks

---
 tooling/car/merge.go | 60 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 47 insertions(+), 13 deletions(-)

diff --git a/tooling/car/merge.go b/tooling/car/merge.go
index 5f422d781..fc9a7ec40 100644
--- a/tooling/car/merge.go
+++ b/tooling/car/merge.go
@@ -15,6 +15,18 @@ import (
 	"github.com/ipld/go-ipld-prime/storage/memstore"
 )
 
+// https://github.com/ipld/go-ipld-prime/blob/65bfa53512f2328d19273e471ce4fd6d964055a2/storage/bsadapter/bsadapter.go#L111C1-L120C2
+func cidFromBinString(key string) (cid.Cid, error) {
+	l, k, err := cid.CidFromBytes([]byte(key))
+	if err != nil {
+		return cid.Undef, fmt.Errorf("bsrvadapter: key was not a cid: %w", err)
+	}
+	if l != len(key) {
+		return cid.Undef, fmt.Errorf("bsrvadapter: key was not a cid: had %d bytes leftover", len(key)-l)
+	}
+	return k, nil
+}
+
 func Merge(inputPaths []string, outputPath string) error {
 	// First list all the roots in our fixtures
 	roots := make([]cid.Cid, 0)
@@ -61,12 +73,23 @@ func Merge(inputPaths []string, outputPath string) error {
 
 	// Adding to a map, they won't accept duplicate, hence the code above
 	node := fluent.MustBuildMap(basicnode.Prototype.Map, int64(len(roots)), func(ma fluent.MapAssembler) {
-		for _, root := range roots {
-			fmt.Println("adding root", root)
-			ma.AssembleEntry(root.String()).AssignLink(cidlink.Link{Cid: root})
-			// getting error:
-			// 2023/07/05 11:13:54 invalid key for map dagpb.PBNode: "bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu": no such field
-		}
+		ma.AssembleEntry("Links").CreateList(int64(len(roots)), func(na fluent.ListAssembler) {
+			for _, root := range roots {
+				na.AssembleValue().CreateMap(3, func(fma fluent.MapAssembler) {
+					// fma.AssembleEntry("Name").AssignString("")
+					// fma.AssembleEntry("Tsize").AssignInt(262158)
+					fma.AssembleEntry("Hash").AssignLink(cidlink.Link{Cid: root})
+				})
+
+			}
+		})
+
+		// this code gives:
+		// 2023/07/05 11:13:54 invalid key for map dagpb.PBNode: "bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu": no such field
+		// for _, root := range roots {
+		// 	fmt.Println("adding root", root)
+		// 	ma.AssembleEntry(root.String()).AssignLink(cidlink.Link{Cid: root})
+		// }
 	})
 
 	// NOTE: by default we generate super large CIDs (from the doc)
@@ -142,13 +165,24 @@ func Merge(inputPaths []string, outputPath string) error {
 			// return err
 		}
 
-		// blk, err := blocks.NewBlockWithCid(v, c)
-		// if err != nil {
-		// 	return err
-		// }
+		// https://github.com/ipld/go-ipld-prime/blob/65bfa53512f2328d19273e471ce4fd6d964055a2/storage/bsadapter/bsadapter.go#L87-L89
+		c, err = cidFromBinString(k)
+		if err != nil {
+			fmt.Println("Error here:", c, err)
+			// return err
+		}
 
-		blk := blocks.NewBlock(v)
-		rout.Put(context.Background(), blk)
+		blk, err := blocks.NewBlockWithCid(v, c)
+		if err != nil {
+			return err
+		}
+
+		// that call doesn't work: the cid produced is not the one we get as "root cid"
+		// blk := blocks.NewBlock(v)
+		err = rout.Put(context.Background(), blk)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Then aggregate all our blocks.
@@ -170,7 +204,7 @@ func Merge(inputPaths []string, outputPath string) error {
 		}
 
 		for c := range cids {
-			fmt.Printf("Adding %s\n", c.String())
+			// fmt.Printf("Adding %s\n", c.String())
 			block, err := robs.Get(context.Background(), c)
 			if err != nil {
 				fmt.Println("Error here:3", err)

From 363822171a0a662b6f7262b7440589dbfa974ec9 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Wed, 5 Jul 2023 17:26:25 +0200
Subject: [PATCH 12/37] go mod tidy

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index e4a132b7b..2aae65d3a 100644
--- a/go.mod
+++ b/go.mod
@@ -47,7 +47,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
-	github.com/ipfs/go-block-format v0.1.2 // indirect
+	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-datastore v0.6.0 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
 	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect

From 05ef7089b839cbb60b260e2571f367f772539ff6 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Wed, 5 Jul 2023 17:45:43 +0200
Subject: [PATCH 13/37] ci: continue after 2 minutes

---
 .github/workflows/test-prod-e2e.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 227d8a3a8..d619db361 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -48,14 +48,21 @@ jobs:
           set -o pipefail
 
           pin_status=""
+          start_time=$(date +%s)
+          timeout=$((2 * 60)) # Timeout set to 2 minutes
 
           while [[ "$pin_status" != "Pinned" ]]; do
             response=$(curl --fail --oauth2-bearer "$W3STORAGE_TOKEN" "https://api.web3.storage/status/$CID")
             pin_status=$(echo $response | jq -r '.pins[0].status')
             echo "Current pin status: $pin_status"
 
-            if [[ "$pin_status" != "Pinned" ]]; then
+            current_time=$(date +%s)
+            elapsed_time=$((current_time - start_time))
+
+            if [[ "$pin_status" != "Pinned" && $elapsed_time -lt $timeout ]]; then
               sleep 10 # Sleep for 10 seconds before trying again
+            else
+              break
             fi
           done
   test:

From ad9de3210b84bf9f635bf877ed118b790b33a659 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Wed, 5 Jul 2023 18:08:20 +0200
Subject: [PATCH 14/37] ci: generate fixtures

---
 .github/workflows/test-prod-e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index d619db361..e417f101a 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -99,5 +99,5 @@ jobs:
         if: (failure() || success())
         uses: actions/upload-artifact@v3
         with:
-          name: conformance.html
+          name: conformance-${{ matrix.target }}.html
           path: ./output.html

From 5f69a672e73f0151c023899c58385c57c9bf1a83 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 6 Jul 2023 12:14:15 +0200
Subject: [PATCH 15/37] ci: wip test artifacts

---
 .github/workflows/test-prod-e2e.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index e417f101a..ceee2bf06 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -101,3 +101,29 @@ jobs:
         with:
           name: conformance-${{ matrix.target }}.html
           path: ./output.html
+      - name: Upload JSON output
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance-${{ matrix.target }}.json
+          path: ./output.json
+  aggregate:
+    runs-on: "ubuntu-latest"
+    needs: [test]
+    strategy:
+      fail-fast: true
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Aggregate results
+        run: |
+          set -e
+          set -o pipefail
+
+          ls ./artifacts
+        working-directory: ./artifacts

From 1d7b92d59979daece73c4f23019b2b6d7851c3c6 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 6 Jul 2023 12:30:03 +0200
Subject: [PATCH 16/37] ci: run always

---
 .github/workflows/test-prod-e2e.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index ceee2bf06..045afffcd 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -68,7 +68,6 @@ jobs:
   test:
     runs-on: "ubuntu-latest"
     strategy:
-      fail-fast: false
       matrix:
         target: ["ipfs.runfission.com", "w3s.link"]
     defaults:
@@ -110,8 +109,8 @@ jobs:
   aggregate:
     runs-on: "ubuntu-latest"
     needs: [test]
-    strategy:
-      fail-fast: true
+    # the tests might have failed
+    if: always()
     defaults:
       run:
         shell: bash

From 77e185a34b5ff67f341567352dbc88fead2211d6 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 10:07:43 +0200
Subject: [PATCH 17/37] ci: tweak artifacts

---
 .github/workflows/test-prod-e2e.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 045afffcd..5ec87c176 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -124,5 +124,7 @@ jobs:
           set -e
           set -o pipefail
 
-          ls ./artifacts
+          echo $PWD
+          ls ./
+          ls ./*.json
         working-directory: ./artifacts

From d279d761db783176724d9bcdcbbffaa31ee512b8 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 10:40:30 +0200
Subject: [PATCH 18/37] ci: add aggregations

---
 .github/workflows/test-prod-e2e.yml | 31 +++++++++-
 aggregate-into-table.js             | 86 ++++++++++++++++++++++++++
 aggregate.js                        | 94 +++++++++++++++++++++++++++++
 3 files changed, 208 insertions(+), 3 deletions(-)
 create mode 100644 aggregate-into-table.js
 create mode 100644 aggregate.js

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 5ec87c176..5dde4ab39 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -70,6 +70,7 @@ jobs:
     strategy:
       matrix:
         target: ["ipfs.runfission.com", "w3s.link"]
+      fail-fast: false
     defaults:
       run:
         shell: bash
@@ -115,16 +116,40 @@ jobs:
       run:
         shell: bash
     steps:
+      - uses: actions/checkout@v3
+        with:
+          path: "gateway-conformance"
       - name: Download Artifacts
         uses: actions/download-artifact@v3
         with:
           path: artifacts
       - name: Aggregate results
+        working-directory: ./artifacts
         run: |
           set -e
           set -o pipefail
+          mkdir aggregates/
 
-          echo $PWD
           ls ./
-          ls ./*.json
-        working-directory: ./artifacts
+          ls ./*
+          ls ./*/*
+
+          # download-artifact downloads artifacts in a directory named after the artifact
+          # details: https://github.com/actions/download-artifact#download-all-artifacts
+          for folder in ./conformance-*.json; do
+            echo $folder;
+            file="${folder}/output.json"
+            new_file="aggregates/${folder#conformance-}"
+            jq -ns 'inputs' "$file" | node ../gateway-conformance/aggregate.js 1 > "${new_file}"
+          done
+
+          node ../gateway-conformance/aggregate-into-table.js ./aggregates/*.json > ./table.md
+      - name: Set summary
+        if: (failure() || success())
+        run: cat ./artifacts/table.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload one-page MD report
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance.md
+          path: ./table.md
diff --git a/aggregate-into-table.js b/aggregate-into-table.js
new file mode 100644
index 000000000..67ea9012a
--- /dev/null
+++ b/aggregate-into-table.js
@@ -0,0 +1,86 @@
+const fs = require("fs");
+
+// retrieve the list of input files from the command line
+const files = process.argv.slice(2);
+
+// read all input files (json)
+const inputs = files.map((file) => {
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+);
+
+// merge all the unique keys from all the inputs
+let keys = new Set();
+inputs.forEach((input) => {
+    Object.keys(input).forEach((key) => {
+        keys.add(key);
+    });
+});
+keys = Array.from(keys).sort();
+
+// generate a table
+const columns = [];
+
+// add the leading column ("gateway", "key1", "key2", ... "keyN")
+const leading = ["gateway"];
+keys.forEach((key) => {
+    // Skip the "Test" prefix
+    const niceKey = key.replace(/^Test/, '');
+    leading.push(niceKey);
+});
+columns.push(leading);
+
+// add the data for every input
+const cellRender = (cell) => {
+    if (cell === null) {
+        return '';
+    }
+
+    if (cell['fail'] > 0) {
+        return `:red_circle: (${cell['pass']} / ${cell['total']})`;
+    }
+    if (cell['skip'] > 0) {
+        return `:yellow_circle: (skipped)`;
+    }
+    if (cell['pass'] > 0) {
+        return `:green_circle: (${cell['pass']} / ${cell['total']})`;
+    }
+
+    throw new Error(`Unhandled cell value: ${JSON.stringify(cell)}`);
+}
+
+inputs.forEach((input, index) => {
+    // clean name (remove path and extension)
+    let name = files[index].replace(/\.json$/, '').replace(/^.*\//, '');
+
+    const col = [name];
+    keys.forEach((key) => {
+        col.push(cellRender(input[key] || null));
+    });
+    columns.push(col);
+});
+
+// # Rotate the table
+// it's easier to create the table by column, but we want to render it by row
+let rows = columns[0].map((_, i) => columns.map(col => col[i]));
+
+// # Render the table into a markdown table
+
+// add the hyphen header row after the first row
+const hyphenated = rows[0].map((x, i) => {
+    if (i === 0) {
+        return '-'.repeat(Math.max(0, x.length - 2)) + '-:'
+    }
+    return ':-' + '-'.repeat(Math.max(0, x.length - 2));
+})
+
+rows = [
+    rows[0],
+    hyphenated,
+    ...rows.slice(1),
+]
+
+let markdown = rows.map(row => '| ' + row.join(' | ') + ' |').join('\n');
+
+// output the table to stdout
+fs.writeFileSync(1, markdown);
diff --git a/aggregate.js b/aggregate.js
new file mode 100644
index 000000000..347096409
--- /dev/null
+++ b/aggregate.js
@@ -0,0 +1,94 @@
+const fs = require("fs");
+
+// # read json from stdin:
+let lines = fs.readFileSync(0, "utf-8");
+lines = JSON.parse(lines);
+
+// # clean input
+lines = lines.filter((line) => {
+  const { Test } = line;
+  return Test !== undefined;
+});
+
+lines = lines.filter((line) => {
+  const { Action } = line;
+  return ["pass", "fail", "skip"].includes(Action);
+});
+
+// # add "Path" field by parsing "Name" and split by "/"
+//   also update the name to make it readable
+//   also remove "Time" field while we're at it
+lines = lines.map((line) => {
+  const { Test, Time, ...rest } = line;
+  const path = Test.split("/").map((name) => {
+    return name.replace(/_/g, " ");
+  });
+
+  return { ...rest, Path: path };
+});
+
+// # Aggregate all known "Path" values, use a tree structure to represent it
+//   {
+//       child1: {
+//           child2: {
+//               ...,
+//           }
+//       }
+//   }
+const testTree = {};
+
+lines.forEach((line) => {
+  const { Path } = line;
+  let current = testTree;
+
+  Path.forEach((path) => {
+    if (!current[path]) {
+      current[path] = {};
+    }
+    current = current[path];
+  });
+})
+
+// # Drop all lines where the Test "Path" does not point to a leaf
+//   if the test has children then we don't really care about it's pass / fail / skip status,
+//   we'll aggregate its children results'
+lines = lines.filter((line) => {
+  const { Path } = line;
+  let current = testTree;
+
+  Path.forEach((path) => {
+    if (!current[path]) {
+      return false;
+    }
+    current = current[path];
+  });
+
+  // if current has children, it is not a leaf
+  return Object.keys(current).length === 0;
+});
+
+// # Aggregate by Path and count actions
+
+const depth = process.argv[2] && parseInt(process.argv[2], 10) || 1;
+
+// test result is a map { [path_str]: { [path], [action]: count } }
+const testResults = {};
+
+lines.forEach((line) => {
+  const { Path, Action } = line;
+  let current = testResults;
+
+  const path = Path.slice(0, depth)
+  const key = path.join(" > ");
+
+  if (!current[key]) {
+    current[key] = {Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0};
+  }
+  current = current[key];
+
+  current[Action] += 1;
+  current["total"] += 1;
+});
+
+// output result to stdout
+fs.writeFileSync(1, JSON.stringify(testResults, null, 2));

From b29dbab6b1733d9c65d9eed124ac7849a3ed9335 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 14:45:32 +0200
Subject: [PATCH 19/37] tooling: fix merge

---
 tooling/car/merge.go | 104 +++++++------------------------------------
 1 file changed, 16 insertions(+), 88 deletions(-)

diff --git a/tooling/car/merge.go b/tooling/car/merge.go
index fc9a7ec40..659aec748 100644
--- a/tooling/car/merge.go
+++ b/tooling/car/merge.go
@@ -28,9 +28,8 @@ func cidFromBinString(key string) (cid.Cid, error) {
 }
 
 func Merge(inputPaths []string, outputPath string) error {
-	// First list all the roots in our fixtures
-	roots := make([]cid.Cid, 0)
-
+	// First list all the unique roots in our fixtures
+	uniqRoots := make(map[string]cid.Cid)
 	for _, path := range inputPaths {
 		fmt.Printf("processing %s\n", path)
 		robs, err := blockstore.OpenReadOnly(path,
@@ -45,7 +44,14 @@ func Merge(inputPaths []string, outputPath string) error {
 			return err
 		}
 
-		roots = append(roots, r...)
+		for _, root := range r {
+			uniqRoots[root.String()] = root
+		}
+	}
+
+	roots := make([]cid.Cid, 0)
+	for _, root := range uniqRoots {
+		roots = append(roots, root)
 	}
 
 	// Then aggregate all roots under a single one
@@ -54,122 +60,50 @@ func Merge(inputPaths []string, outputPath string) error {
 	lsys.SetWriteStorage(&store)
 	lsys.SetReadStorage(&store)
 
-	// THIS was not compatible with dag pb I think,
-	// node := fluent.MustBuildList(basicnode.Prototype.List, int64(len(roots)), func(na fluent.ListAssembler) {
-	// 	for _, root := range roots {
-	// 		na.AssembleValue().AssignLink(cidlink.Link{Cid: root})
-	// 	}
-	// })
-
-	// creating list of uniq roots
-	uniqRoots := make(map[string]cid.Cid)
-	for _, root := range roots {
-		uniqRoots[root.String()] = root
-	}
-	roots = make([]cid.Cid, 0)
-	for _, root := range uniqRoots {
-		roots = append(roots, root)
-	}
-
-	// Adding to a map, they won't accept duplicate, hence the code above
+	// Adding to a map, they won't accept duplicate, hence the need for the uniqRoots
 	node := fluent.MustBuildMap(basicnode.Prototype.Map, int64(len(roots)), func(ma fluent.MapAssembler) {
 		ma.AssembleEntry("Links").CreateList(int64(len(roots)), func(na fluent.ListAssembler) {
 			for _, root := range roots {
 				na.AssembleValue().CreateMap(3, func(fma fluent.MapAssembler) {
-					// fma.AssembleEntry("Name").AssignString("")
-					// fma.AssembleEntry("Tsize").AssignInt(262158)
 					fma.AssembleEntry("Hash").AssignLink(cidlink.Link{Cid: root})
 				})
-
 			}
 		})
-
-		// this code gives:
-		// 2023/07/05 11:13:54 invalid key for map dagpb.PBNode: "bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu": no such field
-		// for _, root := range roots {
-		// 	fmt.Println("adding root", root)
-		// 	ma.AssembleEntry(root.String()).AssignLink(cidlink.Link{Cid: root})
-		// }
 	})
 
-	// NOTE: by default we generate super large CIDs (from the doc)
-	// which are not compatible with dns.
-	// lp := cidlink.LinkPrototype{Prefix: cid.Prefix{
-	// 	Version:  1,    // Usually '1'.
-	// 	Codec:    0x71, // 0x71 means "dag-cbor" -- See the multicodecs table: https://github.com/multiformats/multicodec/
-	// 	MhType:   0x13, // 0x20 means "sha2-512" -- See the multicodecs table: https://github.com/multiformats/multicodec/
-	// 	MhLength: 64,   // sha2-512 hash has a 64-byte sum.
-	// }}
-	// cid: bafyrgqhpkthtuhnrvrnzobebylknmj4ayxac2f3kfm7pxm5ywmhu65ztzuyz4mmrhwf4sjliwntozivctgwk6qxiquospjybg37o4aiyvzt64
-	// So I switch to sah2-256:
-
-	// lp := cidlink.LinkPrototype{Prefix: cid.Prefix{
-	// 	Version:  1,
-	// 	Codec:    0x71,
-	// 	MhType:   0x12,  // sha2-256
-	// 	MhLength: 32,
-	// }}
-	// Trying to publish that configuratino:
-	/*
-		curl --oauth2-bearer "$W3STORAGE_TOKEN" --data-binary @fixtures.car "https://api.web3.storage/car"
-		{"code":"HTTP_ERROR","message":"protobuf: (PBNode) invalid wireType, expected 2, got 0"}%
-
-		What is that wireType? Maybe we can only publish dag-pb, so let's try.
-	*/
-
 	lp := cidlink.LinkPrototype{Prefix: cid.Prefix{
 		Version:  1,
 		Codec:    0x70, // dag-pb
 		MhType:   0x12,
 		MhLength: 32, // sha2-256
 	}}
-	/**
-	Trying to run with that code, no the code doesn't complete:
-	2023/07/05 10:57:36 func called on wrong kind: "AssignNode" called on a dagpb.PBNode node (kind: list), but only makes sense on map
-	*/
 
 	lnk, err := lsys.Store(
 		linking.LinkContext{},
 		lp,
 		node)
-
 	if err != nil {
 		return err
 	}
 
 	rootCid := lnk.(cidlink.Link).Cid
-	fmt.Printf("Root CID: %s\n", rootCid.String())
 
 	// Now prepare our new CAR file
-	fmt.Printf("Opening the %s file, with roots: %v\n", outputPath, roots)
+	fmt.Printf("Opening the %s file, with root: %v\n", outputPath, rootCid)
 	options := []carv2.Option{blockstore.WriteAsCarV1(true)}
 	rout, err := blockstore.OpenReadWrite(outputPath, []cid.Cid{rootCid}, options...)
 	if err != nil {
-		fmt.Println("Error here0:", err)
+		fmt.Println("Error here:a", err)
 		return err
 	}
 
 	// Add blocks from our store (root block)
 	for k, v := range store.Bag {
-		fmt.Println("Adding block", k)
-		c, err := cid.Parse(k)
-		if err != nil {
-			fmt.Println("Error here:", c, err)
-		}
-
-		c, err = cid.Decode(k)
-		if err != nil {
-			fmt.Println("Error here:", c, err)
-			// Adding block q@T:['-j+>Ow311=%hf좢B'6g
-			// 2023/07/05 10:24:43 invalid cid: selected encoding not supported
-			// return err
-		}
-
+		// cid.Parse and cid.Decode does not work here, using:
 		// https://github.com/ipld/go-ipld-prime/blob/65bfa53512f2328d19273e471ce4fd6d964055a2/storage/bsadapter/bsadapter.go#L87-L89
-		c, err = cidFromBinString(k)
+		c, err := cidFromBinString(k)
 		if err != nil {
-			fmt.Println("Error here:", c, err)
-			// return err
+			return err
 		}
 
 		blk, err := blocks.NewBlockWithCid(v, c)
@@ -177,8 +111,6 @@ func Merge(inputPaths []string, outputPath string) error {
 			return err
 		}
 
-		// that call doesn't work: the cid produced is not the one we get as "root cid"
-		// blk := blocks.NewBlock(v)
 		err = rout.Put(context.Background(), blk)
 		if err != nil {
 			return err
@@ -193,21 +125,17 @@ func Merge(inputPaths []string, outputPath string) error {
 			blockstore.UseWholeCIDs(true),
 		)
 		if err != nil {
-			fmt.Println("Error here:1", err)
 			return err
 		}
 
 		cids, err := robs.AllKeysChan(context.Background())
 		if err != nil {
-			fmt.Println("Error here:2", err)
 			return err
 		}
 
 		for c := range cids {
-			// fmt.Printf("Adding %s\n", c.String())
 			block, err := robs.Get(context.Background(), c)
 			if err != nil {
-				fmt.Println("Error here:3", err)
 				return err
 			}
 

From 86fc45e59a1915fe3dc0b5752ac43bba9d6543f7 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 14:47:47 +0200
Subject: [PATCH 20/37] CHANGELOG: add merged note

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 18df60519..e728d1094 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - finalized port of Kubo's sharness tests. [PR](https://github.com/ipfs/gateway-conformance/pull/92)
+- `extract-fixtures --merged` generates a car version 1 with a single root now
 
 ## [0.2.0] - 2023-06-26
 ### Added

From 62dce94712e296e9ae00b52718098b58e8ab4c2a Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 14:51:28 +0200
Subject: [PATCH 21/37] ci: simplif & clean

---
 .github/workflows/test-prod-e2e.yml | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 5dde4ab39..6492e941e 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -92,9 +92,6 @@ jobs:
           xml: output.xml
           html: output.html
           markdown: output.md
-      - name: Set summary
-        if: (failure() || success())
-        run: cat ./output.md >> $GITHUB_STEP_SUMMARY
       - name: Upload one-page HTML report
         if: (failure() || success())
         uses: actions/upload-artifact@v3
@@ -128,16 +125,11 @@ jobs:
         run: |
           set -e
           set -o pipefail
-          mkdir aggregates/
-
-          ls ./
-          ls ./*
-          ls ./*/*
+          mkdir ./aggregates
 
           # download-artifact downloads artifacts in a directory named after the artifact
           # details: https://github.com/actions/download-artifact#download-all-artifacts
           for folder in ./conformance-*.json; do
-            echo $folder;
             file="${folder}/output.json"
             new_file="aggregates/${folder#conformance-}"
             jq -ns 'inputs' "$file" | node ../gateway-conformance/aggregate.js 1 > "${new_file}"

From 7e4db53df3621bae81828815931323d09babf1a7 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 16:49:34 +0200
Subject: [PATCH 22/37] link to dashboard

---
 .github/workflows/update-badge.yml | 57 ++++++++++++++++++++++++++++++
 README.md                          |  2 ++
 2 files changed, 59 insertions(+)
 create mode 100644 .github/workflows/update-badge.yml

diff --git a/.github/workflows/update-badge.yml b/.github/workflows/update-badge.yml
new file mode 100644
index 000000000..a8c17bf9c
--- /dev/null
+++ b/.github/workflows/update-badge.yml
@@ -0,0 +1,57 @@
+# based on https://github.com/libp2p/test-plans/pull/86
+name: Update Badge
+
+on:
+  workflow_run:
+    workflows:
+      - Test Production (e2e)
+    types:
+      - completed
+    branches:
+      - main
+      - feat/text-production-gateways # temporary
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  update-badge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: pl-strflt/job-summary-url-action@v1
+        name: get-metadata
+        with:
+          workflow: test-prod-e2e.yml # ${{ github.event.workflow.path }}
+          run_id: ${{ github.event.workflow_run.id }}
+          run_attempt: ${{ github.event.workflow_run.run_attempt }}
+          job: aggregate
+      - uses: actions/checkout@v3
+      # https://github.com/orgs/community/discussions/26560
+      - run: |
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config user.name "github-actions[bot]"
+      - run: |
+          GITHUB_JOB_SUMMARY_URL="${{ steps.get-metadata.outputs.job_summary_url }}"
+
+          IN='[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](.*)'
+          ESCAPED_IN=$(printf '%s\n' "$IN" | sed -e 's/[][\/!&]/\\&/g')
+
+          OUT="[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](${GITHUB_JOB_SUMMARY_URL})"
+
+          sed -i "s;${ESCAPED_IN};${OUT};" README.md
+      - run: |
+          if [[ -n $(git diff --shortstat 2> /dev/null | tail -n1) ]]; then
+            echo "GIT_DIRTY=1" >> $GITHUB_ENV
+          else
+            echo "GIT_DIRTY=0" >> $GITHUB_ENV
+          fi
+      - if: env.GIT_DIRTY == '1'
+        run: |
+          git add README.md
+          git commit -m 'chore: update the link to the dashboard [skip ci]'
+          git push
diff --git a/README.md b/README.md
index 6bbb7e873..4b0b78f1f 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,8 @@
 
 `gateway-conformance` is a tool designed to test if an IPFS Gateway implementation complies with the IPFS Gateway Specification correctly. The tool is distributed as a Docker image, as well as a GitHub Action(s).
 
+[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](something-something)
+
 ## Table of Contents
 
 - [Commands](#commands)

From 5ba0765d73d92a29e7dc44a1fe2798b44059dd9d Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 17:01:42 +0200
Subject: [PATCH 23/37] ci: simplif

---
 .github/workflows/test-prod-e2e.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 6492e941e..e7d44626a 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -138,10 +138,4 @@ jobs:
           node ../gateway-conformance/aggregate-into-table.js ./aggregates/*.json > ./table.md
       - name: Set summary
         if: (failure() || success())
-        run: cat ./artifacts/table.md >> $GITHUB_STEP_SUMMARY
-      - name: Upload one-page MD report
-        if: (failure() || success())
-        uses: actions/upload-artifact@v3
-        with:
-          name: conformance.md
-          path: ./table.md
+        run: cat ./artifacts/table.md >> $GITHUB_STEP_SUMMARY
\ No newline at end of file

From d757d0f1b9f29cb401f195b8cc24a3c85ef21830 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 17:03:02 +0200
Subject: [PATCH 24/37] attempt with completed-with-failure

---
 .github/workflows/update-badge.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/update-badge.yml b/.github/workflows/update-badge.yml
index a8c17bf9c..fa6b87a73 100644
--- a/.github/workflows/update-badge.yml
+++ b/.github/workflows/update-badge.yml
@@ -7,6 +7,7 @@ on:
       - Test Production (e2e)
     types:
       - completed
+      - completed-with-failure
     branches:
       - main
       - feat/text-production-gateways # temporary

From 84a85278ba218fad3cf134239713ff45936af733 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 17:30:51 +0200
Subject: [PATCH 25/37] tweaks

---
 .github/workflows/update-badge.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/update-badge.yml b/.github/workflows/update-badge.yml
index fa6b87a73..977560c54 100644
--- a/.github/workflows/update-badge.yml
+++ b/.github/workflows/update-badge.yml
@@ -7,10 +7,9 @@ on:
       - Test Production (e2e)
     types:
       - completed
-      - completed-with-failure
-    branches:
-      - main
-      - feat/text-production-gateways # temporary
+    # branches:
+    #   - main
+    #   - feat/text-production-gateways # temporary
 
 defaults:
   run:

From 270219cb1d9e901e2c91461b986672e84044dffc Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 18:00:24 +0200
Subject: [PATCH 26/37] tmp

---
 .github/workflows/update-badge.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/update-badge.yml b/.github/workflows/update-badge.yml
index 977560c54..dbb30e9f7 100644
--- a/.github/workflows/update-badge.yml
+++ b/.github/workflows/update-badge.yml
@@ -1,7 +1,7 @@
-# based on https://github.com/libp2p/test-plans/pull/86
 name: Update Badge
 
 on:
+  pull_request:
   workflow_run:
     workflows:
       - Test Production (e2e)
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: pl-strflt/job-summary-url-action@v1
-        name: get-metadata
+        name: metadata
         with:
           workflow: test-prod-e2e.yml # ${{ github.event.workflow.path }}
           run_id: ${{ github.event.workflow_run.id }}
@@ -36,7 +36,7 @@ jobs:
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config user.name "github-actions[bot]"
       - run: |
-          GITHUB_JOB_SUMMARY_URL="${{ steps.get-metadata.outputs.job_summary_url }}"
+          GITHUB_JOB_SUMMARY_URL="${{ steps.metadata.outputs.job_summary_url }}"
 
           IN='[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](.*)'
           ESCAPED_IN=$(printf '%s\n' "$IN" | sed -e 's/[][\/!&]/\\&/g')

From f5cc719826fd331c3c9825afbd6f1ba62d0a1961 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Tue, 25 Jul 2023 18:04:42 +0200
Subject: [PATCH 27/37] tweak

---
 .github/workflows/update-badge.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/update-badge.yml b/.github/workflows/update-badge.yml
index dbb30e9f7..c3e621b13 100644
--- a/.github/workflows/update-badge.yml
+++ b/.github/workflows/update-badge.yml
@@ -1,7 +1,6 @@
 name: Update Badge
 
 on:
-  pull_request:
   workflow_run:
     workflows:
       - Test Production (e2e)

From 49eb24361f0023de7d46bde633bbc3152facff0f Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Fri, 30 Jun 2023 17:26:46 +0200
Subject: [PATCH 28/37] feat: wip release fixtures

---
 .github/workflows/release-fixtures.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 .github/workflows/release-fixtures.yml

diff --git a/.github/workflows/release-fixtures.yml b/.github/workflows/release-fixtures.yml
new file mode 100644
index 000000000..7dc03f85f
--- /dev/null
+++ b/.github/workflows/release-fixtures.yml
@@ -0,0 +1,20 @@
+name: Release Fixtures
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: [Release]
+    types:
+      - completed
+    
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - id: generate-fixtures
+        env:
+          W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
+        uses: bash
+        run: |
+          make fixtures.car
+          curl -X POST 'https://api.web3.storage/car' # token should be in W3STORAGE_TOKEN
\ No newline at end of file

From b0a6d6b0172182db04ef67667a56928709ae5875 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 26 Jul 2023 07:28:37 +0000
Subject: [PATCH 29/37] ci: udpate badge

---
 .github/workflows/update-badge.yml | 15 +++++++++------
 README.md                          |  2 +-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/update-badge.yml b/.github/workflows/update-badge.yml
index c3e621b13..f4734515b 100644
--- a/.github/workflows/update-badge.yml
+++ b/.github/workflows/update-badge.yml
@@ -1,3 +1,5 @@
+# Note: this workflow requires the repository to give Write access to Github Workflows.
+# in Settings > Actions > General > Workflow permissions
 name: Update Badge
 
 on:
@@ -6,9 +8,8 @@ on:
       - Test Production (e2e)
     types:
       - completed
-    # branches:
-    #   - main
-    #   - feat/text-production-gateways # temporary
+    branches:
+      - main
 
 defaults:
   run:
@@ -23,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: pl-strflt/job-summary-url-action@v1
-        name: metadata
+        id: metadata
         with:
           workflow: test-prod-e2e.yml # ${{ github.event.workflow.path }}
           run_id: ${{ github.event.workflow_run.id }}
@@ -35,14 +36,16 @@ jobs:
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config user.name "github-actions[bot]"
       - run: |
-          GITHUB_JOB_SUMMARY_URL="${{ steps.metadata.outputs.job_summary_url }}"
-
+          echo GITHUB_JOB_SUMMARY_URL=${GITHUB_JOB_SUMMARY_URL}
           IN='[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](.*)'
           ESCAPED_IN=$(printf '%s\n' "$IN" | sed -e 's/[][\/!&]/\\&/g')
 
           OUT="[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](${GITHUB_JOB_SUMMARY_URL})"
 
           sed -i "s;${ESCAPED_IN};${OUT};" README.md
+        env:
+          GITHUB_JOB_SUMMARY_URL: ${{ steps.metadata.outputs.job_summary_url }}
+          REPOSITORY: ${{ github.repository }}
       - run: |
           if [[ -n $(git diff --shortstat 2> /dev/null | tail -n1) ]]; then
             echo "GIT_DIRTY=1" >> $GITHUB_ENV
diff --git a/README.md b/README.md
index 4b0b78f1f..c0db0ba61 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 `gateway-conformance` is a tool designed to test if an IPFS Gateway implementation complies with the IPFS Gateway Specification correctly. The tool is distributed as a Docker image, as well as a GitHub Action(s).
 
-[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)](something-something)
+[![Conformance Production Dashboard](https://github.com/ipfs/gateway-conformance/actions/workflows/test-prod-e2e.yml/badge.svg?branch=master)]()
 
 ## Table of Contents
 

From eb667ffe3d065839148fad364e233ba668bda0a1 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 27 Jul 2023 13:48:44 +0200
Subject: [PATCH 30/37] ci: use w3 upload

---
 .github/workflows/test-prod-e2e.yml | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index e7d44626a..49a8eb8b7 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -29,16 +29,10 @@ jobs:
       # https://web3.storage/docs/how-tos/store/#storing-ipfs-content-archives
       - name: Upload fixtures
         id: upload
-        env:
-          W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
-        run: |
-          set -e
-          set -o pipefail
-
-          response=$(curl --fail --oauth2-bearer "$W3STORAGE_TOKEN" --data-binary @fixtures.car "https://api.web3.storage/car")
-          cid=$(echo $response | jq -r .cid)
-          echo "Pinned fixtures to w3.storage with CID: $cid"
-          echo "cid=$cid" >> "$GITHUB_OUTPUT"
+        uses: web3-storage/add-to-web3@v2
+        with:
+          web3_token: ${{ secrets.W3STORAGE_TOKEN }}
+          path_to_add: 'fixtures.car'
       - name: Wait for pinning
         env:
           W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}

From 323585d791cde111bd728c115691b1a021e968e6 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 27 Jul 2023 13:59:14 +0200
Subject: [PATCH 31/37] ci: sleep instead of pinning API

---
 .github/workflows/test-prod-e2e.yml | 26 ++------------------------
 1 file changed, 2 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 49a8eb8b7..2ed31b2c2 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -34,31 +34,9 @@ jobs:
           web3_token: ${{ secrets.W3STORAGE_TOKEN }}
           path_to_add: 'fixtures.car'
       - name: Wait for pinning
-        env:
-          W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
-          CID: ${{ steps.upload.outputs.cid }}
         run: |
-          set -e
-          set -o pipefail
-
-          pin_status=""
-          start_time=$(date +%s)
-          timeout=$((2 * 60)) # Timeout set to 2 minutes
-
-          while [[ "$pin_status" != "Pinned" ]]; do
-            response=$(curl --fail --oauth2-bearer "$W3STORAGE_TOKEN" "https://api.web3.storage/status/$CID")
-            pin_status=$(echo $response | jq -r '.pins[0].status')
-            echo "Current pin status: $pin_status"
-
-            current_time=$(date +%s)
-            elapsed_time=$((current_time - start_time))
-
-            if [[ "$pin_status" != "Pinned" && $elapsed_time -lt $timeout ]]; then
-              sleep 10 # Sleep for 10 seconds before trying again
-            else
-              break
-            fi
-          done
+          sleep 180 # 3 minutes
+          # see rational in https://github.com/ipfs/gateway-conformance/pull/108#discussion_r1274628865
   test:
     runs-on: "ubuntu-latest"
     strategy:

From 64b2dcb1b28d274f1d8bbaea517dd6c572c5b5de Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 27 Jul 2023 13:59:29 +0200
Subject: [PATCH 32/37] ci: drop duplicate flags

---
 .github/workflows/test-prod-e2e.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 2ed31b2c2..67a7eedc9 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -95,8 +95,6 @@ jobs:
       - name: Aggregate results
         working-directory: ./artifacts
         run: |
-          set -e
-          set -o pipefail
           mkdir ./aggregates
 
           # download-artifact downloads artifacts in a directory named after the artifact

From 415964a5ae6d0dd75ba9f7c174ff2101b55b698e Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 27 Jul 2023 14:09:20 +0200
Subject: [PATCH 33/37] fix: drop debug log

---
 tooling/car/merge.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tooling/car/merge.go b/tooling/car/merge.go
index 659aec748..ab0ada373 100644
--- a/tooling/car/merge.go
+++ b/tooling/car/merge.go
@@ -93,7 +93,6 @@ func Merge(inputPaths []string, outputPath string) error {
 	options := []carv2.Option{blockstore.WriteAsCarV1(true)}
 	rout, err := blockstore.OpenReadWrite(outputPath, []cid.Cid{rootCid}, options...)
 	if err != nil {
-		fmt.Println("Error here:a", err)
 		return err
 	}
 

From 49fde93f48afc2408ea56502ad7cdca36b70b9d7 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 27 Jul 2023 14:08:33 +0200
Subject: [PATCH 34/37] fix: explicit permission

Co-authored-by: Piotr Galar <piotr.galar@gmail.com>
---
 .github/workflows/update-badge.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/update-badge.yml b/.github/workflows/update-badge.yml
index f4734515b..a8d36b1c9 100644
--- a/.github/workflows/update-badge.yml
+++ b/.github/workflows/update-badge.yml
@@ -1,5 +1,8 @@
 # Note: this workflow requires the repository to give Write access to Github Workflows.
 # in Settings > Actions > General > Workflow permissions
+
+permissions:
+  contents: write
 name: Update Badge
 
 on:

From c8a6a38e5c957d046047479fa7e7499196a8173f Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 27 Jul 2023 14:10:10 +0200
Subject: [PATCH 35/37] ci: use output instead of env

Co-authored-by: Piotr Galar <piotr.galar@gmail.com>
---
 .github/workflows/update-badge.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/update-badge.yml b/.github/workflows/update-badge.yml
index a8d36b1c9..4c647c439 100644
--- a/.github/workflows/update-badge.yml
+++ b/.github/workflows/update-badge.yml
@@ -49,13 +49,14 @@ jobs:
         env:
           GITHUB_JOB_SUMMARY_URL: ${{ steps.metadata.outputs.job_summary_url }}
           REPOSITORY: ${{ github.repository }}
-      - run: |
+      - id: git
+         run: |
           if [[ -n $(git diff --shortstat 2> /dev/null | tail -n1) ]]; then
-            echo "GIT_DIRTY=1" >> $GITHUB_ENV
+            echo "dirty=1" >> $GITHUB_OUTPUT
           else
-            echo "GIT_DIRTY=0" >> $GITHUB_ENV
+            echo "dirty=0" >> $GITHUB_OUTPUT
           fi
-      - if: env.GIT_DIRTY == '1'
+      - if: steps.git.outputs.dirty == '1'
         run: |
           git add README.md
           git commit -m 'chore: update the link to the dashboard [skip ci]'

From 3d60a150e2333b5e3910b714595ea47a58278109 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 27 Jul 2023 14:25:06 +0200
Subject: [PATCH 36/37] ci: use w3 storage actions

---
 .github/workflows/release-fixtures.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release-fixtures.yml b/.github/workflows/release-fixtures.yml
index 7dc03f85f..87e1ce904 100644
--- a/.github/workflows/release-fixtures.yml
+++ b/.github/workflows/release-fixtures.yml
@@ -11,10 +11,13 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - id: generate-fixtures
+      - name: Ganerate Fixtures
         env:
           W3STORAGE_TOKEN: ${{ secrets.W3STORAGE_TOKEN }}
-        uses: bash
         run: |
           make fixtures.car
-          curl -X POST 'https://api.web3.storage/car' # token should be in W3STORAGE_TOKEN
\ No newline at end of file
+      - name: Upload fixtures
+        uses: web3-storage/add-to-web3@v2
+        with:
+          web3_token: ${{ secrets.W3STORAGE_TOKEN }}
+          path_to_add: 'fixtures.car'

From cfd4188c9409a3d7cdb36aca393ae0fcfa528096 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Thu, 27 Jul 2023 14:53:56 +0200
Subject: [PATCH 37/37] ci: concurrency group

---
 .github/workflows/test-prod-e2e.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/test-prod-e2e.yml b/.github/workflows/test-prod-e2e.yml
index 67a7eedc9..8819deec2 100644
--- a/.github/workflows/test-prod-e2e.yml
+++ b/.github/workflows/test-prod-e2e.yml
@@ -7,6 +7,10 @@ on:
       - main
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   upload-fixtures:
     runs-on: "ubuntu-latest"