Skip to content

Commit

Permalink
feat: CAR order and dups parameters from IPIP-412 (#87)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hacdias @laurentsenta
hacdias authored and laurentsenta committed Jul 26, 2023
1 parent eee384f commit a462b4d
Show file tree
Hide file tree
Showing 8 changed files with 247 additions and 36 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ provision-cargateway: ./fixtures.car
car -c ./fixtures.car &

provision-kubo:
find ./fixtures -name '*.car' -exec ipfs dag import {} \;
find ./fixtures -name '*.car' -exec ipfs dag import --stats --pin-roots=false {} \;
find ./fixtures -name '*.ipns-record' -exec sh -c 'ipfs routing put --allow-offline /ipns/$$(basename -s .ipns-record "{}") "{}"' \;

# tools
Expand Down
29 changes: 29 additions & 0 deletions fixtures/trustless_gateway_car/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,33 @@ ipfs dag export $CID > file-3k-and-3-blocks.car
REMOVE_BLOCK=$(ipfs dag get $CID | jq '.Links[1].Hash["/"]' -r)
echo $REMOVE_BLOCK | car filter --version 1 --inverse ./file-3k-and-3-blocks.car ./file-3k-and-3-blocks-missing-block.car
ipfs pin rm $CID; ipfs repo gc
# First and third outputted CIDs are used in the missing blocks tests.
ipfs dag get $CID | jq .Links | jq -r '.[].Hash."/"'
```

### [dir-with-duplicate-files.car](./dir-with-duplicate-files.car)

```sh
ipfs version
# ipfs version 0.21.0
TEXT=$(cat <<-EOF
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc non imperdiet nunc. Proin ac quam ut nibh eleifend aliquet. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Sed ligula dolor, imperdiet sagittis arcu et, semper tincidunt urna. Donec et tempor augue, quis sollicitudin metus. Curabitur semper ullamcorper aliquet. Mauris hendrerit sodales lectus eget fermentum. Proin sollicitudin vestibulum commodo. Vivamus nec lectus eu augue aliquet dignissim nec condimentum justo. In hac habitasse platea dictumst. Mauris vel sem neque.
Vivamus finibus, enim at lacinia semper, arcu erat gravida lacus, sit amet gravida magna orci sit amet est. Sed non leo lacus. Nullam viverra ipsum a tincidunt dapibus. Nulla pulvinar ligula sit amet ante ultrices tempus. Proin purus urna, semper sed lobortis quis, gravida vitae ipsum. Aliquam mi urna, pulvinar eu bibendum quis, convallis ac dolor. In gravida justo sed risus ullamcorper, vitae luctus massa hendrerit. Pellentesque habitant amet.
EOF
)

ASCII_CID=$(echo "hello application/vnd.ipld.car" | ipfs add --cid-version=1 -q)
HELLO_CID=$(echo "hello world" | ipfs add --cid-version=1 -q)
MULTIBLOCK_CID=$(echo -n $TEXT | ipfs add --cid-version=1 --chunker=size-256 -q)
# Print the Multiblock CIDs (required for some tests)
ipfs dag get $MULTIBLOCK_CID | jq .Links | jq -r '.[].Hash."/"'
ipfs files mkdir -p --cid-version 1 /dir-with-duplicate-files
ipfs files cp /ipfs/$ASCII_CID /dir-with-duplicate-files/ascii-copy.txt
ipfs files cp /ipfs/$ASCII_CID /dir-with-duplicate-files/ascii.txt
ipfs files cp /ipfs/$HELLO_CID /dir-with-duplicate-files/hello.txt
ipfs files cp /ipfs/$MULTIBLOCK_CID /dir-with-duplicate-files/multiblock.txt
ipfs files ls -l
# Manually CID of "dir-with-duplicate-files" and then...
ipfs dag export $CID
```
Binary file added fixtures/trustless_gateway_car/dir-with-duplicate-files.car
View file
Binary file not shown.
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ module github.com/ipfs/gateway-conformance
go 1.20

require (
github.com/ipfs/boxo v0.10.0
github.com/ipfs/boxo v0.10.3-0.20230724084731-f6b448b4263a
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-unixfsnode v1.7.1
github.com/ipld/go-car v0.6.1
github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d
github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33
github.com/ipld/go-codec-dagpb v1.6.0
github.com/ipld/go-ipld-prime v0.20.0
github.com/libp2p/go-libp2p v0.26.3
Expand All @@ -22,6 +22,7 @@ require (
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
github.com/ipfs/go-blockservice v0.5.0 // indirect
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
Expand All @@ -32,7 +33,6 @@ require (
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/multiformats/go-multiaddr v0.8.0 // indirect
github.com/multiformats/go-multistream v0.4.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
Expand Down
11 changes: 6 additions & 5 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,13 @@ github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfm
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY=
github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
github.com/ipfs/boxo v0.10.3-0.20230724084731-f6b448b4263a h1:eDO++KVTMF7wNqHHhNCZnW/Ocf2K6zCSizAbWoIMREo=
github.com/ipfs/boxo v0.10.3-0.20230724084731-f6b448b4263a/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
Expand Down Expand Up @@ -107,8 +109,8 @@ github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvT
github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
github.com/ipld/go-car v0.6.1 h1:blWbEHf1j62JMWFIqWE//YR0m7k5ZMw0AuUOU5hjrH8=
github.com/ipld/go-car v0.6.1/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d h1:22g+x1tgWSXK34i25qjs+afr7basaneEkHaglBshd2g=
github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d/go.mod h1:SH2pi/NgfGBsV/CGBAQPxMfghIgwzbh5lQ2N+6dNRI8=
github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 h1:0OZwzSYWIuiKEOXd/2vm5cMcEmmGLFn+1h6lHELCm3s=
github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg=
github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
Expand Down Expand Up @@ -193,7 +195,6 @@ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
Expand Down
172 changes: 170 additions & 2 deletions tests/trustless_gateway_car_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,9 @@ func TestTrustlessCarEntityBytes(t *testing.T) {
IgnoreRoots().
HasBlocks(
missingBlockFixture.MustGetCid(),
missingBlockFixture.MustGetChildrenCids()[0],
// This CID is defined at the SPEC level
// See the recipe for `file-3k-and-3-blocks-missing-block.car`
"QmPKt7ptM2ZYSGPUc8PmPT2VBkLDK3iqpG9TBJY7PCE9rF",
).
Exactly(),
),
Expand All @@ -423,7 +425,9 @@ func TestTrustlessCarEntityBytes(t *testing.T) {
IgnoreRoots().
HasBlocks(
missingBlockFixture.MustGetCid(),
missingBlockFixture.MustGetChildrenCids()[2],
// This CID is defined at the SPEC level
// See the recipe for `file-3k-and-3-blocks-missing-block.car`
"QmWXY482zQdwecnfBsj78poUUuPXvyw2JAFAEMw4tzTavV",
).
Exactly(),
),
Expand Down Expand Up @@ -629,6 +633,170 @@ func TestTrustlessCarEntityBytes(t *testing.T) {
RunWithSpecs(t, helpers.StandardCARTestTransforms(t, tests), specs.TrustlessGatewayCAR)
}

func TestTrustlessCarOrderAndDuplicates(t *testing.T) {
dirWithDuplicateFiles := car.MustOpenUnixfsCar("trustless_gateway_car/dir-with-duplicate-files.car")
// This array is defined at the SPEC level and should not depend on library behavior
// See the recipe for `dir-with-duplicate-files.car`
multiblockCIDs := []string{
"bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm",
"bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq",
"bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue",
"bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe",
"bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm",
}

tests := SugarTests{
{
Name: "GET CAR with order=dfs and dups=y of UnixFS Directory With Duplicate Files",
Hint: `
The response MUST contain all the blocks found during traversal even if they
are duplicate. In this test, a directory that contains duplicate files is
requested. The blocks corresponding to the duplicate files must be returned.
`,
Request: Request().
Path("/ipfs/{{cid}}", dirWithDuplicateFiles.MustGetCid()).
Header("Accept", "application/vnd.ipld.car; version=1; order=dfs; dups=y"),
Response: Expect().
Status(200).
Headers(
Header("Content-Type").Contains("application/vnd.ipld.car"),
Header("Content-Type").Contains("order=dfs"),
Header("Content-Type").Contains("dups=y"),
).
Body(
IsCar().
IgnoreRoots().
HasBlock(dirWithDuplicateFiles.MustGetCid()).
HasBlock(dirWithDuplicateFiles.MustGetCid("ascii.txt")). // ascii.txt = ascii-copy.txt
HasBlock(dirWithDuplicateFiles.MustGetCid("ascii-copy.txt")).
HasBlock(dirWithDuplicateFiles.MustGetCid("hello.txt")).
HasBlock(dirWithDuplicateFiles.MustGetCid("multiblock.txt")).
HasBlocks(multiblockCIDs...).
Exactly().
InThatOrder(),
),
},
{
Name: "GET CAR with order=dfs and dups=n of UnixFS Directory With Duplicate Files",
Hint: `
The response MUST NOT contain duplicate blocks. Tested
directory contains duplicate files. The blocks corresponding to
the duplicate files must be returned only ONCE.
`,
Request: Request().
Path("/ipfs/{{cid}}", dirWithDuplicateFiles.MustGetCid()).
Header("Accept", "application/vnd.ipld.car; version=1; order=dfs; dups=n"),
Response: Expect().
Status(200).
Headers(
Header("Content-Type").Contains("application/vnd.ipld.car"),
Header("Content-Type").Contains("order=dfs"),
Header("Content-Type").Contains("dups=n"),
).
Body(
IsCar().
IgnoreRoots().
HasBlock(dirWithDuplicateFiles.MustGetCid()).
HasBlock(dirWithDuplicateFiles.MustGetCid("ascii.txt")). // ascii.txt = ascii-copy.txt
HasBlock(dirWithDuplicateFiles.MustGetCid("hello.txt")).
HasBlock(dirWithDuplicateFiles.MustGetCid("multiblock.txt")).
HasBlocks(multiblockCIDs...).
Exactly().
InThatOrder(),
),
},
{
Name: "GET CAR smoke-test with order=unk of UnixFS Directory",
Hint: `
The order=unk is usually used by gateway to explicitly indicate
it does not guarantee any block order. In this case, we use it
for basic smoke-test to confirm support of IPIP-412. The
response for request with explicit order=unk MUST include an
explicit order in returned Content-Type and contain all the
blocks required to construct the requested CID. However, the
gateway is free to return default ordering of own choosing,
which means the returned blocks can be in any order and
duplicates MAY occur.
`,
Request: Request().
Path("/ipfs/{{cid}}", dirWithDuplicateFiles.MustGetCid()).
Header("Accept", "application/vnd.ipld.car; version=1; order=unk"),
Response: Expect().
Status(200).
Headers(
Header("Content-Type").Contains("application/vnd.ipld.car"),
Header("Content-Type").Contains("order="),
).
Body(
IsCar().
IgnoreRoots().
HasBlock(dirWithDuplicateFiles.MustGetCid()).
HasBlock(dirWithDuplicateFiles.MustGetCid("ascii.txt")).
HasBlock(dirWithDuplicateFiles.MustGetCid("ascii-copy.txt")).
HasBlock(dirWithDuplicateFiles.MustGetCid("hello.txt")).
HasBlock(dirWithDuplicateFiles.MustGetCid("multiblock.txt")).
HasBlocks(multiblockCIDs...),
),
},
{
Name: "GET CAR with order=dfs and dups=y of identity CID",
Hint: `
Identity hashes MUST never be manifested as read blocks.
These are virtual ones and even when dups=y is set, they never
should be returned in CAR response body.
`,
Request: Request().
Path("/ipfs/{{cid}}", "bafkqaf3imvwgy3zaneqgc3janfxgy2lomvscay3jmqfa").
Header("Accept", "application/vnd.ipld.car; dups=y"),
Response: Expect().
Status(200).
Headers(
Header("Content-Type").Contains("application/vnd.ipld.car"),
Header("Content-Type").Contains("dups=y"),
).
Body(
IsCar().
IgnoreRoots().
Exactly().
InThatOrder(),
),
},
{
Name: "GET CAR with Accept and ?format, specific Accept header is prioritized",
Hint: `
The response MUST contain all the blocks found during traversal even if they
are duplicate. In this test, a directory that contains duplicate files is
requested. The blocks corresponding to the duplicate files must be returned.
`,
Request: Request().
Path("/ipfs/{{cid}}", dirWithDuplicateFiles.MustGetCid()).
Query("format", "car").
Header("Accept", "application/vnd.ipld.car; version=1; order=dfs; dups=y"),
Response: Expect().
Status(200).
Headers(
Header("Content-Type").Contains("application/vnd.ipld.car"),
Header("Content-Type").Contains("order=dfs"),
Header("Content-Type").Contains("dups=y"),
).
Body(
IsCar().
IgnoreRoots().
HasBlock(dirWithDuplicateFiles.MustGetCid()).
HasBlock(dirWithDuplicateFiles.MustGetCid("ascii.txt")). // ascii.txt = ascii-copy.txt
HasBlock(dirWithDuplicateFiles.MustGetCid("ascii-copy.txt")).
HasBlock(dirWithDuplicateFiles.MustGetCid("hello.txt")).
HasBlock(dirWithDuplicateFiles.MustGetCid("multiblock.txt")).
HasBlocks(multiblockCIDs...).
Exactly().
InThatOrder(),
),
},
}

RunWithSpecs(t, tests, specs.TrustlessGatewayCAROptional)
}

func flattenStrings(t *testing.T, values ...interface{}) []string {
var res []string
for _, v := range values {
Expand Down
31 changes: 21 additions & 10 deletions tooling/ipns/record.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,32 @@ import (
"time"

"github.com/ipfs/boxo/ipns"
ipns_pb "github.com/ipfs/boxo/ipns/pb"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p/core/peer"
mbase "github.com/multiformats/go-multibase"
"github.com/multiformats/go-multicodec"
)

type IpnsRecord struct {
pb *ipns_pb.IpnsEntry
rec *ipns.Record
key string
id peer.ID
value string
name ipns.Name
validity time.Time
}

func UnmarshalIpnsRecord(data []byte, pubKey string) (*IpnsRecord, error) {
pb, err := ipns.UnmarshalIpnsEntry(data)
pb, err := ipns.UnmarshalRecord(data)
if err != nil {
return nil, err
}

validity, err := ipns.GetEOL(pb)
validity, err := pb.Validity()
if err != nil {
return nil, err
}

value, err := pb.Value()
if err != nil {
return nil, err
}
Expand All @@ -35,11 +40,17 @@ func UnmarshalIpnsRecord(data []byte, pubKey string) (*IpnsRecord, error) {
return nil, err
}

return &IpnsRecord{pb: pb, key: pubKey, id: id, validity: validity}, nil
return &IpnsRecord{
rec: pb,
key: pubKey,
name: ipns.NameFromPeer(id),
validity: validity,
value: value.String(),
}, nil
}

func (i *IpnsRecord) Value() string {
return string(i.pb.Value)
return i.value
}

func (i *IpnsRecord) Key() string {
Expand All @@ -51,11 +62,11 @@ func (i *IpnsRecord) Validity() time.Time {
}

func (i *IpnsRecord) Valid() error {
return ipns.ValidateWithPeerID(i.id, i.pb)
return ipns.ValidateWithName(i.rec, i.name)
}

func (i *IpnsRecord) idV1(codec multicodec.Code, base mbase.Encoding) (string, error) {
c := peer.ToCid(i.id)
c := i.name.Cid()
c = cid.NewCidV1(uint64(codec), c.Hash())
s, err := c.StringOfBase(base)
if err != nil {
Expand Down Expand Up @@ -85,5 +96,5 @@ func (i *IpnsRecord) IdV1() string {
}

func (i *IpnsRecord) B58MH() string {
return i.id.String()
return i.name.Peer().String()
}
Loading

0 comments on commit a462b4d

Please sign in to comment.