diff --git a/config/gateway.go b/config/gateway.go index 8ae312b59ae..816b1f48d23 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -1,6 +1,9 @@ package config -const DefaultInlineDNSLink = false +const ( + DefaultInlineDNSLink = false + DefaultDeserializedResponses = true +) type GatewaySpec struct { // Paths is explicit list of path prefixes that should be handled by @@ -25,6 +28,11 @@ type GatewaySpec struct { // (FQDN) into a single DNS label in order to interop with wildcard TLS certs // and Origin per CID isolation provided by rules like https://publicsuffix.org InlineDNSLink Flag + + // DeserializedResponses configures this gateway to respond to deserialized + // responses. Disabling this option enables a Trustless Gateway, as per: + // https://specs.ipfs.tech/http-gateways/trustless-gateway/. + DeserializedResponses Flag } // Gateway contains options for the HTTP gateway server. @@ -56,6 +64,12 @@ type Gateway struct { // This flag can be overridden per FQDN in PublicGateways. NoDNSLink bool + // DeserializedResponses configures this gateway to respond to deserialized + // requests. Disabling this option enables a Trustless only gateway, as per: + // https://specs.ipfs.tech/http-gateways/trustless-gateway/. This can + // be overridden per FQDN in PublicGateways. + DeserializedResponses Flag + // PublicGateways configures behavior of known public gateways. // Each key is a fully qualified domain name (FQDN). PublicGateways map[string]*GatewaySpec diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index a9c42f18573..0f86d4da7b1 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -28,22 +28,11 @@ import ( func GatewayOption(paths ...string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - cfg, err := n.Repo.Config() + gwConfig, err := getGatewayConfig(n) if err != nil { return nil, err } - headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders)) - for h, v := range cfg.Gateway.HTTPHeaders { - headers[http.CanonicalHeaderKey(h)] = v - } - - gateway.AddAccessControlHeaders(headers) - - gwConfig := gateway.Config{ - Headers: headers, - } - gwAPI, err := newGatewayBackend(n) if err != nil { return nil, err @@ -65,7 +54,7 @@ func GatewayOption(paths ...string) ServeOption { func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - cfg, err := n.Repo.Config() + gwConfig, err := getGatewayConfig(n) if err != nil { return nil, err } @@ -75,9 +64,8 @@ func HostnameOption() ServeOption { return nil, err } - publicGateways := convertPublicGateways(cfg.Gateway.PublicGateways) childMux := http.NewServeMux() - mux.HandleFunc("/", gateway.WithHostname(childMux, gwAPI, publicGateways, cfg.Gateway.NoDNSLink).ServeHTTP) + mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP) return childMux, nil } } @@ -212,30 +200,49 @@ var defaultKnownGateways = map[string]*gateway.Specification{ "localhost": subdomainGatewaySpec, } -func convertPublicGateways(publicGateways map[string]*config.GatewaySpec) map[string]*gateway.Specification { - gws := map[string]*gateway.Specification{} +func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) { + cfg, err := n.Repo.Config() + if err != nil { + return gateway.Config{}, err + } + + // Parse configuration headers and add the default Access Control Headers. + headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders)) + for h, v := range cfg.Gateway.HTTPHeaders { + headers[http.CanonicalHeaderKey(h)] = v + } + gateway.AddAccessControlHeaders(headers) + + // Initialize gateway configuration, with empty PublicGateways, handled after. + gwCfg := gateway.Config{ + Headers: headers, + DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses), + NoDNSLink: cfg.Gateway.NoDNSLink, + PublicGateways: map[string]*gateway.Specification{}, + } - // First, implicit defaults such as subdomain gateway on localhost + // Add default implicit known gateways, such as subdomain gateway on localhost. for hostname, gw := range defaultKnownGateways { - gws[hostname] = gw + gwCfg.PublicGateways[hostname] = gw } - // Then apply values from Gateway.PublicGateways, if present in the config - for hostname, gw := range publicGateways { + // Apply values from cfg.Gateway.PublicGateways if they exist. + for hostname, gw := range cfg.Gateway.PublicGateways { if gw == nil { // Remove any implicit defaults, if present. This is useful when one - // wants to disable subdomain gateway on localhost etc. - delete(gws, hostname) + // wants to disable subdomain gateway on localhost, etc. + delete(gwCfg.PublicGateways, hostname) continue } - gws[hostname] = &gateway.Specification{ - Paths: gw.Paths, - NoDNSLink: gw.NoDNSLink, - UseSubdomains: gw.UseSubdomains, - InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink), + gwCfg.PublicGateways[hostname] = &gateway.Specification{ + Paths: gw.Paths, + NoDNSLink: gw.NoDNSLink, + UseSubdomains: gw.UseSubdomains, + InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink), + DeserializedResponses: gw.DeserializedResponses.WithDefault(gwCfg.DeserializedResponses), } } - return gws + return gwCfg, nil } diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index 522d92f13c6..cfc2451376a 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -14,6 +14,7 @@ import ( core "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" repo "github.com/ipfs/kubo/repo" + "github.com/stretchr/testify/assert" iface "github.com/ipfs/boxo/coreiface" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" @@ -173,3 +174,42 @@ func TestVersion(t *testing.T) { t.Fatalf("response doesn't contain protocol version:\n%s", s) } } + +func TestDeserializedResponsesInheritance(t *testing.T) { + for _, testCase := range []struct { + globalSetting config.Flag + gatewaySetting config.Flag + expectedGatewaySetting bool + }{ + {config.True, config.Default, true}, + {config.False, config.Default, false}, + {config.False, config.True, true}, + {config.True, config.False, false}, + } { + c := config.Config{ + Identity: config.Identity{ + PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node + }, + Gateway: config.Gateway{ + DeserializedResponses: testCase.globalSetting, + PublicGateways: map[string]*config.GatewaySpec{ + "example.com": { + DeserializedResponses: testCase.gatewaySetting, + }, + }, + }, + } + r := &repo.Mock{ + C: c, + D: syncds.MutexWrap(datastore.NewMapDatastore()), + } + n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r}) + assert.NoError(t, err) + + gwCfg, err := getGatewayConfig(n) + assert.NoError(t, err) + + assert.Contains(t, gwCfg.PublicGateways, "example.com") + assert.Equal(t, testCase.expectedGatewaySetting, gwCfg.PublicGateways["example.com"].DeserializedResponses) + } +} diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index b4af6168b3b..d4a69514ff5 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -7,6 +7,7 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) + - [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -29,6 +30,32 @@ enabled. With this update, the same level of robustness is applied to peers that lack mDNS peers and solely rely on the public DHT. + +#### `Gateway.DeserializedResponses` config flag + +This release introduces the +[`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses) +configuration flag. + +With this flag, one can explicitly configure whether the gateway responds to +deserialized requests or not. By default, this flag is enabled. + +Disabling deserialized responses allows the +gateway to operate +as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) +limited to three [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) +response types: +[application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw), +[application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car), +and [application/vnd.ipfs.ipns-record](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record). + +With deserialized responses disabled, the Kubo gateway can serve as a block +backend for other software (like +[bifrost-gateway](https://github.com/ipfs/bifrost-gateway#readme), +[IPFS in Chromium](https://github.com/little-bear-labs/ipfs-chromium/blob/main/README.md) +etc) without the usual risks associated with hosting deserialized data behind +third-party CIDs. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index c571407e8f5..bf2b7750c64 100644 --- a/docs/config.md +++ b/docs/config.md @@ -50,6 +50,7 @@ config file at runtime. - [`Gateway`](#gateway) - [`Gateway.NoFetch`](#gatewaynofetch) - [`Gateway.NoDNSLink`](#gatewaynodnslink) + - [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses) - [`Gateway.HTTPHeaders`](#gatewayhttpheaders) - [`Gateway.RootRedirect`](#gatewayrootredirect) - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold) @@ -60,6 +61,7 @@ config file at runtime. - [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains) - [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink) - [`Gateway.PublicGateways: InlineDNSLink`](#gatewaypublicgateways-inlinednslink) + - [`Gateway.PublicGateways: DeserializedResponses`](#gatewaypublicgateways-deserializedresponses) - [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways) - [`Gateway` recipes](#gateway-recipes) - [`Identity`](#identity) @@ -236,7 +238,7 @@ documented in `ipfs config profile --help`. smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in your IPFS node, and disk usage is more critical than performance, consider using `flatfs`. - - This datastore uses up to several gigabytes of memory. + - This datastore uses up to several gigabytes of memory. - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. - The current implementation is based on old badger 1.x which is no longer supported by the upstream team. @@ -646,6 +648,16 @@ Default: `false` Type: `bool` +#### `Gateway.DeserializedResponses` + +An optional flag to explicitly configure whether this gateway responds to deserialized +requests, or not. By default, it is enabled. When disabling this option, the gateway +operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/. + +Default: `true` + +Type: `flag` + ### `Gateway.HTTPHeaders` Headers to set on gateway responses. @@ -790,6 +802,16 @@ Default: `false` Type: `flag` +#### `Gateway.PublicGateways: DeserializedResponses` + +An optional flag to explicitly configure whether this gateway responds to deserialized +requests, or not. By default, it is enabled. When disabling this option, the gateway +operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/. + +Default: same as global `Gateway.DeserializedResponses` + +Type: `flag` + #### Implicit defaults of `Gateway.PublicGateways` Default entries for `localhost` hostname and loopback IPs are always present. @@ -895,7 +917,7 @@ Type: `string` (base64 encoded) ## `Internal` -This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo. +This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo. **Be aware that making informed change here requires in-depth knowledge and most users should leave these untouched. All knobs listed here are subject to breaking changes between versions.** @@ -971,7 +993,7 @@ Type: `optionalInteger` (byte count, `null` means default which is 1MB) ### `Internal.Bitswap.ProviderSearchDelay` This parameter determines how long to wait before looking for providers outside of bitswap. -Other routing systems like the DHT are able to provide results in less than a second, so lowering +Other routing systems like the DHT are able to provide results in less than a second, so lowering this number will allow faster peers lookups in some cases. Type: `optionalDuration` (`null` means default which is 1s) @@ -1552,7 +1574,7 @@ another node, even if this other node is on a different network. This may trigger netscan alerts on some hosting providers or cause strain in some setups. The `server` configuration profile fills up this list with sensible defaults, -preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, +preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, which is the multiaddress representation of `192.168.0.0/16`) but you should always check settings against your own network and/or hosting provider. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 9de8c6579d8..e21648bd45b 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 + github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 11d27481621..d6dd8467980 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4= -github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0= +github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 2799171dc9b..006c3ab58e5 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 + github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index c9c874dc5a4..2717767c09c 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4= -github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0= +github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/test/cli/gateway_test.go b/test/cli/gateway_test.go index 972544ae001..eb955b593b3 100644 --- a/test/cli/gateway_test.go +++ b/test/cli/gateway_test.go @@ -513,4 +513,87 @@ func TestGateway(t *testing.T) { }) }) }) + + t.Run("DeserializedResponses", func(t *testing.T) { + type testCase struct { + globalValue config.Flag + gatewayValue config.Flag + deserializedGlobalStatusCode int + deserializedGatewayStaticCode int + message string + } + + setHost := func(r *http.Request) { + r.Host = "example.com" + } + + withAccept := func(accept string) func(r *http.Request) { + return func(r *http.Request) { + r.Header.Set("Accept", accept) + } + } + + withHostAndAccept := func(accept string) func(r *http.Request) { + return func(r *http.Request) { + setHost(r) + withAccept(accept)(r) + } + } + + makeTest := func(test *testCase) func(t *testing.T) { + return func(t *testing.T) { + t.Parallel() + + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Gateway.DeserializedResponses = test.globalValue + cfg.Gateway.PublicGateways = map[string]*config.GatewaySpec{ + "example.com": { + Paths: []string{"/ipfs", "/ipns"}, + DeserializedResponses: test.gatewayValue, + }, + } + }) + node.StartDaemon() + + cidFoo := node.IPFSAddStr("foo") + client := node.GatewayClient() + + deserializedPath := "/ipfs/" + cidFoo + + blockPath := deserializedPath + "?format=raw" + carPath := deserializedPath + "?format=car" + + // Global Check (Gateway.DeserializedResponses) + assert.Equal(t, http.StatusOK, client.Get(blockPath).StatusCode) + assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.raw")).StatusCode) + + assert.Equal(t, http.StatusOK, client.Get(carPath).StatusCode) + assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.car")).StatusCode) + + assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath).StatusCode) + assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath, withAccept("application/json")).StatusCode) + + // Public Gateway (example.com) Check (Gateway.PublicGateways[example.com].DeserializedResponses) + assert.Equal(t, http.StatusOK, client.Get(blockPath, setHost).StatusCode) + assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.raw")).StatusCode) + + assert.Equal(t, http.StatusOK, client.Get(carPath, setHost).StatusCode) + assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.car")).StatusCode) + + assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, setHost).StatusCode) + assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, withHostAndAccept("application/json")).StatusCode) + + } + } + + for _, test := range []*testCase{ + {config.True, config.Default, http.StatusOK, http.StatusOK, "when Gateway.DeserializedResponses is globally enabled, leaving implicit default for Gateway.PublicGateways[example.com] should inherit the global setting (enabled)"}, + {config.False, config.Default, http.StatusNotAcceptable, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally disabled, leaving implicit default on Gateway.PublicGateways[example.com] should inherit the global setting (disabled)"}, + {config.False, config.True, http.StatusNotAcceptable, http.StatusOK, "when Gateway.DeserializedResponses is globally disabled, explicitly enabling on Gateway.PublicGateways[example.com] should override global (enabled)"}, + {config.True, config.False, http.StatusOK, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally enabled, explicitly disabling on Gateway.PublicGateways[example.com] should override global (disabled)"}, + } { + t.Run(test.message, makeTest(test)) + } + }) }