Skip to content

Commit

Permalink
feat: Gateway.DeserializedResponses config flag (#9789)
Browse files Browse the repository at this point in the history
Co-authored-by: Marcin Rataj <[email protected]>
  • Loading branch information
hacdias and lidel authored May 29, 2023
1 parent 38556b8 commit c10b804
Show file tree
Hide file tree
Showing 10 changed files with 233 additions and 40 deletions.
16 changes: 15 additions & 1 deletion config/gateway.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package config

const DefaultInlineDNSLink = false
const (
DefaultInlineDNSLink = false
DefaultDeserializedResponses = true
)

type GatewaySpec struct {
// Paths is explicit list of path prefixes that should be handled by
Expand All @@ -25,6 +28,11 @@ type GatewaySpec struct {
// (FQDN) into a single DNS label in order to interop with wildcard TLS certs
// and Origin per CID isolation provided by rules like https://publicsuffix.org
InlineDNSLink Flag

// DeserializedResponses configures this gateway to respond to deserialized
// responses. Disabling this option enables a Trustless Gateway, as per:
// https://specs.ipfs.tech/http-gateways/trustless-gateway/.
DeserializedResponses Flag
}

// Gateway contains options for the HTTP gateway server.
Expand Down Expand Up @@ -56,6 +64,12 @@ type Gateway struct {
// This flag can be overridden per FQDN in PublicGateways.
NoDNSLink bool

// DeserializedResponses configures this gateway to respond to deserialized
// requests. Disabling this option enables a Trustless only gateway, as per:
// https://specs.ipfs.tech/http-gateways/trustless-gateway/. This can
// be overridden per FQDN in PublicGateways.
DeserializedResponses Flag

// PublicGateways configures behavior of known public gateways.
// Each key is a fully qualified domain name (FQDN).
PublicGateways map[string]*GatewaySpec
Expand Down
65 changes: 36 additions & 29 deletions core/corehttp/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,11 @@ import (

func GatewayOption(paths ...string) ServeOption {
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
cfg, err := n.Repo.Config()
gwConfig, err := getGatewayConfig(n)
if err != nil {
return nil, err
}

headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
for h, v := range cfg.Gateway.HTTPHeaders {
headers[http.CanonicalHeaderKey(h)] = v
}

gateway.AddAccessControlHeaders(headers)

gwConfig := gateway.Config{
Headers: headers,
}

gwAPI, err := newGatewayBackend(n)
if err != nil {
return nil, err
Expand All @@ -65,7 +54,7 @@ func GatewayOption(paths ...string) ServeOption {

func HostnameOption() ServeOption {
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
cfg, err := n.Repo.Config()
gwConfig, err := getGatewayConfig(n)
if err != nil {
return nil, err
}
Expand All @@ -75,9 +64,8 @@ func HostnameOption() ServeOption {
return nil, err
}

publicGateways := convertPublicGateways(cfg.Gateway.PublicGateways)
childMux := http.NewServeMux()
mux.HandleFunc("/", gateway.WithHostname(childMux, gwAPI, publicGateways, cfg.Gateway.NoDNSLink).ServeHTTP)
mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP)
return childMux, nil
}
}
Expand Down Expand Up @@ -212,30 +200,49 @@ var defaultKnownGateways = map[string]*gateway.Specification{
"localhost": subdomainGatewaySpec,
}

func convertPublicGateways(publicGateways map[string]*config.GatewaySpec) map[string]*gateway.Specification {
gws := map[string]*gateway.Specification{}
func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
cfg, err := n.Repo.Config()
if err != nil {
return gateway.Config{}, err
}

// Parse configuration headers and add the default Access Control Headers.
headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
for h, v := range cfg.Gateway.HTTPHeaders {
headers[http.CanonicalHeaderKey(h)] = v
}
gateway.AddAccessControlHeaders(headers)

// Initialize gateway configuration, with empty PublicGateways, handled after.
gwCfg := gateway.Config{
Headers: headers,
DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses),
NoDNSLink: cfg.Gateway.NoDNSLink,
PublicGateways: map[string]*gateway.Specification{},
}

// First, implicit defaults such as subdomain gateway on localhost
// Add default implicit known gateways, such as subdomain gateway on localhost.
for hostname, gw := range defaultKnownGateways {
gws[hostname] = gw
gwCfg.PublicGateways[hostname] = gw
}

// Then apply values from Gateway.PublicGateways, if present in the config
for hostname, gw := range publicGateways {
// Apply values from cfg.Gateway.PublicGateways if they exist.
for hostname, gw := range cfg.Gateway.PublicGateways {
if gw == nil {
// Remove any implicit defaults, if present. This is useful when one
// wants to disable subdomain gateway on localhost etc.
delete(gws, hostname)
// wants to disable subdomain gateway on localhost, etc.
delete(gwCfg.PublicGateways, hostname)
continue
}

gws[hostname] = &gateway.Specification{
Paths: gw.Paths,
NoDNSLink: gw.NoDNSLink,
UseSubdomains: gw.UseSubdomains,
InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
gwCfg.PublicGateways[hostname] = &gateway.Specification{
Paths: gw.Paths,
NoDNSLink: gw.NoDNSLink,
UseSubdomains: gw.UseSubdomains,
InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
DeserializedResponses: gw.DeserializedResponses.WithDefault(gwCfg.DeserializedResponses),
}
}

return gws
return gwCfg, nil
}
40 changes: 40 additions & 0 deletions core/corehttp/gateway_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
core "github.com/ipfs/kubo/core"
"github.com/ipfs/kubo/core/coreapi"
repo "github.com/ipfs/kubo/repo"
"github.com/stretchr/testify/assert"

iface "github.com/ipfs/boxo/coreiface"
nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
Expand Down Expand Up @@ -173,3 +174,42 @@ func TestVersion(t *testing.T) {
t.Fatalf("response doesn't contain protocol version:\n%s", s)
}
}

func TestDeserializedResponsesInheritance(t *testing.T) {
for _, testCase := range []struct {
globalSetting config.Flag
gatewaySetting config.Flag
expectedGatewaySetting bool
}{
{config.True, config.Default, true},
{config.False, config.Default, false},
{config.False, config.True, true},
{config.True, config.False, false},
} {
c := config.Config{
Identity: config.Identity{
PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node
},
Gateway: config.Gateway{
DeserializedResponses: testCase.globalSetting,
PublicGateways: map[string]*config.GatewaySpec{
"example.com": {
DeserializedResponses: testCase.gatewaySetting,
},
},
},
}
r := &repo.Mock{
C: c,
D: syncds.MutexWrap(datastore.NewMapDatastore()),
}
n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
assert.NoError(t, err)

gwCfg, err := getGatewayConfig(n)
assert.NoError(t, err)

assert.Contains(t, gwCfg.PublicGateways, "example.com")
assert.Equal(t, testCase.expectedGatewaySetting, gwCfg.PublicGateways["example.com"].DeserializedResponses)
}
}
27 changes: 27 additions & 0 deletions docs/changelogs/v0.21.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
- [Overview](#overview)
- [🔦 Highlights](#-highlights)
- [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping)
- [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag)
- [📝 Changelog](#-changelog)
- [👨‍👩‍👧‍👦 Contributors](#-contributors)

Expand All @@ -29,6 +30,32 @@ enabled.
With this update, the same level of robustness is applied to peers that lack
mDNS peers and solely rely on the public DHT.


#### `Gateway.DeserializedResponses` config flag

This release introduces the
[`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses)
configuration flag.

With this flag, one can explicitly configure whether the gateway responds to
deserialized requests or not. By default, this flag is enabled.

Disabling deserialized responses allows the
gateway to operate
as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
limited to three [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval)
response types:
[application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw),
[application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car),
and [application/vnd.ipfs.ipns-record](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record).

With deserialized responses disabled, the Kubo gateway can serve as a block
backend for other software (like
[bifrost-gateway](https://github.com/ipfs/bifrost-gateway#readme),
[IPFS in Chromium](https://github.com/little-bear-labs/ipfs-chromium/blob/main/README.md)
etc) without the usual risks associated with hosting deserialized data behind
third-party CIDs.

### 📝 Changelog

### 👨‍👩‍👧‍👦 Contributors
30 changes: 26 additions & 4 deletions docs/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ config file at runtime.
- [`Gateway`](#gateway)
- [`Gateway.NoFetch`](#gatewaynofetch)
- [`Gateway.NoDNSLink`](#gatewaynodnslink)
- [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses)
- [`Gateway.HTTPHeaders`](#gatewayhttpheaders)
- [`Gateway.RootRedirect`](#gatewayrootredirect)
- [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold)
Expand All @@ -60,6 +61,7 @@ config file at runtime.
- [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains)
- [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink)
- [`Gateway.PublicGateways: InlineDNSLink`](#gatewaypublicgateways-inlinednslink)
- [`Gateway.PublicGateways: DeserializedResponses`](#gatewaypublicgateways-deserializedresponses)
- [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways)
- [`Gateway` recipes](#gateway-recipes)
- [`Identity`](#identity)
Expand Down Expand Up @@ -236,7 +238,7 @@ documented in `ipfs config profile --help`.
smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in
your IPFS node, and disk usage is more critical than performance, consider using
`flatfs`.
- This datastore uses up to several gigabytes of memory.
- This datastore uses up to several gigabytes of memory.
- Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte.
- The current implementation is based on old badger 1.x which is no longer supported by the upstream team.

Expand Down Expand Up @@ -646,6 +648,16 @@ Default: `false`

Type: `bool`

#### `Gateway.DeserializedResponses`

An optional flag to explicitly configure whether this gateway responds to deserialized
requests, or not. By default, it is enabled. When disabling this option, the gateway
operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.

Default: `true`

Type: `flag`

### `Gateway.HTTPHeaders`

Headers to set on gateway responses.
Expand Down Expand Up @@ -790,6 +802,16 @@ Default: `false`
Type: `flag`
#### `Gateway.PublicGateways: DeserializedResponses`
An optional flag to explicitly configure whether this gateway responds to deserialized
requests, or not. By default, it is enabled. When disabling this option, the gateway
operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
Default: same as global `Gateway.DeserializedResponses`
Type: `flag`
#### Implicit defaults of `Gateway.PublicGateways`
Default entries for `localhost` hostname and loopback IPs are always present.
Expand Down Expand Up @@ -895,7 +917,7 @@ Type: `string` (base64 encoded)

## `Internal`

This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo.
This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo.

**Be aware that making informed change here requires in-depth knowledge and most users should leave these untouched. All knobs listed here are subject to breaking changes between versions.**

Expand Down Expand Up @@ -971,7 +993,7 @@ Type: `optionalInteger` (byte count, `null` means default which is 1MB)
### `Internal.Bitswap.ProviderSearchDelay`

This parameter determines how long to wait before looking for providers outside of bitswap.
Other routing systems like the DHT are able to provide results in less than a second, so lowering
Other routing systems like the DHT are able to provide results in less than a second, so lowering
this number will allow faster peers lookups in some cases.

Type: `optionalDuration` (`null` means default which is 1s)
Expand Down Expand Up @@ -1552,7 +1574,7 @@ another node, even if this other node is on a different network. This may
trigger netscan alerts on some hosting providers or cause strain in some setups.

The `server` configuration profile fills up this list with sensible defaults,
preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`,
preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`,
which is the multiaddress representation of `192.168.0.0/16`) but you should always
check settings against your own network and/or hosting provider.

Expand Down
2 changes: 1 addition & 1 deletion docs/examples/kubo-as-a-library/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ go 1.18
replace github.com/ipfs/kubo => ./../../..

require (
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.27.3
github.com/multiformats/go-multiaddr v0.9.0
Expand Down
4 changes: 2 additions & 2 deletions docs/examples/kubo-as-a-library/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4=
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0=
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/google/uuid v1.3.0
github.com/hashicorp/go-multierror v1.1.1
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad
github.com/ipfs/go-block-format v0.1.2
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-cidutil v0.1.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4=
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0=
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
Expand Down
Loading

0 comments on commit c10b804

Please sign in to comment.