Skip to content

Commit

Permalink
fix: use %-encoded headers in most compatible way
Browse files Browse the repository at this point in the history
  • Loading branch information
hacdias committed Aug 22, 2023
1 parent 65faf08 commit fd69972
Show file tree
Hide file tree
Showing 14 changed files with 233 additions and 49 deletions.
59 changes: 53 additions & 6 deletions client/rpc/api.go
Original file line number Diff line number Diff line change
@@ -1,18 +1,24 @@
package rpc

import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"

"github.com/blang/semver/v4"
iface "github.com/ipfs/boxo/coreiface"
caopts "github.com/ipfs/boxo/coreiface/options"
"github.com/ipfs/boxo/ipld/merkledag"
"github.com/ipfs/go-cid"
legacy "github.com/ipfs/go-ipld-legacy"
ipfs "github.com/ipfs/kubo"
dagpb "github.com/ipld/go-codec-dagpb"
_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
"github.com/ipld/go-ipld-prime/node/basicnode"
Expand Down Expand Up @@ -42,6 +48,8 @@ type HttpApi struct {
Headers http.Header
applyGlobal func(*requestBuilder)
ipldDecoder *legacy.Decoder
versionMu sync.Mutex
version *semver.Version
}

// NewLocalApi tries to construct new HttpApi instance communicating with local
Expand Down Expand Up @@ -151,6 +159,7 @@ func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) {
api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
return fmt.Errorf("unexpected redirect")
}

return api, nil
}

Expand All @@ -160,14 +169,19 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error)
return nil, err
}

subApi := *api
subApi.applyGlobal = func(req *requestBuilder) {
if options.Offline {
req.Option("offline", options.Offline)
}
subApi := &HttpApi{
url: api.url,
httpcli: api.httpcli,
Headers: api.Headers,
applyGlobal: func(req *requestBuilder) {
if options.Offline {
req.Option("offline", options.Offline)
}
},
ipldDecoder: api.ipldDecoder,
}

return &subApi, nil
return subApi, nil
}

func (api *HttpApi) Request(command string, args ...string) RequestBuilder {
Expand Down Expand Up @@ -228,3 +242,36 @@ func (api *HttpApi) PubSub() iface.PubSubAPI {
func (api *HttpApi) Routing() iface.RoutingAPI {
return (*RoutingAPI)(api)
}

func (api *HttpApi) loadRemoteVersion() (*semver.Version, error) {
api.versionMu.Lock()
defer api.versionMu.Unlock()

if api.version == nil {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
defer cancel()

resp, err := api.Request("version").Send(ctx)
if err != nil {
return nil, err
}
if resp.Error != nil {
return nil, resp.Error
}
defer resp.Close()
var out ipfs.VersionInfo
dec := json.NewDecoder(resp.Output)
if err := dec.Decode(&out); err != nil {
return nil, err
}

remoteVersion, err := semver.New(out.Version)
if err != nil {
return nil, err
}

api.version = remoteVersion
}

return api.version, nil
}
33 changes: 27 additions & 6 deletions client/rpc/requestbuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strconv"
"strings"

"github.com/blang/semver/v4"
"github.com/ipfs/boxo/files"
)

Expand All @@ -23,13 +24,18 @@ type RequestBuilder interface {
Exec(ctx context.Context, res interface{}) error
}

// encodedAbsolutePathVersion is the version from which the absolute path header in
// multipart requests is %-encoded. Before this version, its sent raw.
var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev")

// requestBuilder is an IPFS commands request builder.
type requestBuilder struct {
command string
args []string
opts map[string]string
headers map[string]string
body io.Reader
command string
args []string
opts map[string]string
headers map[string]string
body io.Reader
buildError error

shell *HttpApi
}
Expand Down Expand Up @@ -60,7 +66,18 @@ func (r *requestBuilder) Body(body io.Reader) RequestBuilder {
func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder {
pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil)
d := files.NewMapDirectory(map[string]files.Node{"": pr})
r.body = files.NewMultiFileReader(d, false)

version, err := r.shell.loadRemoteVersion()
if err != nil {
// Unfortunately, we cannot return an error here. Changing this API is also
// not the best since we would otherwise have an inconsistent RequestBuilder.
// We save the error and return it when calling [requestBuilder.Send].
r.buildError = err
return r
}

useEncodedAbsPaths := version.LT(encodedAbsolutePathVersion)
r.body = files.NewMultiFileReader(d, false, useEncodedAbsPaths)

return r
}
Expand Down Expand Up @@ -97,6 +114,10 @@ func (r *requestBuilder) Header(name, value string) RequestBuilder {

// Send sends the request and return the response.
func (r *requestBuilder) Send(ctx context.Context) (*Response, error) {
if r.buildError != nil {
return nil, r.buildError
}

r.shell.applyGlobal(r)

req := NewRequest(ctx, r.shell.url, r.command, r.args...)
Expand Down
8 changes: 7 additions & 1 deletion client/rpc/unixfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,13 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
}

d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side
req.Body(files.NewMultiFileReader(d, false))

version, err := api.core().loadRemoteVersion()
if err != nil {
return nil, err
}
useEncodedAbsPaths := version.LT(encodedAbsolutePathVersion)
req.Body(files.NewMultiFileReader(d, false, useEncodedAbsPaths))

var out addEvent
resp, err := req.Send(ctx)
Expand Down
59 changes: 57 additions & 2 deletions cmd/ipfs/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,27 @@
package main

import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"runtime/pprof"
"strings"
"time"

"github.com/blang/semver/v4"
"github.com/google/uuid"
u "github.com/ipfs/boxo/util"
cmds "github.com/ipfs/go-ipfs-cmds"
"github.com/ipfs/go-ipfs-cmds/cli"
cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
logging "github.com/ipfs/go-log"
ipfs "github.com/ipfs/kubo"
"github.com/ipfs/kubo/cmd/ipfs/util"
oldcmds "github.com/ipfs/kubo/commands"
"github.com/ipfs/kubo/core"
Expand Down Expand Up @@ -224,6 +229,10 @@ func apiAddrOption(req *cmds.Request) (ma.Multiaddr, error) {
return ma.NewMultiaddr(apiAddrStr)
}

// encodedAbsolutePathVersion is the version from which the absolute path header in
// multipart requests is %-encoded. Before this version, its sent raw.
var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev")

func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
exe := tracingWrappedExecutor{cmds.NewExecutor(req.Root)}
cctx := env.(*oldcmds.Context)
Expand Down Expand Up @@ -315,9 +324,18 @@ func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
default:
return nil, fmt.Errorf("unsupported API address: %s", apiAddr)
}
opts = append(opts, cmdhttp.ClientWithHTTPClient(&http.Client{

httpClient := &http.Client{
Transport: otelhttp.NewTransport(tpt),
}))
}
opts = append(opts, cmdhttp.ClientWithHTTPClient(httpClient))

// Fetch remove version, as some feature compatibility might change depending on it.
remoteVersion, err := getRemoteVersion(tracingWrappedExecutor{cmdhttp.NewClient(host, opts...)})
if err != nil {
return nil, err
}
opts = append(opts, cmdhttp.ClientWithRawAbsPath(remoteVersion.LT(encodedAbsolutePathVersion)))

return tracingWrappedExecutor{cmdhttp.NewClient(host, opts...)}, nil
}
Expand Down Expand Up @@ -417,3 +435,40 @@ func resolveAddr(ctx context.Context, addr ma.Multiaddr) (ma.Multiaddr, error) {

return addrs[0], nil
}

type nopWriter struct {
io.Writer
}

func (nw nopWriter) Close() error {
return nil
}

func getRemoteVersion(exe cmds.Executor) (*semver.Version, error) {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
defer cancel()

req, err := cmds.NewRequest(ctx, []string{"version"}, nil, nil, nil, Root)
if err != nil {
return nil, err
}

var buf bytes.Buffer
re, err := cmds.NewWriterResponseEmitter(nopWriter{&buf}, req)
if err != nil {
return nil, err
}

err = exe.Execute(req, re, nil)
if err != nil {
return nil, err
}

var out ipfs.VersionInfo
dec := json.NewDecoder(&buf)
if err := dec.Decode(&out); err != nil {
return nil, err
}

return semver.New(out.Version)
}
16 changes: 16 additions & 0 deletions docs/changelogs/v0.23.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
- [🔦 Highlights](#-highlights)
- [Mplex deprecation](#mplex-deprecation)
- [Gateway: meaningful CAR responses on Not Found errors](#gateway-meaningful-car-responses-on-not-found-errors)
- [Binary characters in file names: no longer works with old clients and new Kubo servers](#binary-characters-in-file-names-no-longer-works-with-old-clients-and-new-kubo-servers)
- [📝 Changelog](#-changelog)
- [👨‍👩‍👧‍👦 Contributors](#-contributors)

Expand Down Expand Up @@ -43,6 +44,21 @@ the path does not exist, a CAR will be sent with a root of `bafkqaaa` (empty CID
This CAR will contain all blocks necessary to validate that the path does not
exist without having to trust the gateway.

#### Binary characters in file names: no longer works with old clients and new Kubo servers

In this version, we updated Kubo to support Go 1.20+. In Go 1.20, a regression
regarding multipart headers was [introduced](https://github.com/golang/go/issues/60674).
This only affects `ipfs add` when a file name has binary characters in its name.
As a consequence, we had to update the encoding of the file name headers. This is
the compatibility table:

| | New Client | Old Client |
|------------|------------|-------------|
| New Server || 🟡* |
| Old Server |||

*Old clients can only send Unicode file paths to the server.

### 📝 Changelog

### 👨‍👩‍👧‍👦 Contributors
4 changes: 2 additions & 2 deletions docs/examples/kubo-as-a-library/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ go 1.20
replace github.com/ipfs/kubo => ./../../..

require (
github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3
github.com/ipfs/boxo v0.12.0
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.29.2
github.com/multiformats/go-multiaddr v0.10.1
Expand Down Expand Up @@ -91,7 +91,7 @@ require (
github.com/ipfs/go-unixfsnode v1.7.1 // indirect
github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/ipld/go-ipld-prime v0.20.0 // indirect
github.com/ipld/go-ipld-prime v0.21.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
Expand Down
12 changes: 6 additions & 6 deletions docs/examples/kubo-as-a-library/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
Expand Down Expand Up @@ -301,8 +301,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 h1:oiMqmivloEHtlCkS+8rCq1Pkz/rf6m3sC45lL4cnwBA=
github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ=
github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
Expand Down Expand Up @@ -398,8 +398,8 @@ github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6
github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
Expand Down Expand Up @@ -737,7 +737,7 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI
github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
github.com/warpfork/go-testmark v0.9.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
Expand Down
Loading

0 comments on commit fd69972

Please sign in to comment.