feat: support UnixFS 1.5 file mode and modification times (#653)

CHANGELOG.md

@@ -16,6 +16,8 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- `files`, `ipld/unixfs`, `mfs` and `tar` now support optional UnixFS 1.5 mode and modification time metadata
+
 ### Changed
 
 ### Removed

examples/go.mod

@@ -3,13 +3,13 @@ module github.com/ipfs/boxo/examples
 go 1.21
 
 require (
-	github.com/ipfs/boxo v0.19.0
+	github.com/ipfs/boxo v0.22.0
 	github.com/ipfs/go-block-format v0.2.0
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipld/go-car/v2 v2.13.1
 	github.com/ipld/go-ipld-prime v0.21.0
-	github.com/libp2p/go-libp2p v0.36.1
+	github.com/libp2p/go-libp2p v0.36.2
 	github.com/libp2p/go-libp2p-routing-helpers v0.7.3
 	github.com/multiformats/go-multiaddr v0.13.0
 	github.com/multiformats/go-multicodec v0.9.0
@@ -122,7 +122,7 @@ require (
 	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/pion/datachannel v1.5.8 // indirect
 	github.com/pion/dtls/v2 v2.2.12 // indirect
-	github.com/pion/ice/v2 v2.3.32 // indirect
+	github.com/pion/ice/v2 v2.3.34 // indirect
 	github.com/pion/interceptor v0.1.29 // indirect
 	github.com/pion/logging v0.2.2 // indirect
 	github.com/pion/mdns v0.0.12 // indirect
@@ -133,9 +133,9 @@ require (
 	github.com/pion/sdp/v3 v3.0.9 // indirect
 	github.com/pion/srtp/v2 v2.0.20 // indirect
 	github.com/pion/stun v0.6.1 // indirect
-	github.com/pion/transport/v2 v2.2.9 // indirect
+	github.com/pion/transport/v2 v2.2.10 // indirect
 	github.com/pion/turn/v2 v2.1.6 // indirect
-	github.com/pion/webrtc/v3 v3.2.50 // indirect
+	github.com/pion/webrtc/v3 v3.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect

examples/go.sum

@@ -271,8 +271,8 @@ github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+
 github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg=
 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
-github.com/libp2p/go-libp2p v0.36.1 h1:piAHesy0/8ifBEBUS8HF2m7ywR5vnktUFv00dTsVKcs=
-github.com/libp2p/go-libp2p v0.36.1/go.mod h1:vHzel3CpRB+vS11fIjZSJAU4ALvieKV9VZHC9VerHj8=
+github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U=
+github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY=
 github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
 github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
 github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ=
@@ -372,8 +372,8 @@ github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu
 github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
 github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
 github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
-github.com/pion/ice/v2 v2.3.32 h1:VwE/uEeqiMm0zUWpdt1DJtnqEkj3UjEbhX92/CurtWI=
-github.com/pion/ice/v2 v2.3.32/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU=
+github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM=
+github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ=
 github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M=
 github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
@@ -399,17 +399,16 @@ github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/
 github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
 github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
 github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
-github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
-github.com/pion/transport/v2 v2.2.9 h1:WEDygVovkJlV2CCunM9KS2kds+kcl7zdIefQA5y/nkE=
-github.com/pion/transport/v2 v2.2.9/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
+github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q=
+github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
 github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0=
 github.com/pion/transport/v3 v3.0.6 h1:k1mQU06bmmX143qSWgXFqSH1KUJceQvIUuVH/K5ELWw=
 github.com/pion/transport/v3 v3.0.6/go.mod h1:HvJr2N/JwNJAfipsRleqwFoR3t/pWyHeZUs89v3+t5s=
 github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
 github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
 github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
-github.com/pion/webrtc/v3 v3.2.50 h1:C/rwL2mBfCxHv6tlLzDAO3krJpQXfVx8A8WHnGJ2j34=
-github.com/pion/webrtc/v3 v3.2.50/go.mod h1:dytYYoSBy7ZUWhJMbndx9UckgYvzNAfL7xgVnrIKxqo=
+github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
+github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

files/file.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"io"
 	"os"
+	"time"
 )
 
 var (
@@ -17,6 +18,14 @@ var (
 type Node interface {
 	io.Closer
 
+	// Mode returns the mode.
+	// Optional, if unknown/unspecified returns zero.
+	Mode() os.FileMode
+
+	// ModTime returns the last modification time. If the last
+	// modification time is unknown/unspecified ModTime returns zero.
+	ModTime() (mtime time.Time)
+
 	// Size returns size of this file (if this file is a directory, total size of
 	// all files stored in the tree should be returned). Some implementations may
 	// choose not to implement this

files/file_test.go

@@ -3,8 +3,10 @@ package files
 import (
 	"io"
 	"mime/multipart"
+	"os"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestSliceFiles(t *testing.T) {
@@ -49,6 +51,21 @@ func TestReaderFiles(t *testing.T) {
 	}
 }
 
+func TestReaderFileStat(t *testing.T) {
+	reader := strings.NewReader("beep boop")
+	mode := os.FileMode(0754)
+	mtime := time.Date(2020, 11, 2, 12, 27, 35, 55555, time.UTC)
+	stat := &mockFileInfo{name: "test", mode: mode, mtime: mtime}
+
+	rf := NewReaderStatFile(reader, stat)
+	if rf.Mode() != mode {
+		t.Fatalf("Expected file mode to be [%v] but got [%v]", mode, rf.Mode())
+	}
+	if rf.ModTime() != mtime {
+		t.Fatalf("Expected file modified time to be [%v] but got [%v]", mtime, rf.ModTime())
+	}
+}
+
 func TestMultipartFiles(t *testing.T) {
 	data := `
 --Boundary!
@@ -141,3 +158,127 @@ implicit file2
 		},
 	})
 }
+
+func TestMultipartFilesWithMode(t *testing.T) {
+	data := `
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="file-0?mode=0754&mtime=1604320500&mtime-nsecs=55555"; filename="%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC"
+Some-Header: beep
+
+beep
+--Boundary!
+Content-Type: application/x-directory
+Content-Disposition: form-data; name="dir-0?mode=755&mtime=1604320500"; ans=42; filename="dir1"
+
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="file"; filename="dir1/nested"
+
+some content
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="file?mode=600"; filename="dir1/nested2"; ans=42
+
+some content
+--Boundary!
+Content-Type: application/symlink
+Content-Disposition: form-data; name="file-5"; filename="dir1/simlynk"
+
+anotherfile
+--Boundary!
+Content-Type: application/symlink
+Content-Disposition: form-data; name="file?mtime=1604320500"; filename="dir1/simlynk2"
+
+anotherfile
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="dir?mode=0644"; filename="implicit1/implicit2/deep_implicit"
+
+implicit file1
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="dir?mode=755&mtime=1604320500"; filename="implicit1/shallow_implicit"
+
+implicit file2
+--Boundary!--
+
+`
+
+	reader := strings.NewReader(data)
+	mpReader := multipart.NewReader(reader, "Boundary!")
+	dir, err := NewFileFromPartReader(mpReader, multipartFormdataType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	CheckDir(t, dir, []Event{
+		{
+			kind:  TFile,
+			name:  "£ẞǑǓÆ æ ♫♬",
+			value: "beep",
+			mode:  0754,
+			mtime: time.Unix(1604320500, 55555),
+		},
+		{
+			kind:  TDirStart,
+			name:  "dir1",
+			mode:  0755,
+			mtime: time.Unix(1604320500, 0),
+		},
+		{
+			kind:  TFile,
+			name:  "nested",
+			value: "some content",
+		},
+		{
+			kind:  TFile,
+			name:  "nested2",
+			value: "some content",
+			mode:  0600,
+		},
+		{
+			kind:  TSymlink,
+			name:  "simlynk",
+			value: "anotherfile",
+			mode:  0777,
+		},
+		{
+			kind:  TSymlink,
+			name:  "simlynk2",
+			value: "anotherfile",
+			mode:  0777,
+			mtime: time.Unix(1604320500, 0),
+		},
+		{
+			kind: TDirEnd,
+		},
+		{
+			kind: TDirStart,
+			name: "implicit1",
+		},
+		{
+			kind: TDirStart,
+			name: "implicit2",
+		},
+		{
+			kind:  TFile,
+			name:  "deep_implicit",
+			value: "implicit file1",
+			mode:  0644,
+		},
+		{
+			kind: TDirEnd,
+		},
+		{
+			kind:  TFile,
+			name:  "shallow_implicit",
+			value: "implicit file2",
+			mode:  0755,
+			mtime: time.Unix(1604320500, 0),
+		},
+		{
+			kind: TDirEnd,
+		},
+	})
+}

files/filter_test.go

@@ -4,17 +4,33 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 )
 
 type mockFileInfo struct {
 	os.FileInfo
-	name string
+	name  string
+	mode  os.FileMode
+	mtime time.Time
+	size  int64
 }
 
 func (m *mockFileInfo) Name() string {
 	return m.name
 }
 
+func (m *mockFileInfo) Mode() os.FileMode {
+	return m.mode
+}
+
+func (m *mockFileInfo) ModTime() time.Time {
+	return m.mtime
+}
+
+func (m *mockFileInfo) Size() int64 {
+	return m.size
+}
+
 func (m *mockFileInfo) Sys() interface{} {
 	return nil
 }

files/helpers_test.go

@@ -2,7 +2,9 @@ package files
 
 import (
 	"io"
+	"os"
 	"testing"
+	"time"
 )
 
 type Kind int
@@ -18,6 +20,8 @@ type Event struct {
 	kind  Kind
 	name  string
 	value string
+	mode  os.FileMode
+	mtime time.Time
 }
 
 func CheckDir(t *testing.T, dir Directory, expected []Event) {
@@ -50,6 +54,14 @@ func CheckDir(t *testing.T, dir Directory, expected []Event) {
 				t.Fatalf("[%d] expected filename to be %q", i, next.name)
 			}
 
+			if next.mode != 0 && it.Node().Mode()&os.ModePerm != next.mode {
+				t.Fatalf("[%d] expected mode for '%s' to be %O, got %O", i, it.Name(), next.mode, it.Node().Mode())
+			}
+
+			if !next.mtime.IsZero() && !it.Node().ModTime().Equal(next.mtime) {
+				t.Fatalf("[%d] expected modification time for '%s' to be %q", i, it.Name(), next.mtime)
+			}
+
 			switch next.kind {
 			case TFile:
 				mf, ok := it.Node().(File)

files/linkfile.go

@@ -3,21 +3,41 @@ package files
 import (
 	"os"
 	"strings"
+	"time"
 )
 
 type Symlink struct {
 	Target string
 
-	stat   os.FileInfo
+	mtime  time.Time
 	reader strings.Reader
 }
 
 func NewLinkFile(target string, stat os.FileInfo) File {
-	lf := &Symlink{Target: target, stat: stat}
+	mtime := time.Time{}
+	if stat != nil {
+		mtime = stat.ModTime()
+	}
+	return NewSymlinkFile(target, mtime)
+}
+
+func NewSymlinkFile(target string, mtime time.Time) File {
+	lf := &Symlink{
+		Target: target,
+		mtime:  mtime,
+	}
 	lf.reader.Reset(lf.Target)
 	return lf
 }
 
+func (lf *Symlink) Mode() os.FileMode {
+	return os.ModeSymlink | os.ModePerm
+}
+
+func (lf *Symlink) ModTime() time.Time {
+	return lf.mtime
+}
+
 func (lf *Symlink) Close() error {
 	return nil
 }