Merge pull request #43 from saracen/support-zstd

Support zstd

.github/workflows/go.yml

@@ -10,7 +10,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go: [1.15.x, 1.16.x, 1.17.x]
+        go: [1.18.x, 1.19.x, 1.20.x]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:

archiver.go

@@ -16,6 +16,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/klauspost/compress/zip"
+	"github.com/klauspost/compress/zstd"
 	"github.com/saracen/fastzip/internal/filepool"
 	"github.com/saracen/zipextra"
 	"golang.org/x/sync/errgroup"
@@ -29,7 +30,10 @@ var bufioReaderPool = sync.Pool{
 	},
 }
 
-var defaultCompressor = FlateCompressor(-1)
+var (
+	defaultCompressor     = FlateCompressor(-1)
+	defaultZstdCompressor = ZstdCompressor(int(zstd.SpeedDefault))
+)
 
 // Archiver is an opinionated Zip archiver.
 //
@@ -78,6 +82,7 @@ func NewArchiver(w io.Writer, chroot string, opts ...ArchiverOption) (*Archiver,
 
 	// register flate compressor
 	a.RegisterCompressor(zip.Deflate, defaultCompressor)
+	a.RegisterCompressor(zstd.ZipMethodWinZip, defaultZstdCompressor)
 
 	return a, nil
 }

archiver_test.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/klauspost/compress/zip"
+	"github.com/klauspost/compress/zstd"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -27,15 +28,15 @@ type testFile struct {
 }
 
 func testCreateFiles(t *testing.T, files map[string]testFile) (map[string]os.FileInfo, string) {
-	dir, err := ioutil.TempDir("", "fastzip-test")
-	require.NoError(t, err)
+	dir := t.TempDir()
 
 	filenames := make([]string, 0, len(files))
 	for path := range files {
 		filenames = append(filenames, path)
 	}
 	sort.Strings(filenames)
 
+	var err error
 	for _, path := range filenames {
 		tf := files[path]
 		path = filepath.Join(dir, path)
@@ -51,7 +52,7 @@ func testCreateFiles(t *testing.T, files map[string]testFile) (map[string]os.Fil
 			err = os.Symlink(tf.contents, path)
 
 		default:
-			err = ioutil.WriteFile(path, []byte(tf.contents), tf.mode)
+			err = os.WriteFile(path, []byte(tf.contents), tf.mode)
 		}
 		require.NoError(t, err)
 		require.NoError(t, lchmod(path, tf.mode))
@@ -244,10 +245,7 @@ func TestArchiveWithStageDirectory(t *testing.T) {
 	files, chroot := testCreateFiles(t, testFiles)
 	defer os.RemoveAll(chroot)
 
-	dir, err := ioutil.TempDir("", "fastzip-benchmark-stage")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
+	dir := t.TempDir()
 	f, err := ioutil.TempFile("", "fastzip-test")
 	require.NoError(t, err)
 	defer os.Remove(f.Name())
@@ -262,7 +260,7 @@ func TestArchiveWithStageDirectory(t *testing.T) {
 	require.EqualValues(t, 0, bytes)
 	require.EqualValues(t, 3, entries)
 
-	stageFiles, err := ioutil.ReadDir(dir)
+	stageFiles, err := os.ReadDir(dir)
 	require.NoError(t, err)
 	require.Zero(t, len(stageFiles))
 
@@ -365,10 +363,7 @@ func TestArchiveWithBufferSize(t *testing.T) {
 }
 
 func TestArchiveChroot(t *testing.T) {
-	dir, err := ioutil.TempDir("", "fastzip-test")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
+	dir := t.TempDir()
 	f, err := os.Create(filepath.Join(dir, "archive.zip"))
 	require.NoError(t, err)
 	defer f.Close()
@@ -450,9 +445,7 @@ func benchmarkArchiveOptions(b *testing.B, stdDeflate bool, options ...ArchiverO
 		return nil
 	})
 
-	dir, err := ioutil.TempDir("", "fastzip-benchmark-archive")
-	require.NoError(b, err)
-	defer os.RemoveAll(dir)
+	dir := b.TempDir()
 
 	options = append(options, WithStageDirectory(dir))
 
@@ -521,5 +514,25 @@ func BenchmarkArchiveNonStandardFlate_8(b *testing.B) {
 }
 
 func BenchmarkArchiveNonStandardFlate_16(b *testing.B) {
-	benchmarkArchiveOptions(b, false, WithArchiverConcurrency(16))
+	benchmarkArchiveOptions(b, false, WithArchiverConcurrency(16), WithArchiverMethod(zstd.ZipMethodWinZip))
+}
+
+func BenchmarkArchiveZstd_1(b *testing.B) {
+	benchmarkArchiveOptions(b, true, WithArchiverConcurrency(1), WithArchiverMethod(zstd.ZipMethodWinZip))
+}
+
+func BenchmarkArchiveZstd_2(b *testing.B) {
+	benchmarkArchiveOptions(b, true, WithArchiverConcurrency(2), WithArchiverMethod(zstd.ZipMethodWinZip))
+}
+
+func BenchmarkArchiveZstd_4(b *testing.B) {
+	benchmarkArchiveOptions(b, true, WithArchiverConcurrency(4), WithArchiverMethod(zstd.ZipMethodWinZip))
+}
+
+func BenchmarkArchiveZstd_8(b *testing.B) {
+	benchmarkArchiveOptions(b, true, WithArchiverConcurrency(8), WithArchiverMethod(zstd.ZipMethodWinZip))
+}
+
+func BenchmarkArchiveZstd_16(b *testing.B) {
+	benchmarkArchiveOptions(b, true, WithArchiverConcurrency(16), WithArchiverMethod(zstd.ZipMethodWinZip))
 }

extractor.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -15,6 +14,7 @@ import (
 	"time"
 
 	"github.com/klauspost/compress/zip"
+	"github.com/klauspost/compress/zstd"
 	"github.com/saracen/zipextra"
 	"golang.org/x/sync/errgroup"
 )
@@ -25,7 +25,10 @@ var bufioWriterPool = sync.Pool{
 	},
 }
 
-var defaultDecompressor = FlateDecompressor()
+var (
+	defaultDecompressor     = FlateDecompressor()
+	defaultZstdDecompressor = ZstdDecompressor()
+)
 
 // Extractor is an opinionated Zip file extractor.
 //
@@ -93,6 +96,7 @@ func newExtractor(r *zip.Reader, c io.Closer, chroot string, opts []ExtractorOpt
 	}
 
 	e.RegisterDecompressor(zip.Deflate, defaultDecompressor)
+	e.RegisterDecompressor(zstd.ZipMethodWinZip, defaultZstdDecompressor)
 
 	return e, nil
 }
@@ -205,7 +209,6 @@ func (e *Extractor) Extract(ctx context.Context) (err error) {
 			if err := e.createSymlink(path, file); err != nil {
 				return err
 			}
-			continue
 		}
 
 		err = e.updateFileMetadata(path, file)
@@ -218,7 +221,7 @@ func (e *Extractor) Extract(ctx context.Context) (err error) {
 }
 
 func (e *Extractor) createDirectory(path string, file *zip.File) error {
-	err := os.Mkdir(path, file.Mode().Perm())
+	err := os.Mkdir(path, 0777)
 	if os.IsExist(err) {
 		err = nil
 	}
@@ -237,7 +240,7 @@ func (e *Extractor) createSymlink(path string, file *zip.File) error {
 	}
 	defer r.Close()
 
-	name, err := ioutil.ReadAll(r)
+	name, err := io.ReadAll(r)
 	if err != nil {
 		return err
 	}