Skip to content

Commit

Permalink
Don't use a data descriptor for symlinks so they can be stream-unzipped
Browse files Browse the repository at this point in the history
This changes the behavior of createSymlink to not use a data descriptor.
Because symlinks are stored uncompressed, them not using a data descriptor
means that the correect size of the symlink must be stored up-front in the
local header of the file, which in turn means that the file can be
stream-unzipped because the unzipper can determine when the data of the symlink
ends without having random access to the zip.

This itself requires a small change to the behaviour of the lower level
function createHeaderRaw - it no longer set the data descriptor flag which
allows calling code, such as in createSymlink, to _not_ set it.

A very small extra benefit is that a few bytes are shaved off the resulting
file because symlinks no longer use a data descriptor.

This is to address the request at #48,
which in turn is inspired by the discussion at
uktrade/stream-unzip#105

(I suspect that all non-compressed files should maybe not use a data descriptor
and so would allow them to be stream-unzipped, but this is left to if/when)
  • Loading branch information
michalc committed Oct 18, 2024
1 parent 35c9d55 commit fd1b38a
Showing 1 changed file with 11 additions and 4 deletions.
15 changes: 11 additions & 4 deletions archiver.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bufio"
"context"
"fmt"
"hash/crc32"
"io"
"os"
"path/filepath"
Expand Down Expand Up @@ -228,12 +229,19 @@ func (a *Archiver) createSymlink(path string, fi os.FileInfo, hdr *zip.FileHeade
a.m.Lock()
defer a.m.Unlock()

w, err := a.createHeader(fi, hdr)
link, err := os.Readlink(path)
if err != nil {
return err
}

link, err := os.Readlink(path)
// Don't use a data descriptor to shave a few bytes and to make sure that the symlink can be stream-unzipped
hdr.Flags &= ^uint16(0x8)
hdr.Method = zip.Store
hdr.CompressedSize64 = uint64(len(link))
hdr.UncompressedSize64 = hdr.CompressedSize64
hdr.CRC32 = crc32.ChecksumIEEE([]byte(link))

w, err := a.createHeaderRaw(fi, hdr)
if err != nil {
return err
}
Expand Down Expand Up @@ -282,6 +290,7 @@ func (a *Archiver) compressFile(ctx context.Context, f *os.File, fi os.FileInfo,
return err
}

hdr.Flags |= 0x8
hdr.CompressedSize64 = tmp.Written()
// if compressed file is larger, use the uncompressed version.
if hdr.CompressedSize64 > hdr.UncompressedSize64 {
Expand Down Expand Up @@ -349,8 +358,6 @@ func (a *Archiver) createHeaderRaw(fi os.FileInfo, fh *zip.FileHeader) (io.Write
fh.Extra = append(fh.Extra, zipextra.NewExtendedTimestamp(fh.Modified).Encode()...)
}

fh.Flags |= 0x8

return a.createRaw(fi, fh)
}

Expand Down

0 comments on commit fd1b38a

Please sign in to comment.