Test against 1.20

.github/workflows/go.yml

@@ -1,32 +1,25 @@
 name: Build and Test
 on: [push, pull_request]
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Go CI
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [1.13, 1.14]
+        go: ['1.19', '1.20']
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 #v4.0.1
         with:
           go-version: ${{ matrix.go }}
       - name: Check out source
-        uses: actions/checkout@v1
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install librsync-dev
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #v3.5.2
       - name: Install Linters
-        run: "curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.25.0"
+        run: "curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.2"
       - name: Build
-        env:
-          GO111MODULE: "on"
         run: go build ./...
       - name: Lint
-        env:
-          GO111MODULE: "on"
-        run: |
-          export PATH=${PATH}:$(go env GOPATH)/bin
-          golangci-lint run --disable-all --deadline=10m --enable=gofmt --enable=golint --enable=govet --enable=gosimple --enable=ineffassign
+        run: golangci-lint run

.golangci.yml

@@ -0,0 +1,28 @@
+run:
+  deadline: 10m
+  build-tags:
+
+linters:
+  disable-all: true
+  enable:
+    - asciicheck
+    - bidichk
+    - bodyclose
+    - durationcheck
+    - errchkjson
+    - exportloopref
+    - gofmt
+    - goimports
+    - gosimple
+    - govet
+    - grouper
+    - ineffassign
+    - misspell
+    - nosprintfhostport
+    - reassign
+    - rowserrcheck
+    - tparallel
+    - typecheck
+    - unconvert
+    - unused
+    - vetshadow

appdata.go

@@ -96,11 +96,12 @@ func appDataDir(goos, appName string, roaming bool) string {
 // (%LOCALAPPDATA%) that is used by default.
 //
 // Example results:
-//  dir := AppDataDir("myapp", false)
-//   POSIX (Linux/BSD): ~/.myapp
-//   Mac OS: $HOME/Library/Application Support/Myapp
-//   Windows: %LOCALAPPDATA%\Myapp
-//   Plan 9: $home/myapp
+//
+//	dir := AppDataDir("myapp", false)
+//	 POSIX (Linux/BSD): ~/.myapp
+//	 Mac OS: $HOME/Library/Application Support/Myapp
+//	 Windows: %LOCALAPPDATA%\Myapp
+//	 Plan 9: $home/myapp
 func AppDataDir(appName string, roaming bool) string {
 	return appDataDir(runtime.GOOS, appName, roaming)
 }

backup.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -34,6 +33,13 @@ func lookupGroup(groupName string) (int, error) {
 	return int(gid), nil
 }
 
+func debugf(format string, a ...interface{}) {
+	if syslogDebug {
+		str := fmt.Sprintf(format, a...)
+		sysLog.Debug(str)
+	}
+}
+
 func removeOld(destDir string, dryRun bool) {
 	files, err := ioutil.ReadDir(destDir)
 	if err != nil {
@@ -45,19 +51,20 @@ func removeOld(destDir string, dryRun bool) {
 		}
 		filePath := filepath.Join(destDir, file.Name())
 		if dryRun {
-			log.Printf("deleting %s (dryrun)", filePath)
+			debugf("deleting %s (dryrun)", filePath)
 		} else {
-			log.Printf("deleting %s", filePath)
+			debugf("deleting %s", filePath)
 			err := os.Remove(filePath)
 			if err != nil {
-				log.Printf("ERROR: %v", err)
+				sysLog.Err(fmt.Sprintf("failed to delete %v: %v", filePath, err))
 				continue
 			}
 		}
 	}
 }
 
 func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
+	sysLog.Info("starting backup")
 	destDir := filepath.Clean(cfg.BackupPath)
 	destDirAbs, err := filepath.Abs(destDir)
 	if err != nil {
@@ -76,13 +83,14 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 	}
 	err = os.Chown(destDir, uid, gid)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to chown %q: %w", destDir, err)
 	}
 
 	sigFile := filepath.Join(destDir, "sig.cache")
 	existingSC, err := LoadSignatureCache(sigFile)
 	if err != nil && !os.IsNotExist(err) {
-		return err
+		sysLog.Err(fmt.Sprintf("failed to load signature file %q: %v", sigFile, err))
+		existingSC = nil
 	}
 
 	var sc *SignatureCache
@@ -93,7 +101,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 		sc, err = NewSignatureCache(filepath.Join(destDir, "sig.cache.inprogress"), existingSC.timeStamp, existingSC.Instance()+1)
 	}
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to create new signature cache: %w", err)
 	}
 
 	pathsToCheck := existingSC.Paths()
@@ -102,7 +110,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 		removeOld(destDir, cfg.DryRun)
 	}
 
-	log.Printf("----------  RUNNING LEVEL %d (%v) -----------", sc.instance, sc.timeStamp)
+	debugf("RUNNING LEVEL %d (%v)", sc.instance, sc.timeStamp)
 
 	snap, err := NewSnapshot(ctx, pubKey, uid, gid, cfg.Backup.GZLevel, destDir, sc.hostname, sc.timeStamp, sc.instance, sc.version)
 	if err != nil {
@@ -134,7 +142,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 			}
 
 			if err != nil {
-				log.Printf("Walk: %v", err)
+				sysLog.Err(fmt.Sprintf("Walk: %v", err))
 				return nil
 			}
 			if ctx.Err() != nil {
@@ -154,7 +162,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 			for _, exclude := range cfg.Backup.rExcludes {
 				if exclude.MatchString(srcPath) {
 					filesExcluded++
-					log.Printf("%q: excluding", srcPath)
+					debugf("%q: excluding", srcPath)
 					return nil
 				}
 			}
@@ -174,7 +182,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 			fileMode := os.FileMode(MD.Attribs.Mode)
 			switch {
 			case isSocket(fileMode):
-				log.Printf("skipping socket file: %v", srcPath)
+				debugf("skipping socket file: %v", srcPath)
 				return nil
 			case isCharDevice(fileMode):
 				fallthrough
@@ -189,9 +197,9 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 				}
 				if !bytes.Equal(currentSig.Bytes(), thisSig.Bytes()) {
 					if currentSig.Len() != 0 {
-						log.Printf("%q changed", srcPath)
+						debugf("%q changed", srcPath)
 					} else {
-						log.Printf("%q new file", srcPath)
+						debugf("%q new file", srcPath)
 					}
 					err = snap.Add(MD, nil, 0)
 					if err != nil {
@@ -202,7 +210,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 						return err
 					}
 				} else {
-					log.Printf("%q no change", srcPath)
+					debugf("%q no change", srcPath)
 					err = sc.Add(srcPath, currentSig.Bytes())
 					if err != nil {
 						return err
@@ -222,7 +230,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 				}
 				if !bytes.Equal(currentSig.Bytes(), thisSig.Bytes()) {
 					if currentSig.Len() != 0 {
-						log.Printf("%q changed", srcPath)
+						debugf("%q changed", srcPath)
 
 						delta.Reset()
 						readBuffer.Reset(currentSig.Bytes())
@@ -232,7 +240,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 						}
 						dataReader.Reset(delta.Bytes())
 					} else {
-						log.Printf("%q new file", srcPath)
+						debugf("%q new file", srcPath)
 					}
 					err = snap.Add(MD, dataReader, int64(dataReader.Len()))
 					if err != nil {
@@ -243,7 +251,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 						return err
 					}
 				} else {
-					log.Printf("%q: no change", srcPath)
+					debugf("%q: no change", srcPath)
 					err = sc.Add(srcPath, currentSig.Bytes())
 					if err != nil {
 						return err
@@ -264,20 +272,59 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 				}
 				if !bytes.Equal(currentSig.Bytes(), thisSig.Bytes()) {
 					if currentSig.Len() != 0 {
-						log.Printf("%q: changed", srcPath)
-						delta.Reset()
+						debugf("%q: changed", srcPath)
 						readBuffer.Reset(currentSig.Bytes())
-						err = rsync.GenDelta(readBuffer, srcFD, info.Size(), delta)
-						if err != nil {
-							srcFD.Close()
-							return err
-						}
-						readBuffer.Reset(delta.Bytes())
 
-						err = snap.Add(MD, readBuffer, int64(readBuffer.Len()))
-						readBuffer.Reset(nil)
+						if info.Size() > memoryLimit*10 {
+							tmpFile, err := os.CreateTemp(cfg.BackupPath, info.Name())
+							if err != nil {
+								srcFD.Close()
+								return err
+							}
+							err = rsync.GenDelta(readBuffer, srcFD, info.Size(), tmpFile)
+							if err != nil {
+								tmpFile.Close()
+								os.Remove(tmpFile.Name())
+								srcFD.Close()
+								return err
+							}
+							if _, err = tmpFile.Seek(0, 0); err != nil {
+								tmpFile.Close()
+								os.Remove(tmpFile.Name())
+								srcFD.Close()
+								return err
+							}
+							tmpFileInfo, err := tmpFile.Stat()
+							if err != nil {
+								tmpFile.Close()
+								os.Remove(tmpFile.Name())
+								srcFD.Close()
+								return err
+							}
+							err = snap.Add(MD, tmpFile, tmpFileInfo.Size())
+							if err != nil {
+								tmpFile.Close()
+								os.Remove(tmpFile.Name())
+								srcFD.Close()
+								return err
+							}
+							tmpFile.Close()
+							os.Remove(tmpFile.Name())
+						} else {
+							delta.Reset()
+							readBuffer.Reset(currentSig.Bytes())
+							err = rsync.GenDelta(readBuffer, srcFD, info.Size(), delta)
+							if err != nil {
+								srcFD.Close()
+								return err
+							}
+							readBuffer.Reset(delta.Bytes())
+
+							err = snap.Add(MD, readBuffer, int64(readBuffer.Len()))
+							readBuffer.Reset(nil)
+						}
 					} else {
-						log.Printf("%q new file", srcPath)
+						debugf("%q new file", srcPath)
 						st, err := srcFD.Stat()
 						if err != nil {
 							srcFD.Close()
@@ -299,7 +346,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 						return err
 					}
 				} else {
-					log.Printf("%q: no change", srcPath)
+					debugf("%q: no change", srcPath)
 					err = sc.Add(srcPath, currentSig.Bytes())
 					if err != nil {
 						srcFD.Close()
@@ -320,7 +367,7 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 
 	// handle deleted files
 	for deletedFilePath := range pathsToCheck {
-		log.Printf("%q: deleted", deletedFilePath)
+		debugf("%q: deleted", deletedFilePath)
 		err = snap.Add(&Metadata{Path: deletedFilePath, Attribs: FileAttributes{}}, nil, 0)
 		if err != nil {
 			snap.Close()
@@ -345,10 +392,10 @@ func backup(ctx context.Context, pubKey *stream.PublicKey, cfg *config) error {
 	}
 	err = os.Chown(sigFile, uid, gid)
 	if err != nil {
-		log.Printf("%v", err)
+		sysLog.Err(fmt.Sprintf("failed to chown signature file %q: %v", sigFile, err))
 	}
 
-	log.Printf("completed: duration:%v bytes written:%d files-skipped:%d",
-		time.Since(startTime), snap.BytesWritten(), filesExcluded)
+	sysLog.Info(fmt.Sprintf("completed: duration:%v bytes written:%d files-skipped:%d",
+		time.Since(startTime), snap.BytesWritten(), filesExcluded))
 	return nil
 }

config.go

@@ -29,6 +29,7 @@ type RestoreConfig struct {
 }
 
 type config struct {
+	Debug      bool
 	DryRun     bool
 	Profile    bool
 	BackupPath string

go.mod

@@ -1,14 +1,22 @@
-module github.com/companyzero/multus
+module multus
 
-go 1.13
+go 1.19
 
 require (
+	github.com/jrick/ss v0.9.1
+	github.com/smtc/rsync v0.0.0-00010101000000-000000000000
+	golang.org/x/sync v0.3.0
+	golang.org/x/term v0.9.0
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/companyzero/sntrup4591761 v0.0.0-20200131011700-2b0d299dbd22 // indirect
 	github.com/dchest/blake2b v1.0.0 // indirect
-	github.com/jrick/ss v0.7.1
 	github.com/smtc/rollsum v0.0.0-20150721100732-39e98d252100 // indirect
-	github.com/smtc/rsync v0.0.0-20151014010438-0a038bb0deb8
 	github.com/smtc/seekbuffer v0.0.0-20151009054628-711359748967 // indirect
-	golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a
-	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
-	gopkg.in/yaml.v2 v2.2.8
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/sys v0.9.0 // indirect
 )
+
+replace github.com/smtc/rsync => github.com/dajohi/rsync v0.0.0-20220210212722-7c40f7496082