From caf9d497ec93494ed6948420ebdabed696573b80 Mon Sep 17 00:00:00 2001 From: Yan Song Date: Wed, 11 May 2022 11:11:28 +0800 Subject: [PATCH] makefile: ensure cross-compilation compatibility We need to ensure the snapshotter can be cross-compiled across all platforms, and although we currently only support linux. We must make sure that external projects (such as buildkit) are not affected by cross-compiling when they import package such as `converter`. Signed-off-by: Yan Song --- .github/workflows/ci.yml | 26 +++++++++++++ Makefile | 7 ++-- go.mod | 2 +- pkg/converter/{convert.go => convert_unix.go} | 30 ++------------- pkg/converter/convert_windows.go | 24 ++++++++++++ pkg/converter/types.go | 38 +++++++++++++++++++ pkg/utils/mount/mount_linux.go | 3 ++ .../mount/{mount_darwin.go => mount_other.go} | 7 +++- snapshot/snapshot.go | 9 +++-- snapshot/snapshot_darwin.go | 8 ---- snapshot/snapshot_linux.go | 15 +++++++- snapshot/snapshot_other.go | 17 +++++++++ 12 files changed, 140 insertions(+), 46 deletions(-) rename pkg/converter/{convert.go => convert_unix.go} (89%) create mode 100644 pkg/converter/convert_windows.go create mode 100644 pkg/converter/types.go rename pkg/utils/mount/{mount_darwin.go => mount_other.go} (61%) delete mode 100644 snapshot/snapshot_darwin.go create mode 100644 snapshot/snapshot_other.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e97b947667..649343d481 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,32 @@ jobs: export PATH=$PATH:$(go env GOPATH)/bin make smoke + cross-build-test: + name: Cross Build Test + timeout-minutes: 10 + runs-on: ubuntu-latest + strategy: + matrix: + GOOS: ['linux', 'windows', 'darwin'] + GOARCH: ['amd64', 'arm64'] + steps: + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.17' + - name: Check out code + uses: actions/checkout@v1 + - name: cache go mod + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} + restore-keys: | + ${{ runner.os }}-go + - name: Build + run: | + make -e GOOS=${{ matrix.GOOS }} GOARCH=${{ matrix.GOARCH }} + coverage: name: Code coverage timeout-minutes: 10 diff --git a/Makefile b/Makefile index e4fafacec8..3d6e3f08ad 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ SUDO = $(shell which sudo) GO_EXECUTABLE_PATH ?= $(shell which go) NYDUS_BUILDER ?= /usr/bin/nydus-image NYDUS_NYDUSD ?= /usr/bin/nydusd-fusedev +GOOS ?= linux +GOARCH ?= amd64 #GOPROXY ?= https://goproxy.io ifdef GOPROXY @@ -15,17 +17,16 @@ endif .PHONY: build build: - GOOS=linux ${PROXY} go build -ldflags="-s -w -X 'main.Version=${VERSION}'" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc + GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags="-s -w -X 'main.Version=${VERSION}'" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc static-release: - CGO_ENABLED=0 ${PROXY} GOOS=linux go build -ldflags '-s -w -X "main.Version=${VERSION}" -extldflags "-static"' -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc + CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-s -w -X "main.Version=${VERSION}" -extldflags "-static"' -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc .PHONY: clear clear: rm -f bin/* rm -rf _out - .PHONY: install install: static-release sudo install -D -m 755 bin/containerd-nydus-grpc /usr/local/bin/containerd-nydus-grpc diff --git a/go.mod b/go.mod index ee9ddadf80..d2b71a32e6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.17 require ( github.com/containerd/containerd v1.6.1 github.com/containerd/continuity v0.2.2 + github.com/containerd/fifo v1.0.0 github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/go-containerregistry v0.5.1 @@ -29,7 +30,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/cgroups v1.0.3 // indirect - github.com/containerd/fifo v1.0.0 // indirect github.com/containerd/ttrpc v1.1.0 // indirect github.com/containerd/typeurl v1.0.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect diff --git a/pkg/converter/convert.go b/pkg/converter/convert_unix.go similarity index 89% rename from pkg/converter/convert.go rename to pkg/converter/convert_unix.go index cdc3c6f5f0..7f7cdb17b9 100644 --- a/pkg/converter/convert.go +++ b/pkg/converter/convert_unix.go @@ -1,3 +1,6 @@ +//go:build !windows +// +build !windows + /* * Copyright (c) 2022. Nydus Developers. All rights reserved. * @@ -20,7 +23,6 @@ import ( "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/content" "github.com/containerd/fifo" - "github.com/opencontainers/go-digest" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -32,32 +34,6 @@ const bootstrapNameInTar = "image.boot" const envNydusBuilder = "NYDUS_BUILDER" const envNydusWorkdir = "NYDUS_WORKDIR" -type Layer struct { - // Digest represents the hash of whole tar blob. - Digest digest.Digest - // ReaderAt holds the reader of whole tar blob. - ReaderAt content.ReaderAt -} - -type ConvertOption struct { - // RafsVersion specifies nydus format version, possible values: - // `5`, `6` (EROFS-compatible), default is `5`. - RafsVersion string - // ChunkDictPath holds the bootstrap path of chunk dict image. - ChunkDictPath string - // PrefetchPatterns holds file path pattern list want to prefetch. - PrefetchPatterns string -} - -type MergeOption struct { - // ChunkDictPath holds the bootstrap path of chunk dict image. - ChunkDictPath string - // PrefetchPatterns holds file path pattern list want to prefetch. - PrefetchPatterns string - // WithTar puts bootstrap into a tar stream (no gzip). - WithTar bool -} - func getBuilder() string { builderPath := os.Getenv(envNydusBuilder) if builderPath == "" { diff --git a/pkg/converter/convert_windows.go b/pkg/converter/convert_windows.go new file mode 100644 index 0000000000..f6661baff3 --- /dev/null +++ b/pkg/converter/convert_windows.go @@ -0,0 +1,24 @@ +//go:build windows +// +build windows + +/* + * Copyright (c) 2022. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package converter + +import ( + "context" + "fmt" + "io" +) + +func Convert(ctx context.Context, dest io.Writer, opt ConvertOption) (io.WriteCloser, error) { + return nil, fmt.Errorf("not implemented") +} + +func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) error { + return fmt.Errorf("not implemented") +} diff --git a/pkg/converter/types.go b/pkg/converter/types.go new file mode 100644 index 0000000000..9465505eea --- /dev/null +++ b/pkg/converter/types.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package converter + +import ( + "github.com/containerd/containerd/content" + "github.com/opencontainers/go-digest" +) + +type Layer struct { + // Digest represents the hash of whole tar blob. + Digest digest.Digest + // ReaderAt holds the reader of whole tar blob. + ReaderAt content.ReaderAt +} + +type ConvertOption struct { + // RafsVersion specifies nydus format version, possible values: + // `5`, `6` (EROFS-compatible), default is `5`. + RafsVersion string + // ChunkDictPath holds the bootstrap path of chunk dict image. + ChunkDictPath string + // PrefetchPatterns holds file path pattern list want to prefetch. + PrefetchPatterns string +} + +type MergeOption struct { + // ChunkDictPath holds the bootstrap path of chunk dict image. + ChunkDictPath string + // PrefetchPatterns holds file path pattern list want to prefetch. + PrefetchPatterns string + // WithTar puts bootstrap into a tar stream (no gzip). + WithTar bool +} diff --git a/pkg/utils/mount/mount_linux.go b/pkg/utils/mount/mount_linux.go index 7d1d8346ea..1aafcb9190 100644 --- a/pkg/utils/mount/mount_linux.go +++ b/pkg/utils/mount/mount_linux.go @@ -1,3 +1,6 @@ +//go:build linux +// +build linux + package mount import ( diff --git a/pkg/utils/mount/mount_darwin.go b/pkg/utils/mount/mount_other.go similarity index 61% rename from pkg/utils/mount/mount_darwin.go rename to pkg/utils/mount/mount_other.go index 6ca390187e..43bf82868d 100644 --- a/pkg/utils/mount/mount_darwin.go +++ b/pkg/utils/mount/mount_other.go @@ -1,14 +1,17 @@ +//go:build !linux // +build !linux package mount +import "fmt" + type Mounter struct { } func (m *Mounter) Umount(target string) error { - return nil + return fmt.Errorf("not implemented") } func (m *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - return true, nil + return false, fmt.Errorf("not implemented") } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 20701d0c86..120a14d1e1 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -16,7 +16,6 @@ import ( "os" "path/filepath" "strings" - "syscall" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" @@ -37,6 +36,10 @@ import ( "github.com/containerd/nydus-snapshotter/pkg/process" "github.com/containerd/nydus-snapshotter/pkg/signature" "github.com/containerd/nydus-snapshotter/pkg/snapshot" + + // Import the converter package so that it can be compiled during + // `go build` to ensure cross-compilation compatibility. + _ "github.com/containerd/nydus-snapshotter/pkg/converter" ) const ( @@ -541,9 +544,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k return storage.Snapshot{}, errors.Wrap(err, "failed to stat parent") } - stat := st.Sys().(*syscall.Stat_t) - - if err := os.Lchown(filepath.Join(td, "fs"), int(stat.Uid), int(stat.Gid)); err != nil { + if err := lchown(filepath.Join(td, "fs"), st); err != nil { if rerr := t.Rollback(); rerr != nil { log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") } diff --git a/snapshot/snapshot_darwin.go b/snapshot/snapshot_darwin.go deleted file mode 100644 index 1526947dcc..0000000000 --- a/snapshot/snapshot_darwin.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build !linux - -package snapshot - -func getSupportsDType(dir string) (bool, error){ - return true, nil -} - diff --git a/snapshot/snapshot_linux.go b/snapshot/snapshot_linux.go index 9e07143de3..e455901e46 100644 --- a/snapshot/snapshot_linux.go +++ b/snapshot/snapshot_linux.go @@ -1,7 +1,20 @@ +//go:build linux +// +build linux + package snapshot -import "github.com/containerd/continuity/fs" +import ( + "os" + "syscall" + + "github.com/containerd/continuity/fs" +) func getSupportsDType(dir string) (bool, error) { return fs.SupportsDType(dir) } + +func lchown(target string, st os.FileInfo) error { + stat := st.Sys().(*syscall.Stat_t) + return os.Lchown(target, int(stat.Uid), int(stat.Gid)) +} diff --git a/snapshot/snapshot_other.go b/snapshot/snapshot_other.go new file mode 100644 index 0000000000..b752828f15 --- /dev/null +++ b/snapshot/snapshot_other.go @@ -0,0 +1,17 @@ +//go:build !linux +// +build !linux + +package snapshot + +import ( + "fmt" + "os" +) + +func getSupportsDType(dir string) (bool, error) { + return false, fmt.Errorf("not implemented") +} + +func lchown(target string, st os.FileInfo) error { + return fmt.Errorf("not implemented") +}