Merge branch 'main' into prefetchfiles-nri

Signed-off-by: billie60 <[email protected]>

.github/codecov.yml

@@ -3,21 +3,20 @@ coverage:
     project:
       default:
         enabled: yes
-        target: auto  # auto compares coverage to the previous base commit
+        target: auto # auto compares coverage to the previous base commit
         # adjust accordingly based on how flaky your tests are
         # this allows a 0.3% drop from the previous base commit coverage
         threshold: 0.3%
 
 comment:
   layout: "reach, diff, flags, files"
   behavior: default
-  require_changes: false # if true: only post the comment if coverage changes
+  require_changes: true # if true: only post the comment if coverage changes
 
 codecov:
   require_ci_to_pass: false
   notify:
-    wait_for_ci: false
-
+    wait_for_ci: true
 # When modifying this file, please validate using
 
-# curl -X POST --data-binary @codecov.yml https://codecov.io/validate
+# curl -X POST --data-binary @codecov.yml https://codecov.io/validate

.github/workflows/e2e.yml

@@ -19,8 +19,37 @@ env:
   REGISTRY: ghcr.io
 
 jobs:
-  run-e2e:
-    runs-on: ubuntu-latest
+  run-e2e-for-cgroups-v1:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.19.6"
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: cache go mod
+        uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go
+      - name: Log in to container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run e2e test
+        run: |
+          TAG=$GITHUB_REF_NAME
+          [ "$TAG" == "main" ] && TAG="latest"
+          [ "$GITHUB_EVENT_NAME" == "pull_request" ] && TAG="local"
+          make integration
+  
+  run-e2e-for-cgroups-v2:
+    runs-on: ubuntu-22.04
     steps:
       - name: Set up Go
         uses: actions/setup-go@v3

.github/workflows/optimizer.yml

@@ -74,18 +74,27 @@ jobs:
           sudo make install-optimizer
           sudo install -D -m 755 misc/example/optimizer-nri-plugin.conf /etc/nri/conf.d/02-optimizer-nri-plugin.conf
           sudo systemctl restart containerd
+          systemctl status containerd --no-pager -l
       - name: Generate accessed files list
         run: |
-          sudo crictl run misc/optimizer/ubuntu.yaml misc/optimizer/sandbox.yaml
+          sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
+          sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
           sleep 20
           sudo crictl rmp  -f --all
           tree /opt/nri/optimizer/results/
-          cat /opt/nri/optimizer/results/library/ubuntu:22.04
-          count=$(cat /opt/nri/optimizer/results/library/ubuntu:22.04 | wc -l)
-          echo $count
-          if [ "$count" != 4 ]; then
-            echo "failed to generate accessed files list for ubuntu:22.04"
+          count=$(cat /opt/nri/optimizer/results/library/nginx:1.23.3 | wc -l)
+          expected=$(cat misc/optimizer/script/file_list.txt | wc -l)
+          echo "count: $count expected minimum value: $expected"
+          if [ $count -lt $expected ]; then
+            echo "failed to generate accessed files list for nginx:1.23.3"
+            cat misc/optimizer/script/file_list.txt
             exit 1
           fi
-          cat /opt/nri/optimizer/results/library/ubuntu:22.04.csv
-    
+          cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv
+      - name: Dump logs
+        if: failure()
+        continue-on-error: true
+        run: |
+          systemctl status containerd --no-pager -l
+          journalctl -xeu containerd --no-pager
+          

.github/workflows/release.yml

@@ -55,18 +55,12 @@ jobs:
     needs: [build]
     steps:
       - uses: actions/checkout@v3
-      - name: install hub
-        run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub'
-          sudo mv hub /usr/local/bin/hub
-      - name: download artifacts
+      - name: Download Artifacts
         uses: actions/download-artifact@v3
         with:
           name: nydus-snapshotter_artifacts
           path: nydus-snapshotter
-      - name: upload artifacts
+      - name: Upload Artifacts
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="nydus-snapshotter-$tag-x86_64.tgz"

MAINTAINERS

@@ -12,3 +12,5 @@ imeoer, Yan Song, [email protected]
 #
 # REVIEWERS
 # GitHub ID, Name, Email address
+sctb512, Bin Tang, [email protected]
+

Makefile

@@ -130,7 +130,7 @@ smoke:
 integration:
 	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
 	$(SUDO) DOCKER_BUILDKIT=1 docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile .
-	$(SUDO) docker run --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \
+	$(SUDO) docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \
 	-v `go env GOCACHE`:/root/.cache/go-build -v `pwd`:/nydus-snapshotter \
 	-v /usr/src/linux-headers-${KERNEL_VER}:/usr/src/linux-headers-${KERNEL_VER} \
 	${ENV_TARGET_IMAGES_FILE}  \

config/config.go

@@ -16,8 +16,11 @@ import (
 
 	"github.com/containerd/nydus-snapshotter/internal/constant"
 	"github.com/containerd/nydus-snapshotter/internal/flags"
+	"github.com/containerd/nydus-snapshotter/pkg/cgroup"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/file"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/parser"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/sysinfo"
 )
 
 func init() {
@@ -97,15 +100,22 @@ func ParseRecoverPolicy(p string) (DaemonRecoverPolicy, error) {
 }
 
 const (
-	FsDriverFusedev string = constant.FsDriverFusedev
-	FsDriverFscache string = constant.FsDriverFscache
+	FsDriverBlockdev string = constant.FsDriverBlockdev
+	FsDriverFusedev  string = constant.FsDriverFusedev
+	FsDriverFscache  string = constant.FsDriverFscache
+	FsDriverNodev    string = constant.FsDriverNodev
 )
 
 type Experimental struct {
 	EnableStargz         bool `toml:"enable_stargz"`
 	EnableReferrerDetect bool `toml:"enable_referrer_detect"`
 }
 
+type CgroupConfig struct {
+	Enable      bool   `toml:"enable"`
+	MemoryLimit string `toml:"memory_limit"`
+}
+
 // Configure how to start and recover nydusd daemons
 type DaemonConfig struct {
 	NydusdPath       string `toml:"nydusd_path"`
@@ -114,6 +124,7 @@ type DaemonConfig struct {
 	RecoverPolicy    string `toml:"recover_policy"`
 	FsDriver         string `toml:"fs_driver"`
 	ThreadsNumber    int    `toml:"threads_number"`
+	LogRotationSize  int    `toml:"log_rotation_size"`
 }
 
 type LoggingConfig struct {
@@ -203,6 +214,7 @@ type SnapshotterConfig struct {
 	ImageConfig            ImageConfig            `toml:"image"`
 	CacheManagerConfig     CacheManagerConfig     `toml:"cache_manager"`
 	LoggingConfig          LoggingConfig          `toml:"log"`
+	CgroupConfig           CgroupConfig           `toml:"cgroup"`
 	Experimental           Experimental           `toml:"experimental"`
 }
 
@@ -216,6 +228,7 @@ func LoadSnapshotterConfig(path string) (*SnapshotterConfig, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "load toml configuration from file %q", path)
 	}
+
 	if err = tree.Unmarshal(&config); err != nil {
 		return nil, errors.Wrap(err, "unmarshal snapshotter configuration")
 	}
@@ -336,3 +349,19 @@ func ParseParameters(args *flags.Args, cfg *SnapshotterConfig) error {
 
 	return nil
 }
+
+func ParseCgroupConfig(config CgroupConfig) (cgroup.Config, error) {
+	totalMemory, err := sysinfo.GetTotalMemoryBytes()
+	if err != nil {
+		return cgroup.Config{}, errors.Wrap(err, "Failed  to get total memory bytes")
+	}
+
+	memoryLimitInBytes, err := parser.MemoryConfigToBytes(config.MemoryLimit, totalMemory)
+	if err != nil {
+		return cgroup.Config{}, err
+	}
+
+	return cgroup.Config{
+		MemoryLimitInBytes: memoryLimitInBytes,
+	}, nil
+}

config/config_test.go

@@ -47,6 +47,7 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) {
 			RecoverPolicy:    "restart",
 			NydusdConfigPath: "/etc/nydus/nydusd-config.fusedev.json",
 			ThreadsNumber:    4,
+			LogRotationSize:  100,
 		},
 		SnapshotsConfig: SnapshotConfig{
 			EnableNydusOverlayFS: false,
@@ -77,12 +78,16 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) {
 			RotateLogLocalTime:  true,
 			RotateLogMaxAge:     7,
 			RotateLogMaxBackups: 5,
-			RotateLogMaxSize:    1,
+			RotateLogMaxSize:    100,
 			LogToStdout:         false,
 		},
 		MetricsConfig: MetricsConfig{
 			Address: ":9110",
 		},
+		CgroupConfig: CgroupConfig{
+			Enable:      true,
+			MemoryLimit: "",
+		},
 	}
 
 	A.EqualValues(cfg, &exampleConfig)

config/default.go

@@ -43,6 +43,7 @@ func (c *SnapshotterConfig) FillUpWithDefaults() error {
 	}
 	daemonConfig.RecoverPolicy = RecoverPolicyRestart.String()
 	daemonConfig.FsDriver = constant.DefaultFsDriver
+	daemonConfig.LogRotationSize = constant.DefaultDaemonRotateLogMaxSize
 
 	// cache configuration
 	cacheConfig := &c.CacheManagerConfig
@@ -54,11 +55,6 @@ func (c *SnapshotterConfig) FillUpWithDefaults() error {
 }
 
 func (c *SnapshotterConfig) SetupNydusBinaryPaths() error {
-	// when using DaemonMode = none, nydusd and nydus-image binaries are not required
-	if c.DaemonMode == string(DaemonModeNone) {
-		return nil
-	}
-
 	// resolve nydusd path
 	if path, err := exec.LookPath(constant.NydusdBinaryName); err == nil {
 		c.DaemonConfig.NydusdPath = path

config/global.go

@@ -40,6 +40,10 @@ type GlobalConfig struct {
 	MirrorsConfig    MirrorsConfig
 }
 
+func IsFusedevSharedModeEnabled() bool {
+	return globalConfig.DaemonMode == DaemonModeShared
+}
+
 func GetDaemonMode() DaemonMode {
 	return globalConfig.DaemonMode
 }
@@ -80,6 +84,10 @@ func GetLogLevel() string {
 	return globalConfig.origin.LoggingConfig.LogLevel
 }
 
+func GetDaemonLogRotationSize() int {
+	return globalConfig.origin.DaemonConfig.LogRotationSize
+}
+
 func GetDaemonThreadsNumber() int {
 	return globalConfig.origin.DaemonConfig.ThreadsNumber
 }

docs/configure_nydus.md

@@ -128,7 +128,10 @@ The Nydus snapshotter will get the new secret and parse the authorization. If yo
 
 ## Metrics
 
-Nydusd records metrics in its own format. The metrics are exported via a HTTP server on top of unix domain socket. Nydus-snapshotter fetches the metrics and convert them in to Prometheus format which is exported via a network address. Nydus-snapshotter by default does not fetch metrics from nydusd. You can enable the nydusd metrics download by assigning an network address to `metrics.address` in nydus-snapshotter's toml [configuration file](../misc/snapshotter/config.toml).
+Nydusd records metrics in its own format. The metrics are exported via a HTTP server on top of unix domain socket. Nydus-snapshotter fetches the metrics and convert them in to Prometheus format which is exported via a network address. Nydus-snapshotter by default does not fetch metrics from nydusd. You can enable the nydusd metrics download by assigning a network address to `metrics.address` in nydus-snapshotter's toml [configuration file](../misc/snapshotter/config.toml).
+
+Once this entry is enabled, not only nydusd metrics, but also some information about the nydus-snapshotter 
+runtime and snapshot related events are exported in Prometheus format as well.
 
 ## Diagnose
 

go.mod

@@ -92,6 +92,13 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )
 
+require (
+	github.com/cilium/ebpf v0.9.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+)
+
 require (
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect

go.sum

@@ -605,6 +605,7 @@ go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a
 go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -634,6 +635,8 @@ golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERs
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

integration/Dockerfile

@@ -23,8 +23,6 @@ ARG NERDCTL_VER
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-backports main contrib non-free\n\
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bullseye-security main contrib non-free\n' > /etc/apt/sources.list
 
-ENV GOPROXY=https://goproxy.cn
-
 RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev sudo psmisc jq lsof net-tools
 
 RUN go install github.com/go-delve/delve/cmd/dlv@latest

integration/entrypoint.sh

@@ -563,6 +563,18 @@ function kill_multiple_nydusd_recover_failover {
     detect_go_race
 }
 
+# Refer to https://github.com/moby/moby/blob/088afc99e4bf8adb78e29733396182417d67ada2/hack/dind#L28-L38
+function enable_nesting_for_cgroup_v2() {
+    if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+        mkdir -p /sys/fs/cgroup/init
+        xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+        sed -e 's/ / +/g' -e 's/^/-/' < /sys/fs/cgroup/cgroup.controllers \
+            > /sys/fs/cgroup/cgroup.subtree_control
+    fi
+}
+
+enable_nesting_for_cgroup_v2
+
 reboot_containerd multiple
 
 start_single_container_multiple_daemons

internal/constant/values.go

@@ -17,8 +17,14 @@ const (
 )
 
 const (
+	// Mount RAFS filesystem by using EROFS over block devices.
+	FsDriverBlockdev string = "blockdev"
+	// Mount RAFS filesystem by using FUSE subsystem
 	FsDriverFusedev string = "fusedev"
+	// Mount RAFS filesystem by using fscache/EROFS.
 	FsDriverFscache string = "fscache"
+	// Only prepare/supply meta/data blobs, do not mount RAFS filesystem.
+	FsDriverNodev string = "nodev"
 )
 
 const (
@@ -38,9 +44,10 @@ const (
 	DefaultSystemControllerAddress = "/run/containerd-nydus/system.sock"
 
 	// Log rotation
-	DefaultRotateLogMaxSize    = 200 // 200 megabytes
-	DefaultRotateLogMaxBackups = 10
-	DefaultRotateLogMaxAge     = 0 // days
-	DefaultRotateLogLocalTime  = true
-	DefaultRotateLogCompress   = true
+	DefaultDaemonRotateLogMaxSize = 100 // 100 megabytes
+	DefaultRotateLogMaxSize       = 200 // 200 megabytes
+	DefaultRotateLogMaxBackups    = 5
+	DefaultRotateLogMaxAge        = 0 // days
+	DefaultRotateLogLocalTime     = true
+	DefaultRotateLogCompress      = true
 )