Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Separate fanotify #588

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Separate fanotify #588

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4970e2f
unawareness runtime prefetch implementation on optimizer side
billie60 Jan 5, 2024
56dfa93
unawareness prefetch implementation on snapshotter side
billie60 Jan 22, 2024
3d0ff68
chore: set default rust version to 1.72.1
billie60 Apr 18, 2024
4315abc
modify optimizer test
billie60 Jul 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 60 additions & 4 deletions .github/workflows/optimizer.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ on:

env:
CARGO_TERM_COLOR: always
REGISTRY: ghcr.io
SNAPSHOTTER_CONFIG: /etc/nydus/config.toml
SOURCE_SNAPSHOTTER_CONFIG: misc/snapshotter/config_optimizer.toml
NYDUSD_CONFIG: /etc/nydus/nydusd-config.fusedev.json
SOURCE_NYDUSD_CONFIG: misc/snapshotter/nydusd-config.fusedev.json
SNAPSHOTTER_SYSTEMD_UNIT_SERVICE: misc/snapshotter/nydus-snapshotter.fusedev.service

jobs:
run_optimizer:
Expand Down Expand Up @@ -58,6 +64,34 @@ jobs:
sudo mkdir -p /opt/cni/bin
sudo tar xzf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/
sudo install -D -m 755 misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist
- name: Setup nydus-snapshotter and nydus service
run: |
GOOS=linux GOARCH=amd64 go build -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
sudo install -D -m 755 bin/containerd-nydus-grpc /usr/local/bin/containerd-nydus-grpc
if [ ! -e "${{ env.NYDUSD_CONFIG }}" ]; then
echo "Installing ${{ env.SOURCE_NYDUSD_CONFIG }} to ${{ env.NYDUSD_CONFIG }}"
sudo install -D -m 664 "${{ env.SOURCE_NYDUSD_CONFIG }}" "${{ env.NYDUSD_CONFIG }}"
fi
if [ ! -e "${{ env.SNAPSHOTTER_CONFIG }}" ]; then
echo "Installing ${{ env.SOURCE_SNAPSHOTTER_CONFIG }} to ${{ env.SNAPSHOTTER_CONFIG }}"
sudo install -D -m 664 "${{ env.SOURCE_SNAPSHOTTER_CONFIG }}" "${{ env.SNAPSHOTTER_CONFIG }}"
fi
sudo ln -f -s /etc/nydus/nydusd-config.fusedev.json /etc/nydus/nydusd-config.json
echo "Installing ${{ env.SNAPSHOTTER_SYSTEMD_UNIT_SERVICE }} to /etc/systemd/system/nydus-snapshotter.service"
sudo install -D -m 644 "${{ env.SNAPSHOTTER_SYSTEMD_UNIT_SERVICE }}" /etc/systemd/system/nydus-snapshotter.service
if command -v systemctl >/dev/null; then
sudo systemctl enable /etc/systemd/system/nydus-snapshotter.service
sudo systemctl restart nydus-snapshotter
fi
sleep 5
NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//')
wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
tar xzvf nydus-static-$NYDUS_VER-linux-amd64.tgz
# sudo systemctl restart nydus-snapshotter.service
sudo install -D -m 755 nydus-static/nydusd /usr/local/bin/nydusd
sudo install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image
sudo install -D -m 755 nydus-static/nydusctl /usr/local/bin/nydusctl

- name: Build and install optimizer
run: |
rustup component add rustfmt clippy
Expand All @@ -84,26 +118,48 @@ jobs:
echo "containerd is not ready"
exit 1
fi
- name: Setup prefetch-distribution http server
run: |
go build -o prefetch-distribution tools/prefetch-distribution/main.go
nohup ./prefetch-distribution &
sleep 5
- name: Log in to container registry
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Generate accessed files list
run: |
sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
sleep 20
sudo crictl rmp -f --all
sudo crictl rmi --all
tree /opt/nri/optimizer/results/
count=$(cat /opt/nri/optimizer/results/library/nginx:1.23.3 | wc -l)
count=$(cat /opt/nri/optimizer/results/dragonflyoss/image-service/nginx:nydus-latest | wc -l)
expected=$(cat misc/optimizer/script/file_list.txt | wc -l)
echo "count: $count expected minimum value: $expected"
if [ $count -lt $expected ]; then
echo "failed to generate accessed files list for nginx:1.23.3"
echo "failed to generate accessed files list for nginx:nydus-latest"
cat misc/optimizer/script/file_list.txt
exit 1
fi
cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv
cat /opt/nri/optimizer/results/dragonflyoss/image-service/nginx:nydus-latest.csv
- name: Transmit the prefetch list to nydusd
run: |
sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
if sudo ps aux | grep "[/]usr/local/bin/nydusd" | grep "prefetch-files"; then
echo "Found --prefetch-files in running processes"
else
echo "Error: --prefetch-files not found in running processes"
exit 1
fi
- name: Dump logs
if: failure()
continue-on-error: true
run: |
systemctl status containerd --no-pager -l
journalctl -xeu containerd --no-pager

145 changes: 55 additions & 90 deletions cmd/optimizer-nri-plugin/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,19 @@ package main

import (
"context"
"fmt"
"io"
"log/syslog"
"os"
"path/filepath"
"strings"
"time"

"github.com/containerd/log"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"

"github.com/containerd/containerd/reference/docker"
"github.com/containerd/nri/pkg/api"
"github.com/containerd/nri/pkg/stub"
"github.com/containerd/nydus-snapshotter/pkg/errdefs"
"github.com/containerd/nydus-snapshotter/pkg/fanotify"
"github.com/containerd/nydus-snapshotter/pkg/optimizer"
"github.com/containerd/nydus-snapshotter/version"
"github.com/pelletier/go-toml"
)
Expand All @@ -33,23 +29,18 @@ const (
defaultEvents = "StartContainer,StopContainer"
defaultServerPath = "/usr/local/bin/optimizer-server"
defaultPersistDir = "/opt/nri/optimizer/results"
defaultServerType = optimizer.FANOTIFY
)

type PluginConfig struct {
Events []string `toml:"events"`

ServerPath string `toml:"server_path"`
PersistDir string `toml:"persist_dir"`
Readable bool `toml:"readable"`
Timeout int `toml:"timeout"`
Overwrite bool `toml:"overwrite"`
Events []string
optimizerCfg optimizer.Config
}

type PluginArgs struct {
PluginName string
PluginIdx string
PluginEvents string
Config PluginConfig
PluginName string
PluginIdx string
Config PluginConfig
}

type Flags struct {
Expand All @@ -70,39 +61,44 @@ func buildFlags(args *PluginArgs) []cli.Flag {
Destination: &args.PluginIdx,
},
&cli.StringFlag{
Name: "events",
Value: defaultEvents,
Usage: "the events that containerd subscribes to. DO NOT CHANGE THIS.",
Destination: &args.PluginEvents,
Name: "server-type",
Value: defaultServerType,
Usage: "the type of optimizer, available value includes [\"fanotify\"]",
Destination: &args.Config.optimizerCfg.ServerType,
},
&cli.StringFlag{
Name: "server-path",
Value: defaultServerPath,
Usage: "the path of optimizer server binary",
Destination: &args.Config.ServerPath,
Destination: &args.Config.optimizerCfg.ServerPath,
},
&cli.StringFlag{
Name: "persist-dir",
Value: defaultPersistDir,
Usage: "the directory to persist accessed files list for container",
Destination: &args.Config.PersistDir,
Destination: &args.Config.optimizerCfg.PersistDir,
},
&cli.BoolFlag{
Name: "readable",
Value: false,
Usage: "whether to make the csv file human readable",
Destination: &args.Config.Readable,
Destination: &args.Config.optimizerCfg.Readable,
},
&cli.IntFlag{
Name: "timeout",
Value: 0,
Usage: "the timeout to kill optimizer server, 0 to disable it",
Destination: &args.Config.Timeout,
Destination: &args.Config.optimizerCfg.Timeout,
},
&cli.BoolFlag{
Name: "overwrite",
Usage: "whether to overwrite the existed persistent files",
Destination: &args.Config.Overwrite,
Destination: &args.Config.optimizerCfg.Overwrite,
},
&cli.StringFlag{
Name: "prefetch-distribution-url",
Usage: "The service url of prefetch distribution, for example: http://localhost:1323",
Destination: &args.Config.optimizerCfg.PrefetchDistributionURL,
},
}
}
Expand All @@ -121,21 +117,16 @@ type plugin struct {
}

var (
cfg PluginConfig
logWriter *syslog.Writer
globalFanotifyServer = make(map[string]*fanotify.Server)

_ = stub.ConfigureInterface(&plugin{})
_ = stub.StartContainerInterface(&plugin{})
_ = stub.StopContainerInterface(&plugin{})
)

const (
imageNameLabel = "io.kubernetes.cri.image-name"
cfg PluginConfig
log *logrus.Logger
logWriter *syslog.Writer
_ = stub.ConfigureInterface(&plugin{})
globalServer = make(map[string]optimizer.Server)
)

func (p *plugin) Configure(ctx context.Context, config, runtime, version string) (stub.EventMask, error) {
log.G(ctx).Infof("got configuration data: %q from runtime %s %s", config, runtime, version)
func (p *plugin) Configure(_ context.Context, config, runtime, version string) (stub.EventMask, error) {
// log.G(ctx).Infof("got configuration data: %q from runtime %s %s", config, runtime, version)
log.Infof("got configuration data: %q from runtime %s %s", config, runtime, version)
if config == "" {
return p.mask, nil
}
Expand All @@ -144,7 +135,7 @@ func (p *plugin) Configure(ctx context.Context, config, runtime, version string)
if err != nil {
return 0, errors.Wrap(err, "parse TOML")
}
if err := tree.Unmarshal(&cfg); err != nil {
if err := tree.Unmarshal(&cfg.optimizerCfg); err != nil {
return 0, err
}

Expand All @@ -153,74 +144,48 @@ func (p *plugin) Configure(ctx context.Context, config, runtime, version string)
return 0, errors.Wrap(err, "parse events in configuration")
}

log.G(ctx).Infof("configuration: %#v", cfg)
log.Infof("configuration: %#v", cfg)

return p.mask, nil
}

func (p *plugin) StartContainer(_ context.Context, _ *api.PodSandbox, container *api.Container) error {
dir, imageName, err := GetImageName(container.Annotations)
imageName, server, err := optimizer.NewServer(cfg.optimizerCfg, container, logWriter)
if err != nil {
return err
}

persistDir := filepath.Join(cfg.PersistDir, dir)
if err := os.MkdirAll(persistDir, os.ModePerm); err != nil {
return err
}

persistFile := filepath.Join(persistDir, imageName)
if cfg.Timeout > 0 {
persistFile = fmt.Sprintf("%s.timeout%ds", persistFile, cfg.Timeout)
if server == nil {
return nil
}

fanotifyServer := fanotify.NewServer(cfg.ServerPath, container.Pid, imageName, persistFile, cfg.Readable, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter)

if err := fanotifyServer.RunServer(); err != nil {
if err := server.Start(); err != nil {
return err
}

globalFanotifyServer[imageName] = fanotifyServer
globalServer[imageName] = server

return nil
}

func (p *plugin) StopContainer(_ context.Context, _ *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) {
var update = []*api.ContainerUpdate{}
_, imageName, err := GetImageName(container.Annotations)
_, imageName, _, err := optimizer.GetImageName(container.Annotations)
if err != nil {
return update, err
}
if fanotifyServer, ok := globalFanotifyServer[imageName]; ok {
fanotifyServer.StopServer()
} else {
return nil, errors.New("can not find fanotify server for container image " + imageName)
}

return update, nil
}

func GetImageName(annotations map[string]string) (string, string, error) {
named, err := docker.ParseDockerRef(annotations[imageNameLabel])
if err != nil {
return "", "", err
if server, ok := globalServer[imageName]; ok {
server.Stop()
}
nameTagged := named.(docker.NamedTagged)
repo := docker.Path(nameTagged)

dir := filepath.Dir(repo)
image := filepath.Base(repo)

imageName := image + ":" + nameTagged.Tag()

return dir, imageName, nil
return update, nil
}

func (p *plugin) onClose() {
for _, fanotifyServer := range globalFanotifyServer {
fanotifyServer.StopServer()
for _, server := range globalServer {
server.Stop()
}
os.Exit(0)
}

func main() {
Expand All @@ -239,13 +204,13 @@ func main() {

cfg = flags.Args.Config

// FIXME(thaJeztah): ucontainerd's log does not set "PadLevelText: true"
_ = log.SetFormat(log.TextFormat)
ctx := log.WithLogger(context.Background(), log.L)

log = logrus.StandardLogger()
log.SetFormatter(&logrus.TextFormatter{
PadLevelText: true,
})
logWriter, err = syslog.New(syslog.LOG_INFO, "optimizer-nri-plugin")
if err == nil {
log.G(ctx).Logger.SetOutput(io.MultiWriter(os.Stdout, logWriter))
log.SetOutput(io.MultiWriter(os.Stdout, logWriter))
}

if flags.Args.PluginName != "" {
Expand All @@ -257,18 +222,18 @@ func main() {

p := &plugin{}

if p.mask, err = api.ParseEventMask(flags.Args.PluginEvents); err != nil {
log.G(ctx).Fatalf("failed to parse events: %v", err)
if p.mask, err = api.ParseEventMask(defaultEvents); err != nil {
log.Fatalf("failed to parse events: %v", err)
}
cfg.Events = strings.Split(flags.Args.PluginEvents, ",")
cfg.Events = strings.Split(defaultEvents, ",")

if p.stub, err = stub.New(p, append(opts, stub.WithOnClose(p.onClose))...); err != nil {
log.G(ctx).Fatalf("failed to create plugin stub: %v", err)
log.Fatalf("failed to create plugin stub: %v", err)
}

err = p.stub.Run(context.Background())
if err != nil {
log.G(ctx).Errorf("plugin exited with error %v", err)
log.Errorf("plugin exited with error %v", err)
os.Exit(1)
}

Expand All @@ -277,9 +242,9 @@ func main() {
}
if err := app.Run(os.Args); err != nil {
if errdefs.IsConnectionClosed(err) {
log.L.Info("optimizer NRI plugin exited")
log.Info("optimizer NRI plugin exited")
} else {
log.L.WithError(err).Fatal("failed to start optimizer NRI plugin")
log.WithError(err).Fatal("failed to start optimizer NRI plugin")
}
}
}
Loading
Loading