From b37efb1fdd4f6d6a16ade8f97f3956163f5bc6da Mon Sep 17 00:00:00 2001 From: Guijie Wang Date: Sun, 1 Oct 2023 11:07:02 +0800 Subject: [PATCH 1/8] config: add enable_backend_source startup option to control persistence of auth information This change introduces a new startup option, enable_backend_source, which controls whether the authentication information is persisted or not. By default, enable_backend_source is set to false, which means that the authentication information will be persisted. Signed-off-by: Guijie Wang --- config/config.go | 1 + config/global.go | 4 ++++ misc/snapshotter/config.toml | 3 +++ pkg/daemon/command/command.go | 7 +++++++ pkg/manager/daemon_adaptor.go | 9 +++++++++ 5 files changed, 24 insertions(+) diff --git a/config/config.go b/config/config.go index c3f4c4f797..143b77f574 100644 --- a/config/config.go +++ b/config/config.go @@ -111,6 +111,7 @@ type Experimental struct { EnableStargz bool `toml:"enable_stargz"` EnableReferrerDetect bool `toml:"enable_referrer_detect"` TarfsConfig TarfsConfig `toml:"tarfs"` + EnableBackendSource bool `toml:"enable_backend_source"` } type TarfsConfig struct { diff --git a/config/global.go b/config/global.go index d866c4dce6..3c1906073a 100644 --- a/config/global.go +++ b/config/global.go @@ -96,6 +96,10 @@ func GetLogToStdout() bool { return globalConfig.origin.LoggingConfig.LogToStdout } +func IsBackendSourceEnabled() bool { + return globalConfig.origin.Experimental.EnableBackendSource && globalConfig.origin.SystemControllerConfig.Enable +} + func IsSystemControllerEnabled() bool { return globalConfig.origin.SystemControllerConfig.Enable } diff --git a/misc/snapshotter/config.toml b/misc/snapshotter/config.toml index 631fca5524..e8d3c118d9 100644 --- a/misc/snapshotter/config.toml +++ b/misc/snapshotter/config.toml @@ -107,6 +107,9 @@ enable_stargz = false # The option enables trying to fetch the Nydus image associated with the OCI image and run it. # Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers enable_referrer_detect = false +# Whether to enable authentication support +# The option enables nydus snapshot to provide backend information to nydusd. +enable_backend_source = false [experimental.tarfs] # Whether to enable nydus tarfs mode. Tarfs is supported by: # - The EROFS filesystem driver since Linux 6.4 diff --git a/pkg/daemon/command/command.go b/pkg/daemon/command/command.go index 968a2ee501..3b92860552 100644 --- a/pkg/daemon/command/command.go +++ b/pkg/daemon/command/command.go @@ -36,6 +36,7 @@ type DaemonCommand struct { Supervisor string `type:"param" name:"supervisor"` LogFile string `type:"param" name:"log-file"` PrefetchFiles string `type:"param" name:"prefetch-files"` + BackendSource string `type:"param" name:"backend-source"` } // Build exec style command line @@ -189,3 +190,9 @@ func WithUpgrade() Opt { cmd.Upgrade = true } } + +func WithBackendSource(source string) Opt { + return func(cmd *DaemonCommand) { + cmd.BackendSource = source + } +} diff --git a/pkg/manager/daemon_adaptor.go b/pkg/manager/daemon_adaptor.go index 24f13be1e2..e1335c1d6b 100644 --- a/pkg/manager/daemon_adaptor.go +++ b/pkg/manager/daemon_adaptor.go @@ -7,6 +7,7 @@ package manager import ( + "fmt" "os" "os/exec" "strings" @@ -25,6 +26,8 @@ import ( "github.com/containerd/nydus-snapshotter/pkg/prefetch" ) +const endpointGetBackend string = "/api/v1/daemons/%s/backend" + // Spawn a nydusd daemon to serve the daemon instance. // // When returning from `StartDaemon()` with out error: @@ -156,6 +159,12 @@ func (m *Manager) BuildDaemonCommand(d *daemon.Daemon, bin string, upgrade bool) command.WithConfig(d.ConfigFile("")), command.WithBootstrap(bootstrap), ) + if config.IsBackendSourceEnabled() { + configAPIPath := fmt.Sprintf(endpointGetBackend, d.States.ID) + cmdOpts = append(cmdOpts, + command.WithBackendSource(config.SystemControllerAddress()+configAPIPath), + ) + } default: return nil, errors.Errorf("invalid daemon mode %s ", d.States.DaemonMode) } From 17efbb47f0677aace625014a82ddfb3df45f5229 Mon Sep 17 00:00:00 2001 From: Guijie Wang Date: Sun, 1 Oct 2023 11:31:46 +0800 Subject: [PATCH 2/8] deamonconfig: add serializeWithSecretFilter function to filter fields with secret tag This commit introduces a new function, serializeWithSecretFilter, which filters fields with the 'secret' tag and prevents them from being persisted to local storage when the value is true. Signed-off-by: Guijie Wang --- config/daemonconfig/daemonconfig.go | 62 ++++++++++++++++++++++-- config/daemonconfig/daemonconfig_test.go | 61 +++++++++++++++++++++++ 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index 0ecb22344c..314cdbdbc1 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -10,6 +10,8 @@ package daemonconfig import ( "encoding/json" "os" + "reflect" + "strings" "github.com/pkg/errors" @@ -75,16 +77,16 @@ type BackendConfig struct { // Registry backend configs Host string `json:"host,omitempty"` Repo string `json:"repo,omitempty"` - Auth string `json:"auth,omitempty"` - RegistryToken string `json:"registry_token,omitempty"` + Auth string `json:"auth,omitempty" secret:"true"` + RegistryToken string `json:"registry_token,omitempty" secret:"true"` BlobURLScheme string `json:"blob_url_scheme,omitempty"` BlobRedirectedHost string `json:"blob_redirected_host,omitempty"` Mirrors []MirrorConfig `json:"mirrors,omitempty"` // OSS backend configs EndPoint string `json:"endpoint,omitempty"` - AccessKeyID string `json:"access_key_id,omitempty"` - AccessKeySecret string `json:"access_key_secret,omitempty"` + AccessKeyID string `json:"access_key_id,omitempty" secret:"true"` + AccessKeySecret string `json:"access_key_secret,omitempty" secret:"true"` BucketName string `json:"bucket_name,omitempty"` ObjectPrefix string `json:"object_prefix,omitempty"` @@ -124,6 +126,9 @@ type DeviceConfig struct { // We don't have to persist configuration file for fscache since its configuration // is passed through HTTP API. func DumpConfigFile(c interface{}, path string) error { + if config.IsBackendSourceEnabled() { + c = serializeWithSecretFilter(c) + } b, err := json.Marshal(c) if err != nil { return errors.Wrapf(err, "marshal config") @@ -179,3 +184,52 @@ func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string, return nil } + +func serializeWithSecretFilter(obj interface{}) map[string]interface{} { + result := make(map[string]interface{}) + value := reflect.ValueOf(obj) + typeOfObj := reflect.TypeOf(obj) + + if value.Kind() == reflect.Ptr { + value = value.Elem() + typeOfObj = typeOfObj.Elem() + } + + for i := 0; i < value.NumField(); i++ { + field := value.Field(i) + fieldType := typeOfObj.Field(i) + secretTag := fieldType.Tag.Get("secret") + jsonTags := strings.Split(fieldType.Tag.Get("json"), ",") + omitemptyTag := false + + for _, tag := range jsonTags { + if tag == "omitempty" { + omitemptyTag = true + break + } + } + + if secretTag == "true" { + continue + } + + if field.Kind() == reflect.Ptr && field.IsNil() { + continue + } + + if omitemptyTag && reflect.DeepEqual(reflect.Zero(field.Type()).Interface(), field.Interface()) { + continue + } + + switch fieldType.Type.Kind() { + case reflect.Struct: + result[jsonTags[0]] = serializeWithSecretFilter(field.Interface()) + case reflect.Ptr: + result[jsonTags[0]] = serializeWithSecretFilter(field.Elem().Interface()) + default: + result[jsonTags[0]] = field.Interface() + } + } + + return result +} diff --git a/config/daemonconfig/daemonconfig_test.go b/config/daemonconfig/daemonconfig_test.go index d18a85f1e1..9631f0d3a5 100644 --- a/config/daemonconfig/daemonconfig_test.go +++ b/config/daemonconfig/daemonconfig_test.go @@ -62,3 +62,64 @@ func TestLoadConfig(t *testing.T) { require.Equal(t, cfg.Device.Backend.Config.SkipVerify, true) require.Equal(t, cfg.Device.Backend.Config.Proxy.CheckInterval, 5) } + +func TestSerializeWithSecretFilter(t *testing.T) { + buf := []byte(`{ + "device": { + "backend": { + "type": "registry", + "config": { + "skip_verify": true, + "host": "acr-nydus-registry-vpc.cn-hangzhou.cr.aliyuncs.com", + "repo": "test/myserver", + "auth": "token_token", + "blob_url_scheme": "http", + "proxy": { + "url": "http://p2p-proxy:65001", + "fallback": true, + "ping_url": "http://p2p-proxy:40901/server/ping", + "check_interval": 5 + }, + "timeout": 5, + "connect_timeout": 5, + "retry_limit": 0 + } + }, + "cache": { + "type": "blobcache", + "config": { + "work_dir": "/cache" + } + } + }, + "mode": "direct", + "digest_validate": true, + "iostats_files": true, + "enable_xattr": true, + "fs_prefetch": { + "enable": true, + "threads_count": 10, + "merging_size": 131072 + } +}`) + var cfg FuseDaemonConfig + _ = json.Unmarshal(buf, &cfg) + filter := serializeWithSecretFilter(&cfg) + jsonData, err := json.Marshal(filter) + require.Nil(t, err) + var newCfg FuseDaemonConfig + err = json.Unmarshal(jsonData, &newCfg) + require.Nil(t, err) + require.Equal(t, newCfg.FSPrefetch, cfg.FSPrefetch) + require.Equal(t, newCfg.Device.Cache.CacheType, cfg.Device.Cache.CacheType) + require.Equal(t, newCfg.Device.Cache.Config, cfg.Device.Cache.Config) + require.Equal(t, newCfg.Mode, cfg.Mode) + require.Equal(t, newCfg.DigestValidate, cfg.DigestValidate) + require.Equal(t, newCfg.IOStatsFiles, cfg.IOStatsFiles) + require.Equal(t, newCfg.Device.Backend.Config.Host, cfg.Device.Backend.Config.Host) + require.Equal(t, newCfg.Device.Backend.Config.Repo, cfg.Device.Backend.Config.Repo) + require.Equal(t, newCfg.Device.Backend.Config.Proxy, cfg.Device.Backend.Config.Proxy) + require.Equal(t, newCfg.Device.Backend.Config.BlobURLScheme, cfg.Device.Backend.Config.BlobURLScheme) + require.Equal(t, newCfg.Device.Backend.Config.Auth, "") + require.NotEqual(t, newCfg.Device.Backend.Config.Auth, cfg.Device.Backend.Config.Auth) +} From b166c69c501f45c8af9849987d30b8f130c45349 Mon Sep 17 00:00:00 2001 From: Guijie Wang Date: Sun, 1 Oct 2023 11:44:18 +0800 Subject: [PATCH 3/8] system: add interface for nydusd to obtain backend config This commit introduces a new interface that allows nydusd to request backend configurations from the snapshotter via a Unix domain socket. Transferring all backend configs is to cope with the addition of different backend in the future. Signed-off-by: Guijie Wang --- pkg/system/system.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/pkg/system/system.go b/pkg/system/system.go index f073604f5b..d07034314e 100644 --- a/pkg/system/system.go +++ b/pkg/system/system.go @@ -42,6 +42,8 @@ const ( endpointDaemonRecords string = "/api/v1/daemons/records" endpointDaemonsUpgrade string = "/api/v1/daemons/upgrade" endpointPrefetch string = "/api/v1/prefetch" + // Provide backend information + endpointGetBackend string = "/api/v1/daemons/{id}/backend" ) const defaultErrorCode string = "Unknown" @@ -171,6 +173,47 @@ func (sc *Controller) registerRouter() { sc.router.HandleFunc(endpointDaemonsUpgrade, sc.upgradeDaemons()).Methods(http.MethodPut) sc.router.HandleFunc(endpointDaemonRecords, sc.getDaemonRecords()).Methods(http.MethodGet) sc.router.HandleFunc(endpointPrefetch, sc.setPrefetchConfiguration()).Methods(http.MethodPut) + sc.router.HandleFunc(endpointGetBackend, sc.getBackend()).Methods(http.MethodGet) +} + +func (sc *Controller) getBackend() func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + var err error + var statusCode int + + defer func() { + if err != nil { + m := newErrorMessage(err.Error()) + http.Error(w, m.encode(), statusCode) + } + }() + + vars := mux.Vars(r) + id := vars["id"] + + for _, ma := range sc.managers { + ma.Lock() + d := ma.GetByDaemonID(id) + + if d != nil { + backendType, backendConfig := d.Config.StorageBackend() + backend := struct { + BackendType string `json:"type"` + Config interface{} `json:"config"` + }{ + backendType, + backendConfig, + } + jsonResponse(w, backend) + ma.Unlock() + return + } + ma.Unlock() + } + + err = errdefs.ErrNotFound + statusCode = http.StatusNotFound + } } func (sc *Controller) setPrefetchConfiguration() func(w http.ResponseWriter, r *http.Request) { From c64c21942b24953a5821ef00b7d42921594072df Mon Sep 17 00:00:00 2001 From: Guijie Wang Date: Tue, 31 Oct 2023 16:15:09 +0800 Subject: [PATCH 4/8] Fixed CI errors Signed-off-by: Guijie Wang --- config/daemonconfig/daemonconfig.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index 314cdbdbc1..c5766e35ee 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -221,12 +221,11 @@ func serializeWithSecretFilter(obj interface{}) map[string]interface{} { continue } - switch fieldType.Type.Kind() { - case reflect.Struct: + if fieldType.Type.Kind() == reflect.Struct { result[jsonTags[0]] = serializeWithSecretFilter(field.Interface()) - case reflect.Ptr: + } else if fieldType.Type.Kind() == reflect.Ptr { result[jsonTags[0]] = serializeWithSecretFilter(field.Elem().Interface()) - default: + } else { result[jsonTags[0]] = field.Interface() } } From a078fd32642b4543971f6e777153ac043ffc90e2 Mon Sep 17 00:00:00 2001 From: Guijie Wang Date: Tue, 31 Oct 2023 16:35:34 +0800 Subject: [PATCH 5/8] Fixed CI error Signed-off-by: Guijie Wang --- config/daemonconfig/daemonconfig.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index c5766e35ee..2360911c4d 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -221,11 +221,14 @@ func serializeWithSecretFilter(obj interface{}) map[string]interface{} { continue } - if fieldType.Type.Kind() == reflect.Struct { + switch fieldType.Type.Kind() { + case reflect.Struct: result[jsonTags[0]] = serializeWithSecretFilter(field.Interface()) - } else if fieldType.Type.Kind() == reflect.Ptr { + case reflect.Ptr: result[jsonTags[0]] = serializeWithSecretFilter(field.Elem().Interface()) - } else { + case reflect.Invalid: + + default: result[jsonTags[0]] = field.Interface() } } From 2954505815831293483a7e8ef3f287d6bd0bc144 Mon Sep 17 00:00:00 2001 From: Guijie Wang Date: Tue, 31 Oct 2023 16:51:36 +0800 Subject: [PATCH 6/8] Fixed CI error Signed-off-by: Guijie Wang --- config/daemonconfig/daemonconfig.go | 50 ++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index 2360911c4d..2a442cfe7d 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -226,7 +226,55 @@ func serializeWithSecretFilter(obj interface{}) map[string]interface{} { result[jsonTags[0]] = serializeWithSecretFilter(field.Interface()) case reflect.Ptr: result[jsonTags[0]] = serializeWithSecretFilter(field.Elem().Interface()) - case reflect.Invalid: + case reflect.Bool: + result[jsonTags[0]] = field.Interface() + case reflect.Int: + result[jsonTags[0]] = field.Interface() + case reflect.Int8: + result[jsonTags[0]] = field.Interface() + case reflect.Int16: + result[jsonTags[0]] = field.Interface() + case reflect.Int32: + result[jsonTags[0]] = field.Interface() + case reflect.Int64: + result[jsonTags[0]] = field.Interface() + case reflect.Uint: + result[jsonTags[0]] = field.Interface() + case reflect.Uint8: + result[jsonTags[0]] = field.Interface() + case reflect.Uint16: + result[jsonTags[0]] = field.Interface() + case reflect.Uint32: + result[jsonTags[0]] = field.Interface() + case reflect.Uint64: + result[jsonTags[0]] = field.Interface() + case reflect.Uintptr: + result[jsonTags[0]] = field.Interface() + case reflect.Float32: + result[jsonTags[0]] = field.Interface() + case reflect.Float64: + result[jsonTags[0]] = field.Interface() + case reflect.Complex64: + result[jsonTags[0]] = field.Interface() + case reflect.Complex128: + result[jsonTags[0]] = field.Interface() + case reflect.Array: + result[jsonTags[0]] = field.Interface() + case reflect.Chan: + result[jsonTags[0]] = field.Interface() + case reflect.Func: + result[jsonTags[0]] = field.Interface() + case reflect.Interface: + result[jsonTags[0]] = field.Interface() + case reflect.Map: + result[jsonTags[0]] = field.Interface() + case reflect.Slice: + result[jsonTags[0]] = field.Interface() + case reflect.String: + result[jsonTags[0]] = field.Interface() + case reflect.UnsafePointer: + result[jsonTags[0]] = field.Interface() + case reflect.Invalid: default: result[jsonTags[0]] = field.Interface() From 1561c9572bf8cbf5c1699ebff0cbcf0ddb0ef628 Mon Sep 17 00:00:00 2001 From: Guijie Wang Date: Tue, 31 Oct 2023 17:20:54 +0800 Subject: [PATCH 7/8] Fixed CI error Signed-off-by: Guijie Wang --- config/daemonconfig/daemonconfig.go | 51 +---------------------------- 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index 2a442cfe7d..63453f9e01 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -221,61 +221,12 @@ func serializeWithSecretFilter(obj interface{}) map[string]interface{} { continue } + //nolint:staticcheck switch fieldType.Type.Kind() { case reflect.Struct: result[jsonTags[0]] = serializeWithSecretFilter(field.Interface()) case reflect.Ptr: result[jsonTags[0]] = serializeWithSecretFilter(field.Elem().Interface()) - case reflect.Bool: - result[jsonTags[0]] = field.Interface() - case reflect.Int: - result[jsonTags[0]] = field.Interface() - case reflect.Int8: - result[jsonTags[0]] = field.Interface() - case reflect.Int16: - result[jsonTags[0]] = field.Interface() - case reflect.Int32: - result[jsonTags[0]] = field.Interface() - case reflect.Int64: - result[jsonTags[0]] = field.Interface() - case reflect.Uint: - result[jsonTags[0]] = field.Interface() - case reflect.Uint8: - result[jsonTags[0]] = field.Interface() - case reflect.Uint16: - result[jsonTags[0]] = field.Interface() - case reflect.Uint32: - result[jsonTags[0]] = field.Interface() - case reflect.Uint64: - result[jsonTags[0]] = field.Interface() - case reflect.Uintptr: - result[jsonTags[0]] = field.Interface() - case reflect.Float32: - result[jsonTags[0]] = field.Interface() - case reflect.Float64: - result[jsonTags[0]] = field.Interface() - case reflect.Complex64: - result[jsonTags[0]] = field.Interface() - case reflect.Complex128: - result[jsonTags[0]] = field.Interface() - case reflect.Array: - result[jsonTags[0]] = field.Interface() - case reflect.Chan: - result[jsonTags[0]] = field.Interface() - case reflect.Func: - result[jsonTags[0]] = field.Interface() - case reflect.Interface: - result[jsonTags[0]] = field.Interface() - case reflect.Map: - result[jsonTags[0]] = field.Interface() - case reflect.Slice: - result[jsonTags[0]] = field.Interface() - case reflect.String: - result[jsonTags[0]] = field.Interface() - case reflect.UnsafePointer: - result[jsonTags[0]] = field.Interface() - case reflect.Invalid: - default: result[jsonTags[0]] = field.Interface() } From 402b9e4a3f2d0e2d1253929d9847113ce9bd1263 Mon Sep 17 00:00:00 2001 From: Guijie Wang Date: Tue, 31 Oct 2023 18:24:06 +0800 Subject: [PATCH 8/8] Fixed CI errors Signed-off-by: Guijie Wang --- config/daemonconfig/daemonconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index 63453f9e01..f515185a32 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -221,7 +221,7 @@ func serializeWithSecretFilter(obj interface{}) map[string]interface{} { continue } - //nolint:staticcheck + //nolint:exhaustive switch fieldType.Type.Kind() { case reflect.Struct: result[jsonTags[0]] = serializeWithSecretFilter(field.Interface())