From 878fea5867e3bb709c4faf512039c8c0cdb5e9b2 Mon Sep 17 00:00:00 2001 From: Suleiman Dibirov Date: Mon, 22 Jul 2024 19:28:30 +0300 Subject: [PATCH] feat: add separate hashes for service configs and secrets, add file folder support Signed-off-by: Suleiman Dibirov --- pkg/api/labels.go | 6 +- pkg/compose/convergence.go | 12 +++- pkg/compose/create.go | 10 ++- pkg/compose/hash.go | 127 +++++++++++++++++++++++++++++++++---- pkg/compose/hash_test.go | 105 +++++++++++++++++++++++++----- pkg/compose/secrets.go | 53 +--------------- pkg/utils/tar.go | 70 ++++++++++++++++++++ 7 files changed, 296 insertions(+), 87 deletions(-) create mode 100644 pkg/utils/tar.go diff --git a/pkg/api/labels.go b/pkg/api/labels.go index 3a6ee60fba..6248d88e40 100644 --- a/pkg/api/labels.go +++ b/pkg/api/labels.go @@ -31,8 +31,10 @@ const ( ServiceLabel = "com.docker.compose.service" // ConfigHashLabel stores configuration hash for a compose service ConfigHashLabel = "com.docker.compose.config-hash" - // ConfigHashDependenciesLabel stores configuration hash for a compose service dependencies - ConfigHashDependenciesLabel = "com.docker.compose.config-hash-dependencies" + // ServiceConfigsHash stores configuration hash for a compose service configs + ServiceConfigsHash = "com.docker.compose.service.configs-hash" + // ServiceSecretsHash stores configuration hash for a compose service secrets + ServiceSecretsHash = "com.docker.compose.service.secrets-hash" // ContainerNumberLabel stores the container index of a replicated service ContainerNumberLabel = "com.docker.compose.container-number" // VolumeLabel allow to track resource related to a compose volume diff --git a/pkg/compose/convergence.go b/pkg/compose/convergence.go index e34b5175fa..fba0fa6e0b 100644 --- a/pkg/compose/convergence.go +++ b/pkg/compose/convergence.go @@ -312,14 +312,20 @@ func mustRecreate(project *types.Project, expected types.ServiceConfig, actual m return true, nil } - serviceDependenciesHash, err := ServiceDependenciesHash(project, expected) + serviceConfigsHash, err := ServiceConfigsHash(project, expected) if err != nil { return false, err } - serviceDependenciesChanged := actual.Labels[api.ConfigHashDependenciesLabel] != serviceDependenciesHash + serviceSecretsHash, err := ServiceSecretsHash(project, expected) + if err != nil { + return false, err + } + + serviceConfigsChanged := actual.Labels[api.ServiceConfigsHash] != serviceConfigsHash + serviceSecretsChanged := actual.Labels[api.ServiceSecretsHash] != serviceSecretsHash - return serviceDependenciesChanged, nil + return serviceConfigsChanged || serviceSecretsChanged, nil } func getContainerName(projectName string, service types.ServiceConfig, number int) string { diff --git a/pkg/compose/create.go b/pkg/compose/create.go index e2f85a7fe4..ef350154fa 100644 --- a/pkg/compose/create.go +++ b/pkg/compose/create.go @@ -504,13 +504,19 @@ func (s *composeService) prepareLabels(labels types.Labels, project *types.Proje return nil, err } - serviceDependenciesHash, err := ServiceDependenciesHash(project, service) + serviceConfigsHash, err := ServiceConfigsHash(project, service) + if err != nil { + return nil, err + } + + serviceSecretsHash, err := ServiceSecretsHash(project, service) if err != nil { return nil, err } labels[api.ConfigHashLabel] = serviceHash - labels[api.ConfigHashDependenciesLabel] = serviceDependenciesHash + labels[api.ServiceConfigsHash] = serviceConfigsHash + labels[api.ServiceSecretsHash] = serviceSecretsHash labels[api.ContainerNumberLabel] = strconv.Itoa(number) var dependencies []string diff --git a/pkg/compose/hash.go b/pkg/compose/hash.go index 7d38d8d8b3..dfc14ab53d 100644 --- a/pkg/compose/hash.go +++ b/pkg/compose/hash.go @@ -17,10 +17,15 @@ package compose import ( + "bytes" "encoding/json" + "fmt" "os" + "path/filepath" + "time" "github.com/compose-spec/compose-go/v2/types" + "github.com/docker/compose/v2/pkg/utils" "github.com/opencontainers/go-digest" ) @@ -41,27 +46,121 @@ func ServiceHash(o types.ServiceConfig) (string, error) { return digest.SHA256.FromBytes(bytes).Encoded(), nil } -// ServiceDependenciesHash computes the configuration hash for service dependencies. -func ServiceDependenciesHash(project *types.Project, o types.ServiceConfig) (string, error) { +// ServiceConfigsHash computes the configuration hash for service configs. +func ServiceConfigsHash(project *types.Project, serviceConfig types.ServiceConfig) (string, error) { bytes := make([]byte, 0) - for _, serviceConfig := range o.Configs { - projectConfig, ok := project.Configs[serviceConfig.Source] + for _, config := range serviceConfig.Configs { + file := project.Configs[config.Source] + b, err := createTarForConfig(project, types.FileReferenceConfig(config), types.FileObjectConfig(file)) + + if err != nil { + return "", err + } + + bytes = append(bytes, b.Bytes()...) + } + + return digest.SHA256.FromBytes(bytes).Encoded(), nil +} + +// ServiceSecretsHash computes the configuration hash for service secrets. +func ServiceSecretsHash(project *types.Project, serviceConfig types.ServiceConfig) (string, error) { + bytes := make([]byte, 0) + for _, config := range serviceConfig.Secrets { + file := project.Secrets[config.Source] + b, err := createTarForConfig(project, types.FileReferenceConfig(config), types.FileObjectConfig(file)) + + if err != nil { + return "", err + } + + bytes = append(bytes, b.Bytes()...) + } + + return digest.SHA256.FromBytes(bytes).Encoded(), nil +} + +func createTarForConfig( + project *types.Project, + serviceConfig types.FileReferenceConfig, + file types.FileObjectConfig, +) (*bytes.Buffer, error) { + // fixed time to ensure the tarball is deterministic + modTime := time.Unix(0, 0) + content := make([]byte, 0) + + if file.Content != "" { + content = []byte(file.Content) + } else if file.Environment != "" { + env, ok := project.Environment[file.Environment] if !ok { - continue + return nil, fmt.Errorf( + "environment variable %q required by file %q is not set", + file.Environment, + file.Name, + ) } + content = []byte(env) + } else if file.File != "" { + var err error + content, err = readPathContent(file.File) - if projectConfig.Content != "" { - bytes = append(bytes, []byte(projectConfig.Content)...) - } else if projectConfig.File != "" { - content, err := os.ReadFile(projectConfig.File) + if err != nil { + return nil, err + } + } + + if len(content) == 0 { + return nil, fmt.Errorf("config %q is empty", file.Name) + } + + if serviceConfig.Target == "" { + serviceConfig.Target = "/" + serviceConfig.Source + } + + b, err := utils.CreateTar(content, serviceConfig, modTime) + if err != nil { + return nil, err + } + + return &b, nil +} + +func readPathContent(path string) ([]byte, error) { + content := make([]byte, 0) + + // Check if the path is a directory + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("error accessing path %q: %v", path, err) + } + + if info.IsDir() { + // If it's a directory, read all files and concatenate their contents + err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { if err != nil { - return "", err + return err + } + if !info.IsDir() { + fileContent, err := os.ReadFile(path) + if err != nil { + return err + } + content = append(content, fileContent...) } - bytes = append(bytes, content...) - } else if projectConfig.Environment != "" { - bytes = append(bytes, []byte(projectConfig.Environment)...) + return nil + }) + if err != nil { + return nil, fmt.Errorf("error reading directory %q: %v", path, err) } + } else { + // If it's a file, read its content + fileContent, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("error reading file %q: %v", path, err) + } + content = fileContent } - return digest.SHA256.FromBytes(bytes).Encoded(), nil + return content, nil } diff --git a/pkg/compose/hash_test.go b/pkg/compose/hash_test.go index 0aef924c8d..631cd569c6 100644 --- a/pkg/compose/hash_test.go +++ b/pkg/compose/hash_test.go @@ -39,56 +39,124 @@ func TestServiceHashWithIgnorableValues(t *testing.T) { assert.Equal(t, hash1, hash2) } -func TestServiceDependenciesHashWithoutChangesContent(t *testing.T) { - hash1, err := ServiceDependenciesHash(projectConfig("myConfigSource", "a", "", ""), serviceConfig("myContext1", "always", 1)) +func TestServiceConfigsHashWithoutChangesContent(t *testing.T) { + hash1, err := ServiceConfigsHash(projectWithConfigs("myConfigSource", "a", "", ""), serviceConfig("myContext1", "always", 1)) assert.NilError(t, err) - hash2, err := ServiceDependenciesHash(projectConfig("myConfigSource", "a", "", ""), serviceConfig("myContext2", "never", 2)) + hash2, err := ServiceConfigsHash(projectWithConfigs("myConfigSource", "a", "", ""), serviceConfig("myContext2", "never", 2)) assert.NilError(t, err) assert.Assert(t, hash1 == hash2) } -func TestServiceDependenciesHashWithChangedConfigContent(t *testing.T) { - hash1, err := ServiceDependenciesHash(projectConfig("myConfigSource", "a", "", ""), serviceConfig("myContext1", "always", 1)) +func TestServiceConfigsHashWithChangedConfigContent(t *testing.T) { + hash1, err := ServiceConfigsHash(projectWithConfigs("myConfigSource", "a", "", ""), serviceConfig("myContext1", "always", 1)) assert.NilError(t, err) - hash2, err := ServiceDependenciesHash(projectConfig("myConfigSource", "b", "", ""), serviceConfig("myContext2", "never", 2)) + hash2, err := ServiceConfigsHash(projectWithConfigs("myConfigSource", "b", "", ""), serviceConfig("myContext2", "never", 2)) assert.NilError(t, err) assert.Assert(t, hash1 != hash2) } -func TestServiceDependenciesHashWithChangedConfigEnvironment(t *testing.T) { - hash1, err := ServiceDependenciesHash(projectConfig("myConfigSource", "", "a", ""), serviceConfig("myContext1", "always", 1)) +func TestServiceConfigsHashWithChangedConfigEnvironment(t *testing.T) { + hash1, err := ServiceConfigsHash(projectWithConfigs("myConfigSource", "", "a", ""), serviceConfig("myContext1", "always", 1)) assert.NilError(t, err) - hash2, err := ServiceDependenciesHash(projectConfig("myConfigSource", "", "b", ""), serviceConfig("myContext2", "never", 2)) + hash2, err := ServiceConfigsHash(projectWithConfigs("myConfigSource", "", "b", ""), serviceConfig("myContext2", "never", 2)) assert.NilError(t, err) assert.Assert(t, hash1 != hash2) } -func TestServiceDependenciesHashWithChangedConfigFile(t *testing.T) { - hash1, err := ServiceDependenciesHash( - projectConfig("myConfigSource", "", "", "./testdata/config1.txt"), +func TestServiceConfigsHashWithChangedConfigFile(t *testing.T) { + hash1, err := ServiceConfigsHash( + projectWithConfigs("myConfigSource", "", "", "./testdata/config1.txt"), serviceConfig("myContext1", "always", 1), ) assert.NilError(t, err) - hash2, err := ServiceDependenciesHash( - projectConfig("myConfigSource", "", "", "./testdata/config2.txt"), + hash2, err := ServiceConfigsHash( + projectWithConfigs("myConfigSource", "", "", "./testdata/config2.txt"), serviceConfig("myContext2", "never", 2), ) assert.NilError(t, err) assert.Assert(t, hash1 != hash2) } -func projectConfig(configName, configContent, configEnvironment, configFile string) *types.Project { +func TestServiceSecretsHashWithoutChangesContent(t *testing.T) { + hash1, err := ServiceSecretsHash(projectWithSecrets("mySecretSource", "a", "", ""), serviceConfig("myContext1", "always", 1)) + assert.NilError(t, err) + hash2, err := ServiceSecretsHash(projectWithSecrets("mySecretSource", "a", "", ""), serviceConfig("myContext2", "never", 2)) + assert.NilError(t, err) + assert.Assert(t, hash1 == hash2) +} + +func TestServiceSecretsHashWithChangedSecretContent(t *testing.T) { + hash1, err := ServiceSecretsHash(projectWithSecrets("mySecretSource", "a", "", ""), serviceConfig("myContext1", "always", 1)) + assert.NilError(t, err) + hash2, err := ServiceSecretsHash(projectWithSecrets("mySecretSource", "b", "", ""), serviceConfig("myContext2", "never", 2)) + assert.NilError(t, err) + assert.Assert(t, hash1 != hash2) +} + +func TestServiceSecretsHashWithChangedSecretEnvironment(t *testing.T) { + hash1, err := ServiceSecretsHash(projectWithSecrets("mySecretSource", "", "a", ""), serviceConfig("myContext1", "always", 1)) + assert.NilError(t, err) + hash2, err := ServiceSecretsHash(projectWithSecrets("mySecretSource", "", "b", ""), serviceConfig("myContext2", "never", 2)) + assert.NilError(t, err) + assert.Assert(t, hash1 != hash2) +} + +func TestServiceSecretsHashWithChangedSecretFile(t *testing.T) { + hash1, err := ServiceSecretsHash( + projectWithSecrets("mySecretSource", "", "", "./testdata/config1.txt"), + serviceConfig("myContext1", "always", 1), + ) + assert.NilError(t, err) + hash2, err := ServiceSecretsHash( + projectWithSecrets("mySecretSource", "", "", "./testdata/config2.txt"), + serviceConfig("myContext2", "never", 2), + ) + assert.NilError(t, err) + assert.Assert(t, hash1 != hash2) +} + +func projectWithConfigs(configName, configContent, configEnvironmentValue, configFile string) *types.Project { + envName := "myEnv" + + if configEnvironmentValue == "" { + envName = "" + } + return &types.Project{ + Environment: types.Mapping{ + envName: configEnvironmentValue, + }, Configs: types.Configs{ configName: types.ConfigObjConfig{ Content: configContent, - Environment: configEnvironment, + Environment: envName, File: configFile, }, }, } } +func projectWithSecrets(secretName, secretContent, secretEnvironmentValue, secretFile string) *types.Project { + envName := "myEnv" + + if secretEnvironmentValue == "" { + envName = "" + } + + return &types.Project{ + Environment: types.Mapping{ + envName: secretEnvironmentValue, + }, + Secrets: types.Secrets{ + secretName: types.SecretConfig{ + Content: secretContent, + Environment: envName, + File: secretFile, + }, + }, + } +} + func serviceConfig(buildContext, pullPolicy string, replicas int) types.ServiceConfig { return types.ServiceConfig{ Build: &types.BuildConfig{ @@ -106,5 +174,10 @@ func serviceConfig(buildContext, pullPolicy string, replicas int) types.ServiceC Source: "myConfigSource", }, }, + Secrets: []types.ServiceSecretConfig{ + { + Source: "mySecretSource", + }, + }, } } diff --git a/pkg/compose/secrets.go b/pkg/compose/secrets.go index 4ba49eed44..fcabbcd68f 100644 --- a/pkg/compose/secrets.go +++ b/pkg/compose/secrets.go @@ -17,14 +17,12 @@ package compose import ( - "archive/tar" - "bytes" "context" "fmt" - "strconv" "time" "github.com/compose-spec/compose-go/v2/types" + "github.com/docker/compose/v2/pkg/utils" "github.com/docker/docker/api/types/container" ) @@ -45,7 +43,7 @@ func (s *composeService) injectSecrets(ctx context.Context, project *types.Proje if !ok { return fmt.Errorf("environment variable %q required by file %q is not set", file.Environment, file.Name) } - b, err := createTar(env, types.FileReferenceConfig(config)) + b, err := utils.CreateTar([]byte(env), types.FileReferenceConfig(config), time.Now()) if err != nil { return err } @@ -79,7 +77,7 @@ func (s *composeService) injectConfigs(ctx context.Context, project *types.Proje config.Target = "/" + config.Source } - b, err := createTar(content, types.FileReferenceConfig(config)) + b, err := utils.CreateTar([]byte(content), types.FileReferenceConfig(config), time.Now()) if err != nil { return err } @@ -93,48 +91,3 @@ func (s *composeService) injectConfigs(ctx context.Context, project *types.Proje } return nil } - -func createTar(env string, config types.FileReferenceConfig) (bytes.Buffer, error) { - value := []byte(env) - b := bytes.Buffer{} - tarWriter := tar.NewWriter(&b) - mode := uint32(0o444) - if config.Mode != nil { - mode = *config.Mode - } - - var uid, gid int - if config.UID != "" { - v, err := strconv.Atoi(config.UID) - if err != nil { - return b, err - } - uid = v - } - if config.GID != "" { - v, err := strconv.Atoi(config.GID) - if err != nil { - return b, err - } - gid = v - } - - header := &tar.Header{ - Name: config.Target, - Size: int64(len(value)), - Mode: int64(mode), - ModTime: time.Now(), - Uid: uid, - Gid: gid, - } - err := tarWriter.WriteHeader(header) - if err != nil { - return bytes.Buffer{}, err - } - _, err = tarWriter.Write(value) - if err != nil { - return bytes.Buffer{}, err - } - err = tarWriter.Close() - return b, err -} diff --git a/pkg/utils/tar.go b/pkg/utils/tar.go new file mode 100644 index 0000000000..49305f52e8 --- /dev/null +++ b/pkg/utils/tar.go @@ -0,0 +1,70 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package utils + +import ( + "archive/tar" + "bytes" + "strconv" + "time" + + "github.com/compose-spec/compose-go/v2/types" +) + +func CreateTar(content []byte, config types.FileReferenceConfig, modTime time.Time) (bytes.Buffer, error) { + b := bytes.Buffer{} + tarWriter := tar.NewWriter(&b) + mode := uint32(0o444) + if config.Mode != nil { + mode = *config.Mode + } + + var uid, gid int + if config.UID != "" { + v, err := strconv.Atoi(config.UID) + if err != nil { + return b, err + } + uid = v + } + if config.GID != "" { + v, err := strconv.Atoi(config.GID) + if err != nil { + return b, err + } + gid = v + } + + header := &tar.Header{ + Name: config.Target, + Size: int64(len(content)), + Mode: int64(mode), + ModTime: modTime, + Uid: uid, + Gid: gid, + } + err := tarWriter.WriteHeader(header) + if err != nil { + return bytes.Buffer{}, err + } + _, err = tarWriter.Write(content) + if err != nil { + return bytes.Buffer{}, err + } + err = tarWriter.Close() + return b, err +}