diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 18e28a016efee..ccded5758a3ab 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,5 +1,6 @@ ## Main +- [11802](https://github.com/grafana/loki/pull/11802) **xperimental**: Add support for running with Azure Workload Identity - [11824](https://github.com/grafana/loki/pull/11824) **xperimental**: Improve messages for errors in storage secret - [11524](https://github.com/grafana/loki/pull/11524) **JoaoBraveCoding**, **periklis**: Add OpenShift cloud credentials support for AWS STS - [11513](https://github.com/grafana/loki/pull/11513) **btaani**: Add a custom metric that collects Lokistacks requiring a schema upgrade diff --git a/operator/internal/handlers/internal/storage/secrets.go b/operator/internal/handlers/internal/storage/secrets.go index e41fa9c2c5b08..76ba037eb89bd 100644 --- a/operator/internal/handlers/internal/storage/secrets.go +++ b/operator/internal/handlers/internal/storage/secrets.go @@ -28,6 +28,9 @@ var ( errSecretHashError = errors.New("error calculating hash for secret") errS3NoAuth = errors.New("missing secret fields for static or sts authentication") + + errAzureNoCredentials = errors.New("azure storage secret does contain neither account_key or client_id") + errAzureMixedCredentials = errors.New("azure storage secret can not contain both account_key and client_id") ) func getSecrets(ctx context.Context, k k8s.Client, stack *lokiv1.LokiStack, fg configv1.FeatureGates) (*corev1.Secret, *corev1.Secret, error) { @@ -165,25 +168,58 @@ func extractAzureConfigSecret(s *corev1.Secret) (*storage.AzureStorageConfig, er if len(container) == 0 { return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageContainerName) } - name := s.Data[storage.KeyAzureStorageAccountName] - if len(name) == 0 { - return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageAccountName) - } - key := s.Data[storage.KeyAzureStorageAccountKey] - if len(key) == 0 { - return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageAccountKey) + workloadIdentity, err := validateAzureCredentials(s) + if err != nil { + return nil, err } // Extract and validate optional fields endpointSuffix := s.Data[storage.KeyAzureStorageEndpointSuffix] return &storage.AzureStorageConfig{ - Env: string(env), - Container: string(container), - EndpointSuffix: string(endpointSuffix), + Env: string(env), + Container: string(container), + EndpointSuffix: string(endpointSuffix), + WorkloadIdentity: workloadIdentity, }, nil } +func validateAzureCredentials(s *corev1.Secret) (workloadIdentity bool, err error) { + accountName := s.Data[storage.KeyAzureStorageAccountName] + accountKey := s.Data[storage.KeyAzureStorageAccountKey] + clientID := s.Data[storage.KeyAzureStorageClientID] + tenantID := s.Data[storage.KeyAzureStorageTenantID] + subscriptionID := s.Data[storage.KeyAzureStorageSubscriptionID] + + if len(accountName) == 0 { + return false, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageAccountName) + } + + if len(accountKey) == 0 && len(clientID) == 0 { + return false, errAzureNoCredentials + } + + if len(accountKey) > 0 && len(clientID) > 0 { + return false, errAzureMixedCredentials + } + + if len(accountKey) > 0 { + // have both account_name and account_key -> no workload identity federation + return false, nil + } + + // assume workload-identity from here on + if len(tenantID) == 0 { + return false, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageTenantID) + } + + if len(subscriptionID) == 0 { + return false, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageSubscriptionID) + } + + return true, nil +} + func extractGCSConfigSecret(s *corev1.Secret) (*storage.GCSStorageConfig, error) { // Extract and validate mandatory fields bucket := s.Data[storage.KeyGCPStorageBucketName] diff --git a/operator/internal/handlers/internal/storage/secrets_test.go b/operator/internal/handlers/internal/storage/secrets_test.go index 70aebd18afc53..9d32a594e1874 100644 --- a/operator/internal/handlers/internal/storage/secrets_test.go +++ b/operator/internal/handlers/internal/storage/secrets_test.go @@ -101,7 +101,7 @@ func TestAzureExtract(t *testing.T) { wantError: "missing secret field: account_name", }, { - name: "missing account_key", + name: "no account_key or client_id", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Data: map[string][]byte{ @@ -110,10 +110,51 @@ func TestAzureExtract(t *testing.T) { "account_name": []byte("id"), }, }, - wantError: "missing secret field: account_key", + wantError: errAzureNoCredentials.Error(), }, { - name: "all mandatory set", + name: "both account_key and client_id set", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "account_key": []byte("test-account-key"), + "client_id": []byte("test-client-id"), + }, + }, + wantError: errAzureMixedCredentials.Error(), + }, + { + name: "missing tenant_id", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "client_id": []byte("test-client-id"), + }, + }, + wantError: "missing secret field: tenant_id", + }, + { + name: "missing subscription_id", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "client_id": []byte("test-client-id"), + "tenant_id": []byte("test-tenant-id"), + }, + }, + wantError: "missing secret field: subscription_id", + }, + { + name: "mandatory for normal authentication set", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Data: map[string][]byte{ @@ -124,6 +165,21 @@ func TestAzureExtract(t *testing.T) { }, }, }, + { + name: "mandatory for workload-identity set", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "client_id": []byte("test-client-id"), + "tenant_id": []byte("test-tenant-id"), + "subscription_id": []byte("test-subscription-id"), + "region": []byte("test-region"), + }, + }, + }, { name: "all set including optional", secret: &corev1.Secret{ diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index 61c0de401dc10..f908253a0c228 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -13,7 +13,11 @@ common: environment: {{ .Env }} container_name: {{ .Container }} account_name: ${AZURE_STORAGE_ACCOUNT_NAME} + {{- if .WorkloadIdentity }} + use_federated_token: true + {{- else }} account_key: ${AZURE_STORAGE_ACCOUNT_KEY} + {{- end }} {{- with .EndpointSuffix }} endpoint_suffix: {{ . }} {{- end }} diff --git a/operator/internal/manifests/storage/configure.go b/operator/internal/manifests/storage/configure.go index 6f7b22c4bd8ce..b4ff697b1fe4a 100644 --- a/operator/internal/manifests/storage/configure.go +++ b/operator/internal/manifests/storage/configure.go @@ -56,7 +56,6 @@ func ConfigureStatefulSet(d *appsv1.StatefulSet, opts Options) error { // With this, the deployment will expose credentials specific environment variables. func configureDeployment(d *appsv1.Deployment, opts Options) error { p := ensureObjectStoreCredentials(&d.Spec.Template.Spec, opts) - if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithOverride); err != nil { return kverrors.Wrap(err, "failed to merge gcs object storage spec ") } @@ -83,7 +82,6 @@ func configureDeploymentCA(d *appsv1.Deployment, tls *TLSConfig) error { // With this, the statefulset will expose credentials specific environment variable. func configureStatefulSet(s *appsv1.StatefulSet, opts Options) error { p := ensureObjectStoreCredentials(&s.Spec.Template.Spec, opts) - if err := mergo.Merge(&s.Spec.Template.Spec, p, mergo.WithOverride); err != nil { return kverrors.Wrap(err, "failed to merge gcs object storage spec ") } @@ -195,6 +193,14 @@ func managedAuthCredentials(opts Options) []corev1.EnvVar { envVarFromValue(EnvAWSWebIdentityTokenFile, path.Join(opts.S3.WebIdentityTokenFile, "token")), } } + case lokiv1.ObjectStorageSecretAzure: + return []corev1.EnvVar{ + envVarFromSecret(EnvAzureStorageAccountName, opts.SecretName, KeyAzureStorageAccountName), + envVarFromSecret(EnvAzureClientID, opts.SecretName, KeyAzureStorageClientID), + envVarFromSecret(EnvAzureTenantID, opts.SecretName, KeyAzureStorageTenantID), + envVarFromSecret(EnvAzureSubscriptionID, opts.SecretName, KeyAzureStorageSubscriptionID), + envVarFromValue(EnvAzureFederatedTokenFile, path.Join(azureTokenVolumeDirectory, "token")), + } default: return []corev1.EnvVar{} } @@ -273,6 +279,8 @@ func managedAuthEnabled(opts Options) bool { switch opts.SharedStore { case lokiv1.ObjectStorageSecretS3: return opts.S3 != nil && opts.S3.STS + case lokiv1.ObjectStorageSecretAzure: + return opts.Azure != nil && opts.Azure.WorkloadIdentity default: return false } @@ -293,6 +301,8 @@ func saTokenVolumeMount(opts Options) corev1.VolumeMount { switch opts.SharedStore { case lokiv1.ObjectStorageSecretS3: tokenPath = opts.S3.WebIdentityTokenFile + case lokiv1.ObjectStorageSecretAzure: + tokenPath = azureTokenVolumeDirectory } return corev1.VolumeMount{ Name: saTokenVolumeName, @@ -312,6 +322,8 @@ func saTokenVolume(opts Options) corev1.Volume { if opts.OpenShift.Enabled { audience = AWSOpenShiftAudience } + case lokiv1.ObjectStorageSecretAzure: + audience = azureDefaultAudience } return corev1.Volume{ Name: saTokenVolumeName, diff --git a/operator/internal/manifests/storage/configure_test.go b/operator/internal/manifests/storage/configure_test.go index 3b3029733554d..0b64a8eb8328e 100644 --- a/operator/internal/manifests/storage/configure_test.go +++ b/operator/internal/manifests/storage/configure_test.go @@ -168,6 +168,130 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, + { + desc: "object storage Azure with WIF", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretAzure, + Azure: &AzureStorageConfig{ + WorkloadIdentity: true, + }, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: "/var/run/secrets/azure/serviceaccount", + }, + }, + Env: []corev1.EnvVar{ + { + Name: EnvAzureStorageAccountName, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageAccountName, + }, + }, + }, + { + Name: EnvAzureClientID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageClientID, + }, + }, + }, + { + Name: EnvAzureTenantID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageTenantID, + }, + }, + }, + { + Name: EnvAzureSubscriptionID, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + Key: KeyAzureStorageSubscriptionID, + }, + }, + }, + { + Name: EnvAzureFederatedTokenFile, + Value: "/var/run/secrets/azure/serviceaccount/token", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: azureDefaultAudience, + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { desc: "object storage GCS", opts: Options{ diff --git a/operator/internal/manifests/storage/options.go b/operator/internal/manifests/storage/options.go index 80efb24f62c8b..e1348297ab59c 100644 --- a/operator/internal/manifests/storage/options.go +++ b/operator/internal/manifests/storage/options.go @@ -25,9 +25,10 @@ type Options struct { // AzureStorageConfig for Azure storage config type AzureStorageConfig struct { - Env string - Container string - EndpointSuffix string + Env string + Container string + EndpointSuffix string + WorkloadIdentity bool } // GCSStorageConfig for GCS storage config diff --git a/operator/internal/manifests/storage/var.go b/operator/internal/manifests/storage/var.go index d77de3262d314..048af9e8a88e8 100644 --- a/operator/internal/manifests/storage/var.go +++ b/operator/internal/manifests/storage/var.go @@ -13,7 +13,7 @@ const ( EnvAWSSseKmsEncryptionContext = "AWS_SSE_KMS_ENCRYPTION_CONTEXT" // EnvAWSRoleArn is the environment variable to specify the AWS role ARN secret for the federated identity workflow. EnvAWSRoleArn = "AWS_ROLE_ARN" - // EnvAWSWebIdentityToken is the environment variable to specify the path to the web identity token file used in the federated identity workflow. + // EnvAWSWebIdentityTokenFile is the environment variable to specify the path to the web identity token file used in the federated identity workflow. EnvAWSWebIdentityTokenFile = "AWS_WEB_IDENTITY_TOKEN_FILE" // EnvAWSCredentialsFile is the environment variable to specify the path to the shared credentials file EnvAWSCredentialsFile = "AWS_SHARED_CREDENTIALS_FILE" @@ -23,6 +23,14 @@ const ( EnvAzureStorageAccountName = "AZURE_STORAGE_ACCOUNT_NAME" // EnvAzureStorageAccountKey is the environment variable to specify the Azure storage account key to access the container. EnvAzureStorageAccountKey = "AZURE_STORAGE_ACCOUNT_KEY" + // EnvAzureClientID is the environment variable used to pass the Managed Identity client-ID to the container. + EnvAzureClientID = "AZURE_CLIENT_ID" + // EnvAzureTenantID is the environment variable used to pass the Managed Identity tenant-ID to the container. + EnvAzureTenantID = "AZURE_TENANT_ID" + // EnvAzureSubscriptionID is the environment variable used to pass the Managed Identity subscription-ID to the container. + EnvAzureSubscriptionID = "AZURE_SUBSCRIPTION_ID" + // EnvAzureFederatedTokenFile is the environment variable used to store the path to the Managed Identity token. + EnvAzureFederatedTokenFile = "AZURE_FEDERATED_TOKEN_FILE" // EnvGoogleApplicationCredentials is the environment variable to specify path to key.json EnvGoogleApplicationCredentials = "GOOGLE_APPLICATION_CREDENTIALS" // EnvSwiftPassword is the environment variable to specify the OpenStack Swift password. @@ -66,6 +74,12 @@ const ( KeyAzureStorageAccountKey = "account_key" // KeyAzureStorageAccountName is the secret data key for the Azure storage account name. KeyAzureStorageAccountName = "account_name" + // KeyAzureStorageClientID contains the UUID of the Managed Identity accessing the storage. + KeyAzureStorageClientID = "client_id" + // KeyAzureStorageTenantID contains the UUID of the Tenant hosting the Managed Identity. + KeyAzureStorageTenantID = "tenant_id" + // KeyAzureStorageSubscriptionID contains the UUID of the subscription hosting the Managed Identity. + KeyAzureStorageSubscriptionID = "subscription_id" // KeyAzureStorageContainerName is the secret data key for the Azure storage container name. KeyAzureStorageContainerName = "container" // KeyAzureStorageEndpointSuffix is the secret data key for the Azure storage endpoint URL suffix. @@ -100,11 +114,11 @@ const ( KeySwiftRegion = "region" // KeySwiftUserDomainID is the secret data key for the OpenStack Swift user domain id. KeySwiftUserDomainID = "user_domain_id" - // KeySwiftUserDomainID is the secret data key for the OpenStack Swift user domain name. + // KeySwiftUserDomainName is the secret data key for the OpenStack Swift user domain name. KeySwiftUserDomainName = "user_domain_name" // KeySwiftUserID is the secret data key for the OpenStack Swift user id. KeySwiftUserID = "user_id" - // KeySwiftPassword is the secret data key for the OpenStack Swift password. + // KeySwiftUsername is the secret data key for the OpenStack Swift password. KeySwiftUsername = "username" saTokenVolumeK8sDirectory = "/var/run/secrets/kubernetes.io/serviceaccount" @@ -120,5 +134,8 @@ const ( awsDefaultAudience = "sts.amazonaws.com" AWSOpenShiftAudience = "openshift" + azureDefaultAudience = "api://AzureADTokenExchange" + azureTokenVolumeDirectory = "/var/run/secrets/azure/serviceaccount" + AnnotationCredentialsRequestsSecretRef = "loki.grafana.com/credentials-request-secret-ref" )