diff --git a/api/v1alpha1/olsconfig_types.go b/api/v1alpha1/olsconfig_types.go index 28880f4c..991eab92 100644 --- a/api/v1alpha1/olsconfig_types.go +++ b/api/v1alpha1/olsconfig_types.go @@ -80,6 +80,9 @@ type OLSSpec struct { // User data collection switches // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="User Data Collection" UserDataCollection UserDataCollectionSpec `json:"userDataCollection,omitempty"` + // Additional CA certificates for TLS communication between OLS service and LLM Provider + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Additional CA Configmap",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"} + AdditionalCAConfigMapRef *corev1.LocalObjectReference `json:"additionalCAConfigMapRef,omitempty"` } // DeploymentConfig defines the schema for overriding deployment of OLS instance. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e3880bff..21c3b486 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -332,6 +332,11 @@ func (in *OLSSpec) DeepCopyInto(out *OLSSpec) { copy(*out, *in) } out.UserDataCollection = in.UserDataCollection + if in.AdditionalCAConfigMapRef != nil { + in, out := &in.AdditionalCAConfigMapRef, &out.AdditionalCAConfigMapRef + *out = new(corev1.LocalObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OLSSpec. diff --git a/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml b/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml index 51599177..9ff73d10 100644 --- a/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml +++ b/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml @@ -114,6 +114,11 @@ spec: path: llm.providers[0].type - displayName: OLS Settings path: ols + - description: Additional CA certificates for TLS communication between OLS service and LLM Provider + displayName: Additional CA Configmap + path: ols.additionalCAConfigMapRef + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - displayName: Redis path: ols.conversationCache.redis - description: Secret that holds redis credentials diff --git a/bundle/manifests/ols.openshift.io_olsconfigs.yaml b/bundle/manifests/ols.openshift.io_olsconfigs.yaml index a7335875..d3c08f7b 100644 --- a/bundle/manifests/ols.openshift.io_olsconfigs.yaml +++ b/bundle/manifests/ols.openshift.io_olsconfigs.yaml @@ -130,6 +130,18 @@ spec: ols: description: OLSSpec defines the desired state of OLS deployment. properties: + additionalCAConfigMapRef: + description: Additional CA certificates for TLS communication + between OLS service and LLM Provider + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic conversationCache: description: Conversation cache settings properties: diff --git a/config/crd/bases/ols.openshift.io_olsconfigs.yaml b/config/crd/bases/ols.openshift.io_olsconfigs.yaml index a520b856..7314270d 100644 --- a/config/crd/bases/ols.openshift.io_olsconfigs.yaml +++ b/config/crd/bases/ols.openshift.io_olsconfigs.yaml @@ -130,6 +130,18 @@ spec: ols: description: OLSSpec defines the desired state of OLS deployment. properties: + additionalCAConfigMapRef: + description: Additional CA certificates for TLS communication + between OLS service and LLM Provider + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic conversationCache: description: Conversation cache settings properties: diff --git a/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml b/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml index dc832a0a..ff5f330c 100644 --- a/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml @@ -81,6 +81,12 @@ spec: path: llm.providers[0].type - displayName: OLS Settings path: ols + - description: Additional CA certificates for TLS communication between OLS + service and LLM Provider + displayName: Additional CA Configmap + path: ols.additionalCAConfigMapRef + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - displayName: Redis path: ols.conversationCache.redis - description: Secret that holds redis credentials diff --git a/internal/controller/constants.go b/internal/controller/constants.go index 93a2a66a..fa725c23 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -57,6 +57,14 @@ const ( AppServerPrometheusRuleName = "lightspeed-app-server-prometheus-rule" // AppServerMetricsPath is the path of the metrics endpoint of the OLS application server AppServerMetricsPath = "/metrics" + // AppAdditionalCACertDir is the directory for storing additional CA certificates in the app server container under OLSAppCertsMountRoot + AppAdditionalCACertDir = "ols-additional-ca" + // AdditionalCAVolumeName is the name of the volume for additional CA certificates provided by the user + AdditionalCAVolumeName = "additional-ca" + // CertBundleVolumeName is the name of the volume for the certificate bundle + CertBundleVolumeName = "cert-bundle" + // CertBundleDir is the path of the volume for the certificate bundle + CertBundleDir = "cert-bundle" // Image of the OLS application redis server // OLSConfigHashKey is the key of the hash value of the OLSConfig configmap @@ -68,6 +76,8 @@ const ( OLSAppTLSHashKey = "hash/olstls" // OLSConsoleTLSHashKey is the key of the hash value of the OLS Console TLS certificates OLSConsoleTLSHashKey = "hash/olsconsoletls" + // AdditionalCAHashKey is the key of the hash value of the additional CA certificates in the deployment annotations + AdditionalCAHashKey = "hash/additionalca" // OLSAppServerContainerPort is the port number of the lightspeed-service-api container exposes OLSAppServerContainerPort = 8443 // OLSAppServerServicePort is the port number for OLS application server service. @@ -90,6 +100,8 @@ const ( LLMProviderHashStateCacheKey = "llmprovider-hash" // AzureOpenAIType is the name of the Azure OpenAI provider type AzureOpenAIType = "azure_openai" + // AdditionalCAHashStateCacheKey is the key of the hash value of the additional CA certificates in the state cache + AdditionalCAHashStateCacheKey = "additionalca-hash" /*** console UI plugin ***/ // ConsoleUIConfigMapName is the name of the console UI nginx configmap ConsoleUIConfigMapName = "lightspeed-console-plugin" diff --git a/internal/controller/constants_test.go b/internal/controller/constants_test.go new file mode 100644 index 00000000..a9857a10 --- /dev/null +++ b/internal/controller/constants_test.go @@ -0,0 +1,28 @@ +package controller + +const testCACert = `-----BEGIN CERTIFICATE----- +MIIEMDCCAxigAwIBAgIJANqb7HHzA7AZMA0GCSqGSIb3DQEBCwUAMIGkMQswCQYD +VQQGEwJQQTEPMA0GA1UECAwGUGFuYW1hMRQwEgYDVQQHDAtQYW5hbWEgQ2l0eTEk +MCIGA1UECgwbVHJ1c3RDb3IgU3lzdGVtcyBTLiBkZSBSLkwuMScwJQYDVQQLDB5U +cnVzdENvciBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFlRydXN0Q29y +IFJvb3RDZXJ0IENBLTEwHhcNMTYwMjA0MTIzMjE2WhcNMjkxMjMxMTcyMzE2WjCB +pDELMAkGA1UEBhMCUEExDzANBgNVBAgMBlBhbmFtYTEUMBIGA1UEBwwLUGFuYW1h +IENpdHkxJDAiBgNVBAoMG1RydXN0Q29yIFN5c3RlbXMgUy4gZGUgUi5MLjEnMCUG +A1UECwweVHJ1c3RDb3IgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MR8wHQYDVQQDDBZU +cnVzdENvciBSb290Q2VydCBDQS0xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAv463leLCJhJrMxnHQFgKq1mqjQCj/IDHUHuO1CAmujIS2CNUSSUQIpid +RtLByZ5OGy4sDjjzGiVoHKZaBeYei0i/mJZ0PmnK6bV4pQa81QBeCQryJ3pS/C3V +seq0iWEk8xoT26nPUu0MJLq5nux+AHT6k61sKZKuUbS701e/s/OojZz0JEsq1pme +9J7+wH5COucLlVPat2gOkEz7cD+PSiyU8ybdY2mplNgQTsVHCJCZGxdNuWxu72CV +EY4hgLW9oHPY0LJ3xEXqWib7ZnZ2+AYfYW0PVcWDtxBWcgYHpfOxGgMFZA6dWorW +hnAbJN7+KIor0Gqw/Hqi3LJ5DotlDwIDAQABo2MwYTAdBgNVHQ4EFgQU7mtJPHo/ +DeOxCbeKyKsZn3MzUOcwHwYDVR0jBBgwFoAU7mtJPHo/DeOxCbeKyKsZn3MzUOcw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggEBACUY1JGPE+6PHh0RU9otRCkZoB5rMZ5NDp6tPVxBb5UrJKF5mDo4Nvu7Zp5I +/5CQ7z3UuJu0h3U/IJvOcs+hVcFNZKIZBqEHMwwLKeXx6quj7LUKdJDHfXLy11yf +ke+Ri7fc7Waiz45mO7yfOgLgJ90WmMCV1Aqk5IGadZQ1nJBfiDcGrVmVCrDRZ9MZ +yonnMlo2HD6CqFqTvsbQZJG2z9m2GM/bftJlo6bEjhcxwft+dtvTheNYsnd6djts +L1Ac59v2Z3kf9YKVmgenFK+P3CghZwnS1k1aHBkcjndcw5QkPTJrS37UeJSDvjdN +zl/HHk484IkzlQsPpTLWPFp5LBk= +-----END CERTIFICATE----- +` diff --git a/internal/controller/errors.go b/internal/controller/errors.go index d5b0bb3e..f328b025 100644 --- a/internal/controller/errors.go +++ b/internal/controller/errors.go @@ -1,6 +1,7 @@ package controller const ( + ErrCreateAdditionalCACM = "failed to create additional CA configmap" ErrCreateAPIConfigmap = "failed to create OLS configmap" ErrCreateAPIDeployment = "failed to create OLS deployment" ErrCreateAPIService = "failed to create OLS service" @@ -14,6 +15,8 @@ const ( ErrCreateServiceMonitor = "failed to create ServiceMonitor" ErrCreatePrometheusRule = "failed to create PrometheusRule" ErrDeleteConsolePlugin = "failed to delete Console Plugin" + ErrDeleteAdditionalCACM = "failed to delete additional CA configmap" + ErrGenerateAdditionalCACM = "failed to generate additional CA configmap" ErrGenerateAPIConfigmap = "failed to generate OLS configmap" ErrGenerateAPIDeployment = "failed to generate OLS deployment" ErrGenerateAPIService = "failed to generate OLS service" @@ -27,6 +30,7 @@ const ( ErrGenerateSARClusterRoleBinding = "failed to generate SAR cluster role binding" ErrGenerateServiceMonitor = "failed to generate ServiceMonitor" ErrGeneratePrometheusRule = "failed to generate PrometheusRule" + ErrGetAdditionalCACM = "failed to get additional CA configmap" ErrGetAPIConfigmap = "failed to get OLS configmap" ErrGetAPIDeployment = "failed to get OLS deployment" ErrGetAPIService = "failed to get OLS service" @@ -45,6 +49,7 @@ const ( ErrUpdateAPIConfigmap = "failed to update OLS configmap" ErrUpdateAPIDeployment = "failed to update OLS deployment" ErrUpdateAPIService = "failed to update OLS service" + ErrUpdateAdditionalCACM = "failed to update additional CA configmap" ErrUpdateConsole = "failed to update Console" ErrUpdateConsolePlugin = "failed to update Console Plugin" ErrUpdateConsolePluginConfigMap = "failed to update Console Plugin configmap" diff --git a/internal/controller/ols_app_server_assets.go b/internal/controller/ols_app_server_assets.go index 1a88bc02..de0a7d0c 100644 --- a/internal/controller/ols_app_server_assets.go +++ b/internal/controller/ols_app_server_assets.go @@ -215,6 +215,20 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv }, } + if cr.Spec.OLSConfig.AdditionalCAConfigMapRef != nil { + caFileNames, err := r.getAdditionalCAFileNames(cr) + if err != nil { + return nil, fmt.Errorf("failed to generate OLS config file, additional CA error: %w", err) + } + + olsConfig.ExtraCAs = make([]string, len(caFileNames)) + for i, caFileName := range caFileNames { + olsConfig.ExtraCAs[i] = path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir, caFileName) + } + + olsConfig.CertificateDirectory = path.Join(OLSAppCertsMountRoot, CertBundleDir) + } + if queryFilters := getQueryFilters(cr); queryFilters != nil { olsConfig.QueryFilters = queryFilters } @@ -280,6 +294,31 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv return &cm, nil } +func (r *OLSConfigReconciler) getAdditionalCAFileNames(cr *olsv1alpha1.OLSConfig) ([]string, error) { + if cr.Spec.OLSConfig.AdditionalCAConfigMapRef == nil { + return nil, nil + } + // get data from the referenced configmap + cm := &corev1.ConfigMap{} + err := r.Get(context.TODO(), client.ObjectKey{Name: cr.Spec.OLSConfig.AdditionalCAConfigMapRef.Name, Namespace: r.Options.Namespace}, cm) + if err != nil { + return nil, fmt.Errorf("failed to get additional CA configmap %s/%s: %v", r.Options.Namespace, cr.Spec.OLSConfig.AdditionalCAConfigMapRef.Name, err) + } + + filenames := []string{} + + for key, caStr := range cm.Data { + err = validateCertificateFormat([]byte(caStr)) + if err != nil { + return nil, fmt.Errorf("failed to validate additional CA certificate %s: %v", key, err) + } + filenames = append(filenames, key) + } + + return filenames, nil + +} + func (r *OLSConfigReconciler) generateService(cr *olsv1alpha1.OLSConfig) (*corev1.Service, error) { service := corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/controller/ols_app_server_assets_test.go b/internal/controller/ols_app_server_assets_test.go index 01f637bf..853da05a 100644 --- a/internal/controller/ols_app_server_assets_test.go +++ b/internal/controller/ols_app_server_assets_test.go @@ -86,7 +86,6 @@ var _ = Describe("App server assets", func() { olsconfigGenerated := AppSrvConfigFile{} err = yaml.Unmarshal([]byte(cm.Data[OLSConfigFilename]), &olsconfigGenerated) Expect(err).NotTo(HaveOccurred()) - olsConfigExpected := AppSrvConfigFile{ OLSConfig: OLSConfig{ DefaultModel: "testModel", @@ -592,6 +591,7 @@ var _ = Describe("App server assets", func() { }, )) }) + }) Context("empty custom resource", func() { @@ -871,6 +871,138 @@ user_data_collector_config: {} )) }) }) + + Context("Additional CA", func() { + + const caConfigMapName = "test-ca-configmap" + const certFilename = "additional-ca.crt" + var additionalCACm *corev1.ConfigMap + + BeforeEach(func() { + rOptions = &OLSConfigReconcilerOptions{ + LightspeedServiceImage: "lightspeed-service:latest", + Namespace: OLSNamespaceDefault, + } + cr = getDefaultOLSConfigCR() + r = &OLSConfigReconciler{ + Options: *rOptions, + logger: logf.Log.WithName("olsconfig.reconciler"), + Client: k8sClient, + Scheme: k8sClient.Scheme(), + stateCache: make(map[string]string), + } + By("create the provider secret") + secret, _ = generateRandomSecret() + secret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID", + Name: "test-secret", + }, + }) + err := r.Create(ctx, secret) + Expect(err).NotTo(HaveOccurred()) + By("create the additional CA configmap") + additionalCACm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: caConfigMapName, + Namespace: OLSNamespaceDefault, + }, + Data: map[string]string{ + certFilename: testCACert, + }, + } + err = r.Create(ctx, additionalCACm) + Expect(err).NotTo(HaveOccurred()) + + }) + + AfterEach(func() { + By("Delete the provider secret") + err := r.Delete(ctx, secret) + Expect(err).NotTo(HaveOccurred()) + By("Delete the additional CA configmap") + err = r.Delete(ctx, additionalCACm) + Expect(err).NotTo(HaveOccurred()) + + }) + + It("should update OLS config and mount volumes for additional CA", func() { + olsCm, err := r.generateOLSConfigMap(ctx, cr) + Expect(err).NotTo(HaveOccurred()) + Expect(olsCm.Data[OLSConfigFilename]).NotTo(ContainSubstring("extra_ca:")) + + dep, err := r.generateOLSDeployment(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Spec.Template.Spec.Volumes).NotTo(ContainElement( + corev1.Volume{ + Name: AdditionalCAVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: caConfigMapName, + }, + DefaultMode: &defaultVolumeMode, + }, + }, + })) + Expect(dep.Spec.Template.Spec.Volumes).NotTo(ContainElement( + corev1.Volume{ + Name: CertBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + })) + + cr.Spec.OLSConfig.AdditionalCAConfigMapRef = &corev1.LocalObjectReference{ + Name: caConfigMapName, + } + + olsCm, err = r.generateOLSConfigMap(ctx, cr) + Expect(err).NotTo(HaveOccurred()) + Expect(olsCm.Data[OLSConfigFilename]).To(ContainSubstring("extra_ca:\n - /etc/certs/ols-additional-ca/additional-ca.crt")) + Expect(olsCm.Data[OLSConfigFilename]).To(ContainSubstring("certificate_directory: /etc/certs/cert-bundle")) + + dep, err = r.generateOLSDeployment(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElements( + corev1.Volume{ + Name: AdditionalCAVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: caConfigMapName, + }, + DefaultMode: &defaultVolumeMode, + }, + }, + }, + corev1.Volume{ + Name: CertBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + )) + + }) + + It("should return error if the CA text is malformed", func() { + additionalCACm.Data[certFilename] = "malformed certificate" + err := r.Update(ctx, additionalCACm) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.OLSConfig.AdditionalCAConfigMapRef = &corev1.LocalObjectReference{ + Name: caConfigMapName, + } + _, err = r.generateOLSConfigMap(ctx, cr) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to validate additional CA certificate")) + + }) + + }) }) func generateRandomSecret() (*corev1.Secret, error) { diff --git a/internal/controller/ols_app_server_deployment.go b/internal/controller/ols_app_server_deployment.go index aa63642c..c07fe0dd 100644 --- a/internal/controller/ols_app_server_deployment.go +++ b/internal/controller/ols_app_server_deployment.go @@ -61,6 +61,7 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( const OLSConfigVolumeName = "cm-olsconfig" const OLSUserDataVolumeName = "ols-user-data" const OLSUserDataMountPath = "/app-root/ols-user-data" + revisionHistoryLimit := int32(1) volumeDefaultMode := int32(420) @@ -84,6 +85,7 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( // TLS volume tlsSecretNameMountPath := path.Join(OLSAppCertsMountRoot, OLSCertsSecretName) secretMounts[OLSCertsSecretName] = tlsSecretNameMountPath + AdditionalCAMountPath := path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir) // Container ports ports := []corev1.ContainerPort{ @@ -129,6 +131,27 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( if dataCollectorEnabled { volumes = append(volumes, olsUserDataVolume) } + + // User provided additional CA certificates + if cr.Spec.OLSConfig.AdditionalCAConfigMapRef != nil { + additionalCAVolume := corev1.Volume{ + Name: AdditionalCAVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: *cr.Spec.OLSConfig.AdditionalCAConfigMapRef, + DefaultMode: &volumeDefaultMode, + }, + }, + } + certBundleVolume := corev1.Volume{ + Name: CertBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + volumes = append(volumes, additionalCAVolume, certBundleVolume) + } + // TODO: Update DB //volumes = append(volumes, olsConfigVolume, olsUserDataVolume, getRedisCAConfigVolume()) @@ -156,6 +179,19 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( if dataCollectorEnabled { volumeMounts = append(volumeMounts, olsUserDataVolumeMount) } + if cr.Spec.OLSConfig.AdditionalCAConfigMapRef != nil { + additionalCAVolumeMount := corev1.VolumeMount{ + Name: AdditionalCAVolumeName, + MountPath: AdditionalCAMountPath, + ReadOnly: true, + } + certBundleVolumeMount := corev1.VolumeMount{ + Name: CertBundleVolumeName, + MountPath: path.Join(OLSAppCertsMountRoot, CertBundleDir), + } + volumeMounts = append(volumeMounts, additionalCAVolumeMount, certBundleVolumeMount) + } + // TODO: Update DB //volumeMounts = append(volumeMounts, olsConfigVolumeMount, olsUserDataVolumeMount, getRedisCAVolumeMount(path.Join(OLSAppCertsMountRoot, RedisCertsSecretName, RedisCAVolume))) @@ -183,7 +219,7 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( { Name: "lightspeed-service-api", Image: r.Options.LightspeedServiceImage, - ImagePullPolicy: corev1.PullAlways, + ImagePullPolicy: corev1.PullIfNotPresent, Ports: ports, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: &[]bool{false}[0], @@ -272,19 +308,22 @@ func (r *OLSConfigReconciler) updateOLSDeployment(ctx context.Context, existingD if existingDeployment.Annotations == nil || existingDeployment.Annotations[OLSConfigHashKey] != r.stateCache[OLSConfigHashStateCacheKey] || existingDeployment.Annotations[OLSAppTLSHashKey] != r.stateCache[OLSAppTLSHashStateCacheKey] || - existingDeployment.Annotations[LLMProviderHashKey] != r.stateCache[LLMProviderHashStateCacheKey] { + existingDeployment.Annotations[LLMProviderHashKey] != r.stateCache[LLMProviderHashStateCacheKey] || + existingDeployment.Annotations[AdditionalCAHashKey] != r.stateCache[AdditionalCAHashStateCacheKey] { updateDeploymentAnnotations(existingDeployment, map[string]string{ - OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], - OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], - LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], + OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], + LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], // TODO: Update DB //RedisSecretHashKey: r.stateCache[RedisSecretHashStateCacheKey], }) // update the deployment template annotation triggers the rolling update updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{ - OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], - OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], - LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], + OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], + LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], // TODO: Update DB //RedisSecretHashKey: r.stateCache[RedisSecretHashStateCacheKey], }) diff --git a/internal/controller/ols_app_server_reconciliator.go b/internal/controller/ols_app_server_reconciliator.go index 0602b357..3a23f65b 100644 --- a/internal/controller/ols_app_server_reconciliator.go +++ b/internal/controller/ols_app_server_reconciliator.go @@ -37,6 +37,10 @@ func (r *OLSConfigReconciler) reconcileAppServer(ctx context.Context, olsconfig Name: "reconcile OLSConfigMap", Task: r.reconcileOLSConfigMap, }, + { + Name: "reconcile Additional CA ConfigMap", + Task: r.reconcileOLSAdditionalCAConfigMap, + }, { Name: "reconcile App Service", Task: r.reconcileService, @@ -121,6 +125,49 @@ func (r *OLSConfigReconciler) reconcileOLSConfigMap(ctx context.Context, cr *ols return nil } +func (r *OLSConfigReconciler) reconcileOLSAdditionalCAConfigMap(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + if cr.Spec.OLSConfig.AdditionalCAConfigMapRef == nil { + // no additional CA certs, skip + r.logger.Info("Additional CA not configured, reconciliation skipped") + return nil + } + + // annotate the configmap for watcher + cm := &corev1.ConfigMap{} + + err := r.Client.Get(ctx, client.ObjectKey{Name: cr.Spec.OLSConfig.AdditionalCAConfigMapRef.Name, Namespace: r.Options.Namespace}, cm) + + if err != nil { + return fmt.Errorf("%s: %w", ErrGetAdditionalCACM, err) + } + + annotateConfigMapWatcher(cm) + + err = r.Update(ctx, cm) + if err != nil { + return fmt.Errorf("%s: %w", ErrUpdateAdditionalCACM, err) + } + + certBytes := []byte{} + for key, value := range cm.Data { + certBytes = append(certBytes, []byte(key)...) + certBytes = append(certBytes, []byte(value)...) + } + + foundCmHash, err := hashBytes(certBytes) + if err != nil { + return fmt.Errorf("failed to generate additional CA certs hash %w", err) + } + if foundCmHash == r.stateCache[AdditionalCAHashStateCacheKey] { + r.logger.Info("Additional CA reconciliation skipped", "hash", foundCmHash) + return nil + } + r.stateCache[AdditionalCAHashStateCacheKey] = foundCmHash + + r.logger.Info("additional CA configmap reconciled", "configmap", cm.Name, "hash", foundCmHash) + return nil +} + func (r *OLSConfigReconciler) reconcileServiceAccount(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { sa, err := r.generateServiceAccount(cr) if err != nil { @@ -200,16 +247,18 @@ func (r *OLSConfigReconciler) reconcileDeployment(ctx context.Context, cr *olsv1 err = r.Client.Get(ctx, client.ObjectKey{Name: OLSAppServerDeploymentName, Namespace: r.Options.Namespace}, existingDeployment) if err != nil && errors.IsNotFound(err) { updateDeploymentAnnotations(desiredDeployment, map[string]string{ - OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], - OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], - LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], + OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], + LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], // TODO: Update DB //RedisSecretHashKey: r.stateCache[RedisSecretHashStateCacheKey], }) updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{ - OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], - OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], - LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], + OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], + LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], // TODO: Update DB //RedisSecretHashKey: r.stateCache[RedisSecretHashStateCacheKey], }) diff --git a/internal/controller/ols_app_server_reconciliator_test.go b/internal/controller/ols_app_server_reconciliator_test.go index c67d50c7..3aec438c 100644 --- a/internal/controller/ols_app_server_reconciliator_test.go +++ b/internal/controller/ols_app_server_reconciliator_test.go @@ -1,8 +1,12 @@ package controller import ( + "fmt" + "path" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" @@ -48,6 +52,12 @@ var _ = Describe("App server reconciliator", Ordered, func() { }) secretCreationErr = reconciler.Create(ctx, tlsSecret) Expect(secretCreationErr).NotTo(HaveOccurred()) + + By("Set OLSConfig CR to default") + err := k8sClient.Get(ctx, crNamespacedName, cr) + Expect(err).NotTo(HaveOccurred()) + crDefault := getDefaultOLSConfigCR() + cr.Spec = crDefault.Spec }) AfterEach(func() { @@ -491,4 +501,184 @@ var _ = Describe("App server reconciliator", Ordered, func() { }) }) + + Context("User CA Certs", Ordered, func() { + var secret *corev1.Secret + var volumeDefaultMode = int32(420) + var cmCACert1 *corev1.ConfigMap + var cmCACert2 *corev1.ConfigMap + const cmCACert1Name = "ca-cert-1" + const cmCACert2Name = "ca-cert-2" + const caCert1FileName = "ca-cert-1.crt" + const caCert2FileName = "ca-cert-2.crt" + BeforeEach(func() { + By("create the provider secret") + secret, _ = generateRandomSecret() + secret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID", + Name: "test-secret", + }, + }) + secretCreationErr := reconciler.Create(ctx, secret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + + By("create the tls secret") + tlsSecret, _ = generateRandomSecret() + tlsSecret.Name = OLSCertsSecretName + tlsSecret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID", + Name: OLSCertsSecretName, + }, + }) + secretCreationErr = reconciler.Create(ctx, tlsSecret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + + By("create the config map for CA cert 1") + cmCACert1 = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmCACert1Name, + Namespace: OLSNamespaceDefault, + }, + Data: map[string]string{ + caCert1FileName: testCACert, + }, + } + err := reconciler.Create(ctx, cmCACert1) + Expect(err).NotTo(HaveOccurred()) + + By("create the config map for CA cert 2") + cmCACert2 = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmCACert2Name, + Namespace: OLSNamespaceDefault, + }, + Data: map[string]string{ + caCert2FileName: testCACert, + }, + } + err = reconciler.Create(ctx, cmCACert2) + Expect(err).NotTo(HaveOccurred()) + + By("Generate default CR") + cr = getDefaultOLSConfigCR() + }) + + AfterEach(func() { + By("Delete the provider secret") + secretDeletionErr := reconciler.Delete(ctx, secret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) + + By("Delete the tls secret") + secretDeletionErr = reconciler.Delete(ctx, tlsSecret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) + + By("Delete the config map for CA cert 1") + err := reconciler.Delete(ctx, cmCACert1) + Expect(err).NotTo(HaveOccurred()) + + By("Delete the config map for CA cert 2") + err = reconciler.Delete(ctx, cmCACert2) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should update the configmap and deployment when changing the additional CA cert", func() { + By("Set up an additional CA cert") + cr.Spec.OLSConfig.AdditionalCAConfigMapRef = &corev1.LocalObjectReference{ + Name: cmCACert1Name, + } + err := reconciler.reconcileAppServer(ctx, cr) + Expect(err).NotTo(HaveOccurred()) + + By("check OLS configmap has extra_ca section") + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: OLSConfigCmName, Namespace: OLSNamespaceDefault}, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(HaveKey(OLSConfigFilename)) + Expect(cm.Data[OLSConfigFilename]).To(ContainSubstring(fmt.Sprintf("extra_ca:\n - %s", path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir, caCert1FileName)))) + Expect(cm.Data[OLSConfigFilename]).To(ContainSubstring("certificate_directory: /etc/certs/cert-bundle")) + + By("check the additional CA configmap has watcher annotation") + err = k8sClient.Get(ctx, types.NamespacedName{Name: cmCACert1Name, Namespace: OLSNamespaceDefault}, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Annotations).To(HaveKeyWithValue(WatcherAnnotationKey, OLSConfigName)) + + By("Get app deployment and check the volume mount") + deployment := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: OLSAppServerDeploymentName, Namespace: OLSNamespaceDefault}, deployment) + Expect(err).NotTo(HaveOccurred()) + Expect(deployment.Spec.Template.Spec.Volumes).To(ContainElements( + corev1.Volume{ + Name: AdditionalCAVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmCACert1Name, + }, + DefaultMode: &volumeDefaultMode, + }, + }, + }, + corev1.Volume{ + Name: CertBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + )) + Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(corev1.VolumeMount{ + Name: AdditionalCAVolumeName, + MountPath: path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir), + ReadOnly: true, + })) + Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(corev1.VolumeMount{ + Name: CertBundleVolumeName, + MountPath: path.Join(OLSAppCertsMountRoot, CertBundleDir), + })) + }) + + It("should not generate additional CA related settings if additional CA is not defined", func() { + By("Set no additional CA cert") + cr.Spec.OLSConfig.AdditionalCAConfigMapRef = nil + err := reconciler.reconcileAppServer(ctx, cr) + Expect(err).NotTo(HaveOccurred()) + + By("Check app deployment does not have additional CA volumes and volume mounts") + deployment := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: OLSAppServerDeploymentName, Namespace: OLSNamespaceDefault}, deployment) + Expect(err).NotTo(HaveOccurred()) + Expect(deployment.Spec.Template.Spec.Volumes).NotTo(ContainElement(corev1.Volume{ + Name: AdditionalCAVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmCACert1Name, + }, + DefaultMode: &volumeDefaultMode, + }, + }, + })) + + Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).NotTo(ContainElement(corev1.VolumeMount{ + Name: AdditionalCAVolumeName, + MountPath: path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir), + ReadOnly: true, + })) + + By("Check OLS configmap does not have extra_ca section") + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: OLSConfigCmName, Namespace: OLSNamespaceDefault}, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(HaveKey(OLSConfigFilename)) + Expect(cm.Data[OLSConfigFilename]).NotTo(ContainSubstring("extra_ca:")) + + }) + + }) + }) diff --git a/internal/controller/ols_console_reconciliator.go b/internal/controller/ols_console_reconciliator.go index c2a2dc51..53695838 100644 --- a/internal/controller/ols_console_reconciliator.go +++ b/internal/controller/ols_console_reconciliator.go @@ -279,6 +279,10 @@ func (r *OLSConfigReconciler) deleteConsoleUIPlugin(ctx context.Context) error { } err = r.Delete(ctx, plugin) if err != nil { + if errors.IsNotFound(err) { + r.logger.Info("Console Plugin not found, consider deletion successful") + return nil + } return fmt.Errorf("%s: %w", ErrDeleteConsolePlugin, err) } r.logger.Info("Console Plugin deleted") diff --git a/internal/controller/ols_console_reconciliator_test.go b/internal/controller/ols_console_reconciliator_test.go index 9fde3114..419ab600 100644 --- a/internal/controller/ols_console_reconciliator_test.go +++ b/internal/controller/ols_console_reconciliator_test.go @@ -44,6 +44,12 @@ var _ = Describe("Console UI reconciliator", Ordered, func() { }) secretCreationErr := reconciler.Create(ctx, tlsSecret) Expect(secretCreationErr).NotTo(HaveOccurred()) + + By("set the OLSConfig custom resource to default") + err = k8sClient.Get(ctx, crNamespacedName, cr) + Expect(err).NotTo(HaveOccurred()) + crDefault := getDefaultOLSConfigCR() + cr.Spec = crDefault.Spec }) AfterAll(func() { diff --git a/internal/controller/olsconfig_controller.go b/internal/controller/olsconfig_controller.go index 80c7bb89..081144b9 100644 --- a/internal/controller/olsconfig_controller.go +++ b/internal/controller/olsconfig_controller.go @@ -224,6 +224,7 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(secretWatcherFilter)). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(telemetryPullSecretWatcherFilter)). + Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(configMapWatcherFilter)). Owns(&consolev1.ConsolePlugin{}). Owns(&monv1.ServiceMonitor{}). Owns(&monv1.PrometheusRule{}). diff --git a/internal/controller/resource_watchers.go b/internal/controller/resource_watchers.go index 8e43bb92..c54747c1 100644 --- a/internal/controller/resource_watchers.go +++ b/internal/controller/resource_watchers.go @@ -44,3 +44,28 @@ func telemetryPullSecretWatcherFilter(ctx context.Context, obj client.Object) [] }}, } } + +func configMapWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request { + annotations := obj.GetAnnotations() + if annotations == nil { + return nil + } + crName, exist := annotations[WatcherAnnotationKey] + if !exist { + return nil + } + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: crName, + }}, + } +} + +func annotateConfigMapWatcher(cm *corev1.ConfigMap) { + annotations := cm.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[WatcherAnnotationKey] = OLSConfigName + cm.SetAnnotations(annotations) +} diff --git a/internal/controller/resource_watchers_test.go b/internal/controller/resource_watchers_test.go index 4adf48cd..730b2ebf 100644 --- a/internal/controller/resource_watchers_test.go +++ b/internal/controller/resource_watchers_test.go @@ -27,4 +27,20 @@ var _ = Describe("Watchers", func() { }) }) + Context("configmap", Ordered, func() { + ctx := context.Background() + It("should identify watched configmap by annotations", func() { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "test-configmap"}, + } + requests := configMapWatcherFilter(ctx, configMap) + Expect(requests).To(BeEmpty()) + + annotateConfigMapWatcher(configMap) + requests = configMapWatcherFilter(ctx, configMap) + Expect(requests).To(HaveLen(1)) + Expect(requests[0].Name).To(Equal(OLSConfigName)) + }) + }) + }) diff --git a/internal/controller/types.go b/internal/controller/types.go index 80b02c67..5402a14b 100644 --- a/internal/controller/types.go +++ b/internal/controller/types.go @@ -91,6 +91,10 @@ type OLSConfig struct { ReferenceContent ReferenceContent `json:"reference_content,omitempty"` // User data collection configuration UserDataCollection UserDataCollectionConfig `json:"user_data_collection,omitempty"` + // List of Paths to files containing additional CA certificates in the app server container. + ExtraCAs []string `json:"extra_ca,omitempty"` + // Path to the directory containing the certificates bundle in the app server container. + CertificateDirectory string `json:"certificate_directory,omitempty"` } type LoggingConfig struct { diff --git a/internal/controller/utils.go b/internal/controller/utils.go index d01c8729..f5b81334 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -3,6 +3,8 @@ package controller import ( "context" "crypto/sha256" + "crypto/x509" + "encoding/pem" "fmt" "os" "sort" @@ -397,3 +399,24 @@ func getProxyEnvVars() []corev1.EnvVar { } return envVars } + +// validate the x509 certificate syntax +func validateCertificateFormat(cert []byte) error { + if len(cert) == 0 { + return fmt.Errorf("certificate is empty") + } + block, _ := pem.Decode(cert) + if block == nil { + return fmt.Errorf("failed to decode PEM certificate") + } + if block.Type != "CERTIFICATE" { + return fmt.Errorf("block type is not certificate but %s", block.Type) + } + // check the CA is correctly formatted + _, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("failed to parse certificate: %v", err) + } + + return nil +} diff --git a/test/e2e/client.go b/test/e2e/client.go index 71f6178f..93e71be3 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -192,7 +192,7 @@ func (c *Client) WaitForConfigMapContainString(cm *corev1.ConfigMap, key, substr return true, nil }) if err != nil { - return fmt.Errorf("WaitForConfigMapContainString - waiting for the ConfigMap %s/%s containing the string \"\": %w ; last error: %w", cm.GetNamespace(), cm.GetName(), err, lastErr) + return fmt.Errorf("WaitForConfigMapContainString - waiting for the ConfigMap %s/%s containing the string \"%s\": %w ; last error: %w", cm.GetNamespace(), cm.GetName(), substr, err, lastErr) } return nil diff --git a/test/e2e/constants.go b/test/e2e/constants.go index 3bbbe380..227f81cd 100644 --- a/test/e2e/constants.go +++ b/test/e2e/constants.go @@ -54,4 +54,44 @@ const ( AppServerTLSSecretName = "lightspeed-tls" // #nosec G101 // ConditionTimeoutEnvVar is the environment variable containing the condition check timeout in seconds ConditionTimeoutEnvVar = "CONDITION_TIMEOUT" + + // TestCACert is for testing additional CA certificate + TestCACert = `-----BEGIN CERTIFICATE----- +MIIEMDCCAxigAwIBAgIJANqb7HHzA7AZMA0GCSqGSIb3DQEBCwUAMIGkMQswCQYD +VQQGEwJQQTEPMA0GA1UECAwGUGFuYW1hMRQwEgYDVQQHDAtQYW5hbWEgQ2l0eTEk +MCIGA1UECgwbVHJ1c3RDb3IgU3lzdGVtcyBTLiBkZSBSLkwuMScwJQYDVQQLDB5U +cnVzdENvciBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFlRydXN0Q29y +IFJvb3RDZXJ0IENBLTEwHhcNMTYwMjA0MTIzMjE2WhcNMjkxMjMxMTcyMzE2WjCB +pDELMAkGA1UEBhMCUEExDzANBgNVBAgMBlBhbmFtYTEUMBIGA1UEBwwLUGFuYW1h +IENpdHkxJDAiBgNVBAoMG1RydXN0Q29yIFN5c3RlbXMgUy4gZGUgUi5MLjEnMCUG +A1UECwweVHJ1c3RDb3IgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MR8wHQYDVQQDDBZU +cnVzdENvciBSb290Q2VydCBDQS0xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAv463leLCJhJrMxnHQFgKq1mqjQCj/IDHUHuO1CAmujIS2CNUSSUQIpid +RtLByZ5OGy4sDjjzGiVoHKZaBeYei0i/mJZ0PmnK6bV4pQa81QBeCQryJ3pS/C3V +seq0iWEk8xoT26nPUu0MJLq5nux+AHT6k61sKZKuUbS701e/s/OojZz0JEsq1pme +9J7+wH5COucLlVPat2gOkEz7cD+PSiyU8ybdY2mplNgQTsVHCJCZGxdNuWxu72CV +EY4hgLW9oHPY0LJ3xEXqWib7ZnZ2+AYfYW0PVcWDtxBWcgYHpfOxGgMFZA6dWorW +hnAbJN7+KIor0Gqw/Hqi3LJ5DotlDwIDAQABo2MwYTAdBgNVHQ4EFgQU7mtJPHo/ +DeOxCbeKyKsZn3MzUOcwHwYDVR0jBBgwFoAU7mtJPHo/DeOxCbeKyKsZn3MzUOcw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggEBACUY1JGPE+6PHh0RU9otRCkZoB5rMZ5NDp6tPVxBb5UrJKF5mDo4Nvu7Zp5I +/5CQ7z3UuJu0h3U/IJvOcs+hVcFNZKIZBqEHMwwLKeXx6quj7LUKdJDHfXLy11yf +ke+Ri7fc7Waiz45mO7yfOgLgJ90WmMCV1Aqk5IGadZQ1nJBfiDcGrVmVCrDRZ9MZ +yonnMlo2HD6CqFqTvsbQZJG2z9m2GM/bftJlo6bEjhcxwft+dtvTheNYsnd6djts +L1Ac59v2Z3kf9YKVmgenFK+P3CghZwnS1k1aHBkcjndcw5QkPTJrS37UeJSDvjdN +zl/HHk484IkzlQsPpTLWPFp5LBk= +-----END CERTIFICATE----- +` + // OLSAppCertsMountRoot is the directory hosting the cert files in the container + OLSAppCertsMountRoot = "/etc/certs" + // AdditionalCAVolumeName is the name of the additional CA volume in the app server container + AdditionalCAVolumeName = "additional-ca" + // AppAdditionalCACertDir is the directory for storing additional CA certificates in the app server container under OLSAppCertsMountRoot + AppAdditionalCACertDir = "ols-additional-ca" + // AdditionalCAHashKey is the key of the hash value of the additional CA certificates + AdditionalCAHashKey = "hash/additionalca" + // CertBundleVolumeName is the name of the volume for the certificate bundle + CertBundleVolumeName = "cert-bundle" + // CertBundleDir is the path of the volume for the certificate bundle + CertBundleDir = "cert-bundle" ) diff --git a/test/e2e/reconciliation_test.go b/test/e2e/reconciliation_test.go index 15e2048f..1dc95bf9 100644 --- a/test/e2e/reconciliation_test.go +++ b/test/e2e/reconciliation_test.go @@ -10,6 +10,7 @@ import ( olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -236,4 +237,119 @@ var _ = Describe("Reconciliation From OLSConfig CR", Ordered, func() { }) + It("should setup CA cert volumes and app configs after setting additional CA", func() { + const ( + cmCACert1Name = "ca-cert-1" + caCert1FileName = "ca-cert-1.crt" + caCert2FileName = "ca-cert-2.crt" + ) + By("create additional CA configmap") + caCertConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmCACert1Name, + Namespace: OLSNameSpace, + }, + Data: map[string]string{ + caCert1FileName: TestCACert, + }, + } + err = client.Create(caCertConfigMap) + Expect(err).NotTo(HaveOccurred()) + defer func() { + err = client.Delete(caCertConfigMap) + Expect(err).NotTo(HaveOccurred()) + }() + + By("update additional CA in the OLSConfig CR") + err = client.Get(cr) + Expect(err).NotTo(HaveOccurred()) + cr.Spec.OLSConfig.AdditionalCAConfigMapRef = &corev1.LocalObjectReference{ + Name: cmCACert1Name, + } + err = client.Update(cr) + Expect(err).NotTo(HaveOccurred()) + + By("check the OLS configmap to contain the additional CA cert") + olsConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: AppServerConfigMapName, + Namespace: OLSNameSpace, + }, + } + err = client.WaitForConfigMapContainString(olsConfigMap, AppServerConfigMapKey, "/etc/certs/ols-additional-ca/"+caCert1FileName) + Expect(err).NotTo(HaveOccurred()) + err = client.WaitForConfigMapContainString(olsConfigMap, AppServerConfigMapKey, "certificate_directory: /etc/certs/cert-bundle") + Expect(err).NotTo(HaveOccurred()) + + By("check the app deployment to mount the additional CA cert configmap") + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: AppServerDeploymentName, + Namespace: OLSNameSpace, + }, + } + volumeDefaultMode := int32(420) + err = client.WaitForDeploymentCondition(deployment, func(dep *appsv1.Deployment) (bool, error) { + var certVolumeExist, certBundleVolumeExist, certVolumeMountExist, certBundleVolumeMountExist bool + for _, volume := range dep.Spec.Template.Spec.Volumes { + if apiequality.Semantic.DeepEqual(volume, corev1.Volume{ + Name: AdditionalCAVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmCACert1Name, + }, + DefaultMode: &volumeDefaultMode, + }, + }, + }) { + certVolumeExist = true + } + if apiequality.Semantic.DeepEqual(volume, corev1.Volume{ + Name: CertBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) { + certBundleVolumeExist = true + } + + } + for _, volumeMount := range dep.Spec.Template.Spec.Containers[0].VolumeMounts { + if apiequality.Semantic.DeepEqual(volumeMount, corev1.VolumeMount{ + Name: AdditionalCAVolumeName, + MountPath: path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir), + ReadOnly: true, + }) { + certVolumeMountExist = true + } + if apiequality.Semantic.DeepEqual(volumeMount, corev1.VolumeMount{ + Name: CertBundleVolumeName, + MountPath: path.Join(OLSAppCertsMountRoot, CertBundleDir), + ReadOnly: false, + }) { + certBundleVolumeMountExist = true + } + } + + return certVolumeExist && certBundleVolumeExist && certVolumeMountExist && certBundleVolumeMountExist, nil + }) + Expect(err).NotTo(HaveOccurred()) + firstCmHash := deployment.Spec.Template.Annotations[AdditionalCAHashKey] + + By("check the app deployment and OLS config adapted to modified CA cert configmap") + err = client.Get(caCertConfigMap) + Expect(err).NotTo(HaveOccurred()) + caCertConfigMap.Data[caCert2FileName] = TestCACert + err = client.Update(caCertConfigMap) + Expect(err).NotTo(HaveOccurred()) + err = client.WaitForConfigMapContainString(olsConfigMap, AppServerConfigMapKey, "/etc/certs/ols-additional-ca/"+caCert2FileName) + Expect(err).NotTo(HaveOccurred()) + err = client.WaitForDeploymentCondition(deployment, func(dep *appsv1.Deployment) (bool, error) { + newCmHash := dep.Spec.Template.Annotations[AdditionalCAHashKey] + return newCmHash != firstCmHash, nil + }) + + }) + }) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index 95db0966..256581d6 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -1,3 +1,17 @@ package e2e +import ( + "crypto/sha256" + "fmt" +) + func Ptr[T any](v T) *T { return &v } + +func hashBytes(sourceStr []byte) (string, error) { // nolint:unused + hashFunc := sha256.New() + _, err := hashFunc.Write(sourceStr) + if err != nil { + return "", fmt.Errorf("failed to generate hash %w", err) + } + return fmt.Sprintf("%x", hashFunc.Sum(nil)), nil +}