feat: additional CA certificate for application server

Signed-off-by: Haoyu Sun <[email protected]>

api/v1alpha1/olsconfig_types.go

@@ -80,6 +80,9 @@ type OLSSpec struct {
 	// User data collection switches
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="User Data Collection"
 	UserDataCollection UserDataCollectionSpec `json:"userDataCollection,omitempty"`
+	// Additional CA certificates for TLS communication between OLS service and LLM Provider
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Additional CA Configmap",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
+	AdditionalCAConfigMapRef *corev1.LocalObjectReference `json:"additionalCAConfigMapRef,omitempty"`
 }
 
 // DeploymentConfig defines the schema for overriding deployment of OLS instance.

bundle/manifests/lightspeed-operator.clusterserviceversion.yaml

@@ -114,6 +114,11 @@ spec:
             path: llm.providers[0].type
           - displayName: OLS Settings
             path: ols
+          - description: Additional CA certificates for TLS communication between OLS service and LLM Provider
+            displayName: Additional CA Configmap
+            path: ols.additionalCAConfigMapRef
+            x-descriptors:
+              - urn:alm:descriptor:com.tectonic.ui:advanced
           - displayName: Redis
             path: ols.conversationCache.redis
           - description: Secret that holds redis credentials

bundle/manifests/ols.openshift.io_olsconfigs.yaml

@@ -130,6 +130,18 @@ spec:
               ols:
                 description: OLSSpec defines the desired state of OLS deployment.
                 properties:
+                  additionalCAConfigMapRef:
+                    description: Additional CA certificates for TLS communication
+                      between OLS service and LLM Provider
+                    properties:
+                      name:
+                        description: |-
+                          Name of the referent.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                          TODO: Add other useful fields. apiVersion, kind, uid?
+                        type: string
+                    type: object
+                    x-kubernetes-map-type: atomic
                   conversationCache:
                     description: Conversation cache settings
                     properties:

config/crd/bases/ols.openshift.io_olsconfigs.yaml

@@ -130,6 +130,18 @@ spec:
               ols:
                 description: OLSSpec defines the desired state of OLS deployment.
                 properties:
+                  additionalCAConfigMapRef:
+                    description: Additional CA certificates for TLS communication
+                      between OLS service and LLM Provider
+                    properties:
+                      name:
+                        description: |-
+                          Name of the referent.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                          TODO: Add other useful fields. apiVersion, kind, uid?
+                        type: string
+                    type: object
+                    x-kubernetes-map-type: atomic
                   conversationCache:
                     description: Conversation cache settings
                     properties:

config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml

@@ -81,6 +81,12 @@ spec:
         path: llm.providers[0].type
       - displayName: OLS Settings
         path: ols
+      - description: Additional CA certificates for TLS communication between OLS
+          service and LLM Provider
+        displayName: Additional CA Configmap
+        path: ols.additionalCAConfigMapRef
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
       - displayName: Redis
         path: ols.conversationCache.redis
       - description: Secret that holds redis credentials

internal/controller/constants.go

@@ -57,6 +57,14 @@ const (
 	AppServerPrometheusRuleName = "lightspeed-app-server-prometheus-rule"
 	// AppServerMetricsPath is the path of the metrics endpoint of the OLS application server
 	AppServerMetricsPath = "/metrics"
+	// AppAdditionalCACertDir is the directory for storing additional CA certificates in the app server container under OLSAppCertsMountRoot
+	AppAdditionalCACertDir = "ols-additional-ca"
+	// AdditionalCAVolumeName is the name of the volume for additional CA certificates provided by the user
+	AdditionalCAVolumeName = "additional-ca"
+	// CertBundleVolumeName is the name of the volume for the certificate bundle
+	CertBundleVolumeName = "cert-bundle"
+	// CertBundleDir is the path of the volume for the certificate bundle
+	CertBundleDir = "cert-bundle"
 
 	// Image of the OLS application redis server
 	// OLSConfigHashKey is the key of the hash value of the OLSConfig configmap
@@ -68,6 +76,8 @@ const (
 	OLSAppTLSHashKey = "hash/olstls"
 	// OLSConsoleTLSHashKey is the key of the hash value of the OLS Console TLS certificates
 	OLSConsoleTLSHashKey = "hash/olsconsoletls"
+	// AdditionalCAHashKey is the key of the hash value of the additional CA certificates in the deployment annotations
+	AdditionalCAHashKey = "hash/additionalca"
 	// OLSAppServerContainerPort is the port number of the lightspeed-service-api container exposes
 	OLSAppServerContainerPort = 8443
 	// OLSAppServerServicePort is the port number for OLS application server service.
@@ -90,6 +100,8 @@ const (
 	LLMProviderHashStateCacheKey = "llmprovider-hash"
 	// AzureOpenAIType is the name of the Azure OpenAI provider type
 	AzureOpenAIType = "azure_openai"
+	// AdditionalCAHashStateCacheKey is the key of the hash value of the additional CA certificates in the state cache
+	AdditionalCAHashStateCacheKey = "additionalca-hash"
 	/*** console UI plugin ***/
 	// ConsoleUIConfigMapName is the name of the console UI nginx configmap
 	ConsoleUIConfigMapName = "lightspeed-console-plugin"

internal/controller/constants_test.go

@@ -0,0 +1,28 @@
+package controller
+
+const testCACert = `-----BEGIN CERTIFICATE-----
+MIIEMDCCAxigAwIBAgIJANqb7HHzA7AZMA0GCSqGSIb3DQEBCwUAMIGkMQswCQYD
+VQQGEwJQQTEPMA0GA1UECAwGUGFuYW1hMRQwEgYDVQQHDAtQYW5hbWEgQ2l0eTEk
+MCIGA1UECgwbVHJ1c3RDb3IgU3lzdGVtcyBTLiBkZSBSLkwuMScwJQYDVQQLDB5U
+cnVzdENvciBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFlRydXN0Q29y
+IFJvb3RDZXJ0IENBLTEwHhcNMTYwMjA0MTIzMjE2WhcNMjkxMjMxMTcyMzE2WjCB
+pDELMAkGA1UEBhMCUEExDzANBgNVBAgMBlBhbmFtYTEUMBIGA1UEBwwLUGFuYW1h
+IENpdHkxJDAiBgNVBAoMG1RydXN0Q29yIFN5c3RlbXMgUy4gZGUgUi5MLjEnMCUG
+A1UECwweVHJ1c3RDb3IgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MR8wHQYDVQQDDBZU
+cnVzdENvciBSb290Q2VydCBDQS0xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAv463leLCJhJrMxnHQFgKq1mqjQCj/IDHUHuO1CAmujIS2CNUSSUQIpid
+RtLByZ5OGy4sDjjzGiVoHKZaBeYei0i/mJZ0PmnK6bV4pQa81QBeCQryJ3pS/C3V
+seq0iWEk8xoT26nPUu0MJLq5nux+AHT6k61sKZKuUbS701e/s/OojZz0JEsq1pme
+9J7+wH5COucLlVPat2gOkEz7cD+PSiyU8ybdY2mplNgQTsVHCJCZGxdNuWxu72CV
+EY4hgLW9oHPY0LJ3xEXqWib7ZnZ2+AYfYW0PVcWDtxBWcgYHpfOxGgMFZA6dWorW
+hnAbJN7+KIor0Gqw/Hqi3LJ5DotlDwIDAQABo2MwYTAdBgNVHQ4EFgQU7mtJPHo/
+DeOxCbeKyKsZn3MzUOcwHwYDVR0jBBgwFoAU7mtJPHo/DeOxCbeKyKsZn3MzUOcw
+DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD
+ggEBACUY1JGPE+6PHh0RU9otRCkZoB5rMZ5NDp6tPVxBb5UrJKF5mDo4Nvu7Zp5I
+/5CQ7z3UuJu0h3U/IJvOcs+hVcFNZKIZBqEHMwwLKeXx6quj7LUKdJDHfXLy11yf
+ke+Ri7fc7Waiz45mO7yfOgLgJ90WmMCV1Aqk5IGadZQ1nJBfiDcGrVmVCrDRZ9MZ
+yonnMlo2HD6CqFqTvsbQZJG2z9m2GM/bftJlo6bEjhcxwft+dtvTheNYsnd6djts
+L1Ac59v2Z3kf9YKVmgenFK+P3CghZwnS1k1aHBkcjndcw5QkPTJrS37UeJSDvjdN
+zl/HHk484IkzlQsPpTLWPFp5LBk=
+-----END CERTIFICATE-----
+`

internal/controller/errors.go

@@ -1,6 +1,7 @@
 package controller
 
 const (
+	ErrCreateAdditionalCACM            = "failed to create additional CA configmap"
 	ErrCreateAPIConfigmap              = "failed to create OLS configmap"
 	ErrCreateAPIDeployment             = "failed to create OLS deployment"
 	ErrCreateAPIService                = "failed to create OLS service"
@@ -14,6 +15,8 @@ const (
 	ErrCreateServiceMonitor            = "failed to create ServiceMonitor"
 	ErrCreatePrometheusRule            = "failed to create PrometheusRule"
 	ErrDeleteConsolePlugin             = "failed to delete Console Plugin"
+	ErrDeleteAdditionalCACM            = "failed to delete additional CA configmap"
+	ErrGenerateAdditionalCACM          = "failed to generate additional CA configmap"
 	ErrGenerateAPIConfigmap            = "failed to generate OLS configmap"
 	ErrGenerateAPIDeployment           = "failed to generate OLS deployment"
 	ErrGenerateAPIService              = "failed to generate OLS service"
@@ -27,6 +30,7 @@ const (
 	ErrGenerateSARClusterRoleBinding   = "failed to generate SAR cluster role binding"
 	ErrGenerateServiceMonitor          = "failed to generate ServiceMonitor"
 	ErrGeneratePrometheusRule          = "failed to generate PrometheusRule"
+	ErrGetAdditionalCACM               = "failed to get additional CA configmap"
 	ErrGetAPIConfigmap                 = "failed to get OLS configmap"
 	ErrGetAPIDeployment                = "failed to get OLS deployment"
 	ErrGetAPIService                   = "failed to get OLS service"
@@ -45,6 +49,7 @@ const (
 	ErrUpdateAPIConfigmap              = "failed to update OLS configmap"
 	ErrUpdateAPIDeployment             = "failed to update OLS deployment"
 	ErrUpdateAPIService                = "failed to update OLS service"
+	ErrUpdateAdditionalCACM            = "failed to update additional CA configmap"
 	ErrUpdateConsole                   = "failed to update Console"
 	ErrUpdateConsolePlugin             = "failed to update Console Plugin"
 	ErrUpdateConsolePluginConfigMap    = "failed to update Console Plugin configmap"

internal/controller/ols_app_server_assets.go

@@ -215,6 +215,20 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv
 		},
 	}
 
+	if cr.Spec.OLSConfig.AdditionalCAConfigMapRef != nil {
+		caFileNames, err := r.getAdditionalCAFileNames(cr)
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate OLS config file, additional CA error: %w", err)
+		}
+
+		olsConfig.ExtraCAs = make([]string, len(caFileNames))
+		for i, caFileName := range caFileNames {
+			olsConfig.ExtraCAs[i] = path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir, caFileName)
+		}
+
+		olsConfig.CertificateDirectory = path.Join(OLSAppCertsMountRoot, CertBundleDir)
+	}
+
 	if queryFilters := getQueryFilters(cr); queryFilters != nil {
 		olsConfig.QueryFilters = queryFilters
 	}
@@ -280,6 +294,31 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv
 	return &cm, nil
 }
 
+func (r *OLSConfigReconciler) getAdditionalCAFileNames(cr *olsv1alpha1.OLSConfig) ([]string, error) {
+	if cr.Spec.OLSConfig.AdditionalCAConfigMapRef == nil {
+		return nil, nil
+	}
+	// get data from the referenced configmap
+	cm := &corev1.ConfigMap{}
+	err := r.Get(context.TODO(), client.ObjectKey{Name: cr.Spec.OLSConfig.AdditionalCAConfigMapRef.Name, Namespace: r.Options.Namespace}, cm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get additional CA configmap %s/%s: %v", r.Options.Namespace, cr.Spec.OLSConfig.AdditionalCAConfigMapRef.Name, err)
+	}
+
+	filenames := []string{}
+
+	for key, caStr := range cm.Data {
+		err = validateCertificateFormat([]byte(caStr))
+		if err != nil {
+			return nil, fmt.Errorf("failed to validate additional CA certificate %s: %v", key, err)
+		}
+		filenames = append(filenames, key)
+	}
+
+	return filenames, nil
+
+}
+
 func (r *OLSConfigReconciler) generateService(cr *olsv1alpha1.OLSConfig) (*corev1.Service, error) {
 	service := corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{

internal/controller/ols_app_server_assets_test.go

@@ -86,7 +86,6 @@ var _ = Describe("App server assets", func() {
 			olsconfigGenerated := AppSrvConfigFile{}
 			err = yaml.Unmarshal([]byte(cm.Data[OLSConfigFilename]), &olsconfigGenerated)
 			Expect(err).NotTo(HaveOccurred())
-
 			olsConfigExpected := AppSrvConfigFile{
 				OLSConfig: OLSConfig{
 					DefaultModel:    "testModel",
@@ -592,6 +591,7 @@ var _ = Describe("App server assets", func() {
 				},
 			))
 		})
+
 	})
 
 	Context("empty custom resource", func() {
@@ -871,6 +871,138 @@ user_data_collector_config: {}
 			))
 		})
 	})
+
+	Context("Additional CA", func() {
+
+		const caConfigMapName = "test-ca-configmap"
+		const certFilename = "additional-ca.crt"
+		var additionalCACm *corev1.ConfigMap
+
+		BeforeEach(func() {
+			rOptions = &OLSConfigReconcilerOptions{
+				LightspeedServiceImage: "lightspeed-service:latest",
+				Namespace:              OLSNamespaceDefault,
+			}
+			cr = getDefaultOLSConfigCR()
+			r = &OLSConfigReconciler{
+				Options:    *rOptions,
+				logger:     logf.Log.WithName("olsconfig.reconciler"),
+				Client:     k8sClient,
+				Scheme:     k8sClient.Scheme(),
+				stateCache: make(map[string]string),
+			}
+			By("create the provider secret")
+			secret, _ = generateRandomSecret()
+			secret.SetOwnerReferences([]metav1.OwnerReference{
+				{
+					Kind:       "Secret",
+					APIVersion: "v1",
+					UID:        "ownerUID",
+					Name:       "test-secret",
+				},
+			})
+			err := r.Create(ctx, secret)
+			Expect(err).NotTo(HaveOccurred())
+			By("create the additional CA configmap")
+			additionalCACm = &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      caConfigMapName,
+					Namespace: OLSNamespaceDefault,
+				},
+				Data: map[string]string{
+					certFilename: testCACert,
+				},
+			}
+			err = r.Create(ctx, additionalCACm)
+			Expect(err).NotTo(HaveOccurred())
+
+		})
+
+		AfterEach(func() {
+			By("Delete the provider secret")
+			err := r.Delete(ctx, secret)
+			Expect(err).NotTo(HaveOccurred())
+			By("Delete the additional CA configmap")
+			err = r.Delete(ctx, additionalCACm)
+			Expect(err).NotTo(HaveOccurred())
+
+		})
+
+		It("should update OLS config and mount volumes for additional CA", func() {
+			olsCm, err := r.generateOLSConfigMap(ctx, cr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(olsCm.Data[OLSConfigFilename]).NotTo(ContainSubstring("extra_ca:"))
+
+			dep, err := r.generateOLSDeployment(cr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dep.Spec.Template.Spec.Volumes).NotTo(ContainElement(
+				corev1.Volume{
+					Name: AdditionalCAVolumeName,
+					VolumeSource: corev1.VolumeSource{
+						ConfigMap: &corev1.ConfigMapVolumeSource{
+							LocalObjectReference: corev1.LocalObjectReference{
+								Name: caConfigMapName,
+							},
+							DefaultMode: &defaultVolumeMode,
+						},
+					},
+				}))
+			Expect(dep.Spec.Template.Spec.Volumes).NotTo(ContainElement(
+				corev1.Volume{
+					Name: CertBundleVolumeName,
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				}))
+
+			cr.Spec.OLSConfig.AdditionalCAConfigMapRef = &corev1.LocalObjectReference{
+				Name: caConfigMapName,
+			}
+
+			olsCm, err = r.generateOLSConfigMap(ctx, cr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(olsCm.Data[OLSConfigFilename]).To(ContainSubstring("extra_ca:\n  - /etc/certs/ols-additional-ca/additional-ca.crt"))
+			Expect(olsCm.Data[OLSConfigFilename]).To(ContainSubstring("certificate_directory: /etc/certs/cert-bundle"))
+
+			dep, err = r.generateOLSDeployment(cr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dep.Spec.Template.Spec.Volumes).To(ContainElements(
+				corev1.Volume{
+					Name: AdditionalCAVolumeName,
+					VolumeSource: corev1.VolumeSource{
+						ConfigMap: &corev1.ConfigMapVolumeSource{
+							LocalObjectReference: corev1.LocalObjectReference{
+								Name: caConfigMapName,
+							},
+							DefaultMode: &defaultVolumeMode,
+						},
+					},
+				},
+				corev1.Volume{
+					Name: CertBundleVolumeName,
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+			))
+
+		})
+
+		It("should return error if the CA text is malformed", func() {
+			additionalCACm.Data[certFilename] = "malformed certificate"
+			err := r.Update(ctx, additionalCACm)
+			Expect(err).NotTo(HaveOccurred())
+
+			cr.Spec.OLSConfig.AdditionalCAConfigMapRef = &corev1.LocalObjectReference{
+				Name: caConfigMapName,
+			}
+			_, err = r.generateOLSConfigMap(ctx, cr)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("failed to validate additional CA certificate"))
+
+		})
+
+	})
 })
 
 func generateRandomSecret() (*corev1.Secret, error) {