customizable TLS key and cert on application server side

Signed-off-by: Haoyu Sun <[email protected]>

api/v1alpha1/olsconfig_types.go

@@ -80,6 +80,9 @@ type OLSSpec struct {
 	// User data collection switches
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="User Data Collection"
 	UserDataCollection UserDataCollectionSpec `json:"userDataCollection,omitempty"`
+	// TLS configuration of the Lightspeed backend's HTTPS endpoint
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="TLS Configuration"
+	TLSConfig *TLSConfig `json:"tlsConfig,omitempty"`
 }
 
 // DeploymentConfig defines the schema for overriding deployment of OLS instance.
@@ -257,6 +260,12 @@ type OLSDataCollectorSpec struct {
 	LogLevel string `json:"logLevel,omitempty"`
 }
 
+type TLSConfig struct {
+	// KeySecretRef is the secret that holds the TLS key.
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Key Secret"
+	KeyCertSecretRef corev1.LocalObjectReference `json:"keyCertSecretRef,omitempty"`
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster

bundle/manifests/lightspeed-operator.clusterserviceversion.yaml

@@ -38,7 +38,7 @@ metadata:
       ]
     capabilities: Basic Install
     console.openshift.io/operator-monitoring-default: "true"
-    createdAt: "2024-08-30T09:59:16Z"
+    createdAt: "2024-09-17T17:01:48Z"
     features.operators.openshift.io/cnf: "false"
     features.operators.openshift.io/cni: "false"
     features.operators.openshift.io/csi: "false"
@@ -156,6 +156,11 @@ spec:
           - description: Console container settings.
             displayName: Console Container
             path: ols.deployment.console
+          - description: Certificate Authority (CA) certificate used by the console proxy endpoint.
+            displayName: CA Certificate
+            path: ols.deployment.console.caCertificate
+            x-descriptors:
+              - urn:alm:descriptor:com.tectonic.ui:caCertificate
           - displayName: Node Selector
             path: ols.deployment.console.nodeSelector
             x-descriptors:
@@ -195,6 +200,12 @@ spec:
           - description: Replacement for the matched pattern.
             displayName: Replace With
             path: ols.queryFilters[0].replaceWith
+          - description: TLS configuration of the Lightspeed backend's HTTPS endpoint
+            displayName: TLS Configuration
+            path: ols.tlsConfig
+          - description: KeySecretRef is the secret that holds the TLS key.
+            displayName: Key Secret
+            path: ols.tlsConfig.keyCertSecretRef
           - description: User data collection switches
             displayName: User Data Collection
             path: ols.userDataCollection
@@ -350,7 +361,7 @@ spec:
                         valueFrom:
                           fieldRef:
                             fieldPath: metadata.annotations['olm.targetNamespaces']
-                    image: registry.redhat.io/openshift-lightspeed-beta/lightspeed-rhel9-operator@sha256:e62a1e3e618b7bf6a9ca8b1e7b6ae500d6b0e47c9e5f3e4a8eeb259082c9199e
+                    image: registry.redhat.io/openshift-lightspeed-beta/lightspeed-rhel9-operator@sha256:049a1a398ed87e4f35c99b36304055c7f75d0188a4d8c1726df59b5f400561e5
                     imagePullPolicy: Always
                     livenessProbe:
                       httpGet:
@@ -530,8 +541,8 @@ spec:
       type: AllNamespaces
   keywords:
     - ai
-    - openshift
     - assistant
+    - openshift
     - llm
   links:
     - name: Lightspeed Operator

bundle/manifests/ols.openshift.io_olsconfigs.yaml

@@ -274,6 +274,12 @@ spec:
                       console:
                         description: Console container settings.
                         properties:
+                          caCertificate:
+                            description: Certificate Authority (CA) certificate used
+                              by the console proxy endpoint.
+                            pattern: ^-----BEGIN CERTIFICATE-----([\s\S]*)-----END
+                              CERTIFICATE-----\s?$
+                            type: string
                           nodeSelector:
                             additionalProperties:
                               type: string
@@ -471,6 +477,23 @@ spec:
                           type: string
                       type: object
                     type: array
+                  tlsConfig:
+                    description: TLS configuration of the Lightspeed backend's HTTPS
+                      endpoint
+                    properties:
+                      keyCertSecretRef:
+                        description: KeySecretRef is the secret that holds the TLS
+                          key.
+                        properties:
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              TODO: Add other useful fields. apiVersion, kind, uid?
+                            type: string
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
                   userDataCollection:
                     description: User data collection switches
                     properties:

config/crd/bases/ols.openshift.io_olsconfigs.yaml

@@ -477,6 +477,23 @@ spec:
                           type: string
                       type: object
                     type: array
+                  tlsConfig:
+                    description: TLS configuration of the Lightspeed backend's HTTPS
+                      endpoint
+                    properties:
+                      keyCertSecretRef:
+                        description: KeySecretRef is the secret that holds the TLS
+                          key.
+                        properties:
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              TODO: Add other useful fields. apiVersion, kind, uid?
+                            type: string
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
                   userDataCollection:
                     description: User data collection switches
                     properties:

config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml

@@ -123,6 +123,12 @@ spec:
       - description: Console container settings.
         displayName: Console Container
         path: ols.deployment.console
+      - description: Certificate Authority (CA) certificate used by the console proxy
+          endpoint.
+        displayName: CA Certificate
+        path: ols.deployment.console.caCertificate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:caCertificate
       - displayName: Node Selector
         path: ols.deployment.console.nodeSelector
         x-descriptors:
@@ -163,6 +169,12 @@ spec:
       - description: Replacement for the matched pattern.
         displayName: Replace With
         path: ols.queryFilters[0].replaceWith
+      - description: TLS configuration of the Lightspeed backend's HTTPS endpoint
+        displayName: TLS Configuration
+        path: ols.tlsConfig
+      - description: KeySecretRef is the secret that holds the TLS key.
+        displayName: Key Secret
+        path: ols.tlsConfig.keyCertSecretRef
       - description: User data collection switches
         displayName: User Data Collection
         path: ols.userDataCollection

internal/controller/ols_app_server_assets.go

@@ -189,6 +189,16 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv
 
 	dataCollectorEnabled, _ := r.dataCollectorEnabled(cr)
 
+	tlsConfig := TLSConfig{
+		TLSCertificatePath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.crt"),
+		TLSKeyPath:         path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.key"),
+	}
+
+	if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" {
+		tlsConfig.TLSCertificatePath = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name, "tls.crt")
+		tlsConfig.TLSKeyPath = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name, "tls.key")
+	}
+
 	olsConfig := OLSConfig{
 		DefaultModel:    cr.Spec.OLSConfig.DefaultModel,
 		DefaultProvider: cr.Spec.OLSConfig.DefaultProvider,
@@ -198,10 +208,7 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv
 			UvicornLogLevel: cr.Spec.OLSConfig.LogLevel,
 		},
 		ConversationCache: conversationCache,
-		TLSConfig: TLSConfig{
-			TLSCertificatePath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.crt"),
-			TLSKeyPath:         path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.key"),
-		},
+		TLSConfig:         tlsConfig,
 		ReferenceContent: ReferenceContent{
 			ProductDocsIndexPath: "/app-root/vector_db/ocp_product_docs/" + major + "." + minor,
 			ProductDocsIndexId:   "ocp-product-docs-" + major + "_" + minor,
@@ -285,8 +292,9 @@ func (r *OLSConfigReconciler) generateService(cr *olsv1alpha1.OLSConfig) (*corev
 
 	// Let service-ca operator generate a TLS certificate if the user does not provide one
 	if cr.Spec.OLSConfig.DeploymentConfig.ConsoleContainer.CAcertificate == "" {
-		// Add the service-served certs annotations only if the flag is not set
 		annotations[ServingCertSecretAnnotationKey] = OLSCertsSecretName
+	} else {
+		delete(annotations, ServingCertSecretAnnotationKey)
 	}
 
 	service := corev1.Service{

internal/controller/ols_app_server_assets_test.go

@@ -3,8 +3,14 @@ package controller
 import (
 	"context"
 	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/base64"
+	"encoding/pem"
+	"math/big"
 	"path"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -592,6 +598,43 @@ var _ = Describe("App server assets", func() {
 				},
 			))
 		})
+
+		It("should use user provided TLS settings if user provided one", func() {
+			const tlsSecretName = "test-tls-secret"
+			cr.Spec.OLSConfig.TLSConfig = &olsv1alpha1.TLSConfig{
+				KeyCertSecretRef: corev1.LocalObjectReference{
+					Name: tlsSecretName,
+				},
+			}
+			cm, err := r.generateOLSConfigMap(context.TODO(), cr)
+			Expect(err).NotTo(HaveOccurred())
+			olsconfigGenerated := AppSrvConfigFile{}
+			err = yaml.Unmarshal([]byte(cm.Data[OLSConfigFilename]), &olsconfigGenerated)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(olsconfigGenerated.OLSConfig.TLSConfig.TLSCertificatePath).To(Equal(path.Join(OLSAppCertsMountRoot, tlsSecretName, "tls.crt")))
+			Expect(olsconfigGenerated.OLSConfig.TLSConfig.TLSKeyPath).To(Equal(path.Join(OLSAppCertsMountRoot, tlsSecretName, "tls.key")))
+
+			deployment, err := r.generateOLSDeployment(cr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(
+				corev1.VolumeMount{
+					Name:      "secret-" + tlsSecretName,
+					MountPath: path.Join(OLSAppCertsMountRoot, tlsSecretName),
+					ReadOnly:  true,
+				},
+			))
+			Expect(deployment.Spec.Template.Spec.Volumes).To(ContainElement(
+				corev1.Volume{
+					Name: "secret-" + tlsSecretName,
+					VolumeSource: corev1.VolumeSource{
+						Secret: &corev1.SecretVolumeSource{
+							SecretName:  tlsSecretName,
+							DefaultMode: &defaultVolumeMode,
+						},
+					},
+				},
+			))
+		})
 	})
 
 	Context("empty custom resource", func() {
@@ -879,6 +922,38 @@ func generateRandomSecret() (*corev1.Secret, error) {
 	// Encode the password to base64
 	encodedPassword := base64.StdEncoding.EncodeToString(randomPassword)
 	passwordHash, _ := hashBytes([]byte(encodedPassword))
+
+	// Generate RSA key
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, err
+	}
+	privateKeyPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
+	})
+
+	// Generate self-signed certificate
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			Organization: []string{"Test Org"},
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(365 * 24 * time.Hour),
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
+	if err != nil {
+		return nil, err
+	}
+	certPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certDER,
+	})
+
 	secret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        "test-secret",
@@ -888,10 +963,11 @@ func generateRandomSecret() (*corev1.Secret, error) {
 		},
 		Data: map[string][]byte{
 			"client_secret": []byte(passwordHash),
-			"tls.key":       []byte("test tls key"),
-			"tls.crt":       []byte("test tls crt"),
+			"tls.key":       privateKeyPEM,
+			"tls.crt":       certPEM,
 		},
 	}
+
 	return &secret, nil
 }
 

internal/controller/ols_app_server_deployment.go

@@ -82,8 +82,11 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) (
 	//secretMounts[redisSecretName] = redisCredentialsMountPath
 
 	// TLS volume
-	tlsSecretNameMountPath := path.Join(OLSAppCertsMountRoot, OLSCertsSecretName)
-	secretMounts[OLSCertsSecretName] = tlsSecretNameMountPath
+	if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" {
+		secretMounts[cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name] = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name)
+	} else {
+		secretMounts[OLSCertsSecretName] = path.Join(OLSAppCertsMountRoot, OLSCertsSecretName)
+	}
 
 	// Container ports
 	ports := []corev1.ContainerPort{

internal/controller/ols_app_server_reconciliator.go

@@ -371,10 +371,14 @@ func (r *OLSConfigReconciler) reconcileTLSSecret(ctx context.Context, cr *olsv1a
 	foundSecret := &corev1.Secret{}
 	var err, lastErr error
 	var secretValues map[string]string
+	secretName := OLSCertsSecretName
+	if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" {
+		secretName = cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name
+	}
 	err = wait.PollUntilContextTimeout(ctx, 1*time.Second, ResourceCreationTimeout, true, func(ctx context.Context) (bool, error) {
-		secretValues, err = getSecretContent(r.Client, OLSCertsSecretName, r.Options.Namespace, []string{"tls.key", "tls.crt"}, foundSecret)
+		secretValues, err = getSecretContent(r.Client, secretName, r.Options.Namespace, []string{"tls.key", "tls.crt"}, foundSecret)
 		if err != nil {
-			lastErr = fmt.Errorf("secret: %s does not have expected tls.key or tls.crt. error: %w", OLSCertsSecretName, err)
+			lastErr = fmt.Errorf("secret: %s does not have expected tls.key or tls.crt. error: %w", secretName, err)
 			return false, nil
 		}
 		return true, nil