customizable TLS key and cert on application HTTPS endpoint

Signed-off-by: Haoyu Sun <[email protected]>

api/v1alpha1/olsconfig_types.go

@@ -80,8 +80,9 @@ type OLSSpec struct {
 	// User data collection switches
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="User Data Collection"
 	UserDataCollection UserDataCollectionSpec `json:"userDataCollection,omitempty"`
-
-	UseUserProvidedTLSCerts bool `json:"useUserProvidedTLSCerts,omitempty"`
+	// TLS configuration of the Lightspeed backend's HTTPS endpoint
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="TLS Configuration"
+	TLSConfig *TLSConfig `json:"tlsConfig,omitempty"`
 	// Additional CA certificates for TLS communication between OLS service and LLM Provider
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Additional CA Configmap",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
 	AdditionalCAConfigMapRef *corev1.LocalObjectReference `json:"additionalCAConfigMapRef,omitempty"`
@@ -268,6 +269,12 @@ type OLSDataCollectorSpec struct {
 	LogLevel string `json:"logLevel,omitempty"`
 }
 
+type TLSConfig struct {
+	// KeySecretRef is the secret that holds the TLS key.
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Key Secret"
+	KeyCertSecretRef corev1.LocalObjectReference `json:"keyCertSecretRef,omitempty"`
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster

bundle/manifests/lightspeed-operator.clusterserviceversion.yaml

@@ -38,7 +38,7 @@ metadata:
       ]
     capabilities: Basic Install
     console.openshift.io/operator-monitoring-default: "true"
-    createdAt: "2024-10-09T12:51:43Z"
+    createdAt: "2024-10-09T13:05:58Z"
     features.operators.openshift.io/cnf: "false"
     features.operators.openshift.io/cni: "false"
     features.operators.openshift.io/csi: "false"
@@ -210,6 +210,12 @@ spec:
           - description: Replacement for the matched pattern.
             displayName: Replace With
             path: ols.queryFilters[0].replaceWith
+          - description: TLS configuration of the Lightspeed backend's HTTPS endpoint
+            displayName: TLS Configuration
+            path: ols.tlsConfig
+          - description: KeySecretRef is the secret that holds the TLS key.
+            displayName: Key Secret
+            path: ols.tlsConfig.keyCertSecretRef
           - description: User data collection switches
             displayName: User Data Collection
             path: ols.userDataCollection

bundle/manifests/ols.openshift.io_olsconfigs.yaml

@@ -497,8 +497,23 @@ spec:
                           type: string
                       type: object
                     type: array
-                  useUserProvidedTLSCerts:
-                    type: boolean
+                  tlsConfig:
+                    description: TLS configuration of the Lightspeed backend's HTTPS
+                      endpoint
+                    properties:
+                      keyCertSecretRef:
+                        description: KeySecretRef is the secret that holds the TLS
+                          key.
+                        properties:
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              TODO: Add other useful fields. apiVersion, kind, uid?
+                            type: string
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
                   userDataCollection:
                     description: User data collection switches
                     properties:

config/crd/bases/ols.openshift.io_olsconfigs.yaml

@@ -497,8 +497,23 @@ spec:
                           type: string
                       type: object
                     type: array
-                  useUserProvidedTLSCerts:
-                    type: boolean
+                  tlsConfig:
+                    description: TLS configuration of the Lightspeed backend's HTTPS
+                      endpoint
+                    properties:
+                      keyCertSecretRef:
+                        description: KeySecretRef is the secret that holds the TLS
+                          key.
+                        properties:
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              TODO: Add other useful fields. apiVersion, kind, uid?
+                            type: string
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
                   userDataCollection:
                     description: User data collection switches
                     properties:

config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml

@@ -180,6 +180,12 @@ spec:
       - description: Replacement for the matched pattern.
         displayName: Replace With
         path: ols.queryFilters[0].replaceWith
+      - description: TLS configuration of the Lightspeed backend's HTTPS endpoint
+        displayName: TLS Configuration
+        path: ols.tlsConfig
+      - description: KeySecretRef is the secret that holds the TLS key.
+        displayName: Key Secret
+        path: ols.tlsConfig.keyCertSecretRef
       - description: User data collection switches
         displayName: User Data Collection
         path: ols.userDataCollection

hack/custom_certs/cr-with-custom-certs.yaml

@@ -22,7 +22,6 @@ spec:
     defaultModel: ibm/granite-13b-chat-v2
     defaultProvider: bam
     logLevel: INFO
-    useUserProvidedTLSCerts: true
     deployment:
       replicas: 1
       console:

hack/custom_certs/generate-certs.sh

@@ -7,40 +7,42 @@ CA_CERT="$CERT_DIR/ca.crt"
 PRIVATE_KEY="$CERT_DIR/tls.key"
 CERTIFICATE="$CERT_DIR/tls.crt"
 DAYS_VALID=365
-SECRET_NAME="lightspeed-tls"
+SECRET_NAME="user-app-tls"
+CM_NAME="user-app-ca"
 NAMESPACE="openshift-lightspeed"
 
 # Create directory for certificates if it doesn't exist
 mkdir -p "$CERT_DIR"
 
 # Generate CA private key and self-signed CA certificate
 openssl req -x509 -newkey rsa:4096 -sha256 -days "$DAYS_VALID" -nodes \
-    -keyout "$CA_KEY" -out "$CA_CERT" -subj "/CN=MyCA" \
-    -addext "subjectAltName=DNS:MyCA"
+  -keyout "$CA_KEY" -out "$CA_CERT" -subj "/CN=MyCA" \
+  -addext "subjectAltName=DNS:MyCA"
 
 echo "CA certificate and private key have been generated in $CERT_DIR"
 
 # Generate private key and certificate signing request (CSR) for the server
 openssl req -new -newkey rsa:4096 -nodes -keyout "$PRIVATE_KEY" -out "$CERT_DIR/server.csr" \
-    -subj "/CN=lightspeed-app-server" -addext "subjectAltName=DNS:lightspeed-app-server,DNS:lightspeed-app-server.openshift-lightspeed.svc.cluster.local,IP:127.0.0.1,IP:::1"
+  -subj "/CN=lightspeed-app-server" -addext "subjectAltName=DNS:lightspeed-app-server,DNS:lightspeed-app-server.openshift-lightspeed.svc.cluster.local,IP:127.0.0.1,IP:::1"
 
 # Sign the server certificate with the CA certificate
 openssl x509 -req -in "$CERT_DIR/server.csr" -CA "$CA_CERT" -CAkey "$CA_KEY" -CAcreateserial \
-    -out "$CERTIFICATE" -days "$DAYS_VALID" -sha256 -extfile <(echo "subjectAltName=DNS:lightspeed-app-server,DNS:lightspeed-app-server.openshift-lightspeed.svc.cluster.local,IP:127.0.0.1,IP:::1")
+  -out "$CERTIFICATE" -days "$DAYS_VALID" -sha256 -extfile <(echo "subjectAltName=DNS:lightspeed-app-server,DNS:lightspeed-app-server.openshift-lightspeed.svc.cluster.local,IP:127.0.0.1,IP:::1")
 
 echo "Server certificate signed by CA has been generated in $CERT_DIR"
 
 # Generate the Kubernetes Secret YAML manifest for the TLS certificate and key for the ols-server
-cat <<EOF > "$CERT_DIR/$SECRET_NAME.yaml"
+
+cat <<EOF >"$CERT_DIR/$SECRET_NAME.yaml"
 apiVersion: v1
 kind: Secret
 metadata:
   name: $SECRET_NAME
   namespace: $NAMESPACE
 type: kubernetes.io/tls
 data:
-  tls.crt: $(base64 < "$CERTIFICATE")
-  tls.key: $(base64 < "$PRIVATE_KEY")
+  tls.crt: $(base64 <"$CERTIFICATE" | tr -d '\n')
+  tls.key: $(base64 <"$PRIVATE_KEY" | tr -d '\n')
 EOF
 
-echo "Kubernetes Secret manifest for TLS has been generated at $CERT_DIR/$SECRET_NAME.yaml"
+echo "Kubernetes Secret manifest for TLS has been generated at $CERT_DIR/$SECRET_NAME.yaml"

internal/controller/ols_app_server_assets.go

@@ -189,6 +189,16 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv
 
 	dataCollectorEnabled, _ := r.dataCollectorEnabled(cr)
 
+	tlsConfig := TLSConfig{
+		TLSCertificatePath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.crt"),
+		TLSKeyPath:         path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.key"),
+	}
+
+	if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" {
+		tlsConfig.TLSCertificatePath = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name, "tls.crt")
+		tlsConfig.TLSKeyPath = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name, "tls.key")
+	}
+
 	olsConfig := OLSConfig{
 		DefaultModel:    cr.Spec.OLSConfig.DefaultModel,
 		DefaultProvider: cr.Spec.OLSConfig.DefaultProvider,
@@ -198,10 +208,7 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv
 			UvicornLogLevel: cr.Spec.OLSConfig.LogLevel,
 		},
 		ConversationCache: conversationCache,
-		TLSConfig: TLSConfig{
-			TLSCertificatePath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.crt"),
-			TLSKeyPath:         path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.key"),
-		},
+		TLSConfig:         tlsConfig,
 		ReferenceContent: ReferenceContent{
 			ProductDocsIndexPath: "/app-root/vector_db/ocp_product_docs/" + major + "." + minor,
 			ProductDocsIndexId:   "ocp-product-docs-" + major + "_" + minor,
@@ -322,10 +329,11 @@ func (r *OLSConfigReconciler) getAdditionalCAFileNames(cr *olsv1alpha1.OLSConfig
 func (r *OLSConfigReconciler) generateService(cr *olsv1alpha1.OLSConfig) (*corev1.Service, error) {
 	annotations := map[string]string{}
 
-	// Check if the flag for user-provided TLS certs is set
-	if !cr.Spec.OLSConfig.UseUserProvidedTLSCerts {
-		// Add the service-served certs annotations only if the flag is not set
+	// Let service-ca operator generate a TLS certificate if the user does not provide one
+	if cr.Spec.OLSConfig.DeploymentConfig.ConsoleContainer.CAcertificate == "" {
 		annotations[ServingCertSecretAnnotationKey] = OLSCertsSecretName
+	} else {
+		delete(annotations, ServingCertSecretAnnotationKey)
 	}
 
 	service := corev1.Service{

internal/controller/ols_app_server_assets_test.go

@@ -3,8 +3,14 @@ package controller
 import (
 	"context"
 	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/base64"
+	"encoding/pem"
+	"math/big"
 	"path"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -592,6 +598,43 @@ var _ = Describe("App server assets", func() {
 			))
 		})
 
+		It("should use user provided TLS settings if user provided one", func() {
+			const tlsSecretName = "test-tls-secret"
+			cr.Spec.OLSConfig.TLSConfig = &olsv1alpha1.TLSConfig{
+				KeyCertSecretRef: corev1.LocalObjectReference{
+					Name: tlsSecretName,
+				},
+			}
+			cm, err := r.generateOLSConfigMap(context.TODO(), cr)
+			Expect(err).NotTo(HaveOccurred())
+			olsconfigGenerated := AppSrvConfigFile{}
+			err = yaml.Unmarshal([]byte(cm.Data[OLSConfigFilename]), &olsconfigGenerated)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(olsconfigGenerated.OLSConfig.TLSConfig.TLSCertificatePath).To(Equal(path.Join(OLSAppCertsMountRoot, tlsSecretName, "tls.crt")))
+			Expect(olsconfigGenerated.OLSConfig.TLSConfig.TLSKeyPath).To(Equal(path.Join(OLSAppCertsMountRoot, tlsSecretName, "tls.key")))
+
+			deployment, err := r.generateOLSDeployment(cr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(
+				corev1.VolumeMount{
+					Name:      "secret-" + tlsSecretName,
+					MountPath: path.Join(OLSAppCertsMountRoot, tlsSecretName),
+					ReadOnly:  true,
+				},
+			))
+			Expect(deployment.Spec.Template.Spec.Volumes).To(ContainElement(
+				corev1.Volume{
+					Name: "secret-" + tlsSecretName,
+					VolumeSource: corev1.VolumeSource{
+						Secret: &corev1.SecretVolumeSource{
+							SecretName:  tlsSecretName,
+							DefaultMode: &defaultVolumeMode,
+						},
+					},
+				},
+			))
+		})
+
 	})
 
 	Context("empty custom resource", func() {
@@ -1012,6 +1055,38 @@ func generateRandomSecret() (*corev1.Secret, error) {
 	// Encode the password to base64
 	encodedPassword := base64.StdEncoding.EncodeToString(randomPassword)
 	passwordHash, _ := hashBytes([]byte(encodedPassword))
+
+	// Generate RSA key
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, err
+	}
+	privateKeyPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
+	})
+
+	// Generate self-signed certificate
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			Organization: []string{"Test Org"},
+		},
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(365 * 24 * time.Hour),
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
+	if err != nil {
+		return nil, err
+	}
+	certPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certDER,
+	})
+
 	secret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        "test-secret",
@@ -1021,10 +1096,11 @@ func generateRandomSecret() (*corev1.Secret, error) {
 		},
 		Data: map[string][]byte{
 			"client_secret": []byte(passwordHash),
-			"tls.key":       []byte("test tls key"),
-			"tls.crt":       []byte("test tls crt"),
+			"tls.key":       privateKeyPEM,
+			"tls.crt":       certPEM,
 		},
 	}
+
 	return &secret, nil
 }
 

internal/controller/ols_app_server_deployment.go

@@ -83,8 +83,11 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) (
 	//secretMounts[redisSecretName] = redisCredentialsMountPath
 
 	// TLS volume
-	tlsSecretNameMountPath := path.Join(OLSAppCertsMountRoot, OLSCertsSecretName)
-	secretMounts[OLSCertsSecretName] = tlsSecretNameMountPath
+	if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" {
+		secretMounts[cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name] = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name)
+	} else {
+		secretMounts[OLSCertsSecretName] = path.Join(OLSAppCertsMountRoot, OLSCertsSecretName)
+	}
 	AdditionalCAMountPath := path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir)
 
 	// Container ports