From cb520d6eb39a26982b0fc3d1b6f54833f97427a8 Mon Sep 17 00:00:00 2001 From: Haoyu Sun Date: Mon, 16 Sep 2024 18:50:13 +0200 Subject: [PATCH] customizable TLS key and cert on application HTTPS endpoint Signed-off-by: Haoyu Sun --- api/v1alpha1/olsconfig_types.go | 16 +++- api/v1alpha1/zz_generated.deepcopy.go | 21 +++++ ...tspeed-operator.clusterserviceversion.yaml | 15 +++- .../ols.openshift.io_olsconfigs.yaml | 23 ++++++ .../bases/ols.openshift.io_olsconfigs.yaml | 23 +++++- ...tspeed-operator.clusterserviceversion.yaml | 12 +++ hack/custom_certs/cr-with-custom-certs.yaml | 1 - hack/custom_certs/generate-certs.sh | 20 ++--- internal/controller/ols_app_server_assets.go | 22 +++-- .../controller/ols_app_server_assets_test.go | 80 ++++++++++++++++++- .../controller/ols_app_server_deployment.go | 7 +- .../ols_app_server_reconciliator.go | 10 ++- .../ols_app_server_reconciliator_test.go | 47 ++++++++++- .../controller/ols_console_reconciliator.go | 4 +- internal/controller/ols_console_ui_assets.go | 5 +- 15 files changed, 270 insertions(+), 36 deletions(-) diff --git a/api/v1alpha1/olsconfig_types.go b/api/v1alpha1/olsconfig_types.go index 1b9c1e45..7270e64b 100644 --- a/api/v1alpha1/olsconfig_types.go +++ b/api/v1alpha1/olsconfig_types.go @@ -80,8 +80,9 @@ type OLSSpec struct { // User data collection switches // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="User Data Collection" UserDataCollection UserDataCollectionSpec `json:"userDataCollection,omitempty"` - - UseUserProvidedTLSCerts bool `json:"useUserProvidedTLSCerts,omitempty"` + // TLS configuration of the Lightspeed backend's HTTPS endpoint + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="TLS Configuration" + TLSConfig *TLSConfig `json:"tlsConfig,omitempty"` // Additional CA certificates for TLS communication between OLS service and LLM Provider // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Additional CA Configmap",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"} AdditionalCAConfigMapRef *corev1.LocalObjectReference `json:"additionalCAConfigMapRef,omitempty"` @@ -126,7 +127,10 @@ type ConsoleContainerConfig struct { Tolerations []corev1.Toleration `json:"tolerations,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Node Selector",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:nodeSelector"} NodeSelector map[string]string `json:"nodeSelector,omitempty"` - + // Certificate Authority (CA) certificate used by the console proxy endpoint. + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="CA Certificate",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:caCertificate"} + // +kubebuilder:validation:Pattern=`^-----BEGIN CERTIFICATE-----([\s\S]*)-----END CERTIFICATE-----\s?$` + // +optional CAcertificate string `json:"caCertificate,omitempty"` } @@ -259,6 +263,12 @@ type OLSDataCollectorSpec struct { LogLevel string `json:"logLevel,omitempty"` } +type TLSConfig struct { + // KeySecretRef is the secret that holds the TLS key. + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Key Secret" + KeyCertSecretRef corev1.LocalObjectReference `json:"keyCertSecretRef,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 21c3b486..0aa403eb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -332,6 +332,11 @@ func (in *OLSSpec) DeepCopyInto(out *OLSSpec) { copy(*out, *in) } out.UserDataCollection = in.UserDataCollection + if in.TLSConfig != nil { + in, out := &in.TLSConfig, &out.TLSConfig + *out = new(TLSConfig) + **out = **in + } if in.AdditionalCAConfigMapRef != nil { in, out := &in.AdditionalCAConfigMapRef, &out.AdditionalCAConfigMapRef *out = new(corev1.LocalObjectReference) @@ -405,6 +410,22 @@ func (in *RedisSpec) DeepCopy() *RedisSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { + *out = *in + out.KeyCertSecretRef = in.KeyCertSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. +func (in *TLSConfig) DeepCopy() *TLSConfig { + if in == nil { + return nil + } + out := new(TLSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserDataCollectionSpec) DeepCopyInto(out *UserDataCollectionSpec) { *out = *in diff --git a/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml b/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml index f34c0b8d..0381bb68 100644 --- a/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml +++ b/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml @@ -38,7 +38,7 @@ metadata: ] capabilities: Basic Install console.openshift.io/operator-monitoring-default: "true" - createdAt: "2024-08-30T09:59:16Z" + createdAt: "2024-09-17T17:01:48Z" features.operators.openshift.io/cnf: "false" features.operators.openshift.io/cni: "false" features.operators.openshift.io/csi: "false" @@ -161,6 +161,11 @@ spec: - description: Console container settings. displayName: Console Container path: ols.deployment.console + - description: Certificate Authority (CA) certificate used by the console proxy endpoint. + displayName: CA Certificate + path: ols.deployment.console.caCertificate + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:caCertificate - displayName: Node Selector path: ols.deployment.console.nodeSelector x-descriptors: @@ -200,6 +205,12 @@ spec: - description: Replacement for the matched pattern. displayName: Replace With path: ols.queryFilters[0].replaceWith + - description: TLS configuration of the Lightspeed backend's HTTPS endpoint + displayName: TLS Configuration + path: ols.tlsConfig + - description: KeySecretRef is the secret that holds the TLS key. + displayName: Key Secret + path: ols.tlsConfig.keyCertSecretRef - description: User data collection switches displayName: User Data Collection path: ols.userDataCollection @@ -535,8 +546,8 @@ spec: type: AllNamespaces keywords: - ai - - openshift - assistant + - openshift - llm links: - name: Lightspeed Operator diff --git a/bundle/manifests/ols.openshift.io_olsconfigs.yaml b/bundle/manifests/ols.openshift.io_olsconfigs.yaml index d3c08f7b..d6153886 100644 --- a/bundle/manifests/ols.openshift.io_olsconfigs.yaml +++ b/bundle/manifests/ols.openshift.io_olsconfigs.yaml @@ -286,6 +286,12 @@ spec: console: description: Console container settings. properties: + caCertificate: + description: Certificate Authority (CA) certificate used + by the console proxy endpoint. + pattern: ^-----BEGIN CERTIFICATE-----([\s\S]*)-----END + CERTIFICATE-----\s?$ + type: string nodeSelector: additionalProperties: type: string @@ -483,6 +489,23 @@ spec: type: string type: object type: array + tlsConfig: + description: TLS configuration of the Lightspeed backend's HTTPS + endpoint + properties: + keyCertSecretRef: + description: KeySecretRef is the secret that holds the TLS + key. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object userDataCollection: description: User data collection switches properties: diff --git a/config/crd/bases/ols.openshift.io_olsconfigs.yaml b/config/crd/bases/ols.openshift.io_olsconfigs.yaml index 4ccce27e..06558b74 100644 --- a/config/crd/bases/ols.openshift.io_olsconfigs.yaml +++ b/config/crd/bases/ols.openshift.io_olsconfigs.yaml @@ -287,6 +287,10 @@ spec: description: Console container settings. properties: caCertificate: + description: Certificate Authority (CA) certificate used + by the console proxy endpoint. + pattern: ^-----BEGIN CERTIFICATE-----([\s\S]*)-----END + CERTIFICATE-----\s?$ type: string nodeSelector: additionalProperties: @@ -485,8 +489,23 @@ spec: type: string type: object type: array - useUserProvidedTLSCerts: - type: boolean + tlsConfig: + description: TLS configuration of the Lightspeed backend's HTTPS + endpoint + properties: + keyCertSecretRef: + description: KeySecretRef is the secret that holds the TLS + key. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object userDataCollection: description: User data collection switches properties: diff --git a/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml b/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml index ff5f330c..31fbda94 100644 --- a/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml @@ -129,6 +129,12 @@ spec: - description: Console container settings. displayName: Console Container path: ols.deployment.console + - description: Certificate Authority (CA) certificate used by the console proxy + endpoint. + displayName: CA Certificate + path: ols.deployment.console.caCertificate + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:caCertificate - displayName: Node Selector path: ols.deployment.console.nodeSelector x-descriptors: @@ -169,6 +175,12 @@ spec: - description: Replacement for the matched pattern. displayName: Replace With path: ols.queryFilters[0].replaceWith + - description: TLS configuration of the Lightspeed backend's HTTPS endpoint + displayName: TLS Configuration + path: ols.tlsConfig + - description: KeySecretRef is the secret that holds the TLS key. + displayName: Key Secret + path: ols.tlsConfig.keyCertSecretRef - description: User data collection switches displayName: User Data Collection path: ols.userDataCollection diff --git a/hack/custom_certs/cr-with-custom-certs.yaml b/hack/custom_certs/cr-with-custom-certs.yaml index 9f7d20c6..fcc4de78 100644 --- a/hack/custom_certs/cr-with-custom-certs.yaml +++ b/hack/custom_certs/cr-with-custom-certs.yaml @@ -22,7 +22,6 @@ spec: defaultModel: ibm/granite-13b-chat-v2 defaultProvider: bam logLevel: INFO - useUserProvidedTLSCerts: true deployment: replicas: 1 console: diff --git a/hack/custom_certs/generate-certs.sh b/hack/custom_certs/generate-certs.sh index d846766f..083d4ac2 100755 --- a/hack/custom_certs/generate-certs.sh +++ b/hack/custom_certs/generate-certs.sh @@ -7,7 +7,8 @@ CA_CERT="$CERT_DIR/ca.crt" PRIVATE_KEY="$CERT_DIR/tls.key" CERTIFICATE="$CERT_DIR/tls.crt" DAYS_VALID=365 -SECRET_NAME="lightspeed-tls" +SECRET_NAME="user-app-tls" +CM_NAME="user-app-ca" NAMESPACE="openshift-lightspeed" # Create directory for certificates if it doesn't exist @@ -15,23 +16,24 @@ mkdir -p "$CERT_DIR" # Generate CA private key and self-signed CA certificate openssl req -x509 -newkey rsa:4096 -sha256 -days "$DAYS_VALID" -nodes \ - -keyout "$CA_KEY" -out "$CA_CERT" -subj "/CN=MyCA" \ - -addext "subjectAltName=DNS:MyCA" + -keyout "$CA_KEY" -out "$CA_CERT" -subj "/CN=MyCA" \ + -addext "subjectAltName=DNS:MyCA" echo "CA certificate and private key have been generated in $CERT_DIR" # Generate private key and certificate signing request (CSR) for the server openssl req -new -newkey rsa:4096 -nodes -keyout "$PRIVATE_KEY" -out "$CERT_DIR/server.csr" \ - -subj "/CN=lightspeed-app-server" -addext "subjectAltName=DNS:lightspeed-app-server,DNS:lightspeed-app-server.openshift-lightspeed.svc.cluster.local,IP:127.0.0.1,IP:::1" + -subj "/CN=lightspeed-app-server" -addext "subjectAltName=DNS:lightspeed-app-server,DNS:lightspeed-app-server.openshift-lightspeed.svc.cluster.local,IP:127.0.0.1,IP:::1" # Sign the server certificate with the CA certificate openssl x509 -req -in "$CERT_DIR/server.csr" -CA "$CA_CERT" -CAkey "$CA_KEY" -CAcreateserial \ - -out "$CERTIFICATE" -days "$DAYS_VALID" -sha256 -extfile <(echo "subjectAltName=DNS:lightspeed-app-server,DNS:lightspeed-app-server.openshift-lightspeed.svc.cluster.local,IP:127.0.0.1,IP:::1") + -out "$CERTIFICATE" -days "$DAYS_VALID" -sha256 -extfile <(echo "subjectAltName=DNS:lightspeed-app-server,DNS:lightspeed-app-server.openshift-lightspeed.svc.cluster.local,IP:127.0.0.1,IP:::1") echo "Server certificate signed by CA has been generated in $CERT_DIR" # Generate the Kubernetes Secret YAML manifest for the TLS certificate and key for the ols-server -cat < "$CERT_DIR/$SECRET_NAME.yaml" + +cat <"$CERT_DIR/$SECRET_NAME.yaml" apiVersion: v1 kind: Secret metadata: @@ -39,8 +41,8 @@ metadata: namespace: $NAMESPACE type: kubernetes.io/tls data: - tls.crt: $(base64 < "$CERTIFICATE") - tls.key: $(base64 < "$PRIVATE_KEY") + tls.crt: $(base64 <"$CERTIFICATE" | tr -d '\n') + tls.key: $(base64 <"$PRIVATE_KEY" | tr -d '\n') EOF -echo "Kubernetes Secret manifest for TLS has been generated at $CERT_DIR/$SECRET_NAME.yaml" \ No newline at end of file +echo "Kubernetes Secret manifest for TLS has been generated at $CERT_DIR/$SECRET_NAME.yaml" diff --git a/internal/controller/ols_app_server_assets.go b/internal/controller/ols_app_server_assets.go index c07bd377..bbb833fa 100644 --- a/internal/controller/ols_app_server_assets.go +++ b/internal/controller/ols_app_server_assets.go @@ -189,6 +189,16 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv dataCollectorEnabled, _ := r.dataCollectorEnabled(cr) + tlsConfig := TLSConfig{ + TLSCertificatePath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.crt"), + TLSKeyPath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.key"), + } + + if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" { + tlsConfig.TLSCertificatePath = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name, "tls.crt") + tlsConfig.TLSKeyPath = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name, "tls.key") + } + olsConfig := OLSConfig{ DefaultModel: cr.Spec.OLSConfig.DefaultModel, DefaultProvider: cr.Spec.OLSConfig.DefaultProvider, @@ -198,10 +208,7 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv UvicornLogLevel: cr.Spec.OLSConfig.LogLevel, }, ConversationCache: conversationCache, - TLSConfig: TLSConfig{ - TLSCertificatePath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.crt"), - TLSKeyPath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName, "tls.key"), - }, + TLSConfig: tlsConfig, ReferenceContent: ReferenceContent{ ProductDocsIndexPath: "/app-root/vector_db/ocp_product_docs/" + major + "." + minor, ProductDocsIndexId: "ocp-product-docs-" + major + "_" + minor, @@ -322,10 +329,11 @@ func (r *OLSConfigReconciler) getAdditionalCAFileNames(cr *olsv1alpha1.OLSConfig func (r *OLSConfigReconciler) generateService(cr *olsv1alpha1.OLSConfig) (*corev1.Service, error) { annotations := map[string]string{} - // Check if the flag for user-provided TLS certs is set - if !cr.Spec.OLSConfig.UseUserProvidedTLSCerts { - // Add the service-served certs annotations only if the flag is not set + // Let service-ca operator generate a TLS certificate if the user does not provide one + if cr.Spec.OLSConfig.DeploymentConfig.ConsoleContainer.CAcertificate == "" { annotations[ServingCertSecretAnnotationKey] = OLSCertsSecretName + } else { + delete(annotations, ServingCertSecretAnnotationKey) } service := corev1.Service{ diff --git a/internal/controller/ols_app_server_assets_test.go b/internal/controller/ols_app_server_assets_test.go index 853da05a..9be2274b 100644 --- a/internal/controller/ols_app_server_assets_test.go +++ b/internal/controller/ols_app_server_assets_test.go @@ -3,8 +3,14 @@ package controller import ( "context" "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" "encoding/base64" + "encoding/pem" + "math/big" "path" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -592,6 +598,43 @@ var _ = Describe("App server assets", func() { )) }) + It("should use user provided TLS settings if user provided one", func() { + const tlsSecretName = "test-tls-secret" + cr.Spec.OLSConfig.TLSConfig = &olsv1alpha1.TLSConfig{ + KeyCertSecretRef: corev1.LocalObjectReference{ + Name: tlsSecretName, + }, + } + cm, err := r.generateOLSConfigMap(context.TODO(), cr) + Expect(err).NotTo(HaveOccurred()) + olsconfigGenerated := AppSrvConfigFile{} + err = yaml.Unmarshal([]byte(cm.Data[OLSConfigFilename]), &olsconfigGenerated) + Expect(err).NotTo(HaveOccurred()) + Expect(olsconfigGenerated.OLSConfig.TLSConfig.TLSCertificatePath).To(Equal(path.Join(OLSAppCertsMountRoot, tlsSecretName, "tls.crt"))) + Expect(olsconfigGenerated.OLSConfig.TLSConfig.TLSKeyPath).To(Equal(path.Join(OLSAppCertsMountRoot, tlsSecretName, "tls.key"))) + + deployment, err := r.generateOLSDeployment(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement( + corev1.VolumeMount{ + Name: "secret-" + tlsSecretName, + MountPath: path.Join(OLSAppCertsMountRoot, tlsSecretName), + ReadOnly: true, + }, + )) + Expect(deployment.Spec.Template.Spec.Volumes).To(ContainElement( + corev1.Volume{ + Name: "secret-" + tlsSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsSecretName, + DefaultMode: &defaultVolumeMode, + }, + }, + }, + )) + }) + }) Context("empty custom resource", func() { @@ -1011,6 +1054,38 @@ func generateRandomSecret() (*corev1.Secret, error) { // Encode the password to base64 encodedPassword := base64.StdEncoding.EncodeToString(randomPassword) passwordHash, _ := hashBytes([]byte(encodedPassword)) + + // Generate RSA key + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + }) + + // Generate self-signed certificate + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Test Org"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + if err != nil { + return nil, err + } + certPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certDER, + }) + secret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "test-secret", @@ -1020,10 +1095,11 @@ func generateRandomSecret() (*corev1.Secret, error) { }, Data: map[string][]byte{ "client_secret": []byte(passwordHash), - "tls.key": []byte("test tls key"), - "tls.crt": []byte("test tls crt"), + "tls.key": privateKeyPEM, + "tls.crt": certPEM, }, } + return &secret, nil } diff --git a/internal/controller/ols_app_server_deployment.go b/internal/controller/ols_app_server_deployment.go index c07fe0dd..b03b1400 100644 --- a/internal/controller/ols_app_server_deployment.go +++ b/internal/controller/ols_app_server_deployment.go @@ -83,8 +83,11 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( //secretMounts[redisSecretName] = redisCredentialsMountPath // TLS volume - tlsSecretNameMountPath := path.Join(OLSAppCertsMountRoot, OLSCertsSecretName) - secretMounts[OLSCertsSecretName] = tlsSecretNameMountPath + if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" { + secretMounts[cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name] = path.Join(OLSAppCertsMountRoot, cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name) + } else { + secretMounts[OLSCertsSecretName] = path.Join(OLSAppCertsMountRoot, OLSCertsSecretName) + } AdditionalCAMountPath := path.Join(OLSAppCertsMountRoot, AppAdditionalCACertDir) // Container ports diff --git a/internal/controller/ols_app_server_reconciliator.go b/internal/controller/ols_app_server_reconciliator.go index ec197380..2afecb92 100644 --- a/internal/controller/ols_app_server_reconciliator.go +++ b/internal/controller/ols_app_server_reconciliator.go @@ -304,7 +304,7 @@ func (r *OLSConfigReconciler) reconcileService(ctx context.Context, cr *olsv1alp } if serviceEqual(foundService, service) && foundService.ObjectMeta.Annotations != nil { - if cr.Spec.OLSConfig.UseUserProvidedTLSCerts { + if cr.Spec.OLSConfig.DeploymentConfig.ConsoleContainer.CAcertificate != "" { r.logger.Info("OLS service unchanged, reconciliation skipped", "service", service.Name) return nil @@ -421,10 +421,14 @@ func (r *OLSConfigReconciler) reconcileTLSSecret(ctx context.Context, cr *olsv1a foundSecret := &corev1.Secret{} var err, lastErr error var secretValues map[string]string + secretName := OLSCertsSecretName + if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" { + secretName = cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name + } err = wait.PollUntilContextTimeout(ctx, 1*time.Second, ResourceCreationTimeout, true, func(ctx context.Context) (bool, error) { - secretValues, err = getSecretContent(r.Client, OLSCertsSecretName, r.Options.Namespace, []string{"tls.key", "tls.crt"}, foundSecret) + secretValues, err = getSecretContent(r.Client, secretName, r.Options.Namespace, []string{"tls.key", "tls.crt"}, foundSecret) if err != nil { - lastErr = fmt.Errorf("secret: %s does not have expected tls.key or tls.crt. error: %w", OLSCertsSecretName, err) + lastErr = fmt.Errorf("secret: %s does not have expected tls.key or tls.crt. error: %w", secretName, err) return false, nil } return true, nil diff --git a/internal/controller/ols_app_server_reconciliator_test.go b/internal/controller/ols_app_server_reconciliator_test.go index 3aec438c..d1b118f9 100644 --- a/internal/controller/ols_app_server_reconciliator_test.go +++ b/internal/controller/ols_app_server_reconciliator_test.go @@ -25,6 +25,8 @@ var _ = Describe("App server reconciliator", Ordered, func() { Context("Creation logic", Ordered, func() { var secret *corev1.Secret var tlsSecret *corev1.Secret + var tlsUserSecret *corev1.Secret + const tlsUserSecretName = "tls-user-secret" BeforeEach(func() { By("create the provider secret") secret, _ = generateRandomSecret() @@ -39,7 +41,7 @@ var _ = Describe("App server reconciliator", Ordered, func() { secretCreationErr := reconciler.Create(ctx, secret) Expect(secretCreationErr).NotTo(HaveOccurred()) - By("create the tls secret") + By("create the default tls secret") tlsSecret, _ = generateRandomSecret() tlsSecret.Name = OLSCertsSecretName tlsSecret.SetOwnerReferences([]metav1.OwnerReference{ @@ -53,6 +55,12 @@ var _ = Describe("App server reconciliator", Ordered, func() { secretCreationErr = reconciler.Create(ctx, tlsSecret) Expect(secretCreationErr).NotTo(HaveOccurred()) + By("create user provided tls secret") + tlsUserSecret, _ = generateRandomSecret() + tlsUserSecret.Name = tlsUserSecretName + secretCreationErr = reconciler.Create(ctx, tlsUserSecret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + By("Set OLSConfig CR to default") err := k8sClient.Get(ctx, crNamespacedName, cr) Expect(err).NotTo(HaveOccurred()) @@ -68,6 +76,10 @@ var _ = Describe("App server reconciliator", Ordered, func() { By("Delete the tls secret") secretDeletionErr = reconciler.Delete(ctx, tlsSecret) Expect(secretDeletionErr).NotTo(HaveOccurred()) + + By("Delete the user provided tls secret") + secretDeletionErr = reconciler.Delete(ctx, tlsUserSecret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) }) It("should reconcile from OLSConfig custom resource", func() { @@ -283,6 +295,39 @@ var _ = Describe("App server reconciliator", Ordered, func() { Expect(dep.Annotations[OLSAppTLSHashKey]).NotTo(Equal(oldHash)) }) + It("should update the deployment when switching to user provided tls secret", func() { + By("Get the old hash") + dep := &appsv1.Deployment{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: OLSAppServerDeploymentName, Namespace: OLSNamespaceDefault}, dep) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Spec.Template.Annotations).NotTo(BeNil()) + oldHash := dep.Spec.Template.Annotations[OLSAppTLSHashKey] + Expect(oldHash).NotTo(BeEmpty()) + + By("Change OLSConfig to use user provided tls secret and reconcile") + olsConfig := cr.DeepCopy() + olsConfig.Spec.OLSConfig.TLSConfig = &olsv1alpha1.TLSConfig{ + KeyCertSecretRef: corev1.LocalObjectReference{ + Name: tlsUserSecretName, + }, + } + err = reconciler.reconcileAppServer(ctx, olsConfig) + Expect(err).NotTo(HaveOccurred()) + + By("Check new hash is updated") + dep = &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: OLSAppServerDeploymentName, Namespace: OLSNamespaceDefault}, dep) + Expect(err).NotTo(HaveOccurred()) + bytesArr := make([]byte, len(tlsUserSecret.Data["tls.key"])+len(tlsUserSecret.Data["tls.crt"])) + copy(bytesArr, tlsUserSecret.Data["tls.key"]) + copy(bytesArr[len(tlsUserSecret.Data["tls.key"]):], tlsUserSecret.Data["tls.crt"]) + newHash, err := hashBytes(bytesArr) + Expect(err).NotTo(HaveOccurred()) + Expect(newHash).NotTo(Equal(oldHash)) + Expect(dep.Spec.Template.Annotations[OLSAppTLSHashKey]).To(Equal(newHash)) + + }) + It("should trigger rolling update of the deployment when changing LLM secret content", func() { By("Reconcile for LLM Provider Secrets") diff --git a/internal/controller/ols_console_reconciliator.go b/internal/controller/ols_console_reconciliator.go index 94d0ac4d..53695838 100644 --- a/internal/controller/ols_console_reconciliator.go +++ b/internal/controller/ols_console_reconciliator.go @@ -19,7 +19,7 @@ import ( olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" ) -func (r *OLSConfigReconciler) reconcileConsoleUI(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { +func (r *OLSConfigReconciler) reconcileConsoleUI(ctx context.Context, olsconfig *olsv1alpha1.OLSConfig) error { r.logger.Info("reconcileConsoleUI starts") tasks := []ReconcileTask{ { @@ -49,7 +49,7 @@ func (r *OLSConfigReconciler) reconcileConsoleUI(ctx context.Context, cr *olsv1a } for _, task := range tasks { - err := task.Task(ctx, cr) + err := task.Task(ctx, olsconfig) if err != nil { r.logger.Error(err, "reconcileConsoleUI error", "task", task.Name) return fmt.Errorf("failed to %s: %w", task.Name, err) diff --git a/internal/controller/ols_console_ui_assets.go b/internal/controller/ols_console_ui_assets.go index 780512fc..2471258c 100644 --- a/internal/controller/ols_console_ui_assets.go +++ b/internal/controller/ols_console_ui_assets.go @@ -101,6 +101,7 @@ func (r *OLSConfigReconciler) generateConsoleUIService(cr *olsv1alpha1.OLSConfig } func (r *OLSConfigReconciler) generateConsoleUIDeployment(cr *olsv1alpha1.OLSConfig) (*appsv1.Deployment, error) { + const certVolumeName = "lightspeed-console-plugin-cert" replicas := int32(2) val_true := true volumeDefaultMode := int32(420) @@ -137,7 +138,7 @@ func (r *OLSConfigReconciler) generateConsoleUIDeployment(cr *olsv1alpha1.OLSCon Resources: *resources, VolumeMounts: []corev1.VolumeMount{ { - Name: "lightspeed-console-plugin-cert", + Name: certVolumeName, MountPath: "/var/cert", ReadOnly: true, }, @@ -152,7 +153,7 @@ func (r *OLSConfigReconciler) generateConsoleUIDeployment(cr *olsv1alpha1.OLSCon }, Volumes: []corev1.Volume{ { - Name: "lightspeed-console-plugin-cert", + Name: certVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: ConsoleUIServiceCertSecretName,