fix: use datasource as the default source in knowledgebase

Signed-off-by: bjwswang <[email protected]>
nkwangleiGIT · Mar 18, 2024 · aedee1e · aedee1e
1 parent d1aed4d
commit aedee1e
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 6 deletions.
diff --git a/apiserver/pkg/chat/chat_docs.go b/apiserver/pkg/chat/chat_docs.go
@@ -217,8 +217,13 @@ func (cs *ChatServer) BuildConversationKnowledgeBase(ctx context.Context, req Co
 		}
 		// append document path
 		kb.Spec.FileGroups = append(kb.Spec.FileGroups, arcadiav1alpha1.FileGroup{
-			Source: systemDatasource.TypedObjectReference(),
-			Paths:  []string{document.Object},
+			Source: &arcadiav1alpha1.TypedObjectReference{
+				APIGroup:  &arcadiav1alpha1.GroupVersion.Group,
+				Kind:      "Datasource",
+				Name:      systemDatasource.Name,
+				Namespace: &systemDatasource.Namespace,
+			},
+			Paths: []string{document.Object},
 		})
 		return nil
 	})

diff --git a/apiserver/pkg/knowledgebase/knowledgebase.go b/apiserver/pkg/knowledgebase/knowledgebase.go
@@ -160,6 +160,13 @@ func CreateKnowledgeBase(ctx context.Context, c client.Client, input generated.C
 	}
 	if input.FileGroups != nil {
 		for _, f := range input.FileGroups {
+			// use datasource by default
+			if f.Source.Kind == "" {
+				f.Source.Kind = "Datasource"
+			}
+			if f.Source.Namespace == nil {
+				f.Source.Namespace = &input.Namespace
+			}
 			filegroup := v1alpha1.FileGroup{
 				Source: (*v1alpha1.TypedObjectReference)(&f.Source),
 				Paths:  f.Path,

diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml
@@ -19,6 +19,7 @@ resources:
 - bases/chain.arcadia.kubeagi.k8s.com.cn_apichains.yaml
 - bases/prompt.arcadia.kubeagi.k8s.com.cn_prompts.yaml
 - bases/retriever.arcadia.kubeagi.k8s.com.cn_knowledgebaseretrievers.yaml
+- bases/retriever.arcadia.kubeagi.k8s.com.cn_multiqueryretrievers.yaml
 - bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml
 #+kubebuilder:scaffold:crdkustomizeresource
 

diff --git a/controllers/base/knowledgebase_controller.go b/controllers/base/knowledgebase_controller.go
@@ -392,7 +392,7 @@ func (r *KnowledgeBaseReconciler) reconcileFileGroup(ctx context.Context, log lo
 		}
 		// basepath for this versioneddataset
 		vsBasePath = filepath.Join("dataset", versionedDataset.Spec.Dataset.Name, versionedDataset.Spec.Version)
-	case "datasource":
+	case "datasource", "":
 		dsObj := &arcadiav1alpha1.Datasource{}
 		if err = r.Get(ctx, types.NamespacedName{Name: group.Source.Name, Namespace: ns}, dsObj); err != nil {
 			if apierrors.IsNotFound(err) {
@@ -403,14 +403,21 @@ func (r *KnowledgeBaseReconciler) reconcileFileGroup(ctx context.Context, log lo
 		if !dsObj.Status.IsReady() {
 			return errDataSourceNotReady
 		}
-		ds, err = datasource.NewOSS(ctx, r.Client, &dsObj.Spec.Endpoint)
+		// set endpoint's auth secret namespace to current datasource if not set
+		endpoint := dsObj.Spec.Endpoint.DeepCopy()
+		if endpoint != nil && endpoint.AuthSecret != nil {
+			endpoint.AuthSecret.WithNameSpace(dsObj.Namespace)
+		}
+		ds, err = datasource.NewOSS(ctx, r.Client, endpoint)
 		if err != nil {
 			return err
 		}
 		// for none-conversation knowledgebase, bucket is the same as datasource.
 		if kb.Spec.Type != arcadiav1alpha1.KnowledgeBaseTypeConversation {
 			info.Bucket = dsObj.Spec.OSS.Bucket
 		}
+	default:
+		return fmt.Errorf("source type %s not supported yet", group.Source.Kind)
 	}
 
 	if len(kb.Status.FileGroupDetail) == 0 {
@@ -466,8 +473,10 @@ func (r *KnowledgeBaseReconciler) reconcileFileGroup(ctx context.Context, log lo
 		case "versioneddataset":
 			// info.Object has been
 			info.Object = filepath.Join(vsBasePath, path)
-		case "datasource":
+		case "datasource", "":
 			info.Object = path
+		default:
+			return fmt.Errorf("source type %s not supported yet", group.Source.Kind)
 		}
 
 		stat, err := ds.StatFile(ctx, info)

diff --git a/deploy/charts/arcadia/Chart.yaml b/deploy/charts/arcadia/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: arcadia
 description: A Helm chart(Also a KubeBB Component) for KubeAGI Arcadia
 type: application
-version: 0.3.18
+version: 0.3.19
 appVersion: "0.2.0"
 
 keywords:

diff --git a/deploy/charts/arcadia/templates/post-datasource.yaml b/deploy/charts/arcadia/templates/post-datasource.yaml
@@ -15,6 +15,7 @@ spec:
     authSecret:
       kind: Secret
       name: {{ .Release.Name }}-minio
+      namespace: {{ .Release.Namespace }}
     insecure: {{ .Values.minio.ingress.api.insecure }}
   oss:
     # pre-defined buckets for arcadia