feat: add CRD: KnowledgeBase and VectorStore

Signed-off-by: Abirdcfly <[email protected]>

PROJECT

@@ -90,4 +90,22 @@ resources:
   kind: Model
   path: github.com/kubeagi/arcadia/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: kubeagi.k8s.com.cn
+  group: arcadia
+  kind: KnowledgeBase
+  path: github.com/kubeagi/arcadia/api/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: kubeagi.k8s.com.cn
+  group: arcadia
+  kind: VectorStore
+  path: github.com/kubeagi/arcadia/api/v1alpha1
+  version: v1alpha1
 version: "3"

api/v1alpha1/common.go

@@ -16,6 +16,10 @@ limitations under the License.
 
 package v1alpha1
 
+import (
+	"github.com/kubeagi/arcadia/pkg/utils"
+)
+
 const (
 	// Finalizer is the key of the finalizer
 	Finalizer = Group + "/finalizer"
@@ -67,6 +71,13 @@ func (in *TypedObjectReference) WithNameSpace(namespace string) {
 	in.Namespace = &namespace
 }
 
+func (in *TypedObjectReference) GetNamespace() string {
+	if in.Namespace == nil {
+		return utils.GetSelfNamespace()
+	}
+	return *in.Namespace
+}
+
 // Endpoint represents a reachable API endpoint.
 type Endpoint struct {
 	// URL chart repository address
@@ -80,3 +91,14 @@ type Endpoint struct {
 	// Insecure if the endpoint needs a secure connection
 	Insecure bool `json:"insecure,omitempty"`
 }
+
+type CommonSpec struct {
+	// Creator defines datasource creator (AUTO-FILLED by webhook)
+	Creator string `json:"creator,omitempty"`
+
+	// DisplayName defines datasource display name
+	DisplayName string `json:"displayName,omitempty"`
+
+	// Description defines datasource description
+	Description string `json:"description,omitempty"`
+}

api/v1alpha1/condition.go

@@ -207,3 +207,7 @@ func (s *ConditionedStatus) Equal(other *ConditionedStatus) bool {
 
 	return true
 }
+
+func (s *ConditionedStatus) IsReady() bool {
+	return s.GetCondition(TypeReady).Status == corev1.ConditionTrue
+}

api/v1alpha1/dataset_types.go

@@ -25,11 +25,7 @@ import (
 
 // DatasetSpec defines the desired state of Dataset
 type DatasetSpec struct {
-	// Creator defines dataset creator(AUTO-FILLED by webhook)
-	Creator string `json:"creator,omitempty"`
-
-	// DisplayName defines dataset display name
-	DiplayName string `json:"displayName"`
+	CommonSpec `json:",inline"`
 
 	// ContentType defines dataset
 	ContentType string `json:"contentType"`

api/v1alpha1/datasource_types.go

@@ -25,14 +25,7 @@ import (
 
 // DatasourceSpec defines the desired state of Datasource
 type DatasourceSpec struct {
-	// Creator defines datasource creator(AUTO-FILLED by webhook)
-	Creator string `json:"creator,omitempty"`
-
-	// DisplayName defines datasource display name
-	DiplayName string `json:"displayName"`
-
-	// Description defines datasource description
-	Description string `json:"description,omitempty"`
+	CommonSpec `json:",inline"`
 
 	// Enpoint defines connection info
 	Enpoint *Endpoint `json:"endpoint,omitempty"`

api/v1alpha1/embedder_types.go

@@ -27,11 +27,7 @@ import (
 
 // EmbedderSpec defines the desired state of Embedder
 type EmbedderSpec struct {
-	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
-
-	// Name of the Embedding service
-	DisplayName string `json:"displayName,omitempty"`
+	CommonSpec `json:",inline"`
 
 	// ServiceType indicates the source type of embedding service
 	ServiceType embeddings.EmbeddingType `json:"serviceType,omitempty"`
@@ -50,6 +46,7 @@ type EmbedderStatus struct {
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="display-name",type=string,JSONPath=`.spec.displayName`
 
 // Embedder is the Schema for the embeddings API
 type Embedder struct {

api/v1alpha1/knowledgebase.go

@@ -0,0 +1,73 @@
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (kb *KnowledgeBase) VectorStoreCollectionName() string {
+	return kb.Namespace + "_" + kb.Name
+}
+
+func (kb *KnowledgeBase) InitCondition() Condition {
+	return Condition{
+		Type:               TypeReady,
+		Status:             corev1.ConditionUnknown,
+		Reason:             "Init",
+		Message:            "Reconciliation in progress",
+		LastTransitionTime: metav1.Now(),
+		LastSuccessfulTime: metav1.Now(),
+	}
+}
+
+func (kb *KnowledgeBase) PendingCondition(msg string) Condition {
+	return Condition{
+		Type:               TypeReady,
+		Status:             corev1.ConditionFalse,
+		Reason:             "Pending",
+		Message:            msg,
+		LastTransitionTime: metav1.Now(),
+		LastSuccessfulTime: metav1.Now(),
+	}
+}
+
+func (kb *KnowledgeBase) ErrorCondition(msg string) Condition {
+	return Condition{
+		Type:               TypeReady,
+		Status:             corev1.ConditionFalse,
+		Reason:             "Error",
+		Message:            msg,
+		LastTransitionTime: metav1.Now(),
+		LastSuccessfulTime: metav1.Now(),
+	}
+}
+
+func (kb *KnowledgeBase) ReadyCondition() Condition {
+	return Condition{
+		Type:               TypeReady,
+		Status:             corev1.ConditionTrue,
+		LastTransitionTime: metav1.Now(),
+		LastSuccessfulTime: metav1.Now(),
+		Message:            "Success",
+	}
+}
+
+func (f *FileDetails) UpdateErr(err error) {
+	f.LastUpdateTime = metav1.Now()
+	if err != nil {
+		f.ErrMessage = err.Error()
+		f.Phase = FileProcessPhaseFailed
+	} else if f.Phase != FileProcessPhaseSucceeded {
+		f.Phase = FileProcessPhaseSucceeded
+	}
+}
+
+func (f *FileGroupDetail) Init(group FileGroup) {
+	f.Datasource = group.Datasource.DeepCopy()
+	f.FileDetails = make([]FileDetails, len(group.Paths))
+	for i := range group.Paths {
+		f.FileDetails[i].Path = group.Paths[i]
+		f.FileDetails[i].Phase = FileProcessPhasePending
+		f.FileDetails[i].LastUpdateTime = metav1.Now()
+	}
+}

api/v1alpha1/knowledgebase_types.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2023 KubeAGI.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// KnowledgeBaseSpec defines the desired state of KnowledgeBase
+type KnowledgeBaseSpec struct {
+	CommonSpec `json:",inline"`
+
+	// Embedder defines the embedder to embedding files
+	Embedder *TypedObjectReference `json:"embedder,omitempty"`
+
+	// VectorStore defines the vectorstore to store results
+	VectorStore *TypedObjectReference `json:"vectorStore,omitempty"`
+
+	// FileGroups included files Grouped by Datasource
+	FileGroups []FileGroup `json:"fileGroups,omitempty"`
+}
+type FileGroupDetail struct {
+	// From defines the datasource which provides these files
+	Datasource *TypedObjectReference `json:"datasource,omitempty"`
+
+	// FileDetails is the detail files
+	FileDetails []FileDetails `json:"fileDetails,omitempty"`
+}
+
+type FileDetails struct {
+	// Path defines the detail path to get objects from above datasource
+	Path string `json:"path,omitempty"`
+
+	// Checksum defines the checksum of the file
+	Checksum string `json:"checksum,omitempty"`
+
+	// The last time this condition was updated.
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+
+	// Phase defines the process phase
+	Phase FileProcessPhase `json:"phase,omitempty"`
+
+	// ErrMessage defines the error message
+	ErrMessage string `json:"errMessage,omitempty"`
+}
+
+type FileProcessPhase string
+
+const (
+	FileProcessPhasePending    FileProcessPhase = "Pending"
+	FileProcessPhaseProcessing FileProcessPhase = "Processing"
+	FileProcessPhaseSucceeded  FileProcessPhase = "Succeeded"
+	FileProcessPhaseFailed     FileProcessPhase = "Failed"
+)
+
+// KnowledgeBaseStatus defines the observed state of KnowledgeBase
+type KnowledgeBaseStatus struct {
+	// ObservedGeneration is the last observed generation.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// FileGroupDetail is the detail of these files
+	FileGroupDetail []FileGroupDetail `json:"fileGroupDetail,omitempty"`
+
+	// ConditionedStatus is the current status
+	ConditionedStatus `json:",inline"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="display-name",type=string,JSONPath=`.spec.displayName`
+
+// KnowledgeBase is the Schema for the knowledgebases API
+type KnowledgeBase struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   KnowledgeBaseSpec   `json:"spec,omitempty"`
+	Status KnowledgeBaseStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// KnowledgeBaseList contains a list of KnowledgeBase
+type KnowledgeBaseList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KnowledgeBase `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&KnowledgeBase{}, &KnowledgeBaseList{})
+}

api/v1alpha1/llm.go

@@ -40,7 +40,7 @@ func (llmStatus LLMStatus) LLMReady() (string, bool) {
 	if len(llmStatus.Conditions) == 0 {
 		return "No conditions yet", false
 	}
-	if llmStatus.Conditions[0].Type != TypeReady || llmStatus.Conditions[0].Status != corev1.ConditionTrue {
+	if !llmStatus.IsReady() {
 		return "Bad condition", false
 	}
 	return "", true

api/v1alpha1/llm_types.go

@@ -24,7 +24,8 @@ import (
 
 // LLMSpec defines the desired state of LLM
 type LLMSpec struct {
-	DisplayName string `json:"displayName,omitempty"`
+	CommonSpec `json:",inline"`
+
 	// Type defines the type of llm
 	Type llms.LLMType `json:"type"`