diff --git a/PROJECT b/PROJECT index 265a2975e..6f463e7c8 100644 --- a/PROJECT +++ b/PROJECT @@ -2,10 +2,10 @@ componentConfig: true domain: kubeagi.k8s.com.cn layout: - go.kubebuilder.io/v3 +multigroup: true plugins: manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} -multigroup: true projectName: arcadia repo: github.com/kubeagi/arcadia resources: @@ -142,4 +142,13 @@ resources: kind: Prompt path: github.com/kubeagi/arcadia/api/app-node/prompt/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: arcadia.kubeagi.k8s.com.cn + group: evaluation + kind: RAG + path: github.com/kubeagi/arcadia/api/evaluation/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/evaluation/v1alpha1/common.go b/api/evaluation/v1alpha1/common.go new file mode 100644 index 000000000..1c90d833c --- /dev/null +++ b/api/evaluation/v1alpha1/common.go @@ -0,0 +1,79 @@ +/* +Copyright 2024 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +type MetricsKind string + +const ( + // AnswerRelevancy in ragas https://docs.ragas.io/en/stable/concepts/metrics/answer_relevance.html + // Scores the relevancy of the answer according to the given question. + AnswerRelevancy MetricsKind = "answer_relevancy" + + // AnswerSimilarity in ragas https://docs.ragas.io/en/stable/concepts/metrics/semantic_similarity.html + // Scores the semantic similarity of ground truth with generated answer. + AnswerSimilarity MetricsKind = "answer_similarity" + + // AnswerCorrectness in ragas https://docs.ragas.io/en/stable/concepts/metrics/answer_correctness.html + // Measures answer correctness compared to ground truth as a combination(Weighted) of + // - factuality + // - semantic similarity + AnswerCorrectness MetricsKind = "answer_correctness" + + // Faithfulness in ragas https://docs.ragas.io/en/stable/concepts/metrics/faithfulness.html + // Scores the factual consistency of the generated answer against the given context. + Faithfulness MetricsKind = "faithfulness" + + // ContextPrecision in ragas https://docs.ragas.io/en/stable/concepts/metrics/context_precision.html + // Average Precision is a metric that evaluates whether all of the relevant items selected by the model are ranked higher or not. + ContextPrecision MetricsKind = "context_precision" + + // ContextRelevancy in ragas https://docs.ragas.io/en/stable/concepts/metrics/context_relevancy.html + // Gauges the relevancy of the retrieved context + ContextRelevancy MetricsKind = "context_relevancy" + + // ContextRecall in ragas https://docs.ragas.io/en/stable/concepts/metrics/context_recall.html + // Estimates context recall by estimating TP and FN using annotated answer and retrieved context. + ContextRecall MetricsKind = "context_recall" + + // AspectCritique in ragas https://docs.ragas.io/en/stable/concepts/metrics/critique.html + // Designed to assess submissions based on predefined aspects such as harmlessness and correctness + // SUPPORTED_ASPECTS = [ harmfulness, maliciousness, coherence, correctness, conciseness, ] + AspectCritique MetricsKind = "aspect_critique" +) + +type Metric struct { + // Kind of this Metric + Kind MetricsKind `json:"kind,omitempty"` + + // Parameters in this Metrics + Parameters []Parameter `json:"parameters,omitempty"` + + // ToleranceThreshbold on this Metric + // If the evaluation score is smaller than this tolerance threshold,we treat this RAG solution as `Bad` + ToleranceThreshbold int `json:"tolerance_threshold,omitempty"` +} + +// Parameter to metrics which is a key-value pair +type Parameter struct { + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` +} + +// Report is the summarization of evaluation +type Report struct { + // TODO +} diff --git a/api/evaluation/v1alpha1/groupversion_info.go b/api/evaluation/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..d87fb7449 --- /dev/null +++ b/api/evaluation/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2024 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the evaluation.arcadia v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=evaluation.arcadia.kubeagi.k8s.com.cn +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "evaluation.arcadia.kubeagi.k8s.com.cn", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/evaluation/v1alpha1/rag_types.go b/api/evaluation/v1alpha1/rag_types.go new file mode 100644 index 000000000..26ff50968 --- /dev/null +++ b/api/evaluation/v1alpha1/rag_types.go @@ -0,0 +1,83 @@ +/* +Copyright 2024 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + basev1alpha1 "github.com/kubeagi/arcadia/api/base/v1alpha1" +) + +// Dataset stands for the files used to generate ragas test dataset +type Dataset struct { + // From defines the source which provides this QA Files for test dataset + // Only `VersionedDataset` allowed + Source *basev1alpha1.TypedObjectReference `json:"source,omitempty"` + // Files retrieved from Source and used in this testdataset + // - For file with tag `object_type: QA`, will be used directly + // - TODO: For file without special tags, will use `QAGenerationChain` to generate QAs (Not Supported Yet) + Files []string `json:"files,omitempty"` +} + +// RAGSpec defines the desired state of RAG +type RAGSpec struct { + // Application(required) defines the target of this RAG evaluation + Application *basev1alpha1.TypedObjectReference `json:"application"` + + // Datasets defines the dataset which will be used to generate test datasets + Datasets []Dataset `json:"datasets"` + + // JudgeLLM(required) defines the judge which is a LLM to evaluate RAG application against test dataset + JudgeLLM *basev1alpha1.TypedObjectReference `json:"judge_llm"` + + // Metrics that this rag evaluation will do + Metrics []Metric `json:"metrics"` + + // Report defines the evaluation report configurations + Report Report `json:"report,omitempty"` +} + +// RAGStatus defines the observed state of RAG +type RAGStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// RAG is the Schema for the rags API +type RAG struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RAGSpec `json:"spec,omitempty"` + Status RAGStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// RAGList contains a list of RAG +type RAGList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RAG `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RAG{}, &RAGList{}) +} diff --git a/api/evaluation/v1alpha1/zz_generated.deepcopy.go b/api/evaluation/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..d3a2bdae8 --- /dev/null +++ b/api/evaluation/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,216 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + basev1alpha1 "github.com/kubeagi/arcadia/api/base/v1alpha1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Dataset) DeepCopyInto(out *Dataset) { + *out = *in + if in.Source != nil { + in, out := &in.Source, &out.Source + *out = new(basev1alpha1.TypedObjectReference) + (*in).DeepCopyInto(*out) + } + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Dataset. +func (in *Dataset) DeepCopy() *Dataset { + if in == nil { + return nil + } + out := new(Dataset) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metric) DeepCopyInto(out *Metric) { + *out = *in + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make([]Parameter, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metric. +func (in *Metric) DeepCopy() *Metric { + if in == nil { + return nil + } + out := new(Metric) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Parameter) DeepCopyInto(out *Parameter) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Parameter. +func (in *Parameter) DeepCopy() *Parameter { + if in == nil { + return nil + } + out := new(Parameter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RAG) DeepCopyInto(out *RAG) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAG. +func (in *RAG) DeepCopy() *RAG { + if in == nil { + return nil + } + out := new(RAG) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RAG) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RAGList) DeepCopyInto(out *RAGList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RAG, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAGList. +func (in *RAGList) DeepCopy() *RAGList { + if in == nil { + return nil + } + out := new(RAGList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RAGList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RAGSpec) DeepCopyInto(out *RAGSpec) { + *out = *in + if in.Application != nil { + in, out := &in.Application, &out.Application + *out = new(basev1alpha1.TypedObjectReference) + (*in).DeepCopyInto(*out) + } + if in.Datasets != nil { + in, out := &in.Datasets, &out.Datasets + *out = make([]Dataset, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.JudgeLLM != nil { + in, out := &in.JudgeLLM, &out.JudgeLLM + *out = new(basev1alpha1.TypedObjectReference) + (*in).DeepCopyInto(*out) + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]Metric, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.Report = in.Report +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAGSpec. +func (in *RAGSpec) DeepCopy() *RAGSpec { + if in == nil { + return nil + } + out := new(RAGSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RAGStatus) DeepCopyInto(out *RAGStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAGStatus. +func (in *RAGStatus) DeepCopy() *RAGStatus { + if in == nil { + return nil + } + out := new(RAGStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Report) DeepCopyInto(out *Report) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Report. +func (in *Report) DeepCopy() *Report { + if in == nil { + return nil + } + out := new(Report) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml b/config/crd/bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml new file mode 100644 index 000000000..a2380b758 --- /dev/null +++ b/config/crd/bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml @@ -0,0 +1,166 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: rags.evaluation.arcadia.kubeagi.k8s.com.cn +spec: + group: evaluation.arcadia.kubeagi.k8s.com.cn + names: + kind: RAG + listKind: RAGList + plural: rags + singular: rag + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RAG is the Schema for the rags API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RAGSpec defines the desired state of RAG + properties: + application: + description: Application(required) defines the target of this RAG + evaluation + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in + the core API group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being referenced + type: string + required: + - kind + - name + type: object + datasets: + description: Datasets defines the dataset which will be used to generate + test datasets + items: + description: Dataset stands for the files used to generate ragas + test dataset + properties: + files: + description: 'Files retrieved from Source and used in this testdataset + - For file with tag `object_type: QA`, will be used directly + - TODO: For file without special tags, will use `QAGenerationChain` + to generate QAs (Not Supported Yet)' + items: + type: string + type: array + source: + description: From defines the source which provides this QA + Files for test dataset Only `VersionedDataset` allowed + properties: + apiGroup: + description: APIGroup is the group for the resource being + referenced. If APIGroup is not specified, the specified + Kind must be in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being + referenced + type: string + required: + - kind + - name + type: object + type: object + type: array + judge_llm: + description: JudgeLLM(required) defines the judge which is a LLM to + evaluate RAG application against test dataset + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in + the core API group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being referenced + type: string + required: + - kind + - name + type: object + metrics: + description: Metrics that this rag evaluation will do + items: + properties: + kind: + description: Kind of this Metric + type: string + parameters: + description: Parameters in this Metrics + items: + description: Parameter to metrics which is a key-value pair + properties: + key: + type: string + value: + type: string + type: object + type: array + tolerance_threshold: + description: ToleranceThreshbold on this Metric If the evaluation + score is smaller than this tolerance threshold,we treat this + RAG solution as `Bad` + type: integer + type: object + type: array + report: + description: Report defines the evaluation report configurations + type: object + required: + - application + - datasets + - judge_llm + - metrics + type: object + status: + description: RAGStatus defines the observed state of RAG + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 1c84193ef..14257ca7c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -17,6 +17,7 @@ resources: - bases/chain.arcadia.kubeagi.k8s.com.cn_retrievalqachains.yaml - bases/prompt.arcadia.kubeagi.k8s.com.cn_prompts.yaml - bases/retriever.arcadia.kubeagi.k8s.com.cn_knowledgebaseretrievers.yaml +- bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -33,6 +34,7 @@ patchesStrategicMerge: #- patches/webhook_in_workers.yaml #- patches/webhook_in_models.yaml #- patches/webhook_in_applications.yaml +#- patches/webhook_in_rags.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -48,6 +50,7 @@ patchesStrategicMerge: #- patches/cainjection_in_knowledgebases.yaml #- patches/cainjection_in_vectorstores.yaml #- patches/cainjection_in_applications.yaml +#- patches/cainjection_in_rags.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_evaluation.arcadia_rags.yaml b/config/crd/patches/cainjection_in_evaluation.arcadia_rags.yaml new file mode 100644 index 000000000..bae9631eb --- /dev/null +++ b/config/crd/patches/cainjection_in_evaluation.arcadia_rags.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: rags.evaluation.arcadia.kubeagi.k8s.com.cn diff --git a/config/crd/patches/webhook_in_evaluation.arcadia_rags.yaml b/config/crd/patches/webhook_in_evaluation.arcadia_rags.yaml new file mode 100644 index 000000000..0af2037f7 --- /dev/null +++ b/config/crd/patches/webhook_in_evaluation.arcadia_rags.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: rags.evaluation.arcadia.kubeagi.k8s.com.cn +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/evaluation.arcadia_rag_editor_role.yaml b/config/rbac/evaluation.arcadia_rag_editor_role.yaml new file mode 100644 index 000000000..ab81a0791 --- /dev/null +++ b/config/rbac/evaluation.arcadia_rag_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit rags. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rag-editor-role +rules: +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags/status + verbs: + - get diff --git a/config/rbac/evaluation.arcadia_rag_viewer_role.yaml b/config/rbac/evaluation.arcadia_rag_viewer_role.yaml new file mode 100644 index 000000000..773c59070 --- /dev/null +++ b/config/rbac/evaluation.arcadia_rag_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view rags. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rag-viewer-role +rules: +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags + verbs: + - get + - list + - watch +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 11ed6ecfa..3dc8bb6dc 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -460,6 +460,32 @@ rules: - get - patch - update +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags/finalizers + verbs: + - update +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags/status + verbs: + - get + - patch + - update - apiGroups: - networking.k8s.io resources: diff --git a/config/samples/evaluation.arcadia_v1alpha1_rag.yaml b/config/samples/evaluation.arcadia_v1alpha1_rag.yaml new file mode 100644 index 000000000..6745ed24e --- /dev/null +++ b/config/samples/evaluation.arcadia_v1alpha1_rag.yaml @@ -0,0 +1,6 @@ +apiVersion: evaluation.arcadia.kubeagi.k8s.com.cn/v1alpha1 +kind: RAG +metadata: + name: rag-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 73845061e..e89e00d2f 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -12,4 +12,5 @@ resources: - arcadia_v1alpha1_model.yaml - arcadia_v1alpha1_application.yaml - app_llmchain_englishteacher.yaml +- evaluation.arcadia_v1alpha1_rag.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/controllers/application_controller.go b/controllers/base/application_controller.go similarity index 100% rename from controllers/application_controller.go rename to controllers/base/application_controller.go diff --git a/controllers/common.go b/controllers/base/common.go similarity index 100% rename from controllers/common.go rename to controllers/base/common.go diff --git a/controllers/dataset_controller.go b/controllers/base/dataset_controller.go similarity index 100% rename from controllers/dataset_controller.go rename to controllers/base/dataset_controller.go diff --git a/controllers/datasource_controller.go b/controllers/base/datasource_controller.go similarity index 100% rename from controllers/datasource_controller.go rename to controllers/base/datasource_controller.go diff --git a/controllers/embedder_controller.go b/controllers/base/embedder_controller.go similarity index 100% rename from controllers/embedder_controller.go rename to controllers/base/embedder_controller.go diff --git a/controllers/knowledgebase_controller.go b/controllers/base/knowledgebase_controller.go similarity index 100% rename from controllers/knowledgebase_controller.go rename to controllers/base/knowledgebase_controller.go diff --git a/controllers/llm_controller.go b/controllers/base/llm_controller.go similarity index 100% rename from controllers/llm_controller.go rename to controllers/base/llm_controller.go diff --git a/controllers/model_controller.go b/controllers/base/model_controller.go similarity index 100% rename from controllers/model_controller.go rename to controllers/base/model_controller.go diff --git a/controllers/namespace_controller.go b/controllers/base/namespace_controller.go similarity index 100% rename from controllers/namespace_controller.go rename to controllers/base/namespace_controller.go diff --git a/controllers/prompt_controller.go b/controllers/base/prompt_controller.go similarity index 100% rename from controllers/prompt_controller.go rename to controllers/base/prompt_controller.go diff --git a/controllers/suite_test.go b/controllers/base/suite_test.go similarity index 96% rename from controllers/suite_test.go rename to controllers/base/suite_test.go index 4e34c08e2..d58249f2f 100644 --- a/controllers/suite_test.go +++ b/controllers/base/suite_test.go @@ -54,7 +54,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, } diff --git a/controllers/vectorstore_controller.go b/controllers/base/vectorstore_controller.go similarity index 100% rename from controllers/vectorstore_controller.go rename to controllers/base/vectorstore_controller.go diff --git a/controllers/versioneddataset_controller.go b/controllers/base/versioneddataset_controller.go similarity index 100% rename from controllers/versioneddataset_controller.go rename to controllers/base/versioneddataset_controller.go diff --git a/controllers/worker_controller.go b/controllers/base/worker_controller.go similarity index 100% rename from controllers/worker_controller.go rename to controllers/base/worker_controller.go diff --git a/controllers/evaluation/rag_controller.go b/controllers/evaluation/rag_controller.go new file mode 100644 index 000000000..65d442f09 --- /dev/null +++ b/controllers/evaluation/rag_controller.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package evaluationarcadia + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + evaluationarcadiav1alpha1 "github.com/kubeagi/arcadia/api/evaluation/v1alpha1" +) + +// RAGReconciler reconciles a RAG object +type RAGReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=evaluation.arcadia.kubeagi.k8s.com.cn,resources=rags,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=evaluation.arcadia.kubeagi.k8s.com.cn,resources=rags/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=evaluation.arcadia.kubeagi.k8s.com.cn,resources=rags/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the RAG object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile +func (r *RAGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + + // TODO(user): your logic here + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *RAGReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&evaluationarcadiav1alpha1.RAG{}). + Complete(r) +} diff --git a/deploy/charts/arcadia/Chart.lock b/deploy/charts/arcadia/Chart.lock index 547c00fed..04ecfe221 100644 --- a/deploy/charts/arcadia/Chart.lock +++ b/deploy/charts/arcadia/Chart.lock @@ -11,5 +11,5 @@ dependencies: - name: chromadb repository: "" version: 0.1.19 -digest: sha256:e1baef276a6d9d9462cca0f5936c9be9017ac640efe82c13518b235c8f52d964 -generated: "2023-12-20T17:42:06.964973+08:00" +digest: sha256:94bd784921bf4082574331977b211f635fa59972579cb8d8296c2b64901ada2a +generated: "2024-01-09T06:09:53.933153139Z" diff --git a/deploy/charts/arcadia/Chart.yaml b/deploy/charts/arcadia/Chart.yaml index dcb8dec8c..4f91ad87b 100644 --- a/deploy/charts/arcadia/Chart.yaml +++ b/deploy/charts/arcadia/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: arcadia description: A Helm chart(KubeBB Component) for KubeAGI Arcadia type: application -version: 0.2.5 +version: 0.2.6 appVersion: "0.1.0" keywords: diff --git a/deploy/charts/arcadia/crds/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml b/deploy/charts/arcadia/crds/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml new file mode 100644 index 000000000..a2380b758 --- /dev/null +++ b/deploy/charts/arcadia/crds/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml @@ -0,0 +1,166 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: rags.evaluation.arcadia.kubeagi.k8s.com.cn +spec: + group: evaluation.arcadia.kubeagi.k8s.com.cn + names: + kind: RAG + listKind: RAGList + plural: rags + singular: rag + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RAG is the Schema for the rags API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RAGSpec defines the desired state of RAG + properties: + application: + description: Application(required) defines the target of this RAG + evaluation + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in + the core API group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being referenced + type: string + required: + - kind + - name + type: object + datasets: + description: Datasets defines the dataset which will be used to generate + test datasets + items: + description: Dataset stands for the files used to generate ragas + test dataset + properties: + files: + description: 'Files retrieved from Source and used in this testdataset + - For file with tag `object_type: QA`, will be used directly + - TODO: For file without special tags, will use `QAGenerationChain` + to generate QAs (Not Supported Yet)' + items: + type: string + type: array + source: + description: From defines the source which provides this QA + Files for test dataset Only `VersionedDataset` allowed + properties: + apiGroup: + description: APIGroup is the group for the resource being + referenced. If APIGroup is not specified, the specified + Kind must be in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being + referenced + type: string + required: + - kind + - name + type: object + type: object + type: array + judge_llm: + description: JudgeLLM(required) defines the judge which is a LLM to + evaluate RAG application against test dataset + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in + the core API group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being referenced + type: string + required: + - kind + - name + type: object + metrics: + description: Metrics that this rag evaluation will do + items: + properties: + kind: + description: Kind of this Metric + type: string + parameters: + description: Parameters in this Metrics + items: + description: Parameter to metrics which is a key-value pair + properties: + key: + type: string + value: + type: string + type: object + type: array + tolerance_threshold: + description: ToleranceThreshbold on this Metric If the evaluation + score is smaller than this tolerance threshold,we treat this + RAG solution as `Bad` + type: integer + type: object + type: array + report: + description: Report defines the evaluation report configurations + type: object + required: + - application + - datasets + - judge_llm + - metrics + type: object + status: + description: RAGStatus defines the observed state of RAG + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/charts/arcadia/templates/rbac.yaml b/deploy/charts/arcadia/templates/rbac.yaml index 3702f57c2..d3df64aa3 100644 --- a/deploy/charts/arcadia/templates/rbac.yaml +++ b/deploy/charts/arcadia/templates/rbac.yaml @@ -477,6 +477,32 @@ rules: - get - patch - update +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags/finalizers + verbs: + - update +- apiGroups: + - evaluation.arcadia.kubeagi.k8s.com.cn + resources: + - rags/status + verbs: + - get + - patch + - update - apiGroups: - networking.k8s.io resources: diff --git a/main.go b/main.go index 68cf6fcc1..88b6c9da1 100644 --- a/main.go +++ b/main.go @@ -40,10 +40,12 @@ import ( apiprompt "github.com/kubeagi/arcadia/api/app-node/prompt/v1alpha1" apiretriever "github.com/kubeagi/arcadia/api/app-node/retriever/v1alpha1" arcadiav1alpha1 "github.com/kubeagi/arcadia/api/base/v1alpha1" - "github.com/kubeagi/arcadia/controllers" + evaluationarcadiav1alpha1 "github.com/kubeagi/arcadia/api/evaluation/v1alpha1" chaincontrollers "github.com/kubeagi/arcadia/controllers/app-node/chain" promptcontrollers "github.com/kubeagi/arcadia/controllers/app-node/prompt" retrievertrollers "github.com/kubeagi/arcadia/controllers/app-node/retriever" + basecontrollers "github.com/kubeagi/arcadia/controllers/base" + evaluationcontrollers "github.com/kubeagi/arcadia/controllers/evaluation" "github.com/kubeagi/arcadia/pkg/config" "github.com/kubeagi/arcadia/pkg/utils" @@ -65,6 +67,7 @@ func init() { utilruntime.Must(apichain.AddToScheme(scheme)) utilruntime.Must(apiprompt.AddToScheme(scheme)) utilruntime.Must(apiretriever.AddToScheme(scheme)) + utilruntime.Must(evaluationarcadiav1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -146,7 +149,7 @@ func main() { panic(err) } - if err = (&controllers.LLMReconciler{ + if err = (&basecontrollers.LLMReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { @@ -154,56 +157,56 @@ func main() { os.Exit(1) } // Deprecated: will remove later, use promptcontrollers.PromptReconciler and construct a application - if err = (&controllers.PromptReconciler{ + if err = (&basecontrollers.PromptReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Prompt") os.Exit(1) } - if err = (&controllers.DatasourceReconciler{ + if err = (&basecontrollers.DatasourceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Datasource") os.Exit(1) } - if err = (&controllers.EmbedderReconciler{ + if err = (&basecontrollers.EmbedderReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Embedder") os.Exit(1) } - if err = (&controllers.DatasetReconciler{ + if err = (&basecontrollers.DatasetReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Dataset") os.Exit(1) } - if err = (&controllers.VersionedDatasetReconciler{ + if err = (&basecontrollers.VersionedDatasetReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "VersionedDataset") os.Exit(1) } - if err = (&controllers.WorkerReconciler{ + if err = (&basecontrollers.WorkerReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Worker") os.Exit(1) } - if err = (&controllers.ModelReconciler{ + if err = (&basecontrollers.ModelReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Model") os.Exit(1) } - if err = (&controllers.KnowledgeBaseReconciler{ + if err = (&basecontrollers.KnowledgeBaseReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), HasHandledSuccessPath: make(map[string]bool, 0), @@ -211,21 +214,21 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "KnowledgeBase") os.Exit(1) } - if err = (&controllers.VectorStoreReconciler{ + if err = (&basecontrollers.VectorStoreReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "VectorStore") os.Exit(1) } - if err = (&controllers.NamespaceReconciler{ + if err = (&basecontrollers.NamespaceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Namespace") os.Exit(1) } - if err = (&controllers.ApplicationReconciler{ + if err = (&basecontrollers.ApplicationReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { @@ -267,6 +270,13 @@ func main() { os.Exit(1) } } + if err = (&evaluationcontrollers.RAGReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RAG") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/pkg/evaluation/evaluation.go b/pkg/evaluation/evaluation.go new file mode 100644 index 000000000..f209b535f --- /dev/null +++ b/pkg/evaluation/evaluation.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package evaluation + +// TO BE DEFINED