diff --git a/api/evaluation/v1alpha1/common.go b/api/evaluation/v1alpha1/common.go index 1c90d833c..fd6da9669 100644 --- a/api/evaluation/v1alpha1/common.go +++ b/api/evaluation/v1alpha1/common.go @@ -77,3 +77,28 @@ type Parameter struct { type Report struct { // TODO } + +// Define RAG common structure and variables +const ( + EvaluationJobLabels = Group + "/rag" + EvaluationApplicationLabel = Group + "/application" +) + +func RagStatusChanged(a, b RAGStatus) bool { + if !a.CompletionTime.Equal(b.CompletionTime) { + return true + } + if a.Phase != b.Phase { + return true + } + ac, bc := a.Conditions, b.Conditions + la, lb := len(ac), len(bc) + if la != lb { + return true + } + if la == 0 { + return false + } + return ac[0].Type != bc[0].Type || ac[0].Status != bc[0].Status || + ac[0].Reason != bc[0].Reason || ac[0].Message != bc[0].Message +} diff --git a/api/evaluation/v1alpha1/condition.go b/api/evaluation/v1alpha1/condition.go new file mode 100644 index 000000000..1c92c1fd8 --- /dev/null +++ b/api/evaluation/v1alpha1/condition.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package v1alpha1 + +type RAGPhase string + +const ( + InitPvcPhase RAGPhase = "init" + DownloadFilesPhase RAGPhase = "download" + GenerateTestFilesPhase RAGPhase = "generate" + JudgeLLMPhase RAGPhase = "judge" + UploadFilesPhase RAGPhase = "upload" + CompletePhase RAGPhase = "complete" +) diff --git a/api/evaluation/v1alpha1/groupversion_info.go b/api/evaluation/v1alpha1/groupversion_info.go index d87fb7449..8109d526b 100644 --- a/api/evaluation/v1alpha1/groupversion_info.go +++ b/api/evaluation/v1alpha1/groupversion_info.go @@ -24,9 +24,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/scheme" ) +const ( + Group = "evaluation.arcadia.kubeagi.k8s.com.cn" + Version = "v1alpha1" +) + var ( // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "evaluation.arcadia.kubeagi.k8s.com.cn", Version: "v1alpha1"} + GroupVersion = schema.GroupVersion{Group: Group, Version: Version} // SchemeBuilder is used to add go types to the GroupVersionKind scheme SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} diff --git a/api/evaluation/v1alpha1/rag_types.go b/api/evaluation/v1alpha1/rag_types.go index 26ff50968..d3188f7b4 100644 --- a/api/evaluation/v1alpha1/rag_types.go +++ b/api/evaluation/v1alpha1/rag_types.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha1 import ( + v1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" basev1alpha1 "github.com/kubeagi/arcadia/api/base/v1alpha1" @@ -35,6 +37,9 @@ type Dataset struct { // RAGSpec defines the desired state of RAG type RAGSpec struct { + // CommonSpec + basev1alpha1.CommonSpec `json:",inline"` + // Application(required) defines the target of this RAG evaluation Application *basev1alpha1.TypedObjectReference `json:"application"` @@ -49,12 +54,30 @@ type RAGSpec struct { // Report defines the evaluation report configurations Report Report `json:"report,omitempty"` + + // Storage storage must be provided and data needs to be saved throughout the evaluation phase. + Storage *corev1.PersistentVolumeClaimSpec `json:"storage"` + + // ServiceAccountName define the user when the job is run + // +kubebuilder:default=default + ServiceAccountName string `json:"serviceAccountName,omitempty"` + + // Suspend suspension of the evaluation process + // +kubebuilder:default=false + Suspend bool `json:"suspend,omitempty"` } // RAGStatus defines the observed state of RAG type RAGStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file + // CompletionTime Evaluation completion time + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // Phase evaluation current stage, + // init,download,generate,judge,upload,complete + Phase RAGPhase `json:"phase,omitempty"` + + // Conditions show the status of the job in the current stage + Conditions []v1.JobCondition `json:"conditions,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/evaluation/v1alpha1/zz_generated.deepcopy.go b/api/evaluation/v1alpha1/zz_generated.deepcopy.go index d3a2bdae8..ceb3bbb8e 100644 --- a/api/evaluation/v1alpha1/zz_generated.deepcopy.go +++ b/api/evaluation/v1alpha1/zz_generated.deepcopy.go @@ -23,6 +23,8 @@ package v1alpha1 import ( basev1alpha1 "github.com/kubeagi/arcadia/api/base/v1alpha1" + batchv1 "k8s.io/api/batch/v1" + "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -92,7 +94,7 @@ func (in *RAG) DeepCopyInto(out *RAG) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAG. @@ -148,6 +150,7 @@ func (in *RAGList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RAGSpec) DeepCopyInto(out *RAGSpec) { *out = *in + out.CommonSpec = in.CommonSpec if in.Application != nil { in, out := &in.Application, &out.Application *out = new(basev1alpha1.TypedObjectReference) @@ -173,6 +176,11 @@ func (in *RAGSpec) DeepCopyInto(out *RAGSpec) { } } out.Report = in.Report + if in.Storage != nil { + in, out := &in.Storage, &out.Storage + *out = new(v1.PersistentVolumeClaimSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAGSpec. @@ -188,6 +196,17 @@ func (in *RAGSpec) DeepCopy() *RAGSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RAGStatus) DeepCopyInto(out *RAGStatus) { *out = *in + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]batchv1.JobCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAGStatus. diff --git a/config/crd/bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml b/config/crd/bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml index a2380b758..dcd9a9f72 100644 --- a/config/crd/bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml +++ b/config/crd/bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml @@ -58,6 +58,9 @@ spec: - kind - name type: object + creator: + description: Creator defines datasource creator (AUTO-FILLED by webhook) + type: string datasets: description: Datasets defines the dataset which will be used to generate test datasets @@ -99,6 +102,12 @@ spec: type: object type: object type: array + description: + description: Description defines datasource description + type: string + displayName: + description: DisplayName defines datasource display name + type: string judge_llm: description: JudgeLLM(required) defines the judge which is a LLM to evaluate RAG application against test dataset @@ -150,14 +159,229 @@ spec: report: description: Report defines the evaluation report configurations type: object + serviceAccountName: + default: default + description: ServiceAccountName define the user when the job is run + type: string + storage: + description: Storage storage must be provided and data needs to be + saved throughout the evaluation phase. + properties: + accessModes: + description: 'accessModes contains the desired access modes the + volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified + data source. If the AnyVolumeDataSource feature gate is enabled, + this field will always have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the resource being + referenced. If APIGroup is not specified, the specified + Kind must be in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from which to + populate the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. When this field + is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator + or dynamic provisioner. This field will replace the functionality + of the DataSource field and as such if both fields are non-empty, + they must have the same value. For backwards compatibility, + both fields (DataSource and DataSourceRef) will be set to the + same value automatically if one of them is empty and the other + is non-empty. There are two important differences between DataSource + and DataSourceRef: * While DataSource only allows two specific + types of objects, DataSourceRef allows any non-core object, + as well as PersistentVolumeClaim objects. * While DataSource + ignores disallowed values (dropping them), DataSourceRef preserves + all values, and generates an error if a disallowed value is + specified. (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource being + referenced. If APIGroup is not specified, the specified + Kind must be in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: 'resources represents the minimum resources the volume + should have. If RecoverVolumeExpansionFailure feature is enabled + users are allowed to specify resource requirements that are + lower than previous value but must still be higher than capacity + recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is required + by the claim. Value of Filesystem is implied when not included + in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume + backing this claim. + type: string + type: object + suspend: + default: false + description: Suspend suspension of the evaluation process + type: boolean required: - application - datasets - judge_llm - metrics + - storage type: object status: description: RAGStatus defines the observed state of RAG + properties: + completionTime: + description: CompletionTime Evaluation completion time + format: date-time + type: string + conditions: + description: Conditions show the status of the job in the current + stage + items: + description: JobCondition describes current state of a job. + properties: + lastProbeTime: + description: Last time the condition was checked. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transit from one status + to another. + format: date-time + type: string + message: + description: Human readable message indicating details about + last transition. + type: string + reason: + description: (brief) reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of job condition, Complete or Failed. + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase evaluation current stage, init,download,generate,judge,upload,complete + type: string type: object type: object served: true diff --git a/deploy/charts/arcadia/Chart.yaml b/deploy/charts/arcadia/Chart.yaml index e77a98e6a..e3a07eb38 100644 --- a/deploy/charts/arcadia/Chart.yaml +++ b/deploy/charts/arcadia/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: arcadia description: A Helm chart(KubeBB Component) for KubeAGI Arcadia type: application -version: 0.2.18 +version: 0.2.19 appVersion: "0.1.0" keywords: diff --git a/deploy/charts/arcadia/crds/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml b/deploy/charts/arcadia/crds/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml index a2380b758..dcd9a9f72 100644 --- a/deploy/charts/arcadia/crds/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml +++ b/deploy/charts/arcadia/crds/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml @@ -58,6 +58,9 @@ spec: - kind - name type: object + creator: + description: Creator defines datasource creator (AUTO-FILLED by webhook) + type: string datasets: description: Datasets defines the dataset which will be used to generate test datasets @@ -99,6 +102,12 @@ spec: type: object type: object type: array + description: + description: Description defines datasource description + type: string + displayName: + description: DisplayName defines datasource display name + type: string judge_llm: description: JudgeLLM(required) defines the judge which is a LLM to evaluate RAG application against test dataset @@ -150,14 +159,229 @@ spec: report: description: Report defines the evaluation report configurations type: object + serviceAccountName: + default: default + description: ServiceAccountName define the user when the job is run + type: string + storage: + description: Storage storage must be provided and data needs to be + saved throughout the evaluation phase. + properties: + accessModes: + description: 'accessModes contains the desired access modes the + volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified + data source. If the AnyVolumeDataSource feature gate is enabled, + this field will always have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the resource being + referenced. If APIGroup is not specified, the specified + Kind must be in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from which to + populate the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. When this field + is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator + or dynamic provisioner. This field will replace the functionality + of the DataSource field and as such if both fields are non-empty, + they must have the same value. For backwards compatibility, + both fields (DataSource and DataSourceRef) will be set to the + same value automatically if one of them is empty and the other + is non-empty. There are two important differences between DataSource + and DataSourceRef: * While DataSource only allows two specific + types of objects, DataSourceRef allows any non-core object, + as well as PersistentVolumeClaim objects. * While DataSource + ignores disallowed values (dropping them), DataSourceRef preserves + all values, and generates an error if a disallowed value is + specified. (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource being + referenced. If APIGroup is not specified, the specified + Kind must be in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: 'resources represents the minimum resources the volume + should have. If RecoverVolumeExpansionFailure feature is enabled + users are allowed to specify resource requirements that are + lower than previous value but must still be higher than capacity + recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is required + by the claim. Value of Filesystem is implied when not included + in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume + backing this claim. + type: string + type: object + suspend: + default: false + description: Suspend suspension of the evaluation process + type: boolean required: - application - datasets - judge_llm - metrics + - storage type: object status: description: RAGStatus defines the observed state of RAG + properties: + completionTime: + description: CompletionTime Evaluation completion time + format: date-time + type: string + conditions: + description: Conditions show the status of the job in the current + stage + items: + description: JobCondition describes current state of a job. + properties: + lastProbeTime: + description: Last time the condition was checked. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transit from one status + to another. + format: date-time + type: string + message: + description: Human readable message indicating details about + last transition. + type: string + reason: + description: (brief) reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of job condition, Complete or Failed. + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase evaluation current stage, init,download,generate,judge,upload,complete + type: string type: object type: object served: true