Merge remote-tracking branch 'upstream/main'

.github/workflows/example_test.yaml

@@ -78,7 +78,7 @@ jobs:
           mkdir -p ${GITHUB_WORKSPACE}/bin 
           cp /usr/local/bin/kustomize ${GITHUB_WORKSPACE}/bin/kustomize
       - name: Example test
-        run: config/samples/example-test.sh
+        run: tests/example-test.sh
       - name: Upload logs if test fail
         if: failure()
         uses: actions/upload-artifact@v3

.github/workflows/lint_test.yaml

@@ -22,7 +22,7 @@ jobs:
           check-latest: true
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.4.0
+        uses: helm/chart-testing-action@v2.6.0
 
       - name: Run chart-testing (list-changed)
         id: list-changed

Dockerfile

@@ -7,6 +7,7 @@ COPY go.mod go.mod
 COPY go.sum go.sum
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
+RUN go env -w GOPROXY=https://goproxy.cn,direct
 RUN go mod download
 
 # Copy the go source

PROJECT

@@ -1,7 +1,3 @@
-# Code generated by tool. DO NOT EDIT.
-# This file is used to track the info used to scaffold your project
-# and allow the plugins properly work.
-# More info: https://book.kubebuilder.io/reference/project-config.html
 componentConfig: true
 domain: kubeagi.k8s.com.cn
 layout:
@@ -58,4 +54,22 @@ resources:
   kind: Embedders
   path: github.com/kubeagi/arcadia/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: kubeagi.k8s.com.cn
+  group: arcadia
+  kind: Dataset
+  path: github.com/kubeagi/arcadia/api/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: kubeagi.k8s.com.cn
+  group: arcadia
+  kind: VersionedDataset
+  path: github.com/kubeagi/arcadia/api/v1alpha1
+  version: v1alpha1
 version: "3"

README.md

@@ -41,6 +41,13 @@ Our design and development in Arcadia design follows operator pattern which exte
   helm install --namespace arcadia --create-namespace arcadia arcadia/arcadia 
 ```
 
+More conveniently,you can use [kubebb](https://github.com/kubebb) to install and upgrade arcadia automatically:
+> Pre-requsities
+> - [kubebb](https://kubebb.github.io/website/docs/quick-start/core_quickstart)
+```shell
+kubectl apply -f ./kubeagi.yaml
+```
+
 ## CLI
 
 We provide a Command Line Tool `arctl` to interact with `arcadia`. See [here](./arctl/README.md) for more details.

api/v1alpha1/condition.go

@@ -51,6 +51,32 @@ const (
 	ReasonReconcilePaused  ConditionReason = "ReconcilePaused"
 )
 
+// Some Data related Condition Types
+const (
+	// Dataset have 3 phases: load -> process -> publish
+	// TypeLoaded resources are believed to be loaded
+	TypeLoaded ConditionType = "Loaded"
+	// TypeProcessed resources are believed to be processed
+	TypeProcessed ConditionType = "Processed"
+	// TypePublished resources are believed to be published
+	TypePublished ConditionType = "Published"
+)
+
+// Some Dataset related Condition reasons
+const (
+	// Load data
+	ReasonDataLoading     ConditionReason = "DataLoading"
+	ReasonDataLoadError   ConditionReason = "DataLoadError"
+	ReasonDataLoadSuccess ConditionReason = "DataLoadSuccess"
+	// Process data
+	ReasonDataProcessing     ConditionReason = "DataProcessing"
+	ReasonDataProcessError   ConditionReason = "DataProcessError"
+	ReasonDataProcessSuccess ConditionReason = "DataProcessSuccess"
+	// Publish dataset
+	ReasonDatasetUnpublished ConditionReason = "DatasetUnpublished"
+	ReasonDatasetPublished   ConditionReason = "DatasetPublished"
+)
+
 // A Condition that may apply to a resource.
 type Condition struct {
 	// Type of this condition. At most one of each condition type may apply to

api/v1alpha1/dataset.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2023 KubeAGI.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+type DatasetContentType string
+
+const (
+	DatasetContentTypeText  DatasetContentType = "text"
+	DatasetContentTypeImage DatasetContentType = "image"
+	DatasetContentTypeVoice DatasetContentType = "voice"
+	DatasetContentTypeVideo DatasetContentType = "video"
+)
+
+var (
+	// LabelDatasetScene defines the content type of this dataset
+	LabelDatasetContentType = Group + "/content-type"
+	// LabelDatasetBestCase defines the best case to use this dataset
+	LabelDatasetBestCase = Group + "/best-case"
+)

api/v1alpha1/dataset_types.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2023 KubeAGI.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// DatasetSpec defines the desired state of Dataset
+type DatasetSpec struct {
+	// Creator defines dataset creator(AUTO-FILLED by webhook)
+	Creator string `json:"creator,omitempty"`
+
+	// DisplayName defines dataset display name
+	DiplayName string `json:"displayName"`
+
+	// ContentType defines dataset
+	ContentType string `json:"contentType"`
+
+	// bestCase defines the best case to use this dataset
+	BestCase string `json:"bestCase,omitempty"`
+}
+
+// DatasetStatus defines the observed state of Dataset
+type DatasetStatus struct {
+	// ConditionedStatus is the current status
+	ConditionedStatus `json:",inline"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="display-name",type=string,JSONPath=`.spec.displayName`
+//+kubebuilder:printcolumn:name="type",type=string,JSONPath=`.spec.contentType`
+
+// Dataset is the Schema for the datasets API
+type Dataset struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   DatasetSpec   `json:"spec,omitempty"`
+	Status DatasetStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// DatasetList contains a list of Dataset
+type DatasetList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Dataset `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Dataset{}, &DatasetList{})
+}

api/v1alpha1/datasource.go

@@ -23,11 +23,19 @@ const (
 type DatasourceType string
 
 const (
+	DatasourceTypeLocal   DatasourceType = "local"
 	DatasourceTypeOSS     DatasourceType = "oss"
 	DatasourceTypeUnknown DatasourceType = "unknown"
 )
 
 func (ds DatasourceSpec) Type() DatasourceType {
+	// if endpoint is nil,this must be a `Local` datasource which utilize `SystemDatasource` by default to host its data
+	if ds.Enpoint == nil {
+		return DatasourceTypeLocal
+	}
+
+	// For third party datasources
+
 	// Object storage service
 	if ds.OSS != nil {
 		return DatasourceTypeOSS

api/v1alpha1/datasource_types.go

@@ -29,13 +29,13 @@ type DatasourceSpec struct {
 	Creator string `json:"creator,omitempty"`
 
 	// DisplayName defines datasource display name
-	DiplayName string `json:"displayName,omitempty"`
+	DiplayName string `json:"displayName"`
 
 	// Description defines datasource description
 	Description string `json:"description,omitempty"`
 
 	// Enpoint defines connection info
-	Enpoint Endpoint `json:"endpoint"`
+	Enpoint *Endpoint `json:"endpoint,omitempty"`
 
 	// OSS defines info for object storage service
 	OSS *OSS `json:"oss,omitempty"`
@@ -56,6 +56,8 @@ type DatasourceStatus struct {
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
 //+kubebuilder:resource:scope=Namespaced
+//+kubebuilder:printcolumn:name="display-name",type=string,JSONPath=`.spec.displayName`
+//+kubebuilder:printcolumn:name="type",type=string,JSONPath=`.metadata.labels.arcadia\.kubeagi\.k8s\.com\.cn/datasource-type`
 
 // Datasource is the Schema for the datasources API
 type Datasource struct {

api/v1alpha1/versioneddataset.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2023 KubeAGI.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+var (
+	LabelVersionedDatasetVersion = Group + "/version"
+)

api/v1alpha1/versioneddataset_types.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2023 KubeAGI.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+type FileGroup struct {
+	// From defines the datasource which provides this `File`
+	Datasource *TypedObjectReference `json:"datasource"`
+	// Paths defines the detail paths to get objects from above datasource
+	Paths []string `json:"paths"`
+}
+
+// VersionedDatasetSpec defines the desired state of VersionedDataset
+type VersionedDatasetSpec struct {
+	// Dataset which this `VersionedDataset` belongs to
+	Dataset *TypedObjectReference `json:"dataset"`
+
+	// Version
+	Version string `json:"version"`
+
+	// FileGroups included in this `VersionedDataset`
+	// Grouped by Datasource
+	FileGroups []FileGroup `json:"fileGroups,omitempty"`
+}
+
+// VersionedDatasetStatus defines the observed state of VersionedDataset
+type VersionedDatasetStatus struct {
+	// ConditionedStatus is the current status
+	ConditionedStatus `json:",inline"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="dataset",type=string,JSONPath=`.spec.dataset.name`
+//+kubebuilder:printcolumn:name="version",type=string,JSONPath=`.spec.version`
+
+// VersionedDataset is the Schema for the versioneddatasets API
+type VersionedDataset struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   VersionedDatasetSpec   `json:"spec,omitempty"`
+	Status VersionedDatasetStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// VersionedDatasetList contains a list of VersionedDataset
+type VersionedDatasetList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []VersionedDataset `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&VersionedDataset{}, &VersionedDatasetList{})
+}