Merge pull request #180 from kubeagi/bjwswang

feat: add crd Model and define crd Worker

Dockerfile

@@ -19,13 +19,13 @@ COPY graphql-server/ graphql-server/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o go-bff-server graphql-server/go-server/main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o graphql-server graphql-server/go-server/main.go
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
-COPY --from=builder /workspace/go-bff-server .
+COPY --from=builder /workspace/graphql-server .
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

PROJECT

@@ -72,4 +72,22 @@ resources:
   kind: VersionedDataset
   path: github.com/kubeagi/arcadia/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: kubeagi.k8s.com.cn
+  group: arcadia
+  kind: Worker
+  path: github.com/kubeagi/arcadia/api/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: kubeagi.k8s.com.cn
+  group: arcadia
+  kind: Model
+  path: github.com/kubeagi/arcadia/api/v1alpha1
+  version: v1alpha1
 version: "3"

README.md

@@ -41,12 +41,12 @@ Our design and development in Arcadia design follows operator pattern which exte
   helm install --namespace arcadia --create-namespace arcadia arcadia/arcadia 
 ```
 
-More conveniently,you can use [kubebb](https://github.com/kubebb) to install and upgrade arcadia automatically:
-> Pre-requsities
-> - [kubebb](https://kubebb.github.io/website/docs/quick-start/core_quickstart)
-```shell
-kubectl apply -f ./kubeagi.yaml
-```
+> More conveniently,you can use [kubebb](https://github.com/kubebb) to install and upgrade arcadia automatically:
+  > Pre-requsities
+  > - [kubebb](https://kubebb.github.io/website/docs/quick-start/core_quickstart)
+> ```shell
+> kubectl apply -f ./kubeagi.yaml
+> ```
 
 ## CLI
 

api/v1alpha1/datasource_types.go

@@ -44,6 +44,7 @@ type DatasourceSpec struct {
 // OSS defines info for object storage service as datasource
 type OSS struct {
 	Bucket string `json:"bucket,omitempty"`
+	// Object must end with a slash "/" if it is a directory
 	Object string `json:"object,omitempty"`
 }
 

api/v1alpha1/model.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2023 KubeAGI.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import "fmt"
+
+const (
+	// LabelModelType keeps the spec.type field
+	LabelModelType     = Group + "/model-type"
+	LabelModelFullPath = Group + "/full-path"
+)
+
+type ModelType string
+
+const (
+	ModelTypeEmbedding ModelType = "embedding"
+	ModelTypeLLM       ModelType = "llm"
+	ModelTypeUnknown   ModelType = "unknown"
+)
+
+func (model Model) ModelType() ModelType {
+	if model.Spec.Type == "" {
+		return ModelTypeUnknown
+	}
+	return model.Spec.Type
+}
+
+// FullPath with bucket and object path
+func (model Model) FullPath() string {
+	return fmt.Sprintf("%s/%s", model.Namespace, model.ObjectPath())
+}
+
+// ObjectPath is the path where model stored at in a bucket
+func (model Model) ObjectPath() string {
+	return fmt.Sprintf("model/%s/", model.Name)
+}

api/v1alpha1/model_types.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2023 KubeAGI.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// ModelSpec defines the desired state of Model
+type ModelSpec struct {
+	// Creator defines dataset creator(AUTO-FILLED by webhook)
+	Creator string `json:"creator,omitempty"`
+
+	// DisplayName defines dataset display name
+	DiplayName string `json:"displayName"`
+
+	// Description defines datasource description
+	Description string `json:"description,omitempty"`
+
+	// Type defines what kind of model this is
+	Type ModelType `json:"type,omitempty"`
+
+	// TODO: extend model to utilize third party storage sources
+	// Source *TypedObjectReference `json:"source,omitempty"`
+	// // Path(relative to source) to the model files
+	// Path string `json:"path,omitempty"`
+}
+
+// ModelStatus defines the observed state of Model
+type ModelStatus struct {
+	// ConditionedStatus is the current status
+	ConditionedStatus `json:",inline"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// Model is the Schema for the models API
+type Model struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ModelSpec   `json:"spec,omitempty"`
+	Status ModelStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// ModelList contains a list of Model
+type ModelList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Model `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Model{}, &ModelList{})
+}

api/v1alpha1/worker_types.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2023 KubeAGI.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// WorkerSpec defines the desired state of Worker
+type WorkerSpec struct {
+	// Creator defines dataset creator(AUTO-FILLED by webhook)
+	Creator string `json:"creator,omitempty"`
+
+	// DisplayName defines dataset display name
+	DiplayName string `json:"displayName"`
+}
+
+// WorkerStatus defines the observed state of Worker
+type WorkerStatus struct {
+	// ConditionedStatus is the current status
+	ConditionedStatus `json:",inline"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// Worker is the Schema for the workers API
+type Worker struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   WorkerSpec   `json:"spec,omitempty"`
+	Status WorkerStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// WorkerList contains a list of Worker
+type WorkerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Worker `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Worker{}, &WorkerList{})
+}