feat: add RDMA definition to datasource

api/base/v1alpha1/datasource.go

@@ -24,6 +24,7 @@ type DatasourceType string
 
 const (
 	DatasourceTypeOSS     DatasourceType = "oss"
+	DatasourceRDMA        DatasourceType = "RDMA"
 	DatasourceTypeUnknown DatasourceType = "unknown"
 )
 
@@ -32,6 +33,9 @@ func (ds DatasourceSpec) Type() DatasourceType {
 	if ds.OSS != nil {
 		return DatasourceTypeOSS
 	}
+	if ds.RDMA != nil {
+		return DatasourceRDMA
+	}
 
 	return DatasourceTypeUnknown
 }

api/base/v1alpha1/datasource_types.go

@@ -32,6 +32,19 @@ type DatasourceSpec struct {
 
 	// OSS defines info for object storage service
 	OSS *OSS `json:"oss,omitempty"`
+
+	// RDMA configure RDMA pulls the model file directly from the remote service to the host node.
+	RDMA *RDMA `json:"rdma,omitempty"`
+}
+
+type RDMA struct {
+	// Path on a model storage server, the usual storage path is /path/ns/mode-name, and the path field is /path/, which must end in /.
+	// example: /opt/kubeagi/, /opt/, /
+	// +kubebuilder:validation:Pattern=(^\/$)|(^\/[a-zA-Z0-9\_.@-]+(\/[a-zA-Z0-9\_.@-]+)*\/$)
+	Path string `json:"path"`
+
+	// Endpoint trans-core's service address
+	Endpoint string `json:"host"`
 }
 
 // OSS defines info for object storage service as datasource

api/base/v1alpha1/model_types.go

@@ -31,10 +31,8 @@ type ModelSpec struct {
 	// Comma separated field which can be wrapped by {llm,embedding}
 	Types string `json:"types,omitempty"`
 
-	// TODO: extend model to utilize third party storage sources
-	// Source *TypedObjectReference `json:"source,omitempty"`
-	// // Path(relative to source) to the model files
-	// Path string `json:"path,omitempty"`
+	// Source define the source of the model file
+	Source *TypedObjectReference `json:"source,omitempty"`
 }
 
 // ModelStatus defines the observed state of Model

config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml

@@ -99,6 +99,23 @@ spec:
                     description: Object must end with a slash "/" if it is a directory
                     type: string
                 type: object
+              rdma:
+                description: RDMA configure RDMA pulls the model file directly from
+                  the remote service to the host node.
+                properties:
+                  host:
+                    description: Endpoint trans-core's service address
+                    type: string
+                  path:
+                    description: 'Path on a model storage server, the usual storage
+                      path is /path/ns/mode-name, and the path field is /path/, which
+                      must end in /. example: /opt/kubeagi/, /opt/, /'
+                    pattern: (^\/$)|(^\/[a-zA-Z0-9\_.@-]+(\/[a-zA-Z0-9\_.@-]+)*\/$)
+                    type: string
+                required:
+                - host
+                - path
+                type: object
             required:
             - endpoint
             type: object

config/crd/bases/arcadia.kubeagi.k8s.com.cn_models.yaml

@@ -44,6 +44,28 @@ spec:
               displayName:
                 description: DisplayName defines datasource display name
                 type: string
+              source:
+                description: Source define the source of the model file
+                properties:
+                  apiGroup:
+                    description: APIGroup is the group for the resource being referenced.
+                      If APIGroup is not specified, the specified Kind must be in
+                      the core API group. For any other third-party types, APIGroup
+                      is required.
+                    type: string
+                  kind:
+                    description: Kind is the type of resource being referenced
+                    type: string
+                  name:
+                    description: Name is the name of resource being referenced
+                    type: string
+                  namespace:
+                    description: Namespace is the namespace of resource being referenced
+                    type: string
+                required:
+                - kind
+                - name
+                type: object
               types:
                 description: Type defines what kind of model this is Comma separated
                   field which can be wrapped by {llm,embedding}

controllers/model_controller.go

@@ -26,6 +26,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -174,34 +175,40 @@ func (r *ModelReconciler) Initialize(ctx context.Context, logger logr.Logger, in
 // CheckModel to update status
 func (r *ModelReconciler) CheckModel(ctx context.Context, logger logr.Logger, instance *arcadiav1alpha1.Model) error {
 	logger.V(5).Info("check model")
-	var err error
 
-	var ds datasource.Datasource
-	var info any
-
-	system, err := config.GetSystemDatasource(ctx, r.Client, nil)
-	if err != nil {
-		return r.UpdateStatus(ctx, instance, err)
-	}
-	endpoint := system.Spec.Endpoint.DeepCopy()
-	if endpoint != nil && endpoint.AuthSecret != nil {
-		endpoint.AuthSecret.WithNameSpace(system.Namespace)
-	}
-	ds, err = datasource.NewLocal(ctx, r.Client, endpoint)
-	if err != nil {
-		return r.UpdateStatus(ctx, instance, err)
-	}
-	// oss info:
-	// - bucket: same as the instance namespace
-	// - object: path joined with "model/{instance.name}"
-	info = &arcadiav1alpha1.OSS{
-		Bucket: instance.Namespace,
-		Object: instance.ObjectPath(),
-	}
+	var (
+		ds   datasource.Datasource
+		info any
+	)
+
+	// If source is empty, it means that the data is still sourced from the internal minio and a state check is required,
+	// otherwise we consider the model file for the trans-core service to be ready.
+	if instance.Spec.Source == nil {
+		klog.V(5).Infof("model %s source is empty, check minio status.", instance.Name)
+		system, err := config.GetSystemDatasource(ctx, r.Client, nil)
+		if err != nil {
+			return r.UpdateStatus(ctx, instance, err)
+		}
+		endpoint := system.Spec.Endpoint.DeepCopy()
+		if endpoint != nil && endpoint.AuthSecret != nil {
+			endpoint.AuthSecret.WithNameSpace(system.Namespace)
+		}
+		ds, err = datasource.NewLocal(ctx, r.Client, endpoint)
+		if err != nil {
+			return r.UpdateStatus(ctx, instance, err)
+		}
+		// oss info:
+		// - bucket: same as the instance namespace
+		// - object: path joined with "model/{instance.name}"
+		info = &arcadiav1alpha1.OSS{
+			Bucket: instance.Namespace,
+			Object: instance.ObjectPath(),
+		}
 
-	// check datasource against info
-	if err := ds.Stat(ctx, info); err != nil {
-		return r.UpdateStatus(ctx, instance, err)
+		// check datasource against info
+		if err := ds.Stat(ctx, info); err != nil {
+			return r.UpdateStatus(ctx, instance, err)
+		}
 	}
 
 	// update status

deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml

@@ -99,6 +99,23 @@ spec:
                     description: Object must end with a slash "/" if it is a directory
                     type: string
                 type: object
+              rdma:
+                description: RDMA configure RDMA pulls the model file directly from
+                  the remote service to the host node.
+                properties:
+                  host:
+                    description: Endpoint trans-core's service address
+                    type: string
+                  path:
+                    description: 'Path on a model storage server, the usual storage
+                      path is /path/ns/mode-name, and the path field is /path/, which
+                      must end in /. example: /opt/kubeagi/, /opt/, /'
+                    pattern: (^\/$)|(^\/[a-zA-Z0-9\_.@-]+(\/[a-zA-Z0-9\_.@-]+)*\/$)
+                    type: string
+                required:
+                - host
+                - path
+                type: object
             required:
             - endpoint
             type: object

deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_models.yaml

@@ -44,6 +44,28 @@ spec:
               displayName:
                 description: DisplayName defines datasource display name
                 type: string
+              source:
+                description: Source define the source of the model file
+                properties:
+                  apiGroup:
+                    description: APIGroup is the group for the resource being referenced.
+                      If APIGroup is not specified, the specified Kind must be in
+                      the core API group. For any other third-party types, APIGroup
+                      is required.
+                    type: string
+                  kind:
+                    description: Kind is the type of resource being referenced
+                    type: string
+                  name:
+                    description: Name is the name of resource being referenced
+                    type: string
+                  namespace:
+                    description: Namespace is the namespace of resource being referenced
+                    type: string
+                required:
+                - kind
+                - name
+                type: object
               types:
                 description: Type defines what kind of model this is Comma separated
                   field which can be wrapped by {llm,embedding}