From 7f5b2376745f7ae74452e0f1722ad4a8ba837646 Mon Sep 17 00:00:00 2001 From: bjwswang Date: Mon, 22 Jan 2024 11:24:17 +0000 Subject: [PATCH] feat: add new datasource Web Signed-off-by: bjwswang --- api/base/v1alpha1/datasource.go | 3 + api/base/v1alpha1/datasource_types.go | 8 +++ api/base/v1alpha1/zz_generated.deepcopy.go | 20 +++++++ ...rcadia.kubeagi.k8s.com.cn_datasources.yaml | 3 + controllers/base/datasource_controller.go | 6 ++ deploy/charts/arcadia/Chart.yaml | 2 +- ...rcadia.kubeagi.k8s.com.cn_datasources.yaml | 3 + pkg/datasource/web.go | 59 +++++++++++++++++++ 8 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 pkg/datasource/web.go diff --git a/api/base/v1alpha1/datasource.go b/api/base/v1alpha1/datasource.go index 344301fbf..ded3e0380 100644 --- a/api/base/v1alpha1/datasource.go +++ b/api/base/v1alpha1/datasource.go @@ -31,6 +31,7 @@ const ( DatasourceTypeOSS DatasourceType = "oss" DatasourceTypeRDMA DatasourceType = "RDMA" DatasourceTypePostgreSQL DatasourceType = "postgresql" + DatasourceTypeWeb DatasourceType = "web" DatasourceTypeUnknown DatasourceType = "unknown" ) @@ -42,6 +43,8 @@ func (ds DatasourceSpec) Type() DatasourceType { return DatasourceTypeRDMA case ds.PostgreSQL != nil: return DatasourceTypePostgreSQL + case ds.Web != nil: + return DatasourceTypeWeb default: return DatasourceTypeUnknown } diff --git a/api/base/v1alpha1/datasource_types.go b/api/base/v1alpha1/datasource_types.go index 5cf4117fe..9400e870e 100644 --- a/api/base/v1alpha1/datasource_types.go +++ b/api/base/v1alpha1/datasource_types.go @@ -35,6 +35,9 @@ type DatasourceSpec struct { // PostgreSQL defines info for PostgreSQL PostgreSQL *PostgreSQL `json:"postgresql,omitempty"` + + // Web defines info for web pages + Web *Web `json:"web,omitempty"` } type RDMA struct { @@ -85,6 +88,11 @@ const ( PGSSLPASSWORD = "PGSSLPASSWORD" ) +// Web defines a +type Web struct { + // TODO: define more fields later +} + // DatasourceStatus defines the observed state of Datasource type DatasourceStatus struct { // ConditionedStatus is the current status diff --git a/api/base/v1alpha1/zz_generated.deepcopy.go b/api/base/v1alpha1/zz_generated.deepcopy.go index 39435841c..4d238a13a 100644 --- a/api/base/v1alpha1/zz_generated.deepcopy.go +++ b/api/base/v1alpha1/zz_generated.deepcopy.go @@ -365,6 +365,11 @@ func (in *DatasourceSpec) DeepCopyInto(out *DatasourceSpec) { *out = new(PostgreSQL) **out = **in } + if in.Web != nil { + in, out := &in.Web, &out.Web + *out = new(Web) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatasourceSpec. @@ -1408,6 +1413,21 @@ func (in *VersionedDatasetStatus) DeepCopy() *VersionedDatasetStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Web) DeepCopyInto(out *Web) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Web. +func (in *Web) DeepCopy() *Web { + if in == nil { + return nil + } + out := new(Web) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WebConfig) DeepCopyInto(out *WebConfig) { *out = *in diff --git a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml index 9a3e464ce..19653e066 100644 --- a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml +++ b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml @@ -147,6 +147,9 @@ spec: required: - path type: object + web: + description: Web defines info for web pages + type: object required: - endpoint type: object diff --git a/controllers/base/datasource_controller.go b/controllers/base/datasource_controller.go index 59d8dabdb..e13db9f3d 100644 --- a/controllers/base/datasource_controller.go +++ b/controllers/base/datasource_controller.go @@ -174,6 +174,12 @@ func (r *DatasourceReconciler) Checkdatasource(ctx context.Context, logger logr. if err != nil { return r.UpdateStatus(ctx, instance, err) } + case arcadiav1alpha1.DatasourceTypeWeb: + ds, err = datasource.NewWeb(ctx) + if err != nil { + return r.UpdateStatus(ctx, instance, err) + } + info = instance.Spec.Web.DeepCopy() default: ds, err = datasource.NewUnknown(ctx, r.Client) if err != nil { diff --git a/deploy/charts/arcadia/Chart.yaml b/deploy/charts/arcadia/Chart.yaml index 25a462ae7..e77a98e6a 100644 --- a/deploy/charts/arcadia/Chart.yaml +++ b/deploy/charts/arcadia/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: arcadia description: A Helm chart(KubeBB Component) for KubeAGI Arcadia type: application -version: 0.2.17 +version: 0.2.18 appVersion: "0.1.0" keywords: diff --git a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml index 9a3e464ce..19653e066 100644 --- a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml +++ b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml @@ -147,6 +147,9 @@ spec: required: - path type: object + web: + description: Web defines info for web pages + type: object required: - endpoint type: object diff --git a/pkg/datasource/web.go b/pkg/datasource/web.go new file mode 100644 index 000000000..c2c42bec0 --- /dev/null +++ b/pkg/datasource/web.go @@ -0,0 +1,59 @@ +/* +Copyright 2024 KubeAGI. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "context" + "io" +) + +var _ Datasource = (*Web)(nil) + +type Web struct { +} + +func NewWeb(ctx context.Context) (*Web, error) { + return &Web{}, nil +} + +func (w *Web) Stat(ctx context.Context, info any) error { + return nil +} + +func (w *Web) Remove(ctx context.Context, info any) error { + // TODO implement me + panic("implement me") +} + +func (w *Web) ReadFile(ctx context.Context, info any) (io.ReadCloser, error) { + // TODO implement me + panic("implement me") +} + +func (w *Web) StatFile(ctx context.Context, info any) (any, error) { + // TODO implement me + panic("implement me") +} + +func (w *Web) GetTags(ctx context.Context, info any) (map[string]string, error) { + // TODO implement me + panic("implement me") +} + +func (w *Web) ListObjects(ctx context.Context, source string, info any) (any, error) { + // TODO implement me + panic("implement me") +}