From 0f4433d3bc8cd05cc017c93a0d71f06ff0803f1b Mon Sep 17 00:00:00 2001 From: Abirdcfly Date: Mon, 25 Dec 2023 22:36:11 +0800 Subject: [PATCH] feat: add spell test and fix existed misspellings Signed-off-by: Abirdcfly --- .github/.codespellignore | 1 + .github/workflows/codespell.yaml | 19 +++++++++++++++++++ Makefile | 6 ++++++ README.md | 11 ++++++----- api/base/v1alpha1/common.go | 8 ++++---- api/base/v1alpha1/datasource_types.go | 4 ++-- api/base/v1alpha1/embedder.go | 4 ++-- api/base/v1alpha1/llm.go | 4 ++-- api/base/v1alpha1/model_types.go | 2 +- api/base/v1alpha1/vectorstore.go | 2 +- api/base/v1alpha1/vectorstore_types.go | 4 ++-- api/base/v1alpha1/zz_generated.deepcopy.go | 10 +++++----- .../examples/upload-download-file/README.md | 2 +- .../examples/upload-download-file/main.go | 2 +- apiserver/pkg/application/application.go | 12 ++++++------ apiserver/pkg/common/common.go | 2 +- apiserver/pkg/common/endpoint.go | 2 +- apiserver/pkg/datasource/datasource.go | 14 +++++++------- apiserver/pkg/embedder/embedder.go | 6 +++--- apiserver/pkg/knowledgebase/knowledgebase.go | 4 ++-- apiserver/pkg/llm/llm.go | 6 +++--- apiserver/pkg/model/model.go | 2 +- apiserver/pkg/modelservice/modelservice.go | 4 ++-- apiserver/pkg/oidc/auth.go | 2 +- .../pkg/versioneddataset/versioned_dataset.go | 2 +- apiserver/service/minio_server.go | 14 +++++++------- ...rcadia.kubeagi.k8s.com.cn_datasources.yaml | 2 +- .../arcadia.kubeagi.k8s.com.cn_embedders.yaml | 2 +- .../arcadia.kubeagi.k8s.com.cn_llms.yaml | 2 +- .../arcadia.kubeagi.k8s.com.cn_models.yaml | 2 +- ...cadia.kubeagi.k8s.com.cn_vectorstores.yaml | 2 +- controllers/datasource_controller.go | 4 ++-- controllers/embedder_controller.go | 4 ++-- controllers/knowledgebase_controller.go | 8 ++++---- controllers/llm_controller.go | 2 +- controllers/model_controller.go | 8 ++++---- controllers/namespace_controller.go | 2 +- controllers/prompt_controller.go | 2 +- controllers/vectorstore_controller.go | 2 +- controllers/versioneddataset_controller.go | 4 ++-- .../data_process_detail_db_operate.py | 4 ++-- .../file_handle/common_handle.py | 4 ++-- .../service/data_process_service.py | 2 +- deploy/charts/arcadia/Chart.yaml | 2 +- ...rcadia.kubeagi.k8s.com.cn_datasources.yaml | 2 +- .../arcadia.kubeagi.k8s.com.cn_embedders.yaml | 2 +- .../crds/arcadia.kubeagi.k8s.com.cn_llms.yaml | 2 +- .../arcadia.kubeagi.k8s.com.cn_models.yaml | 2 +- ...cadia.kubeagi.k8s.com.cn_vectorstores.yaml | 2 +- examples/beijing_gjj_bot/pkg/embedding.go | 4 ++-- pkg/application/chain/llmchain.go | 2 +- pkg/application/chain/retrievalqachain.go | 2 +- pkg/application/llm/llm.go | 2 +- .../retriever/knowledgebaseretriever.go | 4 ++-- pkg/arctl/dataset.go | 4 ++-- pkg/arctl/datasource.go | 2 +- pkg/cache/{memeory.go => memory.go} | 0 pkg/cache/{memeory_test.go => memory_test.go} | 0 pkg/datasource/datasource.go | 2 +- pkg/datasource/oss.go | 6 +++--- pkg/langchainwrap/llm.go | 2 +- pkg/llms/dashscope/api.go | 2 +- pkg/scheduler/scheduler.go | 6 +++--- pkg/worker/devices.go | 2 +- pkg/worker/runner.go | 6 +++--- pkg/worker/worker.go | 4 ++-- tests/deploy-values.yaml | 4 ++-- 67 files changed, 149 insertions(+), 122 deletions(-) create mode 100644 .github/.codespellignore create mode 100644 .github/workflows/codespell.yaml rename pkg/cache/{memeory.go => memory.go} (100%) rename pkg/cache/{memeory_test.go => memory_test.go} (100%) diff --git a/.github/.codespellignore b/.github/.codespellignore new file mode 100644 index 000000000..78cf86273 --- /dev/null +++ b/.github/.codespellignore @@ -0,0 +1 @@ +ue diff --git a/.github/workflows/codespell.yaml b/.github/workflows/codespell.yaml new file mode 100644 index 000000000..53007dfe6 --- /dev/null +++ b/.github/workflows/codespell.yaml @@ -0,0 +1,19 @@ +name: Common misspellings + +on: pull_request + +jobs: + misspellings: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up misspell + uses: codespell-project/actions-codespell@v2 + # plese update Makefile as well + with: + ignore_words_file: .github/.codespellignore + check_filenames: true + skip: go.*,**/*.drawio,./deploy/charts/*,./config/crd/* + diff --git a/Makefile b/Makefile index 4ee318113..617e8671c 100644 --- a/Makefile +++ b/Makefile @@ -300,3 +300,9 @@ DATA_PROCESSING_IMAGE ?= kubebb/dp-base .PHONY: docker-build-dp-base docker-build-dp-base: docker build -f ./data-process/Dockerfile.base -t $(DATA_PROCESSING_IMAGE):$(VERSION) ./data-process/ + +.PHONY: codespell +codespell: + echo "you need: pip install codespell" + # please update .github/workflows/codespell.yaml as well + codespell -w --ignore-words ".github/.codespellignore" --check-filenames --skip "go.*,**/*.drawio,./deploy/charts/*,./config/crd/*" diff --git a/README.md b/README.md index b93cf28b7..d32d6a0ac 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,16 @@ ## What is Arcadia? -**Arcadia** comes from [Greek mythology](https://www.greekmythology.com/Myths/Places/Arcadia/arcadia.html)(a tranquil and idyllic region, representing harmony, serenity, and natural beauty).We aim to help everyone find a more perfect integration between humans and AI. +**Arcadia** comes from [Greek mythology](https://www.greekmythology.com/Myths/Places/Arcadia/arcadia.html)(a tranquil and idyllic region, representing harmony, serenity, and natural beauty). We aim to help everyone find a more perfect integration between humans and AI. -To archieve this goal, we provides this one-stop LLMOps solution: +To achieve this goal, we provide this one-stop LLMOps solution: - Dataset Management: storage/real-time data,multimodal,pre-processing,vectorization - Models Management: local/online LLMs(development,training,deployment),inference acceleration - Application Management: development,optimization,deployment with visual editor -Furthermore,we can easily host **Arcadia** at any kubernetes cluster as production ready by integrating [kubebb](https://github.com/kubebb)(A kubernetes building blocks), +Furthermore, we can easily host **Arcadia** at any kubernetes cluster as production ready +by integrating [kubebb](https://github.com/kubebb)(A kubernetes building blocks). ## Architecture @@ -37,7 +38,7 @@ Our design and development in Arcadia design follows operator pattern which exte 2. [Kubernetes](https://kubernetes.io/) -If you don't have a kubernetes cluster, you can schedule a [kind cluster](https://kind.sigs.k8s.io/). Depends on your choice on CPU or GPU when running LLM worker,you can choose to: +If you don't have a kubernetes cluster, you can schedule a [kind cluster](https://kind.sigs.k8s.io/). Depends on your choice on CPU or GPU when running LLM worker, you can choose to: - [Create a kind cluster with/without GPU enabled](http://kubeagi.k8s.com.cn/docs/Quick%20Start/create-cluster-using-kind) @@ -54,7 +55,7 @@ We provide a Command Line Tool `arctl` to interact with `arcadia`. See [here](./ ## Pure Go Toolchains -To enhace the AI capability in Golang, we developed some packages.Here are the examples of how to use them. +To enhance the AI capability in Golang, we developed some packages.Here are the examples of how to use them. - [chat_with_document](https://github.com/kubeagi/arcadia/tree/main/examples/chat_with_document): a chat server which allows you to chat with your document - [embedding](https://github.com/kubeagi/arcadia/tree/main/examples/embedding) shows how to embedes your document to vector store with embedding service diff --git a/api/base/v1alpha1/common.go b/api/base/v1alpha1/common.go index 1dee34785..d0c6f6e3f 100644 --- a/api/base/v1alpha1/common.go +++ b/api/base/v1alpha1/common.go @@ -53,18 +53,18 @@ const ( // Provider defines how to prvoide the service type Provider struct { - // Enpoint defines connection info - Enpoint *Endpoint `json:"endpoint,omitempty"` + // Endpoint defines connection info + Endpoint *Endpoint `json:"endpoint,omitempty"` // Worker defines the worker info // Means this LLM is provided by a arcadia worker Worker *TypedObjectReference `json:"worker,omitempty"` } -// GetType returnes the type of this provider +// GetType returns the type of this provider func (p Provider) GetType() ProviderType { // if endpoint provided, then 3rd_party - if p.Enpoint != nil { + if p.Endpoint != nil { return ProviderType3rdParty } // if worker provided, then worker diff --git a/api/base/v1alpha1/datasource_types.go b/api/base/v1alpha1/datasource_types.go index 328871715..4fa84a80b 100644 --- a/api/base/v1alpha1/datasource_types.go +++ b/api/base/v1alpha1/datasource_types.go @@ -27,8 +27,8 @@ import ( type DatasourceSpec struct { CommonSpec `json:",inline"` - // Enpoint defines connection info - Enpoint Endpoint `json:"endpoint"` + // Endpoint defines connection info + Endpoint Endpoint `json:"endpoint"` // OSS defines info for object storage service OSS *OSS `json:"oss,omitempty"` diff --git a/api/base/v1alpha1/embedder.go b/api/base/v1alpha1/embedder.go index 68f756290..8e74f3d9e 100644 --- a/api/base/v1alpha1/embedder.go +++ b/api/base/v1alpha1/embedder.go @@ -26,10 +26,10 @@ import ( ) func (e Embedder) AuthAPIKey(ctx context.Context, c client.Client, cli dynamic.Interface) (string, error) { - if e.Spec.Enpoint == nil { + if e.Spec.Endpoint == nil { return "", nil } - return e.Spec.Enpoint.AuthAPIKey(ctx, e.GetNamespace(), c, cli) + return e.Spec.Endpoint.AuthAPIKey(ctx, e.GetNamespace(), c, cli) } // GetModelList returns a model list provided by this LLM based on different provider diff --git a/api/base/v1alpha1/llm.go b/api/base/v1alpha1/llm.go index ef1314fb9..13973eb8b 100644 --- a/api/base/v1alpha1/llm.go +++ b/api/base/v1alpha1/llm.go @@ -26,10 +26,10 @@ import ( ) func (llm LLM) AuthAPIKey(ctx context.Context, c client.Client, cli dynamic.Interface) (string, error) { - if llm.Spec.Enpoint == nil { + if llm.Spec.Endpoint == nil { return "", nil } - return llm.Spec.Enpoint.AuthAPIKey(ctx, llm.GetNamespace(), c, cli) + return llm.Spec.Endpoint.AuthAPIKey(ctx, llm.GetNamespace(), c, cli) } func (llmStatus LLMStatus) LLMReady() (string, bool) { diff --git a/api/base/v1alpha1/model_types.go b/api/base/v1alpha1/model_types.go index 635c46250..4c3b26a50 100644 --- a/api/base/v1alpha1/model_types.go +++ b/api/base/v1alpha1/model_types.go @@ -28,7 +28,7 @@ type ModelSpec struct { CommonSpec `json:",inline"` // Type defines what kind of model this is - // Comma separated field which can be wrapped by {llm,embdding} + // Comma separated field which can be wrapped by {llm,embedding} Types string `json:"types,omitempty"` // TODO: extend model to utilize third party storage sources diff --git a/api/base/v1alpha1/vectorstore.go b/api/base/v1alpha1/vectorstore.go index 193923308..604dbc454 100644 --- a/api/base/v1alpha1/vectorstore.go +++ b/api/base/v1alpha1/vectorstore.go @@ -33,7 +33,7 @@ const ( ) func (vs VectorStoreSpec) Type() VectorStoreType { - if vs.Enpoint == nil { + if vs.Endpoint == nil { return VectorStoreTypeUnknown } diff --git a/api/base/v1alpha1/vectorstore_types.go b/api/base/v1alpha1/vectorstore_types.go index 1449c1296..6a5d95920 100644 --- a/api/base/v1alpha1/vectorstore_types.go +++ b/api/base/v1alpha1/vectorstore_types.go @@ -25,8 +25,8 @@ import ( type VectorStoreSpec struct { CommonSpec `json:",inline"` - // Enpoint defines connection info - Enpoint *Endpoint `json:"endpoint,omitempty"` + // Endpoint defines connection info + Endpoint *Endpoint `json:"endpoint,omitempty"` Chroma *Chroma `json:"chroma,omitempty"` } diff --git a/api/base/v1alpha1/zz_generated.deepcopy.go b/api/base/v1alpha1/zz_generated.deepcopy.go index 23a9a8f24..b88b3bc52 100644 --- a/api/base/v1alpha1/zz_generated.deepcopy.go +++ b/api/base/v1alpha1/zz_generated.deepcopy.go @@ -348,7 +348,7 @@ func (in *DatasourceList) DeepCopyObject() runtime.Object { func (in *DatasourceSpec) DeepCopyInto(out *DatasourceSpec) { *out = *in out.CommonSpec = in.CommonSpec - in.Enpoint.DeepCopyInto(&out.Enpoint) + in.Endpoint.DeepCopyInto(&out.Endpoint) if in.OSS != nil { in, out := &in.OSS, &out.OSS *out = new(OSS) @@ -1042,8 +1042,8 @@ func (in *PromptStatus) DeepCopy() *PromptStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Provider) DeepCopyInto(out *Provider) { *out = *in - if in.Enpoint != nil { - in, out := &in.Enpoint, &out.Enpoint + if in.Endpoint != nil { + in, out := &in.Endpoint, &out.Endpoint *out = new(Endpoint) (*in).DeepCopyInto(*out) } @@ -1152,8 +1152,8 @@ func (in *VectorStoreList) DeepCopyObject() runtime.Object { func (in *VectorStoreSpec) DeepCopyInto(out *VectorStoreSpec) { *out = *in out.CommonSpec = in.CommonSpec - if in.Enpoint != nil { - in, out := &in.Enpoint, &out.Enpoint + if in.Endpoint != nil { + in, out := &in.Endpoint, &out.Endpoint *out = new(Endpoint) (*in).DeepCopyInto(*out) } diff --git a/apiserver/examples/upload-download-file/README.md b/apiserver/examples/upload-download-file/README.md index d787e245e..935e4c363 100644 --- a/apiserver/examples/upload-download-file/README.md +++ b/apiserver/examples/upload-download-file/README.md @@ -117,7 +117,7 @@ I1220 14:44:59.444794 12365 main.go:424] [Step 5], Congratulations, the file w I1211 15:16:42.701067 2904044 main.go:410] [Step 1] get file size I1211 15:16:42.703150 2904044 main.go:436] [DEBUG] file size is 51392595 I1211 15:16:42.703168 2904044 main.go:440] [Step 2] create local file tmp.gz -I1211 15:16:42.703225 2904044 main.go:461] [Step 3] start to donwload... +I1211 15:16:42.703225 2904044 main.go:461] [Step 3] start to download... I1211 15:16:42.703303 2904044 main.go:482] [Chunk 41943040-51392595] send request to http://localhost:8099/bff/versioneddataset/files/download?bucket=abc&bucketPath=def&end=51392595&fileName=tmp.tar.gz&from=41943040 I1211 15:16:42.703391 2904044 main.go:482] [Chunk 10485760-20971520] send request to http://localhost:8099/bff/versioneddataset/files/download?bucket=abc&bucketPath=def&end=20971520&fileName=tmp.tar.gz&from=10485760 I1211 15:16:42.703452 2904044 main.go:482] [Chunk 20971520-31457280] send request to http://localhost:8099/bff/versioneddataset/files/download?bucket=abc&bucketPath=def&end=31457280&fileName=tmp.tar.gz&from=20971520 diff --git a/apiserver/examples/upload-download-file/main.go b/apiserver/examples/upload-download-file/main.go index 196952088..f79c238b4 100644 --- a/apiserver/examples/upload-download-file/main.go +++ b/apiserver/examples/upload-download-file/main.go @@ -490,7 +490,7 @@ func downloadFile(bucket, bucketPath, fileName string, transport http.RoundTripp first := true lock := make(chan struct{}, 1) done := true - klog.Infof("[Step 3] start to donwload...") + klog.Infof("[Step 3] start to download...") var wg sync.WaitGroup for i := 0; i < int(parts); i++ { if !first { diff --git a/apiserver/pkg/application/application.go b/apiserver/pkg/application/application.go index f6e9d6da4..b1f643fbd 100644 --- a/apiserver/pkg/application/application.go +++ b/apiserver/pkg/application/application.go @@ -276,7 +276,7 @@ func ListApplicationMeatadatas(ctx context.Context, c dynamic.Interface, input g return res.Items[i].GetCreationTimestamp().After(res.Items[j].GetCreationTimestamp().Time) }) - filterd := make([]generated.PageNode, 0) + filtered := make([]generated.PageNode, 0) for _, u := range res.Items { if keyword != "" { displayName, _, _ := unstructured.NestedString(u.Object, "spec", "displayName") @@ -288,9 +288,9 @@ func ListApplicationMeatadatas(ctx context.Context, c dynamic.Interface, input g if err != nil { return nil, err } - filterd = append(filterd, m) + filtered = append(filtered, m) } - totalCount := len(filterd) + totalCount := len(filtered) end := page * pageSize if end > totalCount { @@ -298,15 +298,15 @@ func ListApplicationMeatadatas(ctx context.Context, c dynamic.Interface, input g } start := (page - 1) * pageSize if start < totalCount { - filterd = filterd[start:end] + filtered = filtered[start:end] } else { - filterd = []generated.PageNode{} + filtered = []generated.PageNode{} } return &generated.PaginatedResult{ TotalCount: totalCount, HasNextPage: end < totalCount, - Nodes: filterd, + Nodes: filtered, Page: &page, PageSize: &pageSize, }, nil diff --git a/apiserver/pkg/common/common.go b/apiserver/pkg/common/common.go index ab304e385..9dd8ca8b9 100644 --- a/apiserver/pkg/common/common.go +++ b/apiserver/pkg/common/common.go @@ -72,7 +72,7 @@ func SystemDatasourceOSS(ctx context.Context, mgrClient client.Client, dynamicCl if err != nil { return nil, err } - endpoint := systemDatasource.Spec.Enpoint.DeepCopy() + endpoint := systemDatasource.Spec.Endpoint.DeepCopy() if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil { endpoint.AuthSecret.WithNameSpace(systemDatasource.Namespace) } diff --git a/apiserver/pkg/common/endpoint.go b/apiserver/pkg/common/endpoint.go index 5a90e5d4a..f44f9ff2f 100644 --- a/apiserver/pkg/common/endpoint.go +++ b/apiserver/pkg/common/endpoint.go @@ -118,7 +118,7 @@ func MakeAuthSecret(ctx context.Context, c dynamic.Interface, secret generated.T _, err = ResouceGet(ctx, c, secret, metav1.GetOptions{}) if err != nil { - // Create is not fount + // Create is not found if apierrors.IsNotFound(err) { _, err = c.Resource(schema.GroupVersionResource{Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, Resource: "secrets"}). Namespace(*secret.Namespace).Create(ctx, &unstructured.Unstructured{Object: unstructuredDatasource}, metav1.CreateOptions{}) diff --git a/apiserver/pkg/datasource/datasource.go b/apiserver/pkg/datasource/datasource.go index 46d4dce4d..2bf994c1d 100644 --- a/apiserver/pkg/datasource/datasource.go +++ b/apiserver/pkg/datasource/datasource.go @@ -58,14 +58,14 @@ func datasource2model(obj *unstructured.Unstructured) *generated.Datasource { // parse endpoint endpoint := generated.Endpoint{ - URL: &datasource.Spec.Enpoint.URL, - Insecure: &datasource.Spec.Enpoint.Insecure, + URL: &datasource.Spec.Endpoint.URL, + Insecure: &datasource.Spec.Endpoint.Insecure, } - if datasource.Spec.Enpoint.AuthSecret != nil { + if datasource.Spec.Endpoint.AuthSecret != nil { endpoint.AuthSecret = &generated.TypedObjectReference{ Kind: "Secret", - Name: datasource.Spec.Enpoint.AuthSecret.Name, - Namespace: datasource.Spec.Enpoint.AuthSecret.Namespace, + Name: datasource.Spec.Endpoint.AuthSecret.Name, + Namespace: datasource.Spec.Endpoint.AuthSecret.Namespace, } } @@ -132,7 +132,7 @@ func CreateDatasource(ctx context.Context, c dynamic.Interface, input generated. if err != nil { return nil, err } - datasource.Spec.Enpoint = endpoint + datasource.Spec.Endpoint = endpoint if input.Ossinput != nil { datasource.Spec.OSS = &v1alpha1.OSS{ @@ -206,7 +206,7 @@ func UpdateDatasource(ctx context.Context, c dynamic.Interface, input *generated if err != nil { return nil, err } - datasource.Spec.Enpoint = endpoint + datasource.Spec.Endpoint = endpoint } // Update ossinput diff --git a/apiserver/pkg/embedder/embedder.go b/apiserver/pkg/embedder/embedder.go index b0bb306a4..f8790b0fe 100644 --- a/apiserver/pkg/embedder/embedder.go +++ b/apiserver/pkg/embedder/embedder.go @@ -61,7 +61,7 @@ func Embedder2model(ctx context.Context, c dynamic.Interface, obj *unstructured. case v1alpha1.ProviderTypeWorker: baseURL, _ = common.GetAPIServer(ctx, c, true) case v1alpha1.ProviderType3rdParty: - baseURL = embedder.Spec.Enpoint.URL + baseURL = embedder.Spec.Endpoint.URL } md := generated.Embedder{ @@ -112,7 +112,7 @@ func CreateEmbedder(ctx context.Context, c dynamic.Interface, input generated.Cr Description: description, }, Provider: v1alpha1.Provider{ - Enpoint: &v1alpha1.Endpoint{ + Endpoint: &v1alpha1.Endpoint{ URL: input.Endpointinput.URL, }, }, @@ -131,7 +131,7 @@ func CreateEmbedder(ctx context.Context, c dynamic.Interface, input generated.Cr if err != nil { return nil, err } - embedder.Spec.Enpoint.AuthSecret = &v1alpha1.TypedObjectReference{ + embedder.Spec.Endpoint.AuthSecret = &v1alpha1.TypedObjectReference{ Kind: "Secret", Name: secret, Namespace: &input.Namespace, diff --git a/apiserver/pkg/knowledgebase/knowledgebase.go b/apiserver/pkg/knowledgebase/knowledgebase.go index da334bd91..6d8afcf46 100644 --- a/apiserver/pkg/knowledgebase/knowledgebase.go +++ b/apiserver/pkg/knowledgebase/knowledgebase.go @@ -116,7 +116,7 @@ func knowledgebase2model(obj *unstructured.Unstructured) *generated.KnowledgeBas return &md } -func CreateKnowledgeBase(ctx context.Context, c dynamic.Interface, name, namespace, displayname, discription, embedder string, vectorstore v1alpha1.TypedObjectReference, filegroups []v1alpha1.FileGroup) (*generated.KnowledgeBase, error) { +func CreateKnowledgeBase(ctx context.Context, c dynamic.Interface, name, namespace, displayname, description, embedder string, vectorstore v1alpha1.TypedObjectReference, filegroups []v1alpha1.FileGroup) (*generated.KnowledgeBase, error) { knowledgebase := v1alpha1.KnowledgeBase{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -129,7 +129,7 @@ func CreateKnowledgeBase(ctx context.Context, c dynamic.Interface, name, namespa Spec: v1alpha1.KnowledgeBaseSpec{ CommonSpec: v1alpha1.CommonSpec{ DisplayName: displayname, - Description: discription, + Description: description, }, Embedder: &v1alpha1.TypedObjectReference{ Kind: "Embedder", diff --git a/apiserver/pkg/llm/llm.go b/apiserver/pkg/llm/llm.go index 5973d623f..86cf1299d 100644 --- a/apiserver/pkg/llm/llm.go +++ b/apiserver/pkg/llm/llm.go @@ -59,7 +59,7 @@ func LLM2model(ctx context.Context, c dynamic.Interface, obj *unstructured.Unstr case v1alpha1.ProviderTypeWorker: baseURL, _ = common.GetAPIServer(ctx, c, true) case v1alpha1.ProviderType3rdParty: - baseURL = llm.Spec.Enpoint.URL + baseURL = llm.Spec.Endpoint.URL } md := generated.Llm{ @@ -188,7 +188,7 @@ func CreateLLM(ctx context.Context, c dynamic.Interface, input generated.CreateL Description: description, }, Provider: v1alpha1.Provider{ - Enpoint: &v1alpha1.Endpoint{ + Endpoint: &v1alpha1.Endpoint{ URL: input.Endpointinput.URL, }, }, @@ -206,7 +206,7 @@ func CreateLLM(ctx context.Context, c dynamic.Interface, input generated.CreateL if err != nil { return nil, err } - llm.Spec.Enpoint.AuthSecret = &v1alpha1.TypedObjectReference{ + llm.Spec.Endpoint.AuthSecret = &v1alpha1.TypedObjectReference{ Kind: "Secret", Name: secret, Namespace: &input.Namespace, diff --git a/apiserver/pkg/model/model.go b/apiserver/pkg/model/model.go index c52c6e90a..44965063e 100644 --- a/apiserver/pkg/model/model.go +++ b/apiserver/pkg/model/model.go @@ -286,7 +286,7 @@ func ModelFiles(ctx context.Context, c dynamic.Interface, modelName, namespace s return nil, err } - endpoint := systemDatasource.Spec.Enpoint.DeepCopy() + endpoint := systemDatasource.Spec.Endpoint.DeepCopy() if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil { endpoint.AuthSecret.WithNameSpace(systemDatasource.Namespace) } diff --git a/apiserver/pkg/modelservice/modelservice.go b/apiserver/pkg/modelservice/modelservice.go index 550e4dd0f..a89390cca 100644 --- a/apiserver/pkg/modelservice/modelservice.go +++ b/apiserver/pkg/modelservice/modelservice.go @@ -364,7 +364,7 @@ func ListModelServices(ctx context.Context, c dynamic.Interface, input *generate klog.V(5).Infof("match check llm: embedder %s has no matching llm's, add modelservice", v.Name) } if ok && (*llm.APIType != *v.Type || *llm.LlmResource.Provider != *v.Provider) { - klog.V(5).Infof("match check llm: embedder %s type: %s, llm apiType: %s, llm provider: %s, embedder proviers: %s. add modelservice", + klog.V(5).Infof("match check llm: embedder %s type: %s, llm apiType: %s, llm provider: %s, embedder provider: %s. add modelservice", v.Name, *v.Type, *llm.APIType, *llm.LlmResource.Provider, *v.Provider) } @@ -423,7 +423,7 @@ func ListModelServices(ctx context.Context, c dynamic.Interface, input *generate klog.V(5).Infof("match check embedder: llm %s has no matching embedder's, add modelservice", v.Name) } if ok && (*embedder.APIType != *v.Type || *embedder.EmbedderResource.Provider != *v.Provider) { - klog.V(5).Infof("match check embedder: llm %s type: %s, embedder apiType: %s, embedder provider: %s, llm proviers: %s. add modelservice", + klog.V(5).Infof("match check embedder: llm %s type: %s, embedder apiType: %s, embedder provider: %s, llm provider: %s. add modelservice", v.Name, *v.Type, *embedder.APIType, *embedder.EmbedderResource.Provider, *v.Provider) } diff --git a/apiserver/pkg/oidc/auth.go b/apiserver/pkg/oidc/auth.go index 1183e425d..488b9709d 100644 --- a/apiserver/pkg/oidc/auth.go +++ b/apiserver/pkg/oidc/auth.go @@ -130,7 +130,7 @@ func newOIDCAuthProvider(clusterAddress string, cfg map[string]string, persister // } if len(cfg[cfgExtraScopes]) > 0 { - klog.V(2).Infof("%s auth provider field depricated, refresh request don't send scopes", + klog.V(2).Infof("%s auth provider field deprecated, refresh request don't send scopes", cfgExtraScopes) } diff --git a/apiserver/pkg/versioneddataset/versioned_dataset.go b/apiserver/pkg/versioneddataset/versioned_dataset.go index 06341dc3b..f4a0e4b43 100644 --- a/apiserver/pkg/versioneddataset/versioned_dataset.go +++ b/apiserver/pkg/versioneddataset/versioned_dataset.go @@ -110,7 +110,7 @@ func VersionFiles(ctx context.Context, c dynamic.Interface, input *generated.Ver if err != nil { return nil, err } - endpoint := systemDatasource.Spec.Enpoint.DeepCopy() + endpoint := systemDatasource.Spec.Endpoint.DeepCopy() if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil { endpoint.AuthSecret.WithNameSpace(systemDatasource.Namespace) } diff --git a/apiserver/service/minio_server.go b/apiserver/service/minio_server.go index 03e0a5337..364f4ebea 100644 --- a/apiserver/service/minio_server.go +++ b/apiserver/service/minio_server.go @@ -171,7 +171,7 @@ func (m *minioAPI) GetSuccessChunks(ctx *gin.Context) { // When checking the existence of a file in MinIO, besides the "file not found" error, there can be other errors as well. klog.Errorf("failed to check for the existence of a resource %s/%s. error %s", bucketName, objectName, err) ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ - "messge": err.Error(), + "message": err.Error(), }) return } @@ -266,7 +266,7 @@ func (m *minioAPI) NewMultipart(ctx *gin.Context) { if err != nil { klog.Errorf("failed to generate uploadid error %s", err) ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ - "message": "fialed to generate uploadid", + "message": "failed to generate uploadid", }) return } @@ -319,7 +319,7 @@ func (m *minioAPI) GetMultipartUploadURL(ctx *gin.Context) { if body.FileName == "" { klog.Errorf("fileName is empty") ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "mesage": "fileName is required", + "message": "fileName is required", }) return } @@ -396,7 +396,7 @@ func (m *minioAPI) CompleteMultipart(ctx *gin.Context) { datasource.WithUploadID(body.UploadID), datasource.WithFileName(body.FileName)) if err != nil { - klog.Errorf("complte multipart error %s", err) + klog.Errorf("complete multipart error %s", err) ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ "message": err.Error(), }) @@ -428,7 +428,7 @@ func (m *minioAPI) DeleteFiles(ctx *gin.Context) { for _, f := range body.Files { go func(fn string) { if err := source.Remove(context.TODO(), &v1alpha1.OSS{Bucket: body.Bucket, Object: fmt.Sprintf("%s/%s", bucketPath, fn)}); err != nil { - klog.Errorf("faile to delete file %s/%s from bucket %s, error %s", bucketPath, fn, body.Bucket, err) + klog.Errorf("failed to delete file %s/%s from bucket %s, error %s", bucketPath, fn, body.Bucket, err) } }(f) } @@ -535,7 +535,7 @@ func (m *minioAPI) Download(ctx *gin.Context) { } info, err := source.Client.GetObject(ctx.Request.Context(), bucket, objectName, opt) if err != nil { - klog.Errorf("failed to get object %s/%s range %d-%d errro %s", bucket, objectName, from, end, err) + klog.Errorf("failed to get object %s/%s range %d-%d error %s", bucket, objectName, from, end, err) ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ "message": err.Error(), }) @@ -584,7 +584,7 @@ func (m *minioAPI) ReadCSVLines(ctx *gin.Context) { } anyStatInfo, err := source.StatFile(ctx.Request.Context(), &v1alpha1.OSS{Bucket: bucket, Object: objectName}) if err != nil { - klog.Errorf("faield to stat filed %s/%s error %s", bucket, objectName, err) + klog.Errorf("failed to stat filed %s/%s error %s", bucket, objectName, err) ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ "message": err.Error(), }) diff --git a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml index 55b3f0968..c2e2e65e0 100644 --- a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml +++ b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml @@ -52,7 +52,7 @@ spec: description: DisplayName defines datasource display name type: string endpoint: - description: Enpoint defines connection info + description: Endpoint defines connection info properties: authSecret: description: AuthSecret if the chart repository requires auth diff --git a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_embedders.yaml b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_embedders.yaml index 3c9fa369a..e79c0bcbb 100644 --- a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_embedders.yaml +++ b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_embedders.yaml @@ -59,7 +59,7 @@ spec: embedder service properties: endpoint: - description: Enpoint defines connection info + description: Endpoint defines connection info properties: authSecret: description: AuthSecret if the chart repository requires auth diff --git a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_llms.yaml b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_llms.yaml index 995cd7afe..687d6f233 100644 --- a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_llms.yaml +++ b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_llms.yaml @@ -59,7 +59,7 @@ spec: llm service properties: endpoint: - description: Enpoint defines connection info + description: Endpoint defines connection info properties: authSecret: description: AuthSecret if the chart repository requires auth diff --git a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_models.yaml b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_models.yaml index 91df43f34..f74e0ed4a 100644 --- a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_models.yaml +++ b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_models.yaml @@ -46,7 +46,7 @@ spec: type: string types: description: Type defines what kind of model this is Comma separated - field which can be wrapped by {llm,embdding} + field which can be wrapped by {llm,embedding} type: string type: object status: diff --git a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_vectorstores.yaml b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_vectorstores.yaml index ca6701460..d64d3a994 100644 --- a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_vectorstores.yaml +++ b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_vectorstores.yaml @@ -55,7 +55,7 @@ spec: description: DisplayName defines datasource display name type: string endpoint: - description: Enpoint defines connection info + description: Endpoint defines connection info properties: authSecret: description: AuthSecret if the chart repository requires auth diff --git a/controllers/datasource_controller.go b/controllers/datasource_controller.go index 094f9987e..4ffa836eb 100644 --- a/controllers/datasource_controller.go +++ b/controllers/datasource_controller.go @@ -159,7 +159,7 @@ func (r *DatasourceReconciler) Checkdatasource(ctx context.Context, logger logr. var info any switch instance.Spec.Type() { case arcadiav1alpha1.DatasourceTypeOSS: - endpoint := instance.Spec.Enpoint.DeepCopy() + endpoint := instance.Spec.Endpoint.DeepCopy() // set auth secret's namespace to the datasource's namespace if endpoint.AuthSecret != nil { endpoint.AuthSecret.WithNameSpace(instance.Namespace) @@ -185,7 +185,7 @@ func (r *DatasourceReconciler) Checkdatasource(ctx context.Context, logger logr. return r.UpdateStatus(ctx, instance, nil) } -// UpdateStatus uppon error +// UpdateStatus upon error func (r *DatasourceReconciler) UpdateStatus(ctx context.Context, instance *arcadiav1alpha1.Datasource, err error) error { instanceCopy := instance.DeepCopy() var newCondition arcadiav1alpha1.Condition diff --git a/controllers/embedder_controller.go b/controllers/embedder_controller.go index 1794e45cd..d488bad0f 100644 --- a/controllers/embedder_controller.go +++ b/controllers/embedder_controller.go @@ -125,7 +125,7 @@ func (r *EmbedderReconciler) SetupWithManager(mgr ctrl.Manager) error { newEmbedder := ue.ObjectNew.(*arcadiav1alpha1.Embedder) return !reflect.DeepEqual(oldEmbedder.Spec, newEmbedder.Spec) || newEmbedder.DeletionTimestamp != nil }, - // for other event handler, we must add the function explictly. + // for other event handler, we must add the function explicitly. CreateFunc: func(event.CreateEvent) bool { return true }, @@ -193,7 +193,7 @@ func (r *EmbedderReconciler) check3rdPartyEmbedder(ctx context.Context, logger l } msg = res.String() case embeddings.OpenAI: - embedClient := openai.NewOpenAI(apiKey, instance.Spec.Enpoint.URL) + embedClient := openai.NewOpenAI(apiKey, instance.Spec.Endpoint.URL) res, err := embedClient.Validate() if err != nil { return r.UpdateStatus(ctx, instance, nil, err) diff --git a/controllers/knowledgebase_controller.go b/controllers/knowledgebase_controller.go index 96edf3a61..76d2e5917 100644 --- a/controllers/knowledgebase_controller.go +++ b/controllers/knowledgebase_controller.go @@ -256,7 +256,7 @@ func (r *KnowledgeBaseReconciler) reconcileFileGroup(ctx context.Context, log lo if err != nil { return err } - endpoint := system.Spec.Enpoint.DeepCopy() + endpoint := system.Spec.Endpoint.DeepCopy() if endpoint != nil && endpoint.AuthSecret != nil { endpoint.AuthSecret.WithNameSpace(system.Namespace) } @@ -428,7 +428,7 @@ func (r *KnowledgeBaseReconciler) handleFile(ctx context.Context, log logr.Logge loader = documentloaders.NewText(dataReader) } - // initliaze text splitter + // initialize text splitter // var split textsplitter.TextSplitter split := textsplitter.NewRecursiveCharacter( textsplitter.WithChunkSize(300), @@ -464,7 +464,7 @@ func (r *KnowledgeBaseReconciler) handleFile(ctx context.Context, log logr.Logge switch store.Spec.Type() { // nolint: gocritic case arcadiav1alpha1.VectorStoreTypeChroma: s, err := chroma.New( - chroma.WithChromaURL(store.Spec.Enpoint.URL), + chroma.WithChromaURL(store.Spec.Endpoint.URL), chroma.WithDistanceFunction(store.Spec.Chroma.DistanceFunction), chroma.WithNameSpace(kb.VectorStoreCollectionName()), chroma.WithEmbedder(em), @@ -496,7 +496,7 @@ func (r *KnowledgeBaseReconciler) reconcileDelete(ctx context.Context, log logr. switch vectorStore.Spec.Type() { // nolint: gocritic case arcadiav1alpha1.VectorStoreTypeChroma: s, err := chroma.New( - chroma.WithChromaURL(vectorStore.Spec.Enpoint.URL), + chroma.WithChromaURL(vectorStore.Spec.Endpoint.URL), chroma.WithNameSpace(kb.VectorStoreCollectionName()), // workaround to fix 'invalid options: missing embedder or openai api key' chroma.WithOpenAiAPIKey("fake-api-key"), diff --git a/controllers/llm_controller.go b/controllers/llm_controller.go index 0a8e32cd0..cf2f7200f 100644 --- a/controllers/llm_controller.go +++ b/controllers/llm_controller.go @@ -179,7 +179,7 @@ func (r *LLMReconciler) check3rdPartyLLM(ctx context.Context, logger logr.Logger } msg = res.String() case llms.OpenAI: - embedClient := openai.NewOpenAI(apiKey, instance.Spec.Enpoint.URL) + embedClient := openai.NewOpenAI(apiKey, instance.Spec.Endpoint.URL) res, err := embedClient.Validate() if err != nil { return r.UpdateStatus(ctx, instance, nil, err) diff --git a/controllers/model_controller.go b/controllers/model_controller.go index 455b9422a..dc224dee1 100644 --- a/controllers/model_controller.go +++ b/controllers/model_controller.go @@ -157,7 +157,7 @@ func (r *ModelReconciler) Initialize(ctx context.Context, logger logr.Logger, in if instanceDeepCopy.Annotations == nil { instanceDeepCopy.Annotations = make(map[string]string) } - // For model's full storeage path + // For model's full storage path currentFullPath := instanceDeepCopy.FullPath() if v := instanceDeepCopy.Annotations[arcadiav1alpha1.LabelModelFullPath]; v != currentFullPath { instanceDeepCopy.Annotations[arcadiav1alpha1.LabelModelFullPath] = currentFullPath @@ -183,7 +183,7 @@ func (r *ModelReconciler) CheckModel(ctx context.Context, logger logr.Logger, in if err != nil { return r.UpdateStatus(ctx, instance, err) } - endpoint := system.Spec.Enpoint.DeepCopy() + endpoint := system.Spec.Endpoint.DeepCopy() if endpoint != nil && endpoint.AuthSecret != nil { endpoint.AuthSecret.WithNameSpace(system.Namespace) } @@ -217,7 +217,7 @@ func (r *ModelReconciler) RemoveModel(ctx context.Context, logger logr.Logger, i if err != nil { return r.UpdateStatus(ctx, instance, err) } - endpoint := system.Spec.Enpoint.DeepCopy() + endpoint := system.Spec.Endpoint.DeepCopy() if endpoint != nil && endpoint.AuthSecret != nil { endpoint.AuthSecret.WithNameSpace(system.Namespace) } @@ -238,7 +238,7 @@ func (r *ModelReconciler) RemoveModel(ctx context.Context, logger logr.Logger, i return ds.Remove(ctx, info) } -// UpdateStatus uppon error +// UpdateStatus upon error func (r *ModelReconciler) UpdateStatus(ctx context.Context, instance *arcadiav1alpha1.Model, err error) error { instanceCopy := instance.DeepCopy() var newCondition arcadiav1alpha1.Condition diff --git a/controllers/namespace_controller.go b/controllers/namespace_controller.go index 8362b847a..0e50a16f8 100644 --- a/controllers/namespace_controller.go +++ b/controllers/namespace_controller.go @@ -128,7 +128,7 @@ func (r *NamespaceReconciler) ossClient(ctx context.Context) (*datasource.OSS, e klog.Errorf("get system datasource error %s", err) return nil, err } - endpoint := systemDatasource.Spec.Enpoint.DeepCopy() + endpoint := systemDatasource.Spec.Endpoint.DeepCopy() if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil { endpoint.AuthSecret.WithNameSpace(systemDatasource.Namespace) } diff --git a/controllers/prompt_controller.go b/controllers/prompt_controller.go index fa1770ebd..1dc1313a6 100644 --- a/controllers/prompt_controller.go +++ b/controllers/prompt_controller.go @@ -130,7 +130,7 @@ func (r *PromptReconciler) CallLLM(ctx context.Context, logger logr.Logger, prom llmClient = llmszhipuai.NewZhiPuAI(apiKey) callData = prompt.Spec.ZhiPuAIParams.Marshal() case llms.OpenAI: - llmClient = openai.NewOpenAI(apiKey, llm.Spec.Enpoint.URL) + llmClient = openai.NewOpenAI(apiKey, llm.Spec.Endpoint.URL) default: llmClient = llms.NewUnknowLLM() } diff --git a/controllers/vectorstore_controller.go b/controllers/vectorstore_controller.go index 3a176fa41..faf4ac3d5 100644 --- a/controllers/vectorstore_controller.go +++ b/controllers/vectorstore_controller.go @@ -122,7 +122,7 @@ func (r *VectorStoreReconciler) CheckVectorStore(ctx context.Context, log logr.L case arcadiav1alpha1.VectorStoreTypeChroma: _, err := chroma.New( chroma.WithOpenAiAPIKey("fake_key_just_for_chroma_heartbeat"), - chroma.WithChromaURL(vs.Spec.Enpoint.URL), + chroma.WithChromaURL(vs.Spec.Endpoint.URL), chroma.WithDistanceFunction(vs.Spec.Chroma.DistanceFunction), ) if err != nil { diff --git a/controllers/versioneddataset_controller.go b/controllers/versioneddataset_controller.go index 8fe4ec351..804a45a7b 100644 --- a/controllers/versioneddataset_controller.go +++ b/controllers/versioneddataset_controller.go @@ -218,7 +218,7 @@ func (r *VersionedDatasetReconciler) checkStatus(ctx context.Context, logger log logger.Error(err, "Failed to get system datasource") return false, nil, err } - endpoint := systemDatasource.Spec.Enpoint.DeepCopy() + endpoint := systemDatasource.Spec.Endpoint.DeepCopy() if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil { endpoint.AuthSecret.WithNameSpace(systemDatasource.Namespace) } @@ -238,7 +238,7 @@ func (r *VersionedDatasetReconciler) removeBucketFiles(ctx context.Context, logg logger.Error(err, "Failed to get system datasource") return err } - endpoint := systemDatasource.Spec.Enpoint.DeepCopy() + endpoint := systemDatasource.Spec.Endpoint.DeepCopy() if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil { endpoint.AuthSecret.WithNameSpace(systemDatasource.Namespace) } diff --git a/data-processing/data_manipulation/database_operate/data_process_detail_db_operate.py b/data-processing/data_manipulation/database_operate/data_process_detail_db_operate.py index 649fe9f5f..689ee6457 100644 --- a/data-processing/data_manipulation/database_operate/data_process_detail_db_operate.py +++ b/data-processing/data_manipulation/database_operate/data_process_detail_db_operate.py @@ -136,7 +136,7 @@ def list_file_name_for_transform( req_json, pool ): - """List file name for tansform in the task detail. + """List file name for transform in the task detail. req_json is a dictionary object. for example: { @@ -169,7 +169,7 @@ def top_n_list_transform_for_preview( pool ): """List transform info with task id, file name and - tansform type for preview. + transform type for preview. req_json is a dictionary object. for example: { diff --git a/data-processing/data_manipulation/file_handle/common_handle.py b/data-processing/data_manipulation/file_handle/common_handle.py index cf89d6ce5..3ab7cccc2 100644 --- a/data-processing/data_manipulation/file_handle/common_handle.py +++ b/data-processing/data_manipulation/file_handle/common_handle.py @@ -537,7 +537,7 @@ def _generate_qa_list( ): """Generate the Question and Answer list. - chunk_size: chunck size; + chunk_size: chunk size; chunk_overlap: chunk overlap; data: the text used to generate QA; name: llms cr name; @@ -569,7 +569,7 @@ def _generate_qa_list( logger.debug(''.join([ f"original text is: \n{data}\n", - f"splitted text is: \n{texts}\n" + f"split text is: \n{texts}\n" ])) # 更新文件状态为开始 diff --git a/data-processing/data_manipulation/service/data_process_service.py b/data-processing/data_manipulation/service/data_process_service.py index 002364eaf..aa7943255 100644 --- a/data-processing/data_manipulation/service/data_process_service.py +++ b/data-processing/data_manipulation/service/data_process_service.py @@ -172,7 +172,7 @@ def info_by_id( conn_pool=pool ) - # convert the conig resule from map to list + # convert the config resule from map to list config_list_for_result = [] for value in config_map_for_result.values(): config_list_for_result.append(value) diff --git a/deploy/charts/arcadia/Chart.yaml b/deploy/charts/arcadia/Chart.yaml index a6ded5cc5..4f48b7008 100644 --- a/deploy/charts/arcadia/Chart.yaml +++ b/deploy/charts/arcadia/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: arcadia description: A Helm chart(KubeBB Component) for KubeAGI Arcadia type: application -version: 0.1.51 +version: 0.1.52 appVersion: "0.0.1" keywords: diff --git a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml index 55b3f0968..c2e2e65e0 100644 --- a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml +++ b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml @@ -52,7 +52,7 @@ spec: description: DisplayName defines datasource display name type: string endpoint: - description: Enpoint defines connection info + description: Endpoint defines connection info properties: authSecret: description: AuthSecret if the chart repository requires auth diff --git a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_embedders.yaml b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_embedders.yaml index 3c9fa369a..e79c0bcbb 100644 --- a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_embedders.yaml +++ b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_embedders.yaml @@ -59,7 +59,7 @@ spec: embedder service properties: endpoint: - description: Enpoint defines connection info + description: Endpoint defines connection info properties: authSecret: description: AuthSecret if the chart repository requires auth diff --git a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_llms.yaml b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_llms.yaml index 995cd7afe..687d6f233 100644 --- a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_llms.yaml +++ b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_llms.yaml @@ -59,7 +59,7 @@ spec: llm service properties: endpoint: - description: Enpoint defines connection info + description: Endpoint defines connection info properties: authSecret: description: AuthSecret if the chart repository requires auth diff --git a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_models.yaml b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_models.yaml index 91df43f34..f74e0ed4a 100644 --- a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_models.yaml +++ b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_models.yaml @@ -46,7 +46,7 @@ spec: type: string types: description: Type defines what kind of model this is Comma separated - field which can be wrapped by {llm,embdding} + field which can be wrapped by {llm,embedding} type: string type: object status: diff --git a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_vectorstores.yaml b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_vectorstores.yaml index ca6701460..d64d3a994 100644 --- a/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_vectorstores.yaml +++ b/deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_vectorstores.yaml @@ -55,7 +55,7 @@ spec: description: DisplayName defines datasource display name type: string endpoint: - description: Enpoint defines connection info + description: Endpoint defines connection info properties: authSecret: description: AuthSecret if the chart repository requires auth diff --git a/examples/beijing_gjj_bot/pkg/embedding.go b/examples/beijing_gjj_bot/pkg/embedding.go index b146ff0ef..887d61160 100644 --- a/examples/beijing_gjj_bot/pkg/embedding.go +++ b/examples/beijing_gjj_bot/pkg/embedding.go @@ -49,10 +49,10 @@ func NewDashScope(apiKey string, chromaURL, namespace string) (*Bot, error) { if err != nil { return nil, err } - dashs := dashscope.NewDashScope(apiKey, false) + dashes := dashscope.NewDashScope(apiKey, false) return &Bot{ DB: chromadb, - AI: dashs, + AI: dashes, Tree: NewTree(""), }, nil } diff --git a/pkg/application/chain/llmchain.go b/pkg/application/chain/llmchain.go index e150b203f..964f7d4c0 100644 --- a/pkg/application/chain/llmchain.go +++ b/pkg/application/chain/llmchain.go @@ -78,7 +78,7 @@ func (l *LLMChain) Run(ctx context.Context, cli dynamic.Interface, args map[stri obj, err := cli.Resource(schema.GroupVersionResource{Group: v1alpha1.GroupVersion.Group, Version: v1alpha1.GroupVersion.Version, Resource: "llmchains"}). Namespace(l.Ref.GetNamespace(ns)).Get(ctx, l.Ref.Name, metav1.GetOptions{}) if err != nil { - return args, fmt.Errorf("cant find the chain in cluster: %w", err) + return args, fmt.Errorf("can't find the chain in cluster: %w", err) } err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), instance) if err != nil { diff --git a/pkg/application/chain/retrievalqachain.go b/pkg/application/chain/retrievalqachain.go index 7fc696216..36a60903c 100644 --- a/pkg/application/chain/retrievalqachain.go +++ b/pkg/application/chain/retrievalqachain.go @@ -87,7 +87,7 @@ func (l *RetrievalQAChain) Run(ctx context.Context, cli dynamic.Interface, args obj, err := cli.Resource(schema.GroupVersionResource{Group: v1alpha1.GroupVersion.Group, Version: v1alpha1.GroupVersion.Version, Resource: "retrievalqachains"}). Namespace(l.Ref.GetNamespace(ns)).Get(ctx, l.Ref.Name, metav1.GetOptions{}) if err != nil { - return args, fmt.Errorf("cant find the chain in cluster: %w", err) + return args, fmt.Errorf("can't find the chain in cluster: %w", err) } err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), instance) if err != nil { diff --git a/pkg/application/llm/llm.go b/pkg/application/llm/llm.go index 757b22889..c16ad9e6f 100644 --- a/pkg/application/llm/llm.go +++ b/pkg/application/llm/llm.go @@ -48,7 +48,7 @@ func (z *LLM) Init(ctx context.Context, cli dynamic.Interface, args map[string]a obj, err := cli.Resource(schema.GroupVersionResource{Group: v1alpha1.GroupVersion.Group, Version: v1alpha1.GroupVersion.Version, Resource: "llms"}). Namespace(z.Ref.GetNamespace(ns)).Get(ctx, z.Ref.Name, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("cant find the llm in cluster: %w", err) + return fmt.Errorf("can't find the llm in cluster: %w", err) } err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), instance) if err != nil { diff --git a/pkg/application/retriever/knowledgebaseretriever.go b/pkg/application/retriever/knowledgebaseretriever.go index af79bd063..c6561c726 100644 --- a/pkg/application/retriever/knowledgebaseretriever.go +++ b/pkg/application/retriever/knowledgebaseretriever.go @@ -64,7 +64,7 @@ func NewKnowledgeBaseRetriever(ctx context.Context, baseNode base.BaseNode, cli obj, err := cli.Resource(schema.GroupVersionResource{Group: apiretriever.GroupVersion.Group, Version: apiretriever.GroupVersion.Version, Resource: "knowledgebaseretrievers"}). Namespace(baseNode.Ref.GetNamespace(ns)).Get(ctx, baseNode.Ref.Name, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("cant find the retriever in cluster: %w", err) + return nil, fmt.Errorf("can't find the retriever in cluster: %w", err) } err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), instance) if err != nil { @@ -116,7 +116,7 @@ func NewKnowledgeBaseRetriever(ctx context.Context, baseNode base.BaseNode, cli switch vectorStore.Spec.Type() { // nolint: gocritic case v1alpha1.VectorStoreTypeChroma: s, err := chroma.New( - chroma.WithChromaURL(vectorStore.Spec.Enpoint.URL), + chroma.WithChromaURL(vectorStore.Spec.Endpoint.URL), chroma.WithDistanceFunction(vectorStore.Spec.Chroma.DistanceFunction), chroma.WithNameSpace(knowledgebase.VectorStoreCollectionName()), chroma.WithEmbedder(em), diff --git a/pkg/arctl/dataset.go b/pkg/arctl/dataset.go index 0e83fa8d7..cbd1f1674 100644 --- a/pkg/arctl/dataset.go +++ b/pkg/arctl/dataset.go @@ -363,7 +363,7 @@ type File struct { Size int64 `json:"size"` // embedding status - // Chunks is the number of splitted chunks + // Chunks is the number of split chunks Chunks int `json:"chunks"` // ChunksLoaded is the number of chunks loaded ChunksLoaded int `json:"chunks_loaded"` @@ -473,7 +473,7 @@ func (cachedDS *Dataset) loadDocument(ctx context.Context, document string) erro return errors.New("unsupported document language") } - // initliaze text splitter + // initialize text splitter var split textsplitter.TextSplitter switch cachedDS.TextSplitter { case "token": diff --git a/pkg/arctl/datasource.go b/pkg/arctl/datasource.go index 34569572e..e12f59fa4 100644 --- a/pkg/arctl/datasource.go +++ b/pkg/arctl/datasource.go @@ -207,7 +207,7 @@ func DatasourceDeleteCmd(kubeClient dynamic.Interface, namespace string) *cobra. Namespace: namespace, }) if err != nil { - return fmt.Errorf("faield to delete datasource: %w", err) + return fmt.Errorf("failed to delete datasource: %w", err) } klog.Infof("Successfully deleted datasource %s\n", name) return err diff --git a/pkg/cache/memeory.go b/pkg/cache/memory.go similarity index 100% rename from pkg/cache/memeory.go rename to pkg/cache/memory.go diff --git a/pkg/cache/memeory_test.go b/pkg/cache/memory_test.go similarity index 100% rename from pkg/cache/memeory_test.go rename to pkg/cache/memory_test.go diff --git a/pkg/datasource/datasource.go b/pkg/datasource/datasource.go index a00e4101f..525e975a5 100644 --- a/pkg/datasource/datasource.go +++ b/pkg/datasource/datasource.go @@ -28,7 +28,7 @@ import ( ) var ( - ErrUnknowDatasourceType = errors.New("unknow datasource type") + ErrUnknowDatasourceType = errors.New("unknown datasource type") ErrBucketNotProvided = errors.New("no bucket provided") ErrOSSNoSuchBucket = errors.New("no such bucket") ErrOSSNoSuchObject = errors.New("no such object in bucket") diff --git a/pkg/datasource/oss.go b/pkg/datasource/oss.go index 6648dada6..c3beb080d 100644 --- a/pkg/datasource/oss.go +++ b/pkg/datasource/oss.go @@ -63,7 +63,7 @@ func NewOSS(ctx context.Context, c client.Client, dc dynamic.Interface, endpoint var accessKeyID, secretAccessKey string if endpoint.AuthSecret != nil { if endpoint.AuthSecret.Namespace == nil { - return nil, errors.New("no namepsace found for endpoint.authsecret") + return nil, errors.New("no namespace found for endpoint.authsecret") } if err := utils.ValidateClient(c, dc); err != nil { return nil, err @@ -125,7 +125,7 @@ func NewOSS(ctx context.Context, c client.Client, dc dynamic.Interface, endpoint return &OSS{Client: mc, Core: core}, nil } -// Check oss agains info() +// Check oss against info() func (oss *OSS) Stat(ctx context.Context, info any) error { if info == nil { return nil @@ -177,7 +177,7 @@ func (oss *OSS) statObject(ctx context.Context, ossInfo *v1alpha1.OSS) error { // check whether object exists if ossInfo.Object != "" { - // The object by `ListObjects` will trim "/" automatically,so we also need to trim "/" to make sure name comparision successful + // The object by `ListObjects` will trim "/" automatically,so we also need to trim "/" to make sure name comparison successful ossInfo.Object = strings.TrimPrefix(ossInfo.Object, "/") // When object contains "/" which means it is a directory,'ListObjects' will show all objects under that directory without object itself // After we remove "/", the objects by `ListObjects` will have object itself included. diff --git a/pkg/langchainwrap/llm.go b/pkg/langchainwrap/llm.go index 5e230f49b..861838047 100644 --- a/pkg/langchainwrap/llm.go +++ b/pkg/langchainwrap/llm.go @@ -51,7 +51,7 @@ func GetLangchainLLM(ctx context.Context, llm *v1alpha1.LLM, c client.Client, cl z := zhipuai.NewZhiPuAI(apiKey) return &zhipuai.ZhiPuAILLM{ZhiPuAI: *z, RetryTimes: 3}, nil case llms.OpenAI: - return openai.NewChat(openai.WithToken(apiKey), openai.WithBaseURL(llm.Spec.Enpoint.URL)) + return openai.NewChat(openai.WithToken(apiKey), openai.WithBaseURL(llm.Spec.Endpoint.URL)) } case v1alpha1.ProviderTypeWorker: gateway, err := config.GetGateway(ctx, c, cli) diff --git a/pkg/llms/dashscope/api.go b/pkg/llms/dashscope/api.go index 397df5115..adab44769 100644 --- a/pkg/llms/dashscope/api.go +++ b/pkg/llms/dashscope/api.go @@ -162,7 +162,7 @@ func (z *DashScope) GetTaskDetail(ctx context.Context, taskID string) (outURL st } data := respData.Output.EmbeddingOutputASync if data == nil { - return "", fmt.Errorf("cant find data in resp:%+v", respData) + return "", fmt.Errorf("can't find data in resp:%+v", respData) } if data.TaskStatus != TaskStatusSucceeded { return "", fmt.Errorf("taskStatus:%s, message:%s", data.TaskStatus, data.Message) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 59b536485..28154a97a 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -55,7 +55,7 @@ func NewScheduler(ctx context.Context, c client.Client, instance *v1alpha1.Versi cancel() return nil, err } - endpoint := systemDatasource.Spec.Enpoint.DeepCopy() + endpoint := systemDatasource.Spec.Endpoint.DeepCopy() if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil { endpoint.AuthSecret.WithNameSpace(systemDatasource.Namespace) } @@ -67,12 +67,12 @@ func NewScheduler(ctx context.Context, c client.Client, instance *v1alpha1.Versi } s := &Scheduler{ctx: ctx1, cancel: cancel, ds: instance, client: c, remove: remove} - exectuor, err := butcher.NewButcher[JobPayload](newExecutor(ctx1, c, oss, instance, fileStatus, remove), butcher.BufferSize(bufSize), butcher.MaxWorker(maxWorkers)) + executor, err := butcher.NewButcher[JobPayload](newExecutor(ctx1, c, oss, instance, fileStatus, remove), butcher.BufferSize(bufSize), butcher.MaxWorker(maxWorkers)) if err != nil { cancel() return nil, err } - s.runner = exectuor + s.runner = executor return s, nil } diff --git a/pkg/worker/devices.go b/pkg/worker/devices.go index 218575e1b..a61e53ccc 100644 --- a/pkg/worker/devices.go +++ b/pkg/worker/devices.go @@ -41,7 +41,7 @@ const ( ResourceNvidiaGPU corev1.ResourceName = "nvidia.com/gpu" ) -// DeviceBasedOnResource returns the devide type based on the resource list +// DeviceBasedOnResource returns the device type based on the resource list func DeviceBasedOnResource(resource corev1.ResourceList) Device { _, ok := resource[ResourceNvidiaGPU] if ok { diff --git a/pkg/worker/runner.go b/pkg/worker/runner.go index d800dadc6..a74033d5d 100644 --- a/pkg/worker/runner.go +++ b/pkg/worker/runner.go @@ -55,7 +55,7 @@ func (runner *RunnerFastchat) Device() Device { return DeviceBasedOnResource(runner.w.Spec.Resources.Limits) } -// NumberOfGPUs utlized by this runner +// NumberOfGPUs utilized by this runner func (runner *RunnerFastchat) NumberOfGPUs() string { return NumberOfGPUs(runner.w.Spec.Resources.Limits) } @@ -120,12 +120,12 @@ func NewRunnerFastchatVLLM(c client.Client, w *arcadiav1alpha1.Worker) (ModelRun }, nil } -// Devicde used by this runner +// Device used by this runner func (runner *RunnerFastchatVLLM) Device() Device { return DeviceBasedOnResource(runner.w.Spec.Resources.Limits) } -// NumberOfGPUs utlized by this runner +// NumberOfGPUs utilized by this runner func (runner *RunnerFastchatVLLM) NumberOfGPUs() string { return NumberOfGPUs(runner.w.Spec.Resources.Limits) } diff --git a/pkg/worker/worker.go b/pkg/worker/worker.go index 081824528..83b4f8e72 100644 --- a/pkg/worker/worker.go +++ b/pkg/worker/worker.go @@ -91,7 +91,7 @@ type Worker interface { var _ Worker = (*PodWorker)(nil) -// PodWorker hosts this worker in a single pod but with different loader and runner based on Worker's confiugration +// PodWorker hosts this worker in a single pod but with different loader and runner based on Worker's configuration type PodWorker struct { c client.Client s *runtime.Scheme @@ -195,7 +195,7 @@ func NewPodWorker(ctx context.Context, c client.Client, s *runtime.Scheme, w *ar podWorker.deployment = deployment // init loader(Only oss supported yet) - endpoint := d.Spec.Enpoint.DeepCopy() + endpoint := d.Spec.Endpoint.DeepCopy() if endpoint.AuthSecret != nil && endpoint.AuthSecret.Namespace == nil { endpoint.AuthSecret.WithNameSpace(d.Namespace) } diff --git a/tests/deploy-values.yaml b/tests/deploy-values.yaml index 749baa12b..9a08a48a9 100644 --- a/tests/deploy-values.yaml +++ b/tests/deploy-values.yaml @@ -101,8 +101,8 @@ dataprocess: llm: qa_retry_count: '2' -# @section postgresql is used to configure postgreslq service -# Posgresql service will be used in two parts: +# @section postgresql is used to configure postgresql service +# Postgresql service will be used in two parts: # - dataprocessing # - llm application # - used as vector store with pgvector enabled