Merge pull request #875 from 0xff-dev/support-list-all

fix: support getting all data
kubeagi · Mar 17, 2024 · 60cf39a · 60cf39a
2 parents c1ebe9b + 368e868
commit 60cf39a
Show file tree

Hide file tree

Showing 12 changed files with 28 additions and 73 deletions.
diff --git a/apiserver/graph/schema/entrypoint.graphqls b/apiserver/graph/schema/entrypoint.graphqls
@@ -35,7 +35,7 @@ input ListCommonInput {
 
     """
     每页数量，
-    规则: 默认10
+    规则: -1,返回全部
     """
     pageSize: Int
 }

diff --git a/apiserver/pkg/application/application.go b/apiserver/pkg/application/application.go
@@ -335,7 +335,7 @@ func GetApplication(ctx context.Context, c client.Client, name, namespace string
 func ListApplicationMeatadatas(ctx context.Context, c client.Client, input generated.ListCommonInput) (*generated.PaginatedResult, error) {
 	keyword := pointer.StringDeref(input.Keyword, "")
 	page := pointer.IntDeref(input.Page, 1)
-	pageSize := pointer.IntDeref(input.PageSize, 10)
+	pageSize := pointer.IntDeref(input.PageSize, -1)
 	res := &v1alpha1.ApplicationList{}
 	opts, err := common.NewListOptions(input)
 	if err != nil {

diff --git a/apiserver/pkg/dataset/dataset.go b/apiserver/pkg/dataset/dataset.go
@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kubeagi/arcadia/api/base/v1alpha1"
@@ -126,13 +127,8 @@ func ListDatasets(ctx context.Context, c client.Client, input *generated.ListDat
 	if err != nil {
 		return nil, err
 	}
-	page, size := 1, 10
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil && *input.PageSize > 0 {
-		size = *input.PageSize
-	}
+	page := pointer.IntDeref(input.Page, 1)
+	size := pointer.IntDeref(input.PageSize, -1)
 	filters := make([]common.ResourceFilter, 0)
 	if input.DisplayName != nil {
 		filters = append(filters, common.FilterDatasetByDisplayName(*input.DisplayName))

diff --git a/apiserver/pkg/datasource/datasource.go b/apiserver/pkg/datasource/datasource.go
@@ -206,16 +206,11 @@ func DeleteDatasources(ctx context.Context, c client.Client, input *generated.De
 
 func ListDatasources(ctx context.Context, c client.Client, input generated.ListCommonInput) (*generated.PaginatedResult, error) {
 	keyword := ""
-	page, pageSize := 1, 10
+	page := pointer.IntDeref(input.Page, 1)
+	pageSize := pointer.IntDeref(input.PageSize, -1)
 	if input.Keyword != nil {
 		keyword = *input.Keyword
 	}
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil && *input.PageSize > 0 {
-		pageSize = *input.PageSize
-	}
 
 	opts, err := common.NewListOptions(input)
 	if err != nil {

diff --git a/apiserver/pkg/embedder/embedder.go b/apiserver/pkg/embedder/embedder.go
@@ -221,16 +221,11 @@ func ListEmbedders(ctx context.Context, c client.Client, input generated.ListCom
 	}
 
 	keyword := ""
-	page, pageSize := 1, 10
+	page := pointer.IntDeref(input.Page, 1)
+	pageSize := pointer.IntDeref(input.PageSize, -1)
 	if input.Keyword != nil {
 		keyword = *input.Keyword
 	}
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil {
-		pageSize = *input.PageSize
-	}
 
 	us := &v1alpha1.EmbedderList{}
 	list, err := common.NewListOptions(input)

diff --git a/apiserver/pkg/knowledgebase/knowledgebase.go b/apiserver/pkg/knowledgebase/knowledgebase.go
@@ -311,20 +311,15 @@ func ReadKnowledgeBase(ctx context.Context, c client.Client, name, namespace str
 }
 
 func ListKnowledgeBases(ctx context.Context, c client.Client, input generated.ListKnowledgeBaseInput) (*generated.PaginatedResult, error) {
-	page, pageSize := 1, 10
 	filter := make([]common.ResourceFilter, 0)
 	if input.Name != nil {
 		filter = append(filter, common.FilterByNameContains(*input.Name))
 	}
 	if input.DisplayName != nil {
 		filter = append(filter, common.FilterKnowledgeByDisplayName(*input.DisplayName))
 	}
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil && *input.PageSize > 0 {
-		pageSize = *input.PageSize
-	}
+	page := pointer.IntDeref(input.Page, 1)
+	pageSize := pointer.IntDeref(input.PageSize, -1)
 
 	us := &v1alpha1.KnowledgeBaseList{}
 	opts, err := common.NewListOptions(generated.ListCommonInput{

diff --git a/apiserver/pkg/llm/llm.go b/apiserver/pkg/llm/llm.go
@@ -22,6 +22,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kubeagi/arcadia/api/base/v1alpha1"
@@ -105,17 +106,12 @@ func ListLLMs(ctx context.Context, c client.Client, input generated.ListCommonIn
 		optFunc(opts)
 	}
 
-	page, pageSize := 1, 10
 	filter := make([]common.ResourceFilter, 0)
 	if input.Keyword != nil {
 		filter = append(filter, common.FilterLLMByKeyword(*input.Keyword))
 	}
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil {
-		pageSize = *input.PageSize
-	}
+	page := pointer.IntDeref(input.Page, 1)
+	pageSize := pointer.IntDeref(input.PageSize, -1)
 
 	us := &v1alpha1.LLMList{}
 	options, err := common.NewListOptions(input)

diff --git a/apiserver/pkg/model/model.go b/apiserver/pkg/model/model.go
@@ -132,16 +132,11 @@ func DeleteModels(ctx context.Context, c client.Client, input *generated.DeleteC
 
 func ListModels(ctx context.Context, c client.Client, input generated.ListModelInput) (*generated.PaginatedResult, error) {
 	filter := make([]common.ResourceFilter, 0)
-	page, pageSize := 1, 10
+	page := pointer.IntDeref(input.Page, 1)
+	pageSize := pointer.IntDeref(input.PageSize, -1)
 	if input.Keyword != nil {
 		filter = append(filter, common.FilterModelByKeyword(*input.Keyword))
 	}
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil && *input.PageSize > 0 {
-		pageSize = *input.PageSize
-	}
 
 	models := &v1alpha1.ModelList{}
 	opts, err := common.NewListOptions(generated.ListCommonInput{

diff --git a/apiserver/pkg/modelservice/modelservice.go b/apiserver/pkg/modelservice/modelservice.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	"github.com/tmc/langchaingo/llms"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kubeagi/arcadia/api/base/v1alpha1"
@@ -195,8 +196,9 @@ func ListModelServices(ctx context.Context, c client.Client, input *generated.Li
 	// use `UnlimitedPageSize` so we can get all llms and embeddings
 	notWorkerSelector := fmt.Sprintf("%s=%s", v1alpha1.ProviderLabel, v1alpha1.ProviderType3rdParty)
 
+	pageOne := 1
 	query := generated.ListCommonInput{
-		Page:          input.Page,
+		Page:          &pageOne,
 		PageSize:      &common.UnlimitedPageSize,
 		Namespace:     input.Namespace,
 		Keyword:       input.Keyword,
@@ -281,13 +283,8 @@ func ListModelServices(ctx context.Context, c client.Client, input *generated.Li
 	})
 
 	// return ModelService with the actual Page and PageSize
-	page, pageSize := 1, 10
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil && *input.PageSize > 0 {
-		pageSize = *input.PageSize
-	}
+	page := pointer.IntDeref(input.Page, 1)
+	pageSize := pointer.IntDeref(input.PageSize, -1)
 
 	totalCount := len(newNodeList)
 	start, end := common.PagePosition(page, pageSize, totalCount)

diff --git a/apiserver/pkg/rag/rag.go b/apiserver/pkg/rag/rag.go
@@ -467,13 +467,8 @@ func UpdateRAG(ctx context.Context, kubeClient client.Client, input *generated.U
 }
 
 func ListRAG(ctx context.Context, kubeClient client.Client, input *generated.ListRAGInput) (*generated.PaginatedResult, error) {
-	page, size := 1, 10
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil && *input.PageSize > 0 {
-		size = *input.PageSize
-	}
+	page := pointer.IntDeref(input.Page, 1)
+	size := pointer.IntDeref(input.PageSize, -1)
 	filter := make([]common.ResourceFilter, 0)
 	if input.Keyword != nil {
 		filter = append(filter, common.FilterByRAGKeyword(*input.Keyword))

diff --git a/apiserver/pkg/versioneddataset/versioned_dataset.go b/apiserver/pkg/versioneddataset/versioned_dataset.go
@@ -186,13 +186,8 @@ func VersionFiles(ctx context.Context, _ client.Client, input *generated.Version
 }
 
 func ListVersionedDatasets(ctx context.Context, c client.Client, input *generated.ListVersionedDatasetInput) (*generated.PaginatedResult, error) {
-	page, size := 1, 10
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil && *input.PageSize > 0 {
-		size = *input.PageSize
-	}
+	page := pointer.IntDeref(input.Page, 1)
+	size := pointer.IntDeref(input.PageSize, -1)
 	ns := "default"
 	if input.Namespace != nil {
 		ns = *input.Namespace

diff --git a/apiserver/pkg/worker/worker.go b/apiserver/pkg/worker/worker.go
@@ -28,6 +28,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kubeagi/arcadia/api/base/v1alpha1"
@@ -355,19 +356,14 @@ func ListWorkers(ctx context.Context, c client.Client, input generated.ListWorke
 	}
 
 	filter := make([]common.ResourceFilter, 0)
-	page, pageSize := 1, 10
+	page := pointer.IntDeref(input.Page, 1)
+	pageSize := pointer.IntDeref(input.PageSize, -1)
 	if input.Keyword != nil {
 		filter = append(filter, common.FilterWorkerByKeyword(*input.Keyword))
 	}
 	if input.ModelTypes != nil {
 		filter = append(filter, common.FilterWorkerByType(c, input.Namespace, *input.ModelTypes))
 	}
-	if input.Page != nil && *input.Page > 0 {
-		page = *input.Page
-	}
-	if input.PageSize != nil {
-		pageSize = *input.PageSize
-	}
 
 	us := &v1alpha1.WorkerList{}
 	options, err := common.NewListOptions(generated.ListCommonInput{