Merge pull request #496 from Abirdcfly/pg

feat: add datasource type postgresql

api/base/v1alpha1/datasource.go

@@ -23,19 +23,21 @@ const (
 type DatasourceType string
 
 const (
-	DatasourceTypeOSS     DatasourceType = "oss"
-	DatasourceTypeRDMA    DatasourceType = "RDMA"
-	DatasourceTypeUnknown DatasourceType = "unknown"
+	DatasourceTypeOSS        DatasourceType = "oss"
+	DatasourceTypeRDMA       DatasourceType = "RDMA"
+	DatasourceTypePostgreSQL DatasourceType = "postgresql"
+	DatasourceTypeUnknown    DatasourceType = "unknown"
 )
 
 func (ds DatasourceSpec) Type() DatasourceType {
-	// Object storage service
-	if ds.OSS != nil {
+	switch {
+	case ds.OSS != nil:
 		return DatasourceTypeOSS
-	}
-	if ds.RDMA != nil {
+	case ds.RDMA != nil:
 		return DatasourceTypeRDMA
+	case ds.PostgreSQL != nil:
+		return DatasourceTypePostgreSQL
+	default:
+		return DatasourceTypeUnknown
 	}
-
-	return DatasourceTypeUnknown
 }

api/base/v1alpha1/datasource_types.go

@@ -20,9 +20,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
-// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
-
 // DatasourceSpec defines the desired state of Datasource
 type DatasourceSpec struct {
 	CommonSpec `json:",inline"`
@@ -35,6 +32,9 @@ type DatasourceSpec struct {
 
 	// RDMA configure RDMA pulls the model file directly from the remote service to the host node.
 	RDMA *RDMA `json:"rdma,omitempty"`
+
+	// PostgreSQL defines info for PostgreSQL
+	PostgreSQL *PostgreSQL `json:"postgresql,omitempty"`
 }
 
 type RDMA struct {
@@ -54,6 +54,37 @@ type OSS struct {
 	Object string `json:"object,omitempty"`
 }
 
+// PostgreSQL defines info for PostgreSQL
+//
+// ref: https://github.com/jackc/pgx/blame/v5.5.1/pgconn/config.go#L409
+// they are common standard PostgreSQL environment variables
+// For convenience, we use the same name.
+//
+// The PGUSER/PGPASSWORD/PGPASSFILE/PGSSLPASSWORD parameters have been intentionally excluded
+// because they contain sensitive information and are stored in the secret pointed to by `endpoint.authSecret`.
+type PostgreSQL struct {
+	Host               string `json:"PGHOST,omitempty"`
+	Port               string `json:"PGPORT,omitempty"`
+	Database           string `json:"PGDATABASE,omitempty"`
+	AppName            string `json:"PGAPPNAME,omitempty"`
+	ConnectTimeout     string `json:"PGCONNECT_TIMEOUT,omitempty"`
+	SSLMode            string `json:"PGSSLMODE,omitempty"`
+	SSLKey             string `json:"PGSSLKEY,omitempty"`
+	SSLCert            string `json:"PGSSLCERT,omitempty"`
+	SSLSni             string `json:"PGSSLSNI,omitempty"`
+	SSLRootCert        string `json:"PGSSLROOTCERT,omitempty"`
+	TargetSessionAttrs string `json:"PGTARGETSESSIONATTRS,omitempty"`
+	Service            string `json:"PGSERVICE,omitempty"`
+	ServiceFile        string `json:"PGSERVICEFILE,omitempty"`
+}
+
+const (
+	PGUSER        = "PGUSER"
+	PGPASSWORD    = "PGPASSWORD"
+	PGPASSFILE    = "PGPASSFILE"
+	PGSSLPASSWORD = "PGSSLPASSWORD"
+)
+
 // DatasourceStatus defines the observed state of Datasource
 type DatasourceStatus struct {
 	// ConditionedStatus is the current status

config/crd/bases/arcadia.kubeagi.k8s.com.cn_datasources.yaml

@@ -99,6 +99,36 @@ spec:
                     description: Object must end with a slash "/" if it is a directory
                     type: string
                 type: object
+              postgresql:
+                description: PostgreSQL defines info for PostgreSQL
+                properties:
+                  PGAPPNAME:
+                    type: string
+                  PGCONNECT_TIMEOUT:
+                    type: string
+                  PGDATABASE:
+                    type: string
+                  PGHOST:
+                    type: string
+                  PGPORT:
+                    type: string
+                  PGSERVICE:
+                    type: string
+                  PGSERVICEFILE:
+                    type: string
+                  PGSSLCERT:
+                    type: string
+                  PGSSLKEY:
+                    type: string
+                  PGSSLMODE:
+                    type: string
+                  PGSSLROOTCERT:
+                    type: string
+                  PGSSLSNI:
+                    type: string
+                  PGTARGETSESSIONATTRS:
+                    type: string
+                type: object
               rdma:
                 description: RDMA configure RDMA pulls the model file directly from
                   the remote service to the host node.

config/samples/arcadia_v1alpha1_datasource_postgresql.yaml

@@ -0,0 +1,24 @@
+## Datasource secret
+apiVersion: v1
+kind: Secret
+metadata:
+  name: datasource-postgresql-sample-authsecret
+  namespace: arcadia
+data:
+  PGUSER: YWRtaW4=
+  PGPASSWORD: UGFzc3cwcmQh
+---
+apiVersion: arcadia.kubeagi.k8s.com.cn/v1alpha1
+kind: Datasource
+metadata:
+  name: datasource-postgresql-sample
+  namespace: arcadia
+spec:
+  displayName: "postgresql 数据源示例"
+  endpoint:
+    url: postgres://arcadia-postgresql.arcadia.svc.cluster.local:5432
+    authSecret:
+      kind: Secret
+      name: datasource-postgresql-sample-authsecret
+  postgresql:
+    PGDATABASE: arcadia

controllers/datasource_controller.go

@@ -154,23 +154,28 @@ func (r *DatasourceReconciler) Checkdatasource(ctx context.Context, logger logr.
 	logger.V(5).Info("check datasource")
 	var err error
 
+	endpoint := instance.Spec.Endpoint.DeepCopy()
+	// set auth secret's namespace to the datasource's namespace
+	if endpoint.AuthSecret != nil {
+		endpoint.AuthSecret.WithNameSpace(instance.Namespace)
+	}
 	// create datasource
 	var ds datasource.Datasource
 	var info any
 	switch instance.Spec.Type() {
 	case arcadiav1alpha1.DatasourceTypeOSS:
-		endpoint := instance.Spec.Endpoint.DeepCopy()
-		// set auth secret's namespace to the datasource's namespace
-		if endpoint.AuthSecret != nil {
-			endpoint.AuthSecret.WithNameSpace(instance.Namespace)
-		}
 		ds, err = datasource.NewOSS(ctx, r.Client, nil, endpoint)
 		if err != nil {
 			return r.UpdateStatus(ctx, instance, err)
 		}
 		info = instance.Spec.OSS.DeepCopy()
 	case arcadiav1alpha1.DatasourceTypeRDMA:
 		return r.UpdateStatus(ctx, instance, nil)
+	case arcadiav1alpha1.DatasourceTypePostgreSQL:
+		ds, err = datasource.NewPostgreSQL(ctx, r.Client, nil, instance.Spec.PostgreSQL, endpoint)
+		if err != nil {
+			return r.UpdateStatus(ctx, instance, err)
+		}
 	default:
 		ds, err = datasource.NewUnknown(ctx, r.Client)
 		if err != nil {

deploy/charts/arcadia/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: arcadia
 description: A Helm chart(KubeBB Component) for KubeAGI Arcadia
 type: application
-version: 0.2.3
+version: 0.2.4
 appVersion: "0.1.0"
 
 keywords:
@@ -13,8 +13,14 @@ sources:
   - https://github.com/kubeagi/arcadia
 
 maintainers:
+  - name: nkwangleiGIT
+    url: https://github.com/nkwangleiGIT
   - name: bjwswang
     url: https://github.com/bjwswang
+  - name: Abirdcfly
+    url: https://github.com/Abirdcfly
+  - name: 0xff-dev
+    url: https://github.com/0xff-dev
   - name: lanture1064
     url: https://github.com/lanture1064
 

deploy/charts/arcadia/crds/arcadia.kubeagi.k8s.com.cn_datasources.yaml

@@ -99,6 +99,36 @@ spec:
                     description: Object must end with a slash "/" if it is a directory
                     type: string
                 type: object
+              postgresql:
+                description: PostgreSQL defines info for PostgreSQL
+                properties:
+                  PGAPPNAME:
+                    type: string
+                  PGCONNECT_TIMEOUT:
+                    type: string
+                  PGDATABASE:
+                    type: string
+                  PGHOST:
+                    type: string
+                  PGPORT:
+                    type: string
+                  PGSERVICE:
+                    type: string
+                  PGSERVICEFILE:
+                    type: string
+                  PGSSLCERT:
+                    type: string
+                  PGSSLKEY:
+                    type: string
+                  PGSSLMODE:
+                    type: string
+                  PGSSLROOTCERT:
+                    type: string
+                  PGSSLSNI:
+                    type: string
+                  PGTARGETSESSIONATTRS:
+                    type: string
+                type: object
               rdma:
                 description: RDMA configure RDMA pulls the model file directly from
                   the remote service to the host node.

deploy/charts/arcadia/templates/post-datasource.yaml

@@ -19,3 +19,36 @@ spec:
   oss:
     # pre-defined buckets for arcadia
     bucket: {{ .Release.Namespace }}
+
+{{- if .Values.postgresql.enabled }}
+---
+apiVersion: arcadia.kubeagi.k8s.com.cn/v1alpha1
+kind: Datasource
+metadata:
+  name:  {{ .Release.Name }}-postgresql
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "1"
+spec:
+  displayName: "内置PG数据源"
+  description: "Arcadia 内置PG数据源"
+  endpoint:
+    url: postgres://{{ .Release.Name }}-postgresql.{{ .Release.Namespace }}:5432
+    internalURL: postgres://{{ .Release.Name }}-postgresql.{{ .Release.Namespace }}:5432
+    authSecret:
+      kind: Secret
+      name: {{ .Release.Name }}-postgresql-datasource-authsecret
+  postgresql:
+    PGDATABASE: {{ .Values.postgresql.global.postgresql.auth.database }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-postgresql-datasource-authsecret
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  PGUSER: {{ .Values.postgresql.global.postgresql.auth.username  | b64enc | quote }}
+  PGPASSWORD: {{ .Values.postgresql.global.postgresql.auth.password  | b64enc | quote }}
+{{- end }}

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/go-logr/logr v1.2.3
 	github.com/gofiber/fiber/v2 v2.49.1
 	github.com/golang-jwt/jwt v3.2.2+incompatible
+	github.com/jackc/pgx/v5 v5.4.1
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.27.3
 	github.com/r3labs/sse/v2 v2.10.0
@@ -54,6 +55,9 @@ require (
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.3 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/puddle/v2 v2.2.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
@@ -74,6 +78,7 @@ require (
 	github.com/yargevad/filepathx v1.0.0 // indirect
 	go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 // indirect
 	golang.org/x/arch v0.6.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )

go.sum

@@ -422,6 +422,14 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE=
+github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
+github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk=
+github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=

pkg/datasource/oss.go

@@ -17,7 +17,6 @@ package datasource
 import (
 	"context"
 	"crypto/tls"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -75,13 +74,8 @@ func NewOSS(ctx context.Context, c client.Client, dc dynamic.Interface, endpoint
 				return nil, err
 			}
 			data, _, _ := unstructured.NestedStringMap(secret.Object, "data")
-
-			if ds, err := base64.StdEncoding.DecodeString(data["rootUser"]); err == nil {
-				accessKeyID = string(ds)
-			}
-			if ds, err := base64.StdEncoding.DecodeString(data["rootPassword"]); err == nil {
-				secretAccessKey = string(ds)
-			}
+			accessKeyID = utils.DecodeBase64Str(data["rootUser"])
+			secretAccessKey = utils.DecodeBase64Str(data["rootPassword"])
 		}
 		if c != nil {
 			secret := corev1.Secret{}