From 29aa5e9625222f68725f11202cbcdc8b7b2f0381 Mon Sep 17 00:00:00 2001 From: wuyan-dushang <1550693033@qq.com> Date: Fri, 15 Sep 2023 08:21:14 +0800 Subject: [PATCH] install --- app/dubboctl/cmd/build.go | 12 +- app/dubboctl/cmd/dashboard_all_cmds.go | 45 +++---- app/dubboctl/cmd/deploy.go | 125 +++++++++++++++++- app/dubboctl/cmd/manifest_generate.go | 11 +- app/dubboctl/cmd/manifest_install.go | 3 +- app/dubboctl/cmd/manifest_uninstall.go | 3 +- app/dubboctl/cmd/repository_test.go | 122 ----------------- app/dubboctl/cmd/root.go | 2 +- app/dubboctl/internal/dubbo/client.go | 25 ++-- app/dubboctl/internal/dubbo/deploy.tpl | 24 +++- app/dubboctl/internal/dubbo/deployer.go | 9 ++ app/dubboctl/internal/dubbo/dubbo.go | 36 ++--- app/dubboctl/internal/kube/client.go | 55 +++++++- app/dubboctl/internal/kube/client_test.go | 13 +- app/dubboctl/internal/kube/common.go | 2 +- .../internal/{operator => kube}/component.go | 2 +- .../{operator => kube}/component_test.go | 6 +- .../internal/{operator => kube}/operator.go | 10 +- ...dmin_component-render_manifest.golden.yaml | 0 ...acos_component-render_manifest.golden.yaml | 0 ...heus_component-render_manifest.golden.yaml | 1 + ...king_component-render_manifest.golden.yaml | 0 ...pkin_component-render_manifest.golden.yaml | 0 ...eper_component-render_manifest.golden.yaml | 0 docs/cmd/dubboctl/deploy.md | 43 ++++++ 25 files changed, 332 insertions(+), 217 deletions(-) rename app/dubboctl/internal/{operator => kube}/component.go (99%) rename app/dubboctl/internal/{operator => kube}/component_test.go (98%) rename app/dubboctl/internal/{operator => kube}/operator.go (96%) rename app/dubboctl/internal/{operator => kube}/testdata/want/admin_component-render_manifest.golden.yaml (100%) rename app/dubboctl/internal/{operator => kube}/testdata/want/nacos_component-render_manifest.golden.yaml (100%) rename app/dubboctl/internal/{operator => kube}/testdata/want/prometheus_component-render_manifest.golden.yaml (99%) rename app/dubboctl/internal/{operator => kube}/testdata/want/skywalking_component-render_manifest.golden.yaml (100%) rename app/dubboctl/internal/{operator => kube}/testdata/want/zipkin_component-render_manifest.golden.yaml (100%) rename app/dubboctl/internal/{operator => kube}/testdata/want/zookeeper_component-render_manifest.golden.yaml (100%) create mode 100644 docs/cmd/dubboctl/deploy.md diff --git a/app/dubboctl/cmd/build.go b/app/dubboctl/cmd/build.go index fd718d68e..c0a96a2f5 100644 --- a/app/dubboctl/cmd/build.go +++ b/app/dubboctl/cmd/build.go @@ -17,6 +17,7 @@ package cmd import ( "fmt" + "os" "strings" "github.com/AlecAivazis/survey/v2" @@ -78,7 +79,7 @@ func runBuildCmd(cmd *cobra.Command, newClient ClientFactory) error { } cfg.Configure(f) - clientOptions, err := cfg.clientOptions() + clientOptions, err := cfg.buildclientOptions() if err != nil { return err } @@ -131,7 +132,7 @@ func (c *buildConfig) Prompt(d *dubbo.Dubbo) (*buildConfig, error) { return c, err } -func (c buildConfig) clientOptions() ([]dubbo.Option, error) { +func (c buildConfig) buildclientOptions() ([]dubbo.Option, error) { var o []dubbo.Option if c.UseDockerfile { @@ -139,6 +140,7 @@ func (c buildConfig) clientOptions() ([]dubbo.Option, error) { } else { o = append(o, dubbo.WithBuilder(pack.NewBuilder())) } + return o, nil } @@ -178,7 +180,11 @@ func newBuildConfig(cmd *cobra.Command) *buildConfig { func (c *buildConfig) Configure(f *dubbo.Dubbo) { if c.Path == "" { - f.Root = "." + root, err := os.Getwd() + if err != nil { + return + } + f.Root = root } else { f.Root = c.Path } diff --git a/app/dubboctl/cmd/dashboard_all_cmds.go b/app/dubboctl/cmd/dashboard_all_cmds.go index e302df434..615436b6b 100644 --- a/app/dubboctl/cmd/dashboard_all_cmds.go +++ b/app/dubboctl/cmd/dashboard_all_cmds.go @@ -29,7 +29,6 @@ import ( "github.com/apache/dubbo-kubernetes/app/dubboctl/identifier" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/kube" - "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/operator" "github.com/apache/dubbo-kubernetes/pkg/core/logger" "github.com/spf13/cobra" "go.uber.org/zap/zapcore" @@ -41,22 +40,22 @@ import ( var ( // TODO: think about a efficient way to change selectors and ports when yaml files change // ports are coming from /deploy/charts and /deploy/kubernetes - ComponentPortMap = map[operator.ComponentName]int{ - operator.Admin: 8080, - operator.Grafana: 3000, - operator.Nacos: 8848, - operator.Prometheus: 9090, - operator.Skywalking: 8080, - operator.Zipkin: 9411, + ComponentPortMap = map[kube.ComponentName]int{ + kube.Admin: 8080, + kube.Grafana: 3000, + kube.Nacos: 8848, + kube.Prometheus: 9090, + kube.Skywalking: 8080, + kube.Zipkin: 9411, } // ComponentSelectorMap selectors are coming from /deploy/charts and /deploy/kubernetes - ComponentSelectorMap = map[operator.ComponentName]string{ - operator.Admin: "app.kubernetes.io/name=dubbo-admin", - operator.Grafana: "app.kubernetes.io/name=grafana", - operator.Nacos: "app.kubernetes.io/name=nacos", - operator.Prometheus: "app=prometheus", - operator.Skywalking: "app=skywalking, component=ui", - operator.Zipkin: "app.kubernetes.io/name=zipkin", + ComponentSelectorMap = map[kube.ComponentName]string{ + kube.Admin: "app.kubernetes.io/name=dubbo-admin", + kube.Grafana: "app.kubernetes.io/name=grafana", + kube.Nacos: "app.kubernetes.io/name=nacos", + kube.Prometheus: "app=prometheus", + kube.Skywalking: "app=skywalking, component=ui", + kube.Zipkin: "app.kubernetes.io/name=zipkin", } ) @@ -82,7 +81,7 @@ func (dca *DashboardCommonArgs) setDefault() { } } -func commonDashboardCmd(baseCmd *cobra.Command, compName operator.ComponentName) { +func commonDashboardCmd(baseCmd *cobra.Command, compName kube.ComponentName) { nameStr := string(compName) lowerNameStr := strings.ToLower(nameStr) dcArgs := &DashboardCommonArgs{} @@ -125,30 +124,30 @@ func commonDashboardCmd(baseCmd *cobra.Command, compName operator.ComponentName) } func ConfigDashboardAdminCmd(baseCmd *cobra.Command) { - commonDashboardCmd(baseCmd, operator.Admin) + commonDashboardCmd(baseCmd, kube.Admin) } func ConfigDashboardGrafanaCmd(baseCmd *cobra.Command) { - commonDashboardCmd(baseCmd, operator.Grafana) + commonDashboardCmd(baseCmd, kube.Grafana) } func ConfigDashboardNacosCmd(baseCmd *cobra.Command) { - commonDashboardCmd(baseCmd, operator.Nacos) + commonDashboardCmd(baseCmd, kube.Nacos) } func ConfigDashboardPrometheusCmd(baseCmd *cobra.Command) { - commonDashboardCmd(baseCmd, operator.Prometheus) + commonDashboardCmd(baseCmd, kube.Prometheus) } func ConfigDashboardSkywalkingCmd(baseCmd *cobra.Command) { - commonDashboardCmd(baseCmd, operator.Skywalking) + commonDashboardCmd(baseCmd, kube.Skywalking) } func ConfigDashboardZipkinCmd(baseCmd *cobra.Command) { - commonDashboardCmd(baseCmd, operator.Zipkin) + commonDashboardCmd(baseCmd, kube.Zipkin) } -func portForward(args *DashboardCommonArgs, compName operator.ComponentName, writer io.Writer) error { +func portForward(args *DashboardCommonArgs, compName kube.ComponentName, writer io.Writer) error { // process args var podPort int podPort = ComponentPortMap[compName] diff --git a/app/dubboctl/cmd/deploy.go b/app/dubboctl/cmd/deploy.go index 929dff1ef..7a8802f82 100644 --- a/app/dubboctl/cmd/deploy.go +++ b/app/dubboctl/cmd/deploy.go @@ -16,12 +16,20 @@ package cmd import ( + "context" "errors" "fmt" "os" "os/exec" "path/filepath" + "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/kube" + corev1 "k8s.io/api/core/v1" + errors2 "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + client2 "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/dubbo" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/util" @@ -50,7 +58,7 @@ SYNOPSIS PreRunE: bindEnv("path", "output", "namespace", "image", "envs", "name", "secret", "replicas", "revisions", "containerPort", "targetPort", "nodePort", "requestCpu", "limitMem", "minReplicas", "serviceAccount", "serviceAccount", "imagePullPolicy", "maxReplicas", "limitCpu", "requestMem", - "apply", "useDockerfile", "push", "force", "builder-image", "nobuild"), + "apply", "useDockerfile", "force", "builder-image", "nobuild", "context", "kubeConfig", "push"), RunE: func(cmd *cobra.Command, args []string) error { return runDeploy(cmd, newClient) }, @@ -103,10 +111,14 @@ SYNOPSIS "Use the dockerfile with the specified path to build") cmd.Flags().StringP("image", "i", "", "Container image( [registry]/[namespace]/[name]:[tag] )") - cmd.Flags().BoolP("push", "", false, + cmd.Flags().BoolP("push", "", true, "Whether to push the image to the registry center by the way") cmd.Flags().BoolP("force", "f", false, "Whether to force push") + cmd.Flags().StringP("context", "", "", + "Context in kubeconfig to use") + cmd.Flags().StringP("kubeConfig", "k", "", + "Path to kubeconfig") addPathFlag(cmd) baseCmd.AddCommand(cmd) @@ -138,12 +150,97 @@ func runDeploy(cmd *cobra.Command, newClient ClientFactory) error { cfg.Configure(f) - clientOptions, err := cfg.clientOptions() + clientOptions, err := cfg.deployclientOptions() if err != nil { return err } client, done := newClient(clientOptions...) defer done() + + key := client2.ObjectKey{ + Namespace: metav1.NamespaceSystem, + Name: cfg.Namespace, + } + + err = client.KubeCtl.Get(context.Background(), key, &corev1.Namespace{}) + if err != nil { + if errors2.IsNotFound(err) { + nsObj := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: cfg.Namespace, + }, + } + if err := client.KubeCtl.Create(context.Background(), nsObj); err != nil { + return err + } + return nil + } else { + return fmt.Errorf("failed to check if namespace %v exists: %v", cfg.Namespace, err) + } + } + + namespaceSelector := client2.MatchingLabels{ + "dubbo-deploy": "enabled", + } + nsList := &corev1.NamespaceList{} + if err = client.KubeCtl.List(context.Background(), nsList, namespaceSelector); err != nil { + return err + } + var namespace string + if len(nsList.Items) > 0 { + namespace = nsList.Items[0].Name + } + + env := os.Getenv("DUBBO_DEPLOY_NS") + if env != "" { + namespace = env + } + + if namespace != "" { + zkSelector := client2.MatchingLabels{ + "dubbo.apache.org/zookeeper": "true", + } + + zkList := &corev1.ServiceList{} + if err := client.KubeCtl.List(context.Background(), zkList, zkSelector, client2.InNamespace(namespace)); err != nil { + return err + } + var name string + var dns string + if len(zkList.Items) > 0 { + name = zkList.Items[0].Name + dns = fmt.Sprintf("%s.%s.svc", name, namespace) + f.Deploy.ZookeeperAddress = dns + } + + nacosSelector := client2.MatchingLabels{ + "dubbo.apache.org/nacos": "true", + } + + nacosList := &corev1.ServiceList{} + if err := client.KubeCtl.List(context.Background(), nacosList, nacosSelector, client2.InNamespace(namespace)); err != nil { + return err + } + if len(nacosList.Items) > 0 { + name = nacosList.Items[0].Name + dns = fmt.Sprintf("%s.%s.svc", name, namespace) + f.Deploy.NacosAddress = dns + } + + promSelector := client2.MatchingLabels{ + "dubbo.apache.org/prometheus": "true", + } + + promList := &corev1.ServiceList{} + if err := client.KubeCtl.List(context.Background(), promList, promSelector, client2.InNamespace(namespace)); err != nil { + return err + } + if len(promList.Items) > 0 { + f.Deploy.UseProm = true + } + } + if !cfg.NoBuild { if f.Built() && !cfg.Force { fmt.Fprintf(cmd.OutOrStdout(), "The Application is up to date, If you still want to build, use `--force true`\n") @@ -177,6 +274,24 @@ func runDeploy(cmd *cobra.Command, newClient ClientFactory) error { return f.Stamp() } +func (d DeployConfig) deployclientOptions() ([]dubbo.Option, error) { + o, err := d.buildclientOptions() + if err != nil { + return o, err + } + var cliOpts []kube.CtlClientOption + cliOpts = []kube.CtlClientOption{ + kube.WithKubeConfigPath(d.KubeConfig), + kube.WithContext(d.Context), + } + cli, err := kube.NewCtlClient(cliOpts...) + if err != nil { + return o, err + } + o = append(o, dubbo.WithKubeClient(cli)) + return o, nil +} + func applyTok8s(cmd *cobra.Command, d *dubbo.Dubbo) error { file := filepath.Join(d.Root, d.Deploy.Output) c := exec.CommandContext(cmd.Context(), "kubectl", "apply", "-f", file) @@ -276,6 +391,8 @@ func (c DeployConfig) Configure(f *dubbo.Dubbo) { type DeployConfig struct { *buildConfig + KubeConfig string + Context string NoBuild bool Apply bool Namespace string @@ -300,6 +417,8 @@ type DeployConfig struct { func newDeployConfig(cmd *cobra.Command) (c *DeployConfig) { c = &DeployConfig{ buildConfig: newBuildConfig(cmd), + KubeConfig: viper.GetString("kubeConfig"), + Context: viper.GetString("context"), NoBuild: viper.GetBool("nobuild"), Apply: viper.GetBool("apply"), Output: viper.GetString("output"), diff --git a/app/dubboctl/cmd/manifest_generate.go b/app/dubboctl/cmd/manifest_generate.go index cc72a0e1d..fee028f04 100644 --- a/app/dubboctl/cmd/manifest_generate.go +++ b/app/dubboctl/cmd/manifest_generate.go @@ -22,6 +22,8 @@ import ( "sort" "strings" + "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/kube" + "github.com/apache/dubbo-kubernetes/pkg/core/logger" "go.uber.org/zap/zapcore" @@ -31,7 +33,6 @@ import ( "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/apis/dubbo.apache.org/v1alpha1" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/manifest" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/manifest/render" - "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/operator" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/util" "github.com/spf13/cobra" @@ -165,7 +166,7 @@ func generateValues(mgArgs *ManifestGenerateArgs) (*v1alpha1.DubboConfig, string } func generateManifests(mgArgs *ManifestGenerateArgs, cfg *v1alpha1.DubboConfig) error { - op, err := operator.NewDubboOperator(cfg.Spec, nil) + op, err := kube.NewDubboOperator(cfg.Spec, nil) if err != nil { return err } @@ -191,7 +192,7 @@ func generateManifests(mgArgs *ManifestGenerateArgs, cfg *v1alpha1.DubboConfig) return nil } -func sortManifests(manifestMap map[operator.ComponentName]string) (string, error) { +func sortManifests(manifestMap map[kube.ComponentName]string) (string, error) { var names []string var resBuilder strings.Builder for name := range manifestMap { @@ -199,7 +200,7 @@ func sortManifests(manifestMap map[operator.ComponentName]string) (string, error } sort.Strings(names) for _, name := range names { - file := manifestMap[operator.ComponentName(name)] + file := manifestMap[kube.ComponentName(name)] if !strings.HasSuffix(file, render.YAMLSeparator) { resBuilder.WriteString(file + render.YAMLSeparator) } else { @@ -209,7 +210,7 @@ func sortManifests(manifestMap map[operator.ComponentName]string) (string, error return resBuilder.String(), nil } -func writeManifests(manifestMap map[operator.ComponentName]string, outputPath string) error { +func writeManifests(manifestMap map[kube.ComponentName]string, outputPath string) error { if err := os.MkdirAll(outputPath, os.ModePerm); err != nil { return err } diff --git a/app/dubboctl/cmd/manifest_install.go b/app/dubboctl/cmd/manifest_install.go index 0db20f929..fda3be8a9 100644 --- a/app/dubboctl/cmd/manifest_install.go +++ b/app/dubboctl/cmd/manifest_install.go @@ -18,7 +18,6 @@ package cmd import ( "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/apis/dubbo.apache.org/v1alpha1" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/kube" - "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/operator" "github.com/apache/dubbo-kubernetes/pkg/core/logger" "github.com/spf13/cobra" "go.uber.org/zap/zapcore" @@ -80,7 +79,7 @@ func installManifests(miArgs *ManifestInstallArgs, cfg *v1alpha1.DubboConfig) er if err != nil { return err } - op, err := operator.NewDubboOperator(cfg.Spec, cli) + op, err := kube.NewDubboOperator(cfg.Spec, cli) if err != nil { return err } diff --git a/app/dubboctl/cmd/manifest_uninstall.go b/app/dubboctl/cmd/manifest_uninstall.go index 7024a3bf4..00c0cfc29 100644 --- a/app/dubboctl/cmd/manifest_uninstall.go +++ b/app/dubboctl/cmd/manifest_uninstall.go @@ -18,7 +18,6 @@ package cmd import ( "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/apis/dubbo.apache.org/v1alpha1" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/kube" - "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/operator" "github.com/apache/dubbo-kubernetes/pkg/core/logger" "github.com/spf13/cobra" "go.uber.org/zap/zapcore" @@ -80,7 +79,7 @@ func uninstallManifests(muArgs *ManifestUninstallArgs, cfg *v1alpha1.DubboConfig if err != nil { return err } - op, err := operator.NewDubboOperator(cfg.Spec, cli) + op, err := kube.NewDubboOperator(cfg.Spec, cli) if err != nil { return err } diff --git a/app/dubboctl/cmd/repository_test.go b/app/dubboctl/cmd/repository_test.go index cd4f9b9c4..169af7332 100644 --- a/app/dubboctl/cmd/repository_test.go +++ b/app/dubboctl/cmd/repository_test.go @@ -17,8 +17,6 @@ package cmd import ( "testing" - - . "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/testing" ) // TestRepository_List ensures that the 'list' subcommand shows the client's @@ -43,123 +41,3 @@ func TestRepository_List(t *testing.T) { t.Fatalf("expected:\n'%v'\ngot:\n'%v'\n", expect, output) } } - -// TestRepository_Add ensures that the 'add' subcommand accepts its positional -// arguments, respects the repositories' path flag, and the expected name is echoed -// upon subsequent 'list'. -func TestRepository_Add(t *testing.T) { - url := ServeRepo("repository.git#main", t) - _ = fromTempDirectory(t) - t.Log(url) - - var ( - add = NewRepositoryAddCmd(NewClient) - list = NewRepositoryListCmd(NewClient) - ) - // Do not use test command args - add.SetArgs([]string{}) - list.SetArgs([]string{}) - - // add [flags] - add.SetArgs([]string{ - "newrepo", - url, - }) - - // Parse flags and args, performing action - if err := add.Execute(); err != nil { - t.Fatal(err) - } - - // List post-add, capturing output from stdout - if err := list.Execute(); err != nil { - t.Fatal(err) - } -} - -// TestRepository_Rename ensures that the 'rename' subcommand accepts its -// positional arguments, respects the repositories' path flag, and the name is -// reflected as having been renamed upon subsequent 'list'. -func TestRepository_Rename(t *testing.T) { - url := ServeRepo("repository.git", t) - _ = fromTempDirectory(t) - - var ( - add = NewRepositoryAddCmd(NewClient) - rename = NewRepositoryRenameCmd(NewClient) - list = NewRepositoryListCmd(NewClient) - ) - // Do not use test command args - add.SetArgs([]string{}) - rename.SetArgs([]string{}) - list.SetArgs([]string{}) - - // add a repo which will be renamed - add.SetArgs([]string{"newrepo", url}) - if err := add.Execute(); err != nil { - t.Fatal(err) - } - - // rename [flags] - rename.SetArgs([]string{ - "newrepo", - "renamed", - }) - - // Parse flags and args, performing action - if err := rename.Execute(); err != nil { - t.Fatal(err) - } - - // List post-rename, capturing output from stdout - if err := list.Execute(); err != nil { - t.Fatal(err) - } -} - -// TestRepository_Remove ensures that the 'remove' subcommand accepts name as -// its argument, respects the repositories' flag, and the entry is removed upon -// subsequent 'list'. -func TestRepository_Remove(t *testing.T) { - url := ServeRepo("repository.git", t) - _ = fromTempDirectory(t) - - var ( - add = NewRepositoryAddCmd(NewClient) - remove = NewRepositoryRemoveCmd(NewClient) - list = NewRepositoryListCmd(NewClient) - stdout = piped(t) - ) - // Do not use test command args - add.SetArgs([]string{}) - remove.SetArgs([]string{}) - list.SetArgs([]string{}) - - // add a repo which will be removed - add.SetArgs([]string{"newrepo", url}) - if err := add.Execute(); err != nil { - t.Fatal(err) - } - - // remove [flags] - remove.SetArgs([]string{ - "newrepo", - }) - - // Parse flags and args, performing action - if err := remove.Execute(); err != nil { - t.Fatal(err) - } - - // List post-remove, capturing output from stdout - if err := list.Execute(); err != nil { - t.Fatal(err) - } - - // Assert the list output now includes the name from args (whitespace trimmed) - expect := "Repository added: newrepo\nRepository removed\ndefault" - output := stdout() - if output != expect { - t.Fatalf("expected:\n'%v'\ngot:\n'%v'\n", expect, output) - } -} diff --git a/app/dubboctl/cmd/root.go b/app/dubboctl/cmd/root.go index 69f991199..23fcce692 100644 --- a/app/dubboctl/cmd/root.go +++ b/app/dubboctl/cmd/root.go @@ -52,7 +52,7 @@ func getRootCmd(args []string) *cobra.Command { // Environment Variables // Evaluated first after static defaults, set all flags to be associated with // a version prefixed by "DUBBO_" - viper.AutomaticEnv() // read in environment variables for FUNC_ + viper.AutomaticEnv() // read in environment variables for DUBBO_ viper.SetEnvPrefix("dubbo") // ensure that all have the prefix newClient := cfg.NewClient if newClient == nil { diff --git a/app/dubboctl/internal/dubbo/client.go b/app/dubboctl/internal/dubbo/client.go index 2afa9bf66..276358c99 100644 --- a/app/dubboctl/internal/dubbo/client.go +++ b/app/dubboctl/internal/dubbo/client.go @@ -27,6 +27,8 @@ import ( "strings" "time" + "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/kube" + "github.com/spf13/cobra" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/util" @@ -42,13 +44,14 @@ const ( ) type Client struct { - repositoriesPath string // path to repositories - repositoriesURI string // repo URI (overrides repositories path) - templates *Templates // Templates management - repositories *Repositories // Repositories management - builder Builder // Builds a runnable image source - pusher Pusher // Pushes function image to a remote - deployer Deployer // Deploys or Updates a function} + repositoriesPath string // path to repositories + repositoriesURI string // repo URI (overrides repositories path) + templates *Templates // Templates management + repositories *Repositories // Repositories management + builder Builder // Builds a runnable image source + pusher Pusher // Pushes function image to a remote + deployer Deployer // Deploys or Updates a function} + KubeCtl *kube.CtlClient // Kube Client } // Builder of function source to runnable image. @@ -120,6 +123,12 @@ func (c *Client) Runtimes() ([]string, error) { // optionally mutates private members at time of instantiation. type Option func(*Client) +func WithKubeClient(client *kube.CtlClient) Option { + return func(c *Client) { + c.KubeCtl = client + } +} + func WithPusher(pusher Pusher) Option { return func(c *Client) { c.pusher = pusher @@ -398,7 +407,7 @@ func (c *Client) Deploy(ctx context.Context, d *Dubbo, opts ...DeployOption) (*D return d, ErrNameRequired } - fmt.Fprintf(os.Stderr, "⬆️ Deploying function to the cluster or generate manifest\n") + fmt.Fprintf(os.Stderr, "⬆️ Deploying application to the cluster or generate manifest\n") result, err := c.deployer.Deploy(ctx, d) if err != nil { fmt.Printf("deploy error: %v\n", err) diff --git a/app/dubboctl/internal/dubbo/deploy.tpl b/app/dubboctl/internal/dubbo/deploy.tpl index 2045ed164..1e582220c 100644 --- a/app/dubboctl/internal/dubbo/deploy.tpl +++ b/app/dubboctl/internal/dubbo/deploy.tpl @@ -17,22 +17,36 @@ spec: metadata: labels: app: {{.Name}} - app-type: dubbo + app-type: dubbo{{if .UseProm}} #helm-charts 配置 https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus annotations: prometheus.io/scrape: "true" prometheus.io/path: /management/prometheus - prometheus.io/port: "18081" + prometheus.io/port: "18081"{{end}} spec:{{if .ServiceAccount}} serviceAccountName: {{.ServiceAccount}}{{end}} containers: - name: {{.Name}} image: {{.Image}} + env: + - name: DUBBO_CTL_VERSION + value: 0.0.1{{if .Zookeeper}} + - name: zookeeper.address + value: {{.Zookeeper}} + - name: ZOOKEEPER_ADDRESS + value: {{.Zookeeper}}{{end}}{{if .Nacos}} + - name: nacos.address + value: {{.Nacos}} + - name: NACOS_ADDRESS + value: {{.Nacos}}{{end}} {{if .ImagePullPolicy}}imagePullPolicy: {{.ImagePullPolicy}} {{end}}ports: - containerPort: {{.Port}} name: dubbo - protocol: TCP + protocol: TCP{{if .UseProm}} + - containerPort: 18081 + name: metrics + protocol: TCP{{end}} readinessProbe: tcpSocket: port: {{.Port}} @@ -66,7 +80,9 @@ spec: protocol: TCP targetPort: {{.TargetPort}} type: NodePort{{else}}- port: {{.Port}} - targetPort: {{.TargetPort}}{{end}} + targetPort: {{.TargetPort}}{{end}}{{if .UseProm}} + - port: 18081 + targetPort: 18081{{end}} selector: app: {{.Name}} diff --git a/app/dubboctl/internal/dubbo/deployer.go b/app/dubboctl/internal/dubbo/deployer.go index 0c17c235b..890acbd2c 100644 --- a/app/dubboctl/internal/dubbo/deployer.go +++ b/app/dubboctl/internal/dubbo/deployer.go @@ -66,6 +66,12 @@ type Deployment struct { MaxReplicas int ServiceAccount string ImagePullPolicy string + Zookeeper string + Nacos string + UseProm bool + PromScratch bool + PromPath string + PromPort int } func (d *DeployApp) Deploy(ctx context.Context, f *Dubbo, option ...DeployOption) (DeploymentResult, error) { @@ -117,6 +123,9 @@ func (d *DeployApp) Deploy(ctx context.Context, f *Dubbo, option ...DeployOption MaxReplicas: f.Deploy.MaxReplicas, ServiceAccount: f.Deploy.ServiceAccount, ImagePullPolicy: f.Deploy.ImagePullPolicy, + Zookeeper: f.Deploy.ZookeeperAddress, + Nacos: f.Deploy.NacosAddress, + UseProm: f.Deploy.UseProm, }) if err != nil { return DeploymentResult{ diff --git a/app/dubboctl/internal/dubbo/dubbo.go b/app/dubboctl/internal/dubbo/dubbo.go index 498042d7e..e52afbd3f 100644 --- a/app/dubboctl/internal/dubbo/dubbo.go +++ b/app/dubboctl/internal/dubbo/dubbo.go @@ -116,22 +116,26 @@ type BuildSpec struct { } type DeploySpec struct { - Namespace string `yaml:"namespace,omitempty"` - Output string `yaml:"output,omitempty"` - Secret string `yaml:"secret,omitempty"` - Replicas int `yaml:"replicas,omitempty"` - Revisions int `yaml:"revisions,omitempty"` - ContainerPort int `yaml:"containerPort"` - TargetPort int `yaml:"targetPort,omitempty"` - NodePort int `yaml:"nodePort,omitempty"` - RequestCpu int `yaml:"requestCpu,omitempty"` - RequestMem int `yaml:"requestMem,omitempty"` - LimitCpu int `yaml:"limitCpu,omitempty"` - LimitMem int `yaml:"limitMem,omitempty"` - MinReplicas int `yaml:"minReplicas,omitempty"` - MaxReplicas int `yaml:"maxReplicas,omitempty"` - ServiceAccount string `yaml:"serviceAccount,omitempty"` - ImagePullPolicy string `yaml:"imagePullPolicy,omitempty"` + Namespace string `yaml:"namespace,omitempty"` + Output string `yaml:"output,omitempty"` + Secret string `yaml:"secret,omitempty"` + Replicas int `yaml:"replicas,omitempty"` + Revisions int `yaml:"revisions,omitempty"` + ContainerPort int `yaml:"containerPort"` + TargetPort int `yaml:"targetPort,omitempty"` + NodePort int `yaml:"nodePort,omitempty"` + RequestCpu int `yaml:"requestCpu,omitempty"` + RequestMem int `yaml:"requestMem,omitempty"` + LimitCpu int `yaml:"limitCpu,omitempty"` + LimitMem int `yaml:"limitMem,omitempty"` + MinReplicas int `yaml:"minReplicas,omitempty"` + MaxReplicas int `yaml:"maxReplicas,omitempty"` + ServiceAccount string `yaml:"serviceAccount,omitempty"` + ImagePullPolicy string `yaml:"imagePullPolicy,omitempty"` + Envs []string `yaml:"-"` + ZookeeperAddress string `yaml:"-"` + NacosAddress string `yaml:"-"` + UseProm bool `yaml:"-"` } func (f Dubbo) Validate() error { diff --git a/app/dubboctl/internal/kube/client.go b/app/dubboctl/internal/kube/client.go index dcddb9619..b6a9f436c 100644 --- a/app/dubboctl/internal/kube/client.go +++ b/app/dubboctl/internal/kube/client.go @@ -68,7 +68,7 @@ func WithCli(cli client.Client) CtlClientOption { // ApplyManifest applies manifest to certain namespace // If there is not this namespace, create it first -func (cli *CtlClient) ApplyManifest(manifest string, ns string) error { +func (cli *CtlClient) ApplyManifest(manifest string, ns string, name ComponentName) error { if err := cli.CreateNamespace(ns); err != nil { return err } @@ -77,12 +77,39 @@ func (cli *CtlClient) ApplyManifest(manifest string, ns string) error { return err } for _, obj := range objs { + o := obj.Unstructured() if obj.Namespace == "" { obj.SetNamespace(ns) } - if err := cli.ApplyObject(obj.Unstructured()); err != nil { + if err := cli.ApplyObject(o); err != nil { return err } + + if name == Zookeeper && o.GetKind() == "Service" { + labels := o.GetLabels() + labels["dubbo.apache.org/zookeeper"] = "true" + o.SetLabels(labels) + err := cli.Update(context.Background(), o) + if err != nil { + return err + } + } else if name == Nacos && o.GetKind() == "Service" { + labels := o.GetLabels() + labels["dubbo.apache.org/nacos"] = "true" + o.SetLabels(labels) + err := cli.Update(context.Background(), o) + if err != nil { + return err + } + } else if name == Prometheus && o.GetKind() == "Service" { + labels := o.GetLabels() + labels["dubbo.apache.org/prometheus"] = "true" + o.SetLabels(labels) + err := cli.Update(context.Background(), o) + if err != nil { + return err + } + } } return nil } @@ -101,6 +128,7 @@ func (cli *CtlClient) ApplyObject(obj *unstructured.Unstructured) error { } return nil } + key := client.ObjectKeyFromObject(obj) receiver := &unstructured.Unstructured{} receiver.SetGroupVersionKind(obj.GroupVersionKind()) @@ -122,6 +150,10 @@ func (cli *CtlClient) ApplyObject(obj *unstructured.Unstructured) error { if err := OverlayObject(receiver, obj); err != nil { return err } + // TODO We really don’t know the reason for this, and we still need to verify it. + if receiver.GetName() == "prometheus-alertmanager-test-connection" { + return nil + } if err := cli.Update(context.Background(), receiver); err != nil { return err } @@ -138,21 +170,36 @@ func (cli *CtlClient) CreateNamespace(ns string) error { Namespace: metav1.NamespaceSystem, Name: ns, } - if err := cli.Get(context.Background(), key, &corev1.Namespace{}); err != nil { + + namespace := &corev1.Namespace{} + + err := cli.Get(context.Background(), key, namespace) + if err != nil { if errors.IsNotFound(err) { nsObj := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Namespace: metav1.NamespaceSystem, Name: ns, + Labels: map[string]string{ + "dubbo-deploy": "enabled", + }, }, } if err := cli.Create(context.Background(), nsObj); err != nil { return err } return nil + } else { + return fmt.Errorf("failed to check if namespace %v exists: %v", ns, err) } + } + + labels := namespace.Labels + labels["dubbo-deploy"] = "enabled" - return fmt.Errorf("failed to check if namespace %v exists: %v", ns, err) + err = cli.Update(context.Background(), namespace) + if err != nil { + return err } return nil diff --git a/app/dubboctl/internal/kube/client_test.go b/app/dubboctl/internal/kube/client_test.go index 79250a726..f3f06ca03 100644 --- a/app/dubboctl/internal/kube/client_test.go +++ b/app/dubboctl/internal/kube/client_test.go @@ -17,7 +17,6 @@ package kube import ( "context" - "os" "path" "testing" @@ -56,7 +55,7 @@ func TestCtlClient_ApplyManifest(t *testing.T) { t.Fatalf("read input manifest %s err: %s", test.input, err) } testNs := "test" - if err := ctlCli.ApplyManifest(inputManifest, testNs); err != nil { + if err := ctlCli.ApplyManifest(inputManifest, testNs, ""); err != nil { t.Errorf("ApplyManifest failed, err: %s", err) return } @@ -160,7 +159,7 @@ func TestCtlClient_RemoveManifest(t *testing.T) { t.Fatalf("read before manifest %s err: %s", test.before, err) } testNs := "test" - if err := ctlCli.ApplyManifest(beforeManifest, testNs); err != nil { + if err := ctlCli.ApplyManifest(beforeManifest, testNs, ""); err != nil { t.Fatalf("ApplyManifest failed, err: %s", err) return } @@ -251,11 +250,3 @@ func newFakeCli(t *testing.T, before string) client.Client { } return fakeCli } - -func readManifest(path string) (string, error) { - content, err := os.ReadFile(path) - if err != nil { - return "", err - } - return string(content), nil -} diff --git a/app/dubboctl/internal/kube/common.go b/app/dubboctl/internal/kube/common.go index d6f9244fc..d0890b050 100644 --- a/app/dubboctl/internal/kube/common.go +++ b/app/dubboctl/internal/kube/common.go @@ -43,7 +43,7 @@ func OverlayObject(base *unstructured.Unstructured, overlay *unstructured.Unstru } overlayUpdated := overlay.DeepCopy() - if strings.EqualFold(base.GetKind(), "service") { + if strings.EqualFold(base.GetKind(), "Service") { if err := saveClusterIP(base, overlayUpdated); err != nil { return err } diff --git a/app/dubboctl/internal/operator/component.go b/app/dubboctl/internal/kube/component.go similarity index 99% rename from app/dubboctl/internal/operator/component.go rename to app/dubboctl/internal/kube/component.go index c72a18cfe..eac53a5cf 100644 --- a/app/dubboctl/internal/operator/component.go +++ b/app/dubboctl/internal/kube/component.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package kube import ( "os" diff --git a/app/dubboctl/internal/operator/component_test.go b/app/dubboctl/internal/kube/component_test.go similarity index 98% rename from app/dubboctl/internal/operator/component_test.go rename to app/dubboctl/internal/kube/component_test.go index b76547c24..059c79735 100644 --- a/app/dubboctl/internal/operator/component_test.go +++ b/app/dubboctl/internal/kube/component_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package kube import ( "os" @@ -25,10 +25,6 @@ import ( "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/util" ) -const ( - wantPath = "./testdata/want" -) - type newComponentFunc func(t *testing.T) Component func TestRenderManifest(t *testing.T) { diff --git a/app/dubboctl/internal/operator/operator.go b/app/dubboctl/internal/kube/operator.go similarity index 96% rename from app/dubboctl/internal/operator/operator.go rename to app/dubboctl/internal/kube/operator.go index 8f72d9b2a..a84cd10f0 100644 --- a/app/dubboctl/internal/operator/operator.go +++ b/app/dubboctl/internal/kube/operator.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package operator +package kube import ( "errors" @@ -21,8 +21,6 @@ import ( "github.com/apache/dubbo-kubernetes/pkg/core/logger" - "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/kube" - "github.com/apache/dubbo-kubernetes/app/dubboctl/identifier" "github.com/apache/dubbo-kubernetes/app/dubboctl/internal/apis/dubbo.apache.org/v1alpha1" ) @@ -31,7 +29,7 @@ type DubboOperator struct { spec *v1alpha1.DubboConfigSpec started bool components map[ComponentName]Component - kubeCli *kube.CtlClient + kubeCli *CtlClient } // Run must be invoked before invoking other functions. @@ -68,7 +66,7 @@ func (do *DubboOperator) ApplyManifest(manifestMap map[ComponentName]string) err } for name, manifest := range manifestMap { logger.CmdSugar().Infof("Start applying bootstrap %s\n", name) - if err := do.kubeCli.ApplyManifest(manifest, do.spec.Namespace); err != nil { + if err := do.kubeCli.ApplyManifest(manifest, do.spec.Namespace, name); err != nil { return fmt.Errorf("bootstrap %s ApplyManifest err: %v", name, err) } logger.CmdSugar().Infof("Applying bootstrap %s successfully\n", name) @@ -93,7 +91,7 @@ func (do *DubboOperator) RemoveManifest(manifestMap map[ComponentName]string) er // NewDubboOperator accepts cli directly for testing and normal use. // For now, every related command needs a dedicated DubboOperator. -func NewDubboOperator(spec *v1alpha1.DubboConfigSpec, cli *kube.CtlClient) (*DubboOperator, error) { +func NewDubboOperator(spec *v1alpha1.DubboConfigSpec, cli *CtlClient) (*DubboOperator, error) { if spec == nil { return nil, errors.New("DubboConfigSpec is empty") } diff --git a/app/dubboctl/internal/operator/testdata/want/admin_component-render_manifest.golden.yaml b/app/dubboctl/internal/kube/testdata/want/admin_component-render_manifest.golden.yaml similarity index 100% rename from app/dubboctl/internal/operator/testdata/want/admin_component-render_manifest.golden.yaml rename to app/dubboctl/internal/kube/testdata/want/admin_component-render_manifest.golden.yaml diff --git a/app/dubboctl/internal/operator/testdata/want/nacos_component-render_manifest.golden.yaml b/app/dubboctl/internal/kube/testdata/want/nacos_component-render_manifest.golden.yaml similarity index 100% rename from app/dubboctl/internal/operator/testdata/want/nacos_component-render_manifest.golden.yaml rename to app/dubboctl/internal/kube/testdata/want/nacos_component-render_manifest.golden.yaml diff --git a/app/dubboctl/internal/operator/testdata/want/prometheus_component-render_manifest.golden.yaml b/app/dubboctl/internal/kube/testdata/want/prometheus_component-render_manifest.golden.yaml similarity index 99% rename from app/dubboctl/internal/operator/testdata/want/prometheus_component-render_manifest.golden.yaml rename to app/dubboctl/internal/kube/testdata/want/prometheus_component-render_manifest.golden.yaml index b73e0bc77..a28399a63 100644 --- a/app/dubboctl/internal/operator/testdata/want/prometheus_component-render_manifest.golden.yaml +++ b/app/dubboctl/internal/kube/testdata/want/prometheus_component-render_manifest.golden.yaml @@ -58,6 +58,7 @@ metadata: app.kubernetes.io/name: alertmanager app.kubernetes.io/version: v0.25.0 helm.sh/chart: alertmanager-0.24.1 + name: prometheus-alertmanager namespace: dubbo-system spec: diff --git a/app/dubboctl/internal/operator/testdata/want/skywalking_component-render_manifest.golden.yaml b/app/dubboctl/internal/kube/testdata/want/skywalking_component-render_manifest.golden.yaml similarity index 100% rename from app/dubboctl/internal/operator/testdata/want/skywalking_component-render_manifest.golden.yaml rename to app/dubboctl/internal/kube/testdata/want/skywalking_component-render_manifest.golden.yaml diff --git a/app/dubboctl/internal/operator/testdata/want/zipkin_component-render_manifest.golden.yaml b/app/dubboctl/internal/kube/testdata/want/zipkin_component-render_manifest.golden.yaml similarity index 100% rename from app/dubboctl/internal/operator/testdata/want/zipkin_component-render_manifest.golden.yaml rename to app/dubboctl/internal/kube/testdata/want/zipkin_component-render_manifest.golden.yaml diff --git a/app/dubboctl/internal/operator/testdata/want/zookeeper_component-render_manifest.golden.yaml b/app/dubboctl/internal/kube/testdata/want/zookeeper_component-render_manifest.golden.yaml similarity index 100% rename from app/dubboctl/internal/operator/testdata/want/zookeeper_component-render_manifest.golden.yaml rename to app/dubboctl/internal/kube/testdata/want/zookeeper_component-render_manifest.golden.yaml diff --git a/docs/cmd/dubboctl/deploy.md b/docs/cmd/dubboctl/deploy.md new file mode 100644 index 000000000..92f02a2db --- /dev/null +++ b/docs/cmd/dubboctl/deploy.md @@ -0,0 +1,43 @@ +#Deploy + +When you use dubboctl to install the required components, we will dynamically adjust the yaml file generated during the +deploy phase based on the components you installed. Specifically reflected in zookeeper, nacos and Prometheus. After you +install zookeeper, we will add information similar to this to the generated yaml file: + +```yaml + env: + - name: zookeeper.address + value: zookeeper.dubbo-system.svc + - name: ZOOKEEPER_ADDRESS + value: zookeeper.dubbo-system.svc +``` + +With these, you can use placeholders to read environment variables in your application to read the address of zookeeper +without having to fill it in manually, for example: + +```yaml +Dubbo: + application: + logger: slf4j + name: DemoApplication + registry: + address: zookeeper://${zookeeper.address:127.0.0.1}:2181 + protocol: + name: tri + port: 50051 + +``` + +The same goes for nacos. + +For Prometheus, we will automatically generate a default list for Prometheus, which you can modify as needed. + +## Advanced + +If you installed related components in the dubbo-system namespace. Then the exact same components are installed in the +dev namespace, then we will give priority to using the addresses of related middleware in the dev namespace, and +everything will be in chronological order. Unless you specify it in an environment variable: + +```sh +export DUBBO_DEPLOY_NS=dubbo-system +``` \ No newline at end of file