Support deploying microservice (#2377)

components/cluster/command/upgrade.go

@@ -22,7 +22,7 @@ import (
 func newUpgradeCmd() *cobra.Command {
 	offlineMode := false
 	ignoreVersionCheck := false
-	var tidbVer, tikvVer, pdVer, tiflashVer, kvcdcVer, dashboardVer, cdcVer, alertmanagerVer, nodeExporterVer, blackboxExporterVer, tiproxyVer string
+	var tidbVer, tikvVer, pdVer, tsoVer, schedulingVer, tiflashVer, kvcdcVer, dashboardVer, cdcVer, alertmanagerVer, nodeExporterVer, blackboxExporterVer, tiproxyVer string
 
 	cmd := &cobra.Command{
 		Use:   "upgrade <cluster-name> <version>",
@@ -47,6 +47,8 @@ func newUpgradeCmd() *cobra.Command {
 				spec.ComponentTiDB:             tidbVer,
 				spec.ComponentTiKV:             tikvVer,
 				spec.ComponentPD:               pdVer,
+				spec.ComponentTSO:              tsoVer,
+				spec.ComponentScheduling:       schedulingVer,
 				spec.ComponentTiFlash:          tiflashVer,
 				spec.ComponentTiKVCDC:          kvcdcVer,
 				spec.ComponentCDC:              cdcVer,
@@ -76,7 +78,9 @@ func newUpgradeCmd() *cobra.Command {
 
 	// cmd.Flags().StringVar(&tidbVer, "tidb-version", "", "Fix the version of tidb and no longer follows the cluster version.")
 	cmd.Flags().StringVar(&tikvVer, "tikv-version", "", "Fix the version of tikv and no longer follows the cluster version.")
-	cmd.Flags().StringVar(&pdVer, "pd-version", "", "Fix the version of pv and no longer follows the cluster version.")
+	cmd.Flags().StringVar(&pdVer, "pd-version", "", "Fix the version of pd and no longer follows the cluster version.")
+	cmd.Flags().StringVar(&tsoVer, "tso-version", "", "Fix the version of tso and no longer follows the cluster version.")
+	cmd.Flags().StringVar(&schedulingVer, "scheduling-version", "", "Fix the version of scheduling and no longer follows the cluster version.")
 	cmd.Flags().StringVar(&tiflashVer, "tiflash-version", "", "Fix the version of tiflash and no longer follows the cluster version.")
 	cmd.Flags().StringVar(&dashboardVer, "tidb-dashboard-version", "", "Fix the version of tidb-dashboard and no longer follows the cluster version.")
 	cmd.Flags().StringVar(&cdcVer, "cdc-version", "", "Fix the version of cdc and no longer follows the cluster version.")

embed/templates/config/prometheus.yml.tpl

@@ -155,6 +155,36 @@ scrape_configs:
     - targets:
 {{- range .PDAddrs}}
       - '{{.}}'
+{{- end}}
+  - job_name: "tso"
+    honor_labels: true # don't overwrite job & instance labels
+{{- if .TLSEnabled}}
+    scheme: https
+    tls_config:
+      insecure_skip_verify: false
+      ca_file: ../tls/ca.crt
+      cert_file: ../tls/prometheus.crt
+      key_file: ../tls/prometheus.pem
+{{- end}}
+    static_configs:
+    - targets:
+{{- range .TSOAddrs}}
+      - '{{.}}'
+{{- end}}
+  - job_name: "scheduling"
+    honor_labels: true # don't overwrite job & instance labels
+{{- if .TLSEnabled}}
+    scheme: https
+    tls_config:
+      insecure_skip_verify: false
+      ca_file: ../tls/ca.crt
+      cert_file: ../tls/prometheus.crt
+      key_file: ../tls/prometheus.pem
+{{- end}}
+    static_configs:
+    - targets:
+{{- range .SchedulingAddrs}}
+      - '{{.}}'
 {{- end}}
 {{- if .TiFlashStatusAddrs}}
   - job_name: "tiflash"

embed/templates/scripts/run_pd.sh.tpl

@@ -7,11 +7,15 @@ DEPLOY_DIR={{.DeployDir}}
 
 cd "${DEPLOY_DIR}" || exit 1
 
+exec \
 {{- if .NumaNode}}
-exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server \
-{{- else}}
-exec env GODEBUG=madvdontneed=1 bin/pd-server \
+    numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} \
 {{- end}}
+    env GODEBUG=madvdontneed=1 \
+{{- if .MSMode}}
+    PD_SERVICE_MODE=api \
+{{- end}}
+    bin/pd-server \
     --name="{{.Name}}" \
     --client-urls="{{.ClientURL}}" \
     --advertise-client-urls="{{.AdvertiseClientURL}}" \

embed/templates/scripts/run_pd_scale.sh.tpl

@@ -7,11 +7,15 @@ DEPLOY_DIR={{.DeployDir}}
 
 cd "${DEPLOY_DIR}" || exit 1
 
+exec \
 {{- if .NumaNode}}
-exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server \
-{{- else}}
-exec env GODEBUG=madvdontneed=1 bin/pd-server \
+    numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} \
 {{- end}}
+    env GODEBUG=madvdontneed=1 \
+{{- if .MSMode}}
+    PD_SERVICE_MODE=api \
+{{- end}}
+    bin/pd-server \
     --name="{{.Name}}" \
     --client-urls="{{.ClientURL}}" \
     --advertise-client-urls="{{.AdvertiseClientURL}}" \

embed/templates/scripts/run_scheduling.sh.tpl

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+# WARNING: This file was auto-generated. Do not edit!
+#          All your edit might be overwritten!
+DEPLOY_DIR={{.DeployDir}}
+
+cd "${DEPLOY_DIR}" || exit 1
+
+{{- if .NumaNode}}
+exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server services scheduling\
+{{- else}}
+exec env GODEBUG=madvdontneed=1 bin/pd-server services scheduling \
+{{- end}}
+    --backend-endpoints="{{.BackendEndpoints}}" \
+    --listen-addr="{{.ListenURL}}" \
+    --advertise-listen-addr="{{.AdvertiseListenURL}}" \
+    --config=conf/scheduling.toml \
+    --log-file="{{.LogDir}}/scheduling.log" 2>> "{{.LogDir}}/scheduling_stderr.log"

embed/templates/scripts/run_tso.sh.tpl

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+# WARNING: This file was auto-generated. Do not edit!
+#          All your edit might be overwritten!
+DEPLOY_DIR={{.DeployDir}}
+
+cd "${DEPLOY_DIR}" || exit 1
+
+{{- if .NumaNode}}
+exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server services tso\
+{{- else}}
+exec env GODEBUG=madvdontneed=1 bin/pd-server services tso \
+{{- end}}
+    --backend-endpoints="{{.BackendEndpoints}}" \
+    --listen-addr="{{.ListenURL}}" \
+    --advertise-listen-addr="{{.AdvertiseListenURL}}" \
+    --config=conf/tso.toml \
+    --log-file="{{.LogDir}}/tso.log" 2>> "{{.LogDir}}/tso_stderr.log"

pkg/cluster/ansible/import_test.go

@@ -124,6 +124,8 @@ server_configs:
         binlog.enable: true
     tikv: {}
     pd: {}
+    tso: {}
+    scheduling: {}
     tidb_dashboard: {}
     tiflash: {}
     tiproxy: {}

pkg/cluster/ansible/test-data/meta.yaml

@@ -19,6 +19,8 @@ topology:
       binlog.enable: true
     tikv: {}
     pd: {}
+    tso: {}
+    scheduling: {}
     tidb_dashboard: {}
     tiflash: {}
     tiproxy: {}

pkg/cluster/api/pdapi.go

@@ -139,6 +139,7 @@ var (
 	pdStoresURI          = "pd/api/v1/stores"
 	pdStoresLimitURI     = "pd/api/v1/stores/limit"
 	pdRegionsCheckURI    = "pd/api/v1/regions/check"
+	pdServicePrimaryURI  = "pd/api/v2/ms/primary"
 	tsoHealthPrefix      = "tso/api/v1/health"
 )
 
@@ -1006,3 +1007,214 @@ func (pc *PDClient) SetAllStoreLimits(value int) error {
 	pc.l().Debugf("setting store limit: %d", value)
 	return pc.updateConfig(pdStoresLimitURI, bytes.NewBuffer(body))
 }
+
+// GetServicePrimary queries for the primary of a service
+func (pc *PDClient) GetServicePrimary(service string) (string, error) {
+	endpoints := pc.getEndpoints(fmt.Sprintf("%s/%s", pdServicePrimaryURI, service))
+
+	var primary string
+	_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
+		body, err := pc.httpClient.Get(pc.ctx, endpoint)
+		if err != nil {
+			return body, err
+		}
+
+		return body, json.Unmarshal(body, &primary)
+	})
+	return primary, err
+}
+
+const (
+	tsoStatusURI        = "status"
+	schedulingStatusURI = "status"
+)
+
+// TSOClient is an HTTP client of the TSO server
+type TSOClient struct {
+	version    string
+	addrs      []string
+	tlsEnabled bool
+	httpClient *utils.HTTPClient
+	ctx        context.Context
+}
+
+// NewTSOClient returns a new TSOClient, the context must have
+// a *logprinter.Logger as value of "logger"
+func NewTSOClient(
+	ctx context.Context,
+	addrs []string,
+	timeout time.Duration,
+	tlsConfig *tls.Config,
+) *TSOClient {
+	enableTLS := false
+	if tlsConfig != nil {
+		enableTLS = true
+	}
+
+	if _, ok := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger); !ok {
+		panic("the context must have logger inside")
+	}
+
+	cli := &TSOClient{
+		addrs:      addrs,
+		tlsEnabled: enableTLS,
+		httpClient: utils.NewHTTPClient(timeout, tlsConfig),
+		ctx:        ctx,
+	}
+
+	cli.tryIdentifyVersion()
+	return cli
+}
+
+// func (tc *TSOClient) l() *logprinter.Logger {
+// 	return tc.ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger)
+// }
+
+func (tc *TSOClient) tryIdentifyVersion() {
+	endpoints := tc.getEndpoints(tsoStatusURI)
+	response := map[string]string{}
+	_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
+		body, err := tc.httpClient.Get(tc.ctx, endpoint)
+		if err != nil {
+			return body, err
+		}
+
+		return body, json.Unmarshal(body, &response)
+	})
+	if err == nil {
+		tc.version = response["version"]
+	}
+}
+
+// GetURL builds the client URL of PDClient
+func (tc *TSOClient) GetURL(addr string) string {
+	httpPrefix := "http"
+	if tc.tlsEnabled {
+		httpPrefix = "https"
+	}
+	return fmt.Sprintf("%s://%s", httpPrefix, addr)
+}
+
+func (tc *TSOClient) getEndpoints(uri string) (endpoints []string) {
+	for _, addr := range tc.addrs {
+		endpoint := fmt.Sprintf("%s/%s", tc.GetURL(addr), uri)
+		endpoints = append(endpoints, endpoint)
+	}
+
+	return
+}
+
+// CheckHealth checks the health of TSO node.
+func (tc *TSOClient) CheckHealth() error {
+	endpoints := tc.getEndpoints(tsoStatusURI)
+
+	_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
+		body, err := tc.httpClient.Get(tc.ctx, endpoint)
+		if err != nil {
+			return body, err
+		}
+
+		return body, nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SchedulingClient is an HTTP client of the scheduling server
+type SchedulingClient struct {
+	version    string
+	addrs      []string
+	tlsEnabled bool
+	httpClient *utils.HTTPClient
+	ctx        context.Context
+}
+
+// NewSchedulingClient returns a new SchedulingClient, the context must have
+// a *logprinter.Logger as value of "logger"
+func NewSchedulingClient(
+	ctx context.Context,
+	addrs []string,
+	timeout time.Duration,
+	tlsConfig *tls.Config,
+) *SchedulingClient {
+	enableTLS := false
+	if tlsConfig != nil {
+		enableTLS = true
+	}
+
+	if _, ok := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger); !ok {
+		panic("the context must have logger inside")
+	}
+
+	cli := &SchedulingClient{
+		addrs:      addrs,
+		tlsEnabled: enableTLS,
+		httpClient: utils.NewHTTPClient(timeout, tlsConfig),
+		ctx:        ctx,
+	}
+
+	cli.tryIdentifyVersion()
+	return cli
+}
+
+// func (tc *SchedulingClient) l() *logprinter.Logger {
+// 	return tc.ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger)
+// }
+
+func (tc *SchedulingClient) tryIdentifyVersion() {
+	endpoints := tc.getEndpoints(schedulingStatusURI)
+	response := map[string]string{}
+	_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
+		body, err := tc.httpClient.Get(tc.ctx, endpoint)
+		if err != nil {
+			return body, err
+		}
+
+		return body, json.Unmarshal(body, &response)
+	})
+	if err == nil {
+		tc.version = response["version"]
+	}
+}
+
+// GetURL builds the client URL of PDClient
+func (tc *SchedulingClient) GetURL(addr string) string {
+	httpPrefix := "http"
+	if tc.tlsEnabled {
+		httpPrefix = "https"
+	}
+	return fmt.Sprintf("%s://%s", httpPrefix, addr)
+}
+
+func (tc *SchedulingClient) getEndpoints(uri string) (endpoints []string) {
+	for _, addr := range tc.addrs {
+		endpoint := fmt.Sprintf("%s/%s", tc.GetURL(addr), uri)
+		endpoints = append(endpoints, endpoint)
+	}
+
+	return
+}
+
+// CheckHealth checks the health of scheduling node.
+func (tc *SchedulingClient) CheckHealth() error {
+	endpoints := tc.getEndpoints(schedulingStatusURI)
+
+	_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
+		body, err := tc.httpClient.Get(tc.ctx, endpoint)
+		if err != nil {
+			return body, err
+		}
+
+		return body, nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/cluster/spec/instance.go

@@ -40,6 +40,8 @@ const (
 	ComponentTiDB             = "tidb"
 	ComponentTiKV             = "tikv"
 	ComponentPD               = "pd"
+	ComponentTSO              = "tso"
+	ComponentScheduling       = "scheduling"
 	ComponentTiFlash          = "tiflash"
 	ComponentTiProxy          = "tiproxy"
 	ComponentGrafana          = "grafana"