Merge branch 'issueL-2237' of https://github.com/srstack/tiup into is…

…sueL-2237

CHANGELOG.md

@@ -1,5 +1,32 @@
 TiUP Changelog
 
+## [1.12.5] 2023-7-17
+
+### Fix
+
+- Fix cannot start tiflash above v7.1.0 in `tiup-cluster` (#2230, @zanmato1984)
+
+## [1.12.4] 2023-7-13
+
+### Fix
+
+- Fix cannot show tiflash uptime in `tiup-cluster` (#2227, @nexustar)
+
+### Improvement
+
+- Remove tcp_port for tiflash in `tiup-cluster` and `tiup-playground` (#2220, @zanmato1984)
+
+## [1.12.3] 2023-6-14
+
+### Fixes
+
+- Fix cannot edit manage_host on an exist cluster in `tiup-cluster` (#2210, @nexustar)
+- Fix still use host instead of manage_host in `tiup-cluster` (#2206 #2207, @nexustar)
+
+### Improvement
+
+- Check if the compnoent exists when uninstall in `tiup` (#2209, @srstack)
+
 ## [1.12.2] 2023-5-19
 
 ### Notes

cmd/uninstall.go

@@ -51,7 +51,7 @@ which is used to uninstall tiup.
 			teleCommand = cmd.CommandPath()
 			env := environment.GlobalEnv()
 			if self {
-				deletable := []string{"bin", "manifest", "manifests", "components", "storage/cluster/packages"}
+				deletable := []string{"storage/cluster/packages", "components", "manifests", "manifest", "bin"}
 				for _, dir := range deletable {
 					if err := os.RemoveAll(env.Profile().Path(dir)); err != nil {
 						return errors.Trace(err)
@@ -86,6 +86,9 @@ func removeComponents(env *environment.Environment, specs []string, all bool) er
 		if strings.Contains(spec, ":") {
 			parts := strings.SplitN(spec, ":", 2)
 			// after this version is deleted, component will have no version left. delete the whole component dir directly
+			if !utils.IsExist(env.LocalPath(localdata.ComponentParentDir, parts[0])) {
+				return errors.Trace(fmt.Errorf("component `%s` is not installed, please use `tiup list %s` to check", parts[0], parts[0]))
+			}
 			dir, err := os.ReadDir(env.LocalPath(localdata.ComponentParentDir, parts[0]))
 			if err != nil {
 				return errors.Trace(err)
@@ -99,6 +102,7 @@ func removeComponents(env *environment.Environment, specs []string, all bool) er
 			} else {
 				paths = append(paths, env.LocalPath(localdata.ComponentParentDir, parts[0], parts[1]))
 			}
+			// if no more version left, delete the whole component dir
 			if len(dir)-len(paths) < 1 {
 				paths = append(paths, env.LocalPath(localdata.ComponentParentDir, parts[0]))
 			}
@@ -110,6 +114,10 @@ func removeComponents(env *environment.Environment, specs []string, all bool) er
 			paths = append(paths, env.LocalPath(localdata.ComponentParentDir, spec))
 		}
 		for _, path := range paths {
+			if !utils.IsExist(path) {
+				return errors.Trace(fmt.Errorf("component `%s` is not installed, please check", spec))
+			}
+			fmt.Println(path)
 			if err := os.RemoveAll(path); err != nil {
 				return errors.Trace(err)
 			}

components/dm/command/prune.go

@@ -76,7 +76,7 @@ func clearOutDatedEtcdInfo(clusterName string, metadata *spec.Metadata, opt oper
 	if err != nil {
 		return err
 	}
-	dmMasterClient := api.NewDMMasterClient(topo.GetMasterList(), 10*time.Second, tlsCfg)
+	dmMasterClient := api.NewDMMasterClient(topo.GetMasterListWithManageHost(), 10*time.Second, tlsCfg)
 	registeredMasters, registeredWorkers, err := dmMasterClient.GetRegisteredMembers()
 	if err != nil {
 		return err

components/dm/spec/logic.go

@@ -82,6 +82,7 @@ func (c *DMMasterComponent) Instances() []Instance {
 				ManageHost:   s.ManageHost,
 				Port:         s.Port,
 				SSHP:         s.SSHPort,
+				Source:       s.GetSource(),
 
 				Ports: []int{
 					s.Port,
@@ -284,6 +285,7 @@ func (c *DMWorkerComponent) Instances() []Instance {
 				ManageHost:   s.ManageHost,
 				Port:         s.Port,
 				SSHP:         s.SSHPort,
+				Source:       s.GetSource(),
 
 				Ports: []int{
 					s.Port,

components/dm/spec/topology_dm.go

@@ -122,7 +122,7 @@ func AllDMComponentNames() (roles []string) {
 // MasterSpec represents the Master topology specification in topology.yaml
 type MasterSpec struct {
 	Host           string `yaml:"host"`
-	ManageHost     string `yaml:"manage_host,omitempty"`
+	ManageHost     string `yaml:"manage_host,omitempty" validate:"manage_host:editable"`
 	SSHPort        int    `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"`
 	Imported       bool   `yaml:"imported,omitempty"`
 	Patched        bool   `yaml:"patched,omitempty"`
@@ -134,6 +134,7 @@ type MasterSpec struct {
 	DeployDir       string          `yaml:"deploy_dir,omitempty"`
 	DataDir         string          `yaml:"data_dir,omitempty"`
 	LogDir          string          `yaml:"log_dir,omitempty"`
+	Source          string          `yaml:"source,omitempty" validate:"source:editable"`
 	NumaNode        string          `yaml:"numa_node,omitempty" validate:"numa_node:editable"`
 	Config          map[string]any  `yaml:"config,omitempty" validate:"config:ignore"`
 	ResourceControl ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"`
@@ -202,10 +203,18 @@ func (s *MasterSpec) GetAdvertisePeerURL(enableTLS bool) string {
 	return fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(s.Host, s.PeerPort))
 }
 
+// GetSource returns source to download the component
+func (s *MasterSpec) GetSource() string {
+	if s.Source == "" {
+		return ComponentDMMaster
+	}
+	return s.Source
+}
+
 // WorkerSpec represents the Master topology specification in topology.yaml
 type WorkerSpec struct {
 	Host           string `yaml:"host"`
-	ManageHost     string `yaml:"manage_host,omitempty"`
+	ManageHost     string `yaml:"manage_host,omitempty" validate:"manage_host:editable"`
 	SSHPort        int    `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"`
 	Imported       bool   `yaml:"imported,omitempty"`
 	Patched        bool   `yaml:"patched,omitempty"`
@@ -216,6 +225,7 @@ type WorkerSpec struct {
 	DeployDir       string          `yaml:"deploy_dir,omitempty"`
 	DataDir         string          `yaml:"data_dir,omitempty"`
 	LogDir          string          `yaml:"log_dir,omitempty"`
+	Source          string          `yaml:"source,omitempty" validate:"source:editable"`
 	NumaNode        string          `yaml:"numa_node,omitempty" validate:"numa_node:editable"`
 	Config          map[string]any  `yaml:"config,omitempty" validate:"config:ignore"`
 	ResourceControl ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"`
@@ -272,6 +282,14 @@ func (s *WorkerSpec) IgnoreMonitorAgent() bool {
 	return s.IgnoreExporter
 }
 
+// GetSource returns source to download the component
+func (s *WorkerSpec) GetSource() string {
+	if s.Source == "" {
+		return ComponentDMWorker
+	}
+	return s.Source
+}
+
 // UnmarshalYAML sets default values when unmarshaling the topology file
 func (s *Specification) UnmarshalYAML(unmarshal func(any) error) error {
 	type topology Specification
@@ -675,7 +693,7 @@ func (s *Specification) BaseTopo() *spec.BaseTopo {
 	return &spec.BaseTopo{
 		GlobalOptions:    &s.GlobalOptions,
 		MonitoredOptions: s.GetMonitoredOptions(),
-		MasterList:       s.GetMasterList(),
+		MasterList:       s.GetMasterListWithManageHost(),
 		Monitors:         s.Monitors,
 		Grafanas:         s.Grafanas,
 		Alertmanagers:    s.Alertmanagers,
@@ -701,12 +719,16 @@ func (s *Specification) MergeTopo(rhs spec.Topology) spec.Topology {
 	return s.Merge(other)
 }
 
-// GetMasterList returns a list of Master API hosts of the current cluster
-func (s *Specification) GetMasterList() []string {
+// GetMasterListWithManageHost returns a list of Master API hosts of the current cluster
+func (s *Specification) GetMasterListWithManageHost() []string {
 	var masterList []string
 
 	for _, master := range s.Masters {
-		masterList = append(masterList, utils.JoinHostPort(master.Host, master.Port))
+		host := master.Host
+		if master.ManageHost != "" {
+			host = master.ManageHost
+		}
+		masterList = append(masterList, utils.JoinHostPort(host, master.Port))
 	}
 
 	return masterList

components/playground/command.go

@@ -102,7 +102,6 @@ func newScaleOut() *cobra.Command {
 	cmd.Flags().IntVarP(&opt.TiKVCDC.Num, "kvcdc", "", opt.TiKVCDC.Num, "TiKV-CDC instance number")
 	cmd.Flags().IntVarP(&opt.Pump.Num, "pump", "", opt.Pump.Num, "Pump instance number")
 	cmd.Flags().IntVarP(&opt.Drainer.Num, "drainer", "", opt.Pump.Num, "Drainer instance number")
-
 	cmd.Flags().StringVarP(&opt.TiDB.Host, "db.host", "", opt.TiDB.Host, "Playground TiDB host. If not provided, TiDB will still use `host` flag as its host")
 	cmd.Flags().StringVarP(&opt.PD.Host, "pd.host", "", opt.PD.Host, "Playground PD host. If not provided, PD will still use `host` flag as its host")
 

components/playground/instance/instance.go

@@ -109,6 +109,9 @@ func logIfErr(err error) {
 func pdEndpoints(pds []*PDInstance, isHTTP bool) []string {
 	var endpoints []string
 	for _, pd := range pds {
+		if pd.Role == PDRoleTSO || pd.Role == PDRoleResourceManager {
+			continue
+		}
 		if isHTTP {
 			endpoints = append(endpoints, "http://"+utils.JoinHostPort(AdvertiseHost(pd.Host), pd.StatusPort))
 		} else {

components/playground/instance/pd.go

@@ -24,16 +24,32 @@ import (
 	"github.com/pingcap/tiup/pkg/utils"
 )
 
+// PDRole is the role of PD.
+type PDRole string
+
+const (
+	// PDRoleNormal is the default role of PD
+	PDRoleNormal PDRole = "pd"
+	// PDRoleAPI is the role of PD API
+	PDRoleAPI PDRole = "api"
+	// PDRoleTSO is the role of PD TSO
+	PDRoleTSO PDRole = "tso"
+	// PDRoleResourceManager is the role of PD resource manager
+	PDRoleResourceManager PDRole = "resource manager"
+)
+
 // PDInstance represent a running pd-server
 type PDInstance struct {
 	instance
+	Role          PDRole
 	initEndpoints []*PDInstance
 	joinEndpoints []*PDInstance
+	pds           []*PDInstance
 	Process
 }
 
 // NewPDInstance return a PDInstance
-func NewPDInstance(binPath, dir, host, configPath string, id, port int) *PDInstance {
+func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int) *PDInstance {
 	if port <= 0 {
 		port = 2379
 	}
@@ -47,6 +63,8 @@ func NewPDInstance(binPath, dir, host, configPath string, id, port int) *PDInsta
 			StatusPort: utils.MustGetFreePort(host, port),
 			ConfigPath: configPath,
 		},
+		Role: role,
+		pds:  pds,
 	}
 }
 
@@ -70,35 +88,67 @@ func (inst *PDInstance) Name() string {
 // Start calls set inst.cmd and Start
 func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error {
 	uid := inst.Name()
-	args := []string{
-		"--name=" + uid,
-		fmt.Sprintf("--data-dir=%s", filepath.Join(inst.Dir, "data")),
-		fmt.Sprintf("--peer-urls=http://%s", utils.JoinHostPort(inst.Host, inst.Port)),
-		fmt.Sprintf("--advertise-peer-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port)),
-		fmt.Sprintf("--client-urls=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)),
-		fmt.Sprintf("--advertise-client-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
-		fmt.Sprintf("--log-file=%s", inst.LogFile()),
-	}
-	if inst.ConfigPath != "" {
-		args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
-	}
-
-	switch {
-	case len(inst.initEndpoints) > 0:
-		endpoints := make([]string, 0)
-		for _, pd := range inst.initEndpoints {
-			uid := fmt.Sprintf("pd-%d", pd.ID)
-			endpoints = append(endpoints, fmt.Sprintf("%s=http://%s", uid, utils.JoinHostPort(AdvertiseHost(inst.Host), pd.Port)))
+	var args []string
+	switch inst.Role {
+	case PDRoleNormal, PDRoleAPI:
+		if inst.Role == PDRoleAPI {
+			args = []string{"services", "api"}
 		}
-		args = append(args, fmt.Sprintf("--initial-cluster=%s", strings.Join(endpoints, ",")))
-	case len(inst.joinEndpoints) > 0:
-		endpoints := make([]string, 0)
-		for _, pd := range inst.joinEndpoints {
-			endpoints = append(endpoints, fmt.Sprintf("http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), pd.Port)))
+		args = append(args, []string{
+			"--name=" + uid,
+			fmt.Sprintf("--data-dir=%s", filepath.Join(inst.Dir, "data")),
+			fmt.Sprintf("--peer-urls=http://%s", utils.JoinHostPort(inst.Host, inst.Port)),
+			fmt.Sprintf("--advertise-peer-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port)),
+			fmt.Sprintf("--client-urls=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)),
+			fmt.Sprintf("--advertise-client-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
+			fmt.Sprintf("--log-file=%s", inst.LogFile()),
+		}...)
+		if inst.ConfigPath != "" {
+			args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
+		}
+		switch {
+		case len(inst.initEndpoints) > 0:
+			endpoints := make([]string, 0)
+			for _, pd := range inst.initEndpoints {
+				uid := fmt.Sprintf("pd-%d", pd.ID)
+				endpoints = append(endpoints, fmt.Sprintf("%s=http://%s", uid, utils.JoinHostPort(AdvertiseHost(inst.Host), pd.Port)))
+			}
+			args = append(args, fmt.Sprintf("--initial-cluster=%s", strings.Join(endpoints, ",")))
+		case len(inst.joinEndpoints) > 0:
+			endpoints := make([]string, 0)
+			for _, pd := range inst.joinEndpoints {
+				endpoints = append(endpoints, fmt.Sprintf("http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), pd.Port)))
+			}
+			args = append(args, fmt.Sprintf("--join=%s", strings.Join(endpoints, ",")))
+		default:
+			return errors.Errorf("must set the init or join instances")
+		}
+	case PDRoleTSO:
+		endpoints := pdEndpoints(inst.pds, true)
+		args = []string{
+			"services",
+			"tso",
+			fmt.Sprintf("--listen-addr=http://%s", utils.JoinHostPort(inst.Host, inst.Port)),
+			fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port)),
+			fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")),
+			fmt.Sprintf("--log-file=%s", inst.LogFile()),
+		}
+		if inst.ConfigPath != "" {
+			args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
+		}
+	case PDRoleResourceManager:
+		endpoints := pdEndpoints(inst.pds, true)
+		args = []string{
+			"services",
+			"resource-manager",
+			fmt.Sprintf("--listen-addr=http://%s", utils.JoinHostPort(inst.Host, inst.Port)),
+			fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port)),
+			fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")),
+			fmt.Sprintf("--log-file=%s", inst.LogFile()),
+		}
+		if inst.ConfigPath != "" {
+			args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
 		}
-		args = append(args, fmt.Sprintf("--join=%s", strings.Join(endpoints, ",")))
-	default:
-		return errors.Errorf("must set the init or join instances")
 	}
 
 	var err error
@@ -113,12 +163,15 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
 
 // Component return the component name.
 func (inst *PDInstance) Component() string {
-	return "pd"
+	if inst.Role == PDRoleNormal {
+		return "pd"
+	}
+	return fmt.Sprintf("pd %s", inst.Role)
 }
 
 // LogFile return the log file.
 func (inst *PDInstance) LogFile() string {
-	return filepath.Join(inst.Dir, "pd.log")
+	return filepath.Join(inst.Dir, fmt.Sprintf("%s.log", string(inst.Role)))
 }
 
 // Addr return the listen address of PD

components/playground/instance/tiflash.go

@@ -138,7 +138,6 @@ func (inst *TiFlashInstance) Start(ctx context.Context, version utils.Version) e
 		fmt.Sprintf("--tmp_path=%s", filepath.Join(inst.Dir, "tmp")),
 		fmt.Sprintf("--path=%s", filepath.Join(inst.Dir, "data")),
 		fmt.Sprintf("--listen_host=%s", inst.Host),
-		fmt.Sprintf("--tcp_port=%d", inst.TCPPort),
 		fmt.Sprintf("--logger.log=%s", inst.LogFile()),
 		fmt.Sprintf("--logger.errorlog=%s", filepath.Join(inst.Dir, "tiflash_error.log")),
 		fmt.Sprintf("--status.metrics_port=%d", inst.StatusPort),

components/playground/instance/tiflash_pre7_config.go

@@ -34,10 +34,10 @@ const tiflashMarkCacheSizeOld = `mark_cache_size = 5368709120`
 const tiflashConfigOld = `
 default_profile = "default"
 display_name = "TiFlash"
-%[2]s
+http_port = %[2]d
 listen_host = "0.0.0.0"
-path = "%[5]s"
 tcp_port = %[3]d
+path = "%[5]s"
 tmp_path = "%[6]s"
 %[14]s
 %[13]s
@@ -109,11 +109,11 @@ func writeTiFlashConfigOld(w io.Writer, version utils.Version, tcpPort, httpPort
 	var conf string
 
 	if tidbver.TiFlashNotNeedSomeConfig(version.String()) {
-		conf = fmt.Sprintf(tiflashConfigOld, pdAddrs, fmt.Sprintf(`http_port = %d`, httpPort), tcpPort,
+		conf = fmt.Sprintf(tiflashConfigOld, pdAddrs, httpPort, tcpPort,
 			deployDir, dataDir, tmpDir, logDir, servicePort, metricsPort,
 			ip, strings.Join(tidbStatusAddrs, ","), clusterManagerPath, "", "")
 	} else {
-		conf = fmt.Sprintf(tiflashConfigOld, pdAddrs, fmt.Sprintf(`http_port = %d`, httpPort), tcpPort,
+		conf = fmt.Sprintf(tiflashConfigOld, pdAddrs, httpPort, tcpPort,
 			deployDir, dataDir, tmpDir, logDir, servicePort, metricsPort,
 			ip, strings.Join(tidbStatusAddrs, ","), clusterManagerPath, tiflashDaemonConfigOld, tiflashMarkCacheSizeOld)
 	}