Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[feat] curvebs: The formatting and deployment phases support separate… #238

Open
wants to merge 3 commits into
base: tombstone
Choose a base branch
from
Open

[feat] curvebs: The formatting and deployment phases support separate… #238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ab13de3
[feat] curvebs: The formatting and deployment phases support separate…
zyb521 Jun 12, 2023
e2eea99
[fix] some bug fix, and support clean walfiles
zyb521 Jul 19, 2023
adc8a1a
Merge branch 'opencurve:develop' into develop
zyb521 Jul 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions internal/configure/format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import (

const (
DEFAULT_CONTAINER_IMAGE = "opencurvedocker/curvebs:v1.2"
FORMAT_TYPE_DATA = "data"
FORMAT_TYPE_WAL = "wal"
)

/*
Expand All @@ -40,14 +42,18 @@ const (
* - machine2
* - machine3
* disk:
* - /dev/sda:/data/chunkserver0:10 # device:mount_path:format_percent
* - /dev/sdb:/data/chunkserver1:10
* - /dev/sdc:/data/chunkserver2:10
* - data:/dev/sda:/data/chunkserver0:10 # fortmat_type:device:mount_path:format_percent
* - data:/dev/sdb:/data/chunkserver1:10
* - data:/dev/sdc:/data/chunkserver2:10
* - wal:/dev/nvme0n1p1:/data/wal/chunkserver0:10
* - wal:/dev/nvme0n1p2:/data/wal/chunkserver1:10
* - wal:/dev/nvme0n1p3:/data/wal/chunkserver2:10
*/
type (
FormatConfig struct {
ContainerIamge string
Host string
Type string
Device string
MountPoint string
FormtPercent int
Expand All @@ -64,12 +70,15 @@ type (

func NewFormatConfig(containerImage, host, disk string) (*FormatConfig, error) {
items := strings.Split(disk, ":")
if len(items) != 3 {
if len(items) != 4 {
return nil, errno.ERR_INVALID_DISK_FORMAT.S(disk)
}

device, mountPoint, percent := items[0], items[1], items[2]
if !strings.HasPrefix(device, "/") {
formatType, device, mountPoint, percent := items[0], items[1], items[2], items[3]
if formatType != FORMAT_TYPE_DATA && formatType != FORMAT_TYPE_WAL {
return nil, errno.ERR_INVALID_FORMAT_TYPE.
F("formatType: %s", formatType)
} else if !strings.HasPrefix(device, "/") {
return nil, errno.ERR_INVALID_DEVICE.
F("device: %s", device)
} else if !strings.HasPrefix(mountPoint, "/") {
Expand All @@ -89,6 +98,7 @@ func NewFormatConfig(containerImage, host, disk string) (*FormatConfig, error) {
return &FormatConfig{
ContainerIamge: containerImage,
Host: host,
Type: formatType,
Device: device,
MountPoint: mountPoint,
FormtPercent: formatPercent,
Expand Down Expand Up @@ -136,6 +146,7 @@ func ParseFormat(filename string) ([]*FormatConfig, error) {

func (fc *FormatConfig) GetContainerImage() string { return fc.ContainerIamge }
func (fc *FormatConfig) GetHost() string { return fc.Host }
func (fc *FormatConfig) GetFormatType() string { return fc.Type }
func (fc *FormatConfig) GetDevice() string { return fc.Device }
func (fc *FormatConfig) GetMountPoint() string { return fc.MountPoint }
func (fc *FormatConfig) GetFormatPercent() int { return fc.FormtPercent }
16 changes: 15 additions & 1 deletion internal/configure/topology/dc_get.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,11 @@ const (
LAYOUT_SERVICE_CONF_DIR = "/conf"
LAYOUT_SERVICE_LOG_DIR = "/logs"
LAYOUT_SERVICE_DATA_DIR = "/data"
LAYOUT_SERVICE_WAL_DIR = "/wal"
LAYOUT_TOOLS_DIR = "/tools"
LAYOUT_TOOLS_V2_DIR = "/tools-v2"
LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR = "chunkfilepool"
LAYOUT_CURVEBS_WALFILE_POOL_DIR = "walfilepool"
LAYOUT_CURVEBS_COPYSETS_DIR = "copysets"
LAYOUT_CURVEBS_RECYCLER_DIR = "recycler"
LAYOUT_CURVEBS_TOOLS_CONFIG_SYSTEM_PATH = "/etc/curve/tools.conf"
Expand All @@ -57,6 +59,7 @@ const (
BINARY_CURVEFS_TOOL = "curvefs_tool"
BINARY_CURVE_TOOL_V2 = "curve"
METAFILE_CHUNKFILE_POOL = "chunkfilepool.meta"
METAFILE_WALFILE_POOL = "walfilepool.meta"
METAFILE_CHUNKSERVER_ID = "chunkserver.dat"
)

Expand Down Expand Up @@ -131,6 +134,7 @@ func (dc *DeployConfig) GetContainerImage() string {
}
func (dc *DeployConfig) GetLogDir() string { return dc.getString(CONFIG_LOG_DIR) }
func (dc *DeployConfig) GetDataDir() string { return dc.getString(CONFIG_DATA_DIR) }
func (dc *DeployConfig) GetWalDir() string { return dc.getString(CONFIG_WAL_DIR) }
func (dc *DeployConfig) GetCoreDir() string { return dc.getString(CONFIG_CORE_DIR) }
func (dc *DeployConfig) GetListenIp() string { return dc.getString(CONFIG_LISTEN_IP) }
func (dc *DeployConfig) GetListenPort() int { return dc.getInt(CONFIG_LISTEN_PORT) }
Expand Down Expand Up @@ -181,7 +185,8 @@ func (dc *DeployConfig) GetListenExternalPort() int {
* │ ├── conf
* │ ├── data
* │ ├── log
* │ └── sbin
* │ ├── sbin
* │ └── wal
* ├── snapshotclone
* │ ├── conf
* │ ├── data
Expand Down Expand Up @@ -212,6 +217,7 @@ type (
ServiceConfDir string // /curvebs/mds/conf
ServiceLogDir string // /curvebs/mds/logs
ServiceDataDir string // /curvebs/mds/data
ServiceWalDir string // /curvebs/chunkserver/wal
ServiceConfPath string // /curvebs/mds/conf/mds.conf
ServiceConfSrcPath string // /curvebs/conf/mds.conf
ServiceConfFiles []ConfFile
Expand All @@ -236,6 +242,9 @@ type (
ChunkfilePoolRootDir string // /curvebs/chunkserver/data
Copy link

@legionxiong legionxiong Jun 21, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

missing WalfilePoolRootDir initialization, else the layout.WalfilePoolRootDir is an empty string.

Copy link

@legionxiong legionxiong Jun 21, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the same as WalfilePoolDir and WalfilePoolMetaPath.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

ChunkfilePoolDir string // /curvebs/chunkserver/data/chunkfilepool
ChunkfilePoolMetaPath string // /curvebs/chunkserver/data/chunkfilepool.meta
WalfilePoolRootDir string // /curvebs/chunkserver/wal
WalfilePoolDir string // /curvebs/chunkserver/wal/walfilepool
WalfilePoolMetaPath string // /curvebs/chunkserver/wal/walfilepool.meta

// core
CoreSystemDir string
Expand Down Expand Up @@ -278,6 +287,7 @@ func (dc *DeployConfig) GetProjectLayout() Layout {

// format
chunkserverDataDir := fmt.Sprintf("%s/%s%s", root, ROLE_CHUNKSERVER, LAYOUT_SERVICE_DATA_DIR)
chunkserverWalDir := fmt.Sprintf("%s/%s%s", root, ROLE_CHUNKSERVER, LAYOUT_SERVICE_WAL_DIR)

return Layout{
// project
Expand All @@ -292,6 +302,7 @@ func (dc *DeployConfig) GetProjectLayout() Layout {
ServiceConfDir: serviceRootDir + LAYOUT_SERVICE_CONF_DIR,
ServiceLogDir: serviceRootDir + LAYOUT_SERVICE_LOG_DIR,
ServiceDataDir: serviceRootDir + LAYOUT_SERVICE_DATA_DIR,
ServiceWalDir: serviceRootDir + LAYOUT_SERVICE_WAL_DIR,
ServiceConfPath: fmt.Sprintf("%s/%s.conf", serviceConfDir, role),
ServiceConfSrcPath: fmt.Sprintf("%s/%s.conf", confSrcDir, role),
ServiceConfFiles: serviceConfFiles,
Expand All @@ -316,6 +327,9 @@ func (dc *DeployConfig) GetProjectLayout() Layout {
ChunkfilePoolRootDir: chunkserverDataDir,
ChunkfilePoolDir: fmt.Sprintf("%s/%s", chunkserverDataDir, LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR),
ChunkfilePoolMetaPath: fmt.Sprintf("%s/%s", chunkserverDataDir, METAFILE_CHUNKFILE_POOL),
WalfilePoolRootDir: chunkserverWalDir,
WalfilePoolDir: fmt.Sprintf("%s/%s", chunkserverWalDir, LAYOUT_CURVEBS_WALFILE_POOL_DIR),
WalfilePoolMetaPath: fmt.Sprintf("%s/%s", chunkserverWalDir, METAFILE_WALFILE_POOL),

// core
CoreSystemDir: LAYOUT_CORE_SYSTEM_DIR,
Expand Down
7 changes: 7 additions & 0 deletions internal/configure/topology/dc_item.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,13 @@ var (
nil,
)

CONFIG_WAL_DIR = itemset.insert(
"wal_dir",
REQUIRE_STRING,
true,
nil,
)

CONFIG_CORE_DIR = itemset.insert(
"core_dir",
REQUIRE_STRING,
Expand Down
1 change: 1 addition & 0 deletions internal/errno/errno.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,7 @@ var (
ERR_MOUNT_POINT_REQUIRE_ABSOLUTE_PATH = EC(341002, "mount point must be an absolute path")
ERR_FORMAT_PERCENT_REQUIRES_INTERGET = EC(341003, "format percentage requires an integer")
ERR_FORMAT_PERCENT_MUST_BE_BETWEEN_1_AND_100 = EC(341004, "format percentage must be between 1 and 100")
ERR_INVALID_FORMAT_TYPE = EC(341005, "format type must be (data) or (wal)")

// 350: configure (client.yaml: parse failed)
ERR_PARSE_CLIENT_CONFIGURE_FAILED = EC(350000, "parse client configure failed")
Expand Down
23 changes: 17 additions & 6 deletions internal/task/task/bs/format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,22 +231,33 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
}

// new task
formatTpye := fc.GetFormatType()
device := fc.GetDevice()
mountPoint := fc.GetMountPoint()
usagePercent := fc.GetFormatPercent()
subname := fmt.Sprintf("host=%s device=%s mountPoint=%s usage=%d%%",
fc.GetHost(), device, mountPoint, usagePercent)
t := task.NewTask("Start Format Chunkfile Pool", subname, hc.GetSSHConfig())
subname := fmt.Sprintf("host=%s formatTpye=%s device=%s mountPoint=%s usage=%d%%",
fc.GetHost(), formatTpye, device, mountPoint, usagePercent)
t := task.NewTask("Start Format filePool", subname, hc.GetSSHConfig())

// add step to task
var oldContainerId, containerId, oldUUID string
var filePoolRootDir, filePoolDir, filePoolMetaPath string
containerName := device2ContainerName(device)
layout := topology.GetCurveBSProjectLayout()
chunkfilePoolRootDir := layout.ChunkfilePoolRootDir

if formatTpye == configure.FORMAT_TYPE_WAL {
filePoolRootDir = layout.WalfilePoolRootDir
filePoolDir = layout.WalfilePoolDir
filePoolMetaPath = layout.WalfilePoolMetaPath
} else {
filePoolRootDir = layout.ChunkfilePoolRootDir
filePoolDir = layout.ChunkfilePoolDir
filePoolMetaPath = layout.ChunkfilePoolMetaPath
}
formatScript := scripts.SCRIPT_FORMAT
formatScriptPath := fmt.Sprintf("%s/format.sh", layout.ToolsBinDir)
formatCommand := fmt.Sprintf("%s %s %d %d %s %s", formatScriptPath, layout.FormatBinaryPath,
usagePercent, DEFAULT_CHUNKFILE_SIZE, layout.ChunkfilePoolDir, layout.ChunkfilePoolMetaPath)
usagePercent, DEFAULT_CHUNKFILE_SIZE, filePoolDir, filePoolMetaPath)

// 1: skip if formating container exist
t.AddStep(&step.ListContainers{
Expand Down Expand Up @@ -320,7 +331,7 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
Entrypoint: "/bin/bash",
Name: containerName,
Remove: true,
Volumes: []step.Volume{{HostPath: mountPoint, ContainerPath: chunkfilePoolRootDir}},
Volumes: []step.Volume{{HostPath: mountPoint, ContainerPath: filePoolRootDir}},
Out: &containerId,
ExecOptions: curveadm.ExecOptions(),
})
Expand Down
17 changes: 17 additions & 0 deletions internal/task/task/common/clean_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (

const (
LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR = topology.LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR
LAYOUT_CURVEBS_WALFILE_POOL_DIR = topology.LAYOUT_CURVEBS_WALFILE_POOL_DIR
LAYOUT_CURVEBS_COPYSETS_DIR = topology.LAYOUT_CURVEBS_COPYSETS_DIR
LAYOUT_CURVEBS_RECYCLER_DIR = topology.LAYOUT_CURVEBS_RECYCLER_DIR
METAFILE_CHUNKSERVER_ID = topology.METAFILE_CHUNKSERVER_ID
Expand Down Expand Up @@ -96,6 +97,17 @@ func (s *step2RecycleChunk) Execute(ctx *context.Context) error {
if err != nil {
errno.ERR_RUN_SCRIPT_FAILED.E(err)
}

walDir := dc.GetWalDir()
if walDir != "" {
source = fmt.Sprintf("%s/%s", walDir, LAYOUT_CURVEBS_COPYSETS_DIR)
dest = fmt.Sprintf("%s/%s", walDir, LAYOUT_CURVEBS_WALFILE_POOL_DIR)
cmd = ctx.Module().Shell().BashScript(s.recycleScriptPath, source, dest, chunkSize)
_, err = cmd.Execute(s.execOptions)
if err != nil {
errno.ERR_RUN_SCRIPT_FAILED.E(err)
}
}
return nil
}

Expand Down Expand Up @@ -132,9 +144,14 @@ func getCleanFiles(clean map[string]bool, dc *topology.DeployConfig, recycle boo
files = append(files, dc.GetDataDir())
} else {
dataDir := dc.GetDataDir()
walDir := dc.GetWalDir()
copysetsDir := fmt.Sprintf("%s/%s", dataDir, LAYOUT_CURVEBS_COPYSETS_DIR)
chunkserverIdMetafile := fmt.Sprintf("%s/%s", dataDir, METAFILE_CHUNKSERVER_ID)
files = append(files, copysetsDir, chunkserverIdMetafile)
if walDir != "" {
walCopysetsDir := fmt.Sprintf("%s/%s", walDir, LAYOUT_CURVEBS_COPYSETS_DIR)
files = append(files, walCopysetsDir)
}
}
}
}
Expand Down
18 changes: 15 additions & 3 deletions internal/task/task/common/create_container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,10 @@ func getArguments(dc *topology.DeployConfig) string {
// only chunkserver need so many arguments, but who cares
layout := dc.GetProjectLayout()
dataDir := layout.ServiceDataDir
walDir := layout.ServiceWalDir
if dc.GetWalDir() == "" {
walDir = dataDir
}
chunkserverArguments := map[string]interface{}{
// chunkserver
"conf": layout.ServiceConfPath,
Expand All @@ -120,11 +124,11 @@ func getArguments(dc *topology.DeployConfig) string {
"chunkServerPort": dc.GetListenPort(),
"chunkFilePoolDir": dataDir,
"chunkFilePoolMetaPath": fmt.Sprintf("%s/chunkfilepool.meta", dataDir),
"walFilePoolDir": dataDir,
"walFilePoolMetaPath": fmt.Sprintf("%s/walfilepool.meta", dataDir),
"walFilePoolDir": walDir,
"walFilePoolMetaPath": fmt.Sprintf("%s/walfilepool.meta", walDir),
"copySetUri": fmt.Sprintf("local://%s/copysets", dataDir),
"recycleUri": fmt.Sprintf("local://%s/recycler", dataDir),
"raftLogUri": fmt.Sprintf("curve://%s/copysets", dataDir),
"raftLogUri": fmt.Sprintf("curve://%s/copysets", walDir),
"raftSnapshotUri": fmt.Sprintf("curve://%s/copysets", dataDir),
"chunkServerStoreUri": fmt.Sprintf("local://%s", dataDir),
"chunkServerMetaUri": fmt.Sprintf("local://%s/chunkserver.dat", dataDir),
Expand Down Expand Up @@ -163,6 +167,7 @@ func getMountVolumes(dc *topology.DeployConfig, serviceMountDevice bool) []step.
layout := dc.GetProjectLayout()
logDir := dc.GetLogDir()
dataDir := dc.GetDataDir()
walDir := dc.GetWalDir()
coreDir := dc.GetCoreDir()

if len(logDir) > 0 {
Expand All @@ -180,6 +185,13 @@ func getMountVolumes(dc *topology.DeployConfig, serviceMountDevice bool) []step.
})
}

if len(walDir) > 0 {
volumes = append(volumes, step.Volume{
HostPath: walDir,
ContainerPath: layout.ServiceWalDir,
})
}

if len(coreDir) > 0 {
volumes = append(volumes, step.Volume{
HostPath: coreDir,
Expand Down