diff --git a/DEPS.bzl b/DEPS.bzl index 1d324eafb3633..11a76e560119f 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -6010,13 +6010,13 @@ def go_deps(): name = "com_github_pingcap_kvproto", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/kvproto", - sha256 = "368b662c8669d91bcd488b780b8ecb273855b8fc54c1043907171bdebc3ffc54", - strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20240904041139-1de8accd5bb7", + sha256 = "701b1e4f0b7c5007ef20ac9e8ae7f97add4edea1308c2f41a50ce6639905c9a7", + strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20240910154453-b242104f8d31", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240904041139-1de8accd5bb7.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240904041139-1de8accd5bb7.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240904041139-1de8accd5bb7.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240904041139-1de8accd5bb7.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240910154453-b242104f8d31.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240910154453-b242104f8d31.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240910154453-b242104f8d31.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240910154453-b242104f8d31.zip", ], ) go_repository( @@ -7206,26 +7206,26 @@ def go_deps(): name = "com_github_tikv_client_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sha256 = "2c26a7a94e44e2aae520f2013f8d738c5c5f1fb9f70b76894843f6827ce945f7", - strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240821073530-75e3705e58f1", + sha256 = "8e1b11f3d4105df1114446700c91aab534967165c1c586ffbb51f94037b3c165", + strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240911041506-e7894a7b27ba", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240821073530-75e3705e58f1.zip", - "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240821073530-75e3705e58f1.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240821073530-75e3705e58f1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240821073530-75e3705e58f1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240911041506-e7894a7b27ba.zip", + "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240911041506-e7894a7b27ba.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240911041506-e7894a7b27ba.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240911041506-e7894a7b27ba.zip", ], ) go_repository( name = "com_github_tikv_pd_client", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/pd/client", - sha256 = "bb4aa99260c2d1b22054d539cb37c18ce276c96b1b7c4a8f14ac4cbcda829654", - strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240805092608-838ee7983b78", + sha256 = "fc2042d3b3c753de90ac2afdfea97b663f3043aa716f1ee56d2fdf98864e3cbd", + strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240914083230-71f6f96816e9", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240805092608-838ee7983b78.zip", - "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240805092608-838ee7983b78.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240805092608-838ee7983b78.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240805092608-838ee7983b78.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", + "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", ], ) go_repository( @@ -10300,13 +10300,13 @@ def go_deps(): name = "org_golang_x_crypto", build_file_proto_mode = "disable_global", importpath = "golang.org/x/crypto", - sha256 = "ec96acfe28be3ff2fb14201c5f51132f0e24c7d0d6f3201a8aa69c84f989d014", - strip_prefix = "golang.org/x/crypto@v0.26.0", + sha256 = "c724b619b457bb1c445a39541449b1348eb3851323a29d2c313ad0139634d0a5", + strip_prefix = "golang.org/x/crypto@v0.27.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.26.0.zip", - "http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.26.0.zip", - "https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.26.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.26.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.27.0.zip", + "http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.27.0.zip", + "https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.27.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.27.0.zip", ], ) go_repository( @@ -10469,26 +10469,26 @@ def go_deps(): name = "org_golang_x_term", build_file_proto_mode = "disable_global", importpath = "golang.org/x/term", - sha256 = "2597a62b487b952c11c89b2001551af1fe1d29c484388ec1c3f5e3be7ff58ba5", - strip_prefix = "golang.org/x/term@v0.23.0", + sha256 = "80b2b247641ca1b8f54769de20d4d6a0c6861de57a3bc1b31e8432a2a7483ab3", + strip_prefix = "golang.org/x/term@v0.24.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.23.0.zip", - "http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.23.0.zip", - "https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.23.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.23.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.24.0.zip", + "http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.24.0.zip", + "https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.24.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.24.0.zip", ], ) go_repository( name = "org_golang_x_text", build_file_proto_mode = "disable_global", importpath = "golang.org/x/text", - sha256 = "48464f2ab2f988ca8b7b0a9d098e3664224c3b128629b5a9cc08025ee4a7e4ec", - strip_prefix = "golang.org/x/text@v0.17.0", + sha256 = "09da08281c6854e695cdffb25569df0abf53fe545c6610be09d58294728e81e5", + strip_prefix = "golang.org/x/text@v0.18.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/text/org_golang_x_text-v0.17.0.zip", - "http://ats.apps.svc/gomod/golang.org/x/text/org_golang_x_text-v0.17.0.zip", - "https://cache.hawkingrei.com/gomod/golang.org/x/text/org_golang_x_text-v0.17.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/text/org_golang_x_text-v0.17.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/text/org_golang_x_text-v0.18.0.zip", + "http://ats.apps.svc/gomod/golang.org/x/text/org_golang_x_text-v0.18.0.zip", + "https://cache.hawkingrei.com/gomod/golang.org/x/text/org_golang_x_text-v0.18.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/text/org_golang_x_text-v0.18.0.zip", ], ) go_repository( diff --git a/OWNERS b/OWNERS index 724a54404e7a0..c79076fedd5d3 100644 --- a/OWNERS +++ b/OWNERS @@ -98,6 +98,7 @@ approvers: - winoros - WizardXiao - wjhuang2016 + - wk989898 - wshwsh12 - xhebox - xiongjiwei @@ -127,6 +128,7 @@ reviewers: - dhysum - fengou1 - fzzf678 + - ghazalfamilyusa - iamxy - JmPotato - js00070 @@ -145,6 +147,7 @@ reviewers: - shihongzhi - spongedu - tangwz + - terry1purcell - Tjianke - TonsnakeLin - tsthght diff --git a/br/pkg/restore/log_client/BUILD.bazel b/br/pkg/restore/log_client/BUILD.bazel index 2351e2908ea64..76a2f7de49a97 100644 --- a/br/pkg/restore/log_client/BUILD.bazel +++ b/br/pkg/restore/log_client/BUILD.bazel @@ -44,6 +44,7 @@ go_library( "//pkg/util/redact", "//pkg/util/table-filter", "@com_github_fatih_color//:color", + "@com_github_gogo_protobuf//proto", "@com_github_opentracing_opentracing_go//:opentracing-go", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", @@ -82,9 +83,10 @@ go_test( ], embed = [":log_client"], flaky = True, - shard_count = 40, + shard_count = 41, deps = [ "//br/pkg/errors", + "//br/pkg/glue", "//br/pkg/gluetidb", "//br/pkg/mock", "//br/pkg/restore/internal/import_client", @@ -97,10 +99,16 @@ go_test( "//br/pkg/utiltest", "//pkg/domain", "//pkg/kv", + "//pkg/planner/core/resolve", + "//pkg/session", + "//pkg/sessionctx", "//pkg/store/pdtypes", "//pkg/tablecodec", + "//pkg/testkit", "//pkg/testkit/testsetup", + "//pkg/util/chunk", "//pkg/util/codec", + "//pkg/util/sqlexec", "//pkg/util/table-filter", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", diff --git a/br/pkg/restore/log_client/client.go b/br/pkg/restore/log_client/client.go index 6c4666725ef4f..9a7ee8ee9cf8d 100644 --- a/br/pkg/restore/log_client/client.go +++ b/br/pkg/restore/log_client/client.go @@ -28,6 +28,7 @@ import ( "time" "github.com/fatih/color" + "github.com/gogo/protobuf/proto" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -95,6 +96,8 @@ type LogClient struct { // Can not use `restoreTS` directly, because schema created in `full backup` maybe is new than `restoreTS`. currentTS uint64 + upstreamClusterID uint64 + *LogFileManager workerPool *tidbutil.WorkerPool @@ -167,6 +170,11 @@ func (rc *LogClient) SetConcurrency(c uint) { rc.workerPool = tidbutil.NewWorkerPool(c, "file") } +func (rc *LogClient) SetUpstreamClusterID(upstreamClusterID uint64) { + log.Info("upstream cluster id", zap.Uint64("cluster id", upstreamClusterID)) + rc.upstreamClusterID = upstreamClusterID +} + func (rc *LogClient) SetStorage(ctx context.Context, backend *backuppb.StorageBackend, opts *storage.ExternalStorageOptions) error { var err error rc.storage, err = storage.New(ctx, backend, opts) @@ -558,24 +566,38 @@ func (rc *LogClient) RestoreKVFiles( func (rc *LogClient) initSchemasMap( ctx context.Context, - clusterID uint64, restoreTS uint64, ) ([]*backuppb.PitrDBMap, error) { - filename := metautil.PitrIDMapsFilename(clusterID, restoreTS) - exist, err := rc.storage.FileExists(ctx, filename) - if err != nil { - return nil, errors.Annotatef(err, "failed to check filename:%s ", filename) - } else if !exist { - log.Info("pitr id maps isn't existed", zap.String("file", filename)) + getPitrIDMapSQL := "SELECT segment_id, id_map FROM mysql.tidb_pitr_id_map WHERE restored_ts = %? and upstream_cluster_id = %? ORDER BY segment_id;" + execCtx := rc.se.GetSessionCtx().GetRestrictedSQLExecutor() + rows, _, errSQL := execCtx.ExecRestrictedSQL( + kv.WithInternalSourceType(ctx, kv.InternalTxnBR), + nil, + getPitrIDMapSQL, + restoreTS, + rc.upstreamClusterID, + ) + if errSQL != nil { + return nil, errors.Annotatef(errSQL, "failed to get pitr id map from mysql.tidb_pitr_id_map") + } + if len(rows) == 0 { + log.Info("pitr id map does not exist", zap.Uint64("restored ts", restoreTS)) return nil, nil } - - metaData, err := rc.storage.ReadFile(ctx, filename) - if err != nil { - return nil, errors.Trace(err) + metaData := make([]byte, 0, len(rows)*PITRIdMapBlockSize) + for i, row := range rows { + elementID := row.GetUint64(0) + if uint64(i) != elementID { + return nil, errors.Errorf("the part(segment_id = %d) of pitr id map is lost", i) + } + d := row.GetBytes(1) + if len(d) == 0 { + return nil, errors.Errorf("get the empty part(segment_id = %d) of pitr id map", i) + } + metaData = append(metaData, d...) } backupMeta := &backuppb.BackupMeta{} - if err = backupMeta.Unmarshal(metaData); err != nil { + if err := backupMeta.Unmarshal(metaData); err != nil { return nil, errors.Trace(err) } @@ -722,7 +744,7 @@ func (rc *LogClient) InitSchemasReplaceForDDL( if !cfg.IsNewTask { log.Info("try to load pitr id maps") needConstructIdMap = false - dbMaps, err = rc.initSchemasMap(ctx, rc.GetClusterID(ctx), rc.restoreTS) + dbMaps, err = rc.initSchemasMap(ctx, rc.restoreTS) if err != nil { return nil, errors.Trace(err) } @@ -733,7 +755,7 @@ func (rc *LogClient) InitSchemasReplaceForDDL( if len(dbMaps) <= 0 && cfg.FullBackupStorage == nil { log.Info("try to load pitr id maps of the previous task", zap.Uint64("start-ts", rc.startTS)) needConstructIdMap = true - dbMaps, err = rc.initSchemasMap(ctx, rc.GetClusterID(ctx), rc.startTS) + dbMaps, err = rc.initSchemasMap(ctx, rc.startTS) if err != nil { return nil, errors.Trace(err) } @@ -887,7 +909,7 @@ func (rc *LogClient) PreConstructAndSaveIDMap( return errors.Trace(err) } - if err := rc.SaveIDMap(ctx, sr); err != nil { + if err := rc.saveIDMap(ctx, sr); err != nil { return errors.Trace(err) } return nil @@ -1491,24 +1513,36 @@ func (rc *LogClient) GetGCRows() []*stream.PreDelRangeQuery { return rc.deleteRangeQuery } -// SaveIDMap saves the id mapping information. -func (rc *LogClient) SaveIDMap( +const PITRIdMapBlockSize int = 524288 + +// saveIDMap saves the id mapping information. +func (rc *LogClient) saveIDMap( ctx context.Context, sr *stream.SchemasReplace, ) error { - idMaps := sr.TidySchemaMaps() - clusterID := rc.GetClusterID(ctx) - metaFileName := metautil.PitrIDMapsFilename(clusterID, rc.restoreTS) - metaWriter := metautil.NewMetaWriter(rc.storage, metautil.MetaFileSize, false, metaFileName, nil) - metaWriter.Update(func(m *backuppb.BackupMeta) { - // save log startTS to backupmeta file - m.ClusterId = clusterID - m.DbMaps = idMaps - }) - - if err := metaWriter.FlushBackupMeta(ctx); err != nil { + backupmeta := &backuppb.BackupMeta{DbMaps: sr.TidySchemaMaps()} + data, err := proto.Marshal(backupmeta) + if err != nil { + return errors.Trace(err) + } + // clean the dirty id map at first + err = rc.se.ExecuteInternal(ctx, "DELETE FROM mysql.tidb_pitr_id_map WHERE restored_ts = %? and upstream_cluster_id = %?;", rc.restoreTS, rc.upstreamClusterID) + if err != nil { return errors.Trace(err) } + replacePitrIDMapSQL := "REPLACE INTO mysql.tidb_pitr_id_map (restored_ts, upstream_cluster_id, segment_id, id_map) VALUES (%?, %?, %?, %?);" + for startIdx, segmentId := 0, 0; startIdx < len(data); segmentId += 1 { + endIdx := startIdx + PITRIdMapBlockSize + if endIdx > len(data) { + endIdx = len(data) + } + err := rc.se.ExecuteInternal(ctx, replacePitrIDMapSQL, rc.restoreTS, rc.upstreamClusterID, segmentId, data[startIdx:endIdx]) + if err != nil { + return errors.Trace(err) + } + startIdx = endIdx + } + if rc.useCheckpoint { var items map[int64]model.TiFlashReplicaInfo if sr.TiflashRecorder != nil { diff --git a/br/pkg/restore/log_client/client_test.go b/br/pkg/restore/log_client/client_test.go index aaa8e8c79a60c..7b00e30e6eaa7 100644 --- a/br/pkg/restore/log_client/client_test.go +++ b/br/pkg/restore/log_client/client_test.go @@ -25,16 +25,22 @@ import ( "github.com/pingcap/errors" backuppb "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/tidb/br/pkg/glue" "github.com/pingcap/tidb/br/pkg/gluetidb" "github.com/pingcap/tidb/br/pkg/mock" logclient "github.com/pingcap/tidb/br/pkg/restore/log_client" "github.com/pingcap/tidb/br/pkg/restore/utils" - "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/br/pkg/stream" "github.com/pingcap/tidb/br/pkg/utils/iter" "github.com/pingcap/tidb/br/pkg/utiltest" "github.com/pingcap/tidb/pkg/domain" + "github.com/pingcap/tidb/pkg/planner/core/resolve" + "github.com/pingcap/tidb/pkg/session" + "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/tablecodec" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/pingcap/tidb/pkg/util/sqlexec" filter "github.com/pingcap/tidb/pkg/util/table-filter" "github.com/stretchr/testify/require" "google.golang.org/grpc/keepalive" @@ -1341,46 +1347,161 @@ func TestLogFilesIterWithSplitHelper(t *testing.T) { } } -type fakeStorage struct { - storage.ExternalStorage +type fakeSession struct { + glue.Session } -func (fs fakeStorage) FileExists(ctx context.Context, name string) (bool, error) { - return false, errors.Errorf("name: %s", name) +func (fs fakeSession) GetSessionCtx() sessionctx.Context { + return fakeSessionContext{} } -type fakeStorageOK struct { - storage.ExternalStorage +type fakeSessionContext struct { + sessionctx.Context } -func (fs fakeStorageOK) FileExists(ctx context.Context, name string) (bool, error) { - return false, nil +func (fsc fakeSessionContext) GetRestrictedSQLExecutor() sqlexec.RestrictedSQLExecutor { + return fakeSQLExecutor{} +} + +type fakeSQLExecutor struct { + sqlexec.RestrictedSQLExecutor +} + +func (fse fakeSQLExecutor) ExecRestrictedSQL(_ context.Context, _ []sqlexec.OptionFuncAlias, query string, args ...any) ([]chunk.Row, []*resolve.ResultField, error) { + return nil, nil, errors.Errorf("name: %s, %v", query, args) } func TestInitSchemasReplaceForDDL(t *testing.T) { ctx := context.Background() { - client := logclient.TEST_NewLogClient(123, 1, 2, fakeStorage{}, domain.NewMockDomain()) + client := logclient.TEST_NewLogClient(123, 1, 2, 1, domain.NewMockDomain(), fakeSession{}) cfg := &logclient.InitSchemaConfig{IsNewTask: false} _, err := client.InitSchemasReplaceForDDL(ctx, cfg) require.Error(t, err) - require.Contains(t, err.Error(), "failed to check filename:pitr_id_maps/pitr_id_map.cluster_id:123.restored_ts:2") + require.Regexp(t, "failed to get pitr id map from mysql.tidb_pitr_id_map.* [2, 1]", err.Error()) } { - client := logclient.TEST_NewLogClient(123, 1, 2, fakeStorage{}, domain.NewMockDomain()) + client := logclient.TEST_NewLogClient(123, 1, 2, 1, domain.NewMockDomain(), fakeSession{}) cfg := &logclient.InitSchemaConfig{IsNewTask: true} _, err := client.InitSchemasReplaceForDDL(ctx, cfg) require.Error(t, err) - require.Contains(t, err.Error(), "failed to check filename:pitr_id_maps/pitr_id_map.cluster_id:123.restored_ts:1") + require.Regexp(t, "failed to get pitr id map from mysql.tidb_pitr_id_map.* [1, 1]", err.Error()) } { - client := logclient.TEST_NewLogClient(123, 1, 2, fakeStorageOK{}, domain.NewMockDomain()) + s := utiltest.CreateRestoreSchemaSuite(t) + tk := testkit.NewTestKit(t, s.Mock.Storage) + tk.Exec(session.CreatePITRIDMap) + g := gluetidb.New() + se, err := g.CreateSession(s.Mock.Storage) + require.NoError(t, err) + client := logclient.TEST_NewLogClient(123, 1, 2, 1, domain.NewMockDomain(), se) cfg := &logclient.InitSchemaConfig{IsNewTask: true} - _, err := client.InitSchemasReplaceForDDL(ctx, cfg) + _, err = client.InitSchemasReplaceForDDL(ctx, cfg) require.Error(t, err) require.Contains(t, err.Error(), "miss upstream table information at `start-ts`(1) but the full backup path is not specified") } } + +func downstreamID(upstreamID int64) int64 { + return upstreamID + 10000000 +} + +func emptyDB(startupID, endupID int64, replaces map[int64]*stream.DBReplace) { + for id := startupID; id < endupID; id += 1 { + replaces[id] = &stream.DBReplace{ + Name: fmt.Sprintf("db_%d", id), + DbID: downstreamID(id), + } + } +} + +func emptyTables(dbupID, startupID, endupID int64, replaces map[int64]*stream.DBReplace) { + tableMap := make(map[int64]*stream.TableReplace) + for id := startupID; id < endupID; id += 1 { + tableMap[id] = &stream.TableReplace{ + Name: fmt.Sprintf("table_%d", id), + TableID: downstreamID(id), + } + } + replaces[dbupID] = &stream.DBReplace{ + Name: fmt.Sprintf("db_%d", dbupID), + DbID: downstreamID(dbupID), + TableMap: tableMap, + } +} + +func partitions(dbupID, tableupID, startupID, endupID int64, replaces map[int64]*stream.DBReplace) { + partitionMap := make(map[int64]int64) + for id := startupID; id < endupID; id += 1 { + partitionMap[id] = downstreamID(id) + } + replaces[dbupID] = &stream.DBReplace{ + Name: fmt.Sprintf("db_%d", dbupID), + DbID: downstreamID(dbupID), + TableMap: map[int64]*stream.TableReplace{ + tableupID: { + Name: fmt.Sprintf("table_%d", tableupID), + TableID: downstreamID(tableupID), + PartitionMap: partitionMap, + }, + }, + } +} + +func getDBMap() map[int64]*stream.DBReplace { + replaces := make(map[int64]*stream.DBReplace) + emptyDB(1, 3000, replaces) + emptyTables(3000, 3001, 8000, replaces) + partitions(8000, 8001, 8002, 12000, replaces) + emptyTables(12000, 12001, 30000, replaces) + return replaces +} + +func TestPITRIDMap(t *testing.T) { + ctx := context.Background() + s := utiltest.CreateRestoreSchemaSuite(t) + tk := testkit.NewTestKit(t, s.Mock.Storage) + tk.Exec(session.CreatePITRIDMap) + g := gluetidb.New() + se, err := g.CreateSession(s.Mock.Storage) + require.NoError(t, err) + client := logclient.TEST_NewLogClient(123, 1, 2, 3, nil, se) + baseSchemaReplaces := &stream.SchemasReplace{ + DbMap: getDBMap(), + } + err = client.TEST_saveIDMap(ctx, baseSchemaReplaces) + require.NoError(t, err) + newSchemaReplaces, err := client.TEST_initSchemasMap(ctx, 1) + require.NoError(t, err) + require.Nil(t, newSchemaReplaces) + client2 := logclient.TEST_NewLogClient(123, 1, 2, 4, nil, se) + newSchemaReplaces, err = client2.TEST_initSchemasMap(ctx, 2) + require.NoError(t, err) + require.Nil(t, newSchemaReplaces) + newSchemaReplaces, err = client.TEST_initSchemasMap(ctx, 2) + require.NoError(t, err) + + require.Equal(t, len(baseSchemaReplaces.DbMap), len(newSchemaReplaces)) + for _, dbMap := range newSchemaReplaces { + baseDbMap := baseSchemaReplaces.DbMap[dbMap.IdMap.UpstreamId] + require.NotNil(t, baseDbMap) + require.Equal(t, baseDbMap.DbID, dbMap.IdMap.DownstreamId) + require.Equal(t, baseDbMap.Name, dbMap.Name) + require.Equal(t, len(baseDbMap.TableMap), len(dbMap.Tables)) + for _, tableMap := range dbMap.Tables { + baseTableMap := baseDbMap.TableMap[tableMap.IdMap.UpstreamId] + require.NotNil(t, baseTableMap) + require.Equal(t, baseTableMap.TableID, tableMap.IdMap.DownstreamId) + require.Equal(t, baseTableMap.Name, tableMap.Name) + require.Equal(t, len(baseTableMap.PartitionMap), len(tableMap.Partitions)) + for _, partitionMap := range tableMap.Partitions { + basePartitionMap, exist := baseTableMap.PartitionMap[partitionMap.UpstreamId] + require.True(t, exist) + require.Equal(t, basePartitionMap, partitionMap.DownstreamId) + } + } + } +} diff --git a/br/pkg/restore/log_client/export_test.go b/br/pkg/restore/log_client/export_test.go index 18db7e61fc2d4..db1324d61f12a 100644 --- a/br/pkg/restore/log_client/export_test.go +++ b/br/pkg/restore/log_client/export_test.go @@ -19,13 +19,29 @@ import ( "github.com/pingcap/errors" backuppb "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/tidb/br/pkg/glue" "github.com/pingcap/tidb/br/pkg/storage" + "github.com/pingcap/tidb/br/pkg/stream" "github.com/pingcap/tidb/br/pkg/utils/iter" "github.com/pingcap/tidb/pkg/domain" ) var FilterFilesByRegion = filterFilesByRegion +func (rc *LogClient) TEST_saveIDMap( + ctx context.Context, + sr *stream.SchemasReplace, +) error { + return rc.saveIDMap(ctx, sr) +} + +func (rc *LogClient) TEST_initSchemasMap( + ctx context.Context, + restoreTS uint64, +) ([]*backuppb.PitrDBMap, error) { + return rc.initSchemasMap(ctx, restoreTS) +} + // readStreamMetaByTS is used for streaming task. collect all meta file by TS, it is for test usage. func (rc *LogFileManager) ReadStreamMeta(ctx context.Context) ([]Meta, error) { metas, err := rc.streamingMeta(ctx) @@ -39,15 +55,16 @@ func (rc *LogFileManager) ReadStreamMeta(ctx context.Context) ([]Meta, error) { return r.Item, nil } -func TEST_NewLogClient(clusterID, startTS, restoreTS uint64, storage storage.ExternalStorage, dom *domain.Domain) *LogClient { +func TEST_NewLogClient(clusterID, startTS, restoreTS, upstreamClusterID uint64, dom *domain.Domain, se glue.Session) *LogClient { return &LogClient{ - dom: dom, + dom: dom, + se: se, + upstreamClusterID: upstreamClusterID, LogFileManager: &LogFileManager{ startTS: startTS, restoreTS: restoreTS, }, clusterID: clusterID, - storage: storage, } } diff --git a/br/pkg/restore/snap_client/systable_restore.go b/br/pkg/restore/snap_client/systable_restore.go index fdc1b88783967..ddc12516268aa 100644 --- a/br/pkg/restore/snap_client/systable_restore.go +++ b/br/pkg/restore/snap_client/systable_restore.go @@ -68,6 +68,8 @@ var unRecoverableTable = map[string]map[string]struct{}{ // replace into view is not supported now "tidb_mdl_view": {}, + + "tidb_pitr_id_map": {}, }, "sys": { // replace into view is not supported now diff --git a/br/pkg/restore/snap_client/systable_restore_test.go b/br/pkg/restore/snap_client/systable_restore_test.go index eb3d6cf2df5f3..608fa7f57392f 100644 --- a/br/pkg/restore/snap_client/systable_restore_test.go +++ b/br/pkg/restore/snap_client/systable_restore_test.go @@ -116,5 +116,5 @@ func TestCheckSysTableCompatibility(t *testing.T) { // // The above variables are in the file br/pkg/restore/systable_restore.go func TestMonitorTheSystemTableIncremental(t *testing.T) { - require.Equal(t, int64(212), session.CurrentBootstrapVersion) + require.Equal(t, int64(213), session.CurrentBootstrapVersion) } diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go index 3463df2cbba97..3e45bf6934f1c 100644 --- a/br/pkg/task/restore.go +++ b/br/pkg/task/restore.go @@ -258,6 +258,7 @@ type RestoreConfig struct { checkpointSnapshotRestoreTaskName string `json:"-" toml:"-"` checkpointLogRestoreTaskName string `json:"-" toml:"-"` checkpointTaskInfoClusterID uint64 `json:"-" toml:"-"` + upstreamClusterID uint64 `json:"-" toml:"-"` WaitTiflashReady bool `json:"wait-tiflash-ready" toml:"wait-tiflash-ready"` // for ebs-based restore diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go index 1704fbc832711..dc0a5f863637d 100644 --- a/br/pkg/task/stream.go +++ b/br/pkg/task/stream.go @@ -1141,6 +1141,7 @@ func RunStreamRestore( if cfg.RestoreTS == 0 { cfg.RestoreTS = logInfo.logMaxTS } + cfg.upstreamClusterID = logInfo.clusterID if len(cfg.FullBackupStorage) > 0 { startTS, fullClusterID, err := getFullBackupTS(ctx, cfg) @@ -1506,6 +1507,7 @@ func createRestoreClient(ctx context.Context, g glue.Glue, cfg *RestoreConfig, m } client.SetCrypter(&cfg.CipherInfo) client.SetConcurrency(uint(cfg.Concurrency)) + client.SetUpstreamClusterID(cfg.upstreamClusterID) client.InitClients(ctx, u) err = client.SetRawKVBatchClient(ctx, cfg.PD, cfg.TLS.ToKVSecurity()) diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go index 385ed4319a06a..fda272606e5c7 100644 --- a/br/pkg/utils/backoff.go +++ b/br/pkg/utils/backoff.go @@ -207,6 +207,11 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration { } } } + failpoint.Inject("set-import-attempt-to-one", func(_ failpoint.Value) { + if bo.attempt > 1 { + bo.attempt = 1 + } + }) if bo.delayTime > bo.maxDelayTime { return bo.maxDelayTime } diff --git a/br/pkg/version/version.go b/br/pkg/version/version.go index 4b0f3c34cdbad..519797e0daafc 100644 --- a/br/pkg/version/version.go +++ b/br/pkg/version/version.go @@ -165,6 +165,14 @@ func CheckVersionForBRPiTR(s *metapb.Store, tikvVersion *semver.Version) error { s.Address, tikvVersion, build.ReleaseVersion) } } + + if BRVersion.Major > 8 || (BRVersion.Major == 8 && BRVersion.Minor >= 4) { + if tikvVersion.Major < 8 || (tikvVersion.Major == 8 && tikvVersion.Minor < 4) { + return errors.Annotatef(berrors.ErrVersionMismatch, + "TiKV node %s version %s is too old because the PITR id map is written into the cluster system table mysql.tidb_pitr_id_map, please use the tikv with version v8.4.0+", + s.Address, tikvVersion) + } + } return nil } diff --git a/br/pkg/version/version_test.go b/br/pkg/version/version_test.go index 4e7a2966e7dac..853c992113526 100644 --- a/br/pkg/version/version_test.go +++ b/br/pkg/version/version_test.go @@ -157,6 +157,26 @@ func TestCheckClusterVersion(t *testing.T) { require.Regexp(t, `^TiKV .* version mismatch when use PiTR v6.1.0, please `, err.Error()) } + { + build.ReleaseVersion = "v8.4.0" + mock.getAllStores = func() []*metapb.Store { + return []*metapb.Store{{Version: `v6.2.0`}} + } + err := CheckClusterVersion(context.Background(), &mock, CheckVersionForBRPiTR) + require.Error(t, err) + require.Regexp(t, `^TiKV .* is too old because the PITR id map is written into`, err.Error()) + } + + { + build.ReleaseVersion = "v8.5.0" + mock.getAllStores = func() []*metapb.Store { + return []*metapb.Store{{Version: `v6.2.0`}} + } + err := CheckClusterVersion(context.Background(), &mock, CheckVersionForBRPiTR) + require.Error(t, err) + require.Regexp(t, `^TiKV .* is too old because the PITR id map is written into`, err.Error()) + } + { build.ReleaseVersion = "v4.0.5" mock.getAllStores = func() []*metapb.Store { diff --git a/br/tests/br_file_corruption/run.sh b/br/tests/br_file_corruption/run.sh new file mode 100644 index 0000000000000..35a7698bb9fef --- /dev/null +++ b/br/tests/br_file_corruption/run.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# +# Copyright 2024 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux + +DB="$TEST_NAME" +TABLE="usertable" +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +run_sql "CREATE DATABASE $DB;" +go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB +run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" + +filename=$(find $TEST_DIR/$DB -regex ".*.sst" | head -n 1) +filename_temp=$filename"_temp" +filename_bak=$filename"_bak" +echo "corruption" > $filename_temp +cat $filename >> $filename_temp + +# file lost +mv $filename $filename_bak +export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)" +restore_fail=0 +run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$DB" || restore_fail=1 +export GO_FAILPOINTS="" +if [ $restore_fail -ne 1 ]; then + echo 'restore success' + exit 1 +fi + +# file corruption +mv $filename_temp $filename +truncate --size=-11 $filename +export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)" +restore_fail=0 +run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$DB" || restore_fail=1 +export GO_FAILPOINTS="" +if [ $restore_fail -ne 1 ]; then + echo 'restore success' + exit 1 +fi diff --git a/br/tests/br_file_corruption/workload b/br/tests/br_file_corruption/workload new file mode 100644 index 0000000000000..e3fadf9a3d068 --- /dev/null +++ b/br/tests/br_file_corruption/workload @@ -0,0 +1,12 @@ +recordcount=10000 +operationcount=0 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform diff --git a/br/tests/br_full_ddl/run.sh b/br/tests/br_full_ddl/run.sh index 1572d0ef6fddd..5f9d67184e7d6 100755 --- a/br/tests/br_full_ddl/run.sh +++ b/br/tests/br_full_ddl/run.sh @@ -23,6 +23,7 @@ RESTORE_LOG=LOG=/$TEST_DIR/restore.log BACKUP_STAT=/$TEST_DIR/backup_stat RESOTRE_STAT=/$TEST_DIR/restore_stat CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +res_file="$TEST_DIR/sql_res.$TEST_NAME.txt" run_sql "CREATE DATABASE $DB;" go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB @@ -39,6 +40,23 @@ for i in $(seq $DDL_COUNT); do fi done +# wait until the index creation/drop is done +retry_cnt=0 +while true; do + run_sql "ADMIN SHOW DDL JOBS WHERE DB_NAME = '$DB' AND TABLE_NAME = '$TABLE' AND STATE != 'synced';" + if grep -Fq "1. row" $res_file; then + cat $res_file + retry_cnt=$((retry_cnt+1)) + if [ "$retry_cnt" -gt 50 ]; then + echo 'the wait lag is too large' + exit 1 + fi + continue + fi + + break +done + # run analyze to generate stats run_sql "analyze table $DB.$TABLE;" # record the stats and remove last_update_version diff --git a/br/tests/br_pitr/run.sh b/br/tests/br_pitr/run.sh index 607d2a4e4e114..994a9ef343c3d 100644 --- a/br/tests/br_pitr/run.sh +++ b/br/tests/br_pitr/run.sh @@ -166,3 +166,37 @@ if [ $restore_fail -ne 1 ]; then echo 'pitr success' exit 1 fi + +# start a new cluster for corruption +echo "restart a services" +restart_services + +echo "corrupt a log file" +filename=$(find $TEST_DIR/$PREFIX/log -regex ".*\.log" | grep -v "schema-meta" | tail -n 1) +filename_temp=$filename"_temp" +filename_bak=$filename"_bak" +echo "corruption" > $filename_temp +cat $filename >> $filename_temp + +# file lost +mv $filename $filename_bak +export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)" +restore_fail=0 +run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1 +export GO_FAILPOINTS="" +if [ $restore_fail -ne 1 ]; then + echo 'pitr success' + exit 1 +fi + +# file corruption +mv $filename_temp $filename +truncate --size=-11 $filename +export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)" +restore_fail=0 +run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1 +export GO_FAILPOINTS="" +if [ $restore_fail -ne 1 ]; then + echo 'pitr success' + exit 1 +fi diff --git a/br/tests/br_txn/run.sh b/br/tests/br_txn/run.sh index ff98bcc8fdb7d..567be9d76e263 100755 --- a/br/tests/br_txn/run.sh +++ b/br/tests/br_txn/run.sh @@ -97,12 +97,22 @@ run_test() { # delete data in range[start-key, end-key) clean "hello" "world" # Ensure the data is deleted - checksum_new=$(checksum "hello" "world") - - if [ "$checksum_new" != "$checksum_empty" ];then - echo "failed to delete data in range after backup" - fail_and_exit - fi + retry_cnt=0 + while true; do + checksum_new=$(checksum "hello" "world") + + if [ "$checksum_new" != "$checksum_empty" ]; then + echo "failed to delete data in range after backup; retry_cnt = $retry_cnt" + retry_cnt=$((retry_cnt+1)) + if [ "$retry_cnt" -gt 50 ]; then + fail_and_exit + fi + sleep 1 + continue + fi + + break + done # restore rawkv echo "restore start..." diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh index ae9f17d7a462b..04ff8c60701d4 100755 --- a/br/tests/run_group_br_tests.sh +++ b/br/tests/run_group_br_tests.sh @@ -28,7 +28,7 @@ groups=( ["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter' ["G06"]='br_tikv_outage br_tikv_outage3' ["G07"]='br_pitr' - ["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom' + ["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption' ) # Get other cases not in groups, to avoid missing any case diff --git a/errors.toml b/errors.toml index 9c02f10d6ac04..0f88b036bb828 100644 --- a/errors.toml +++ b/errors.toml @@ -1936,6 +1936,11 @@ error = ''' Your query has been cancelled due to exceeding the allowed memory limit for the tidb-server instance and this query is currently using the most memory. Please try narrowing your query scope or increase the tidb_server_memory_limit and try again.[conn=%d] ''' +["executor:8177"] +error = ''' +Delete can not find column %s for table %s +''' + ["executor:8212"] error = ''' Failed to split region ranges: %s @@ -2651,6 +2656,11 @@ error = ''' variable %s has no effect in TiDB ''' +["planner:8177"] +error = ''' +Delete can not find column %s for table %s +''' + ["planner:8242"] error = ''' '%s' is unsupported on cache tables. @@ -2931,6 +2941,11 @@ error = ''' Connections using insecure transport are prohibited while --require_secure_transport=ON. ''' +["server:8051"] +error = ''' +unknown field type +''' + ["server:8052"] error = ''' invalid sequence diff --git a/go.mod b/go.mod index d0fd01a92132c..ff3101b085f15 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a github.com/pingcap/fn v1.0.0 - github.com/pingcap/kvproto v0.0.0-20240904041139-1de8accd5bb7 + github.com/pingcap/kvproto v0.0.0-20240910154453-b242104f8d31 github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e @@ -109,8 +109,8 @@ require ( github.com/tdakkota/asciicheck v0.2.0 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tidwall/btree v1.7.0 - github.com/tikv/client-go/v2 v2.0.8-0.20240821073530-75e3705e58f1 - github.com/tikv/pd/client v0.0.0-20240805092608-838ee7983b78 + github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba + github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9 github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a github.com/twmb/murmur3 v1.1.6 github.com/uber/jaeger-client-go v2.22.1+incompatible @@ -135,8 +135,8 @@ require ( golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.8.0 golang.org/x/sys v0.25.0 - golang.org/x/term v0.23.0 - golang.org/x/text v0.17.0 + golang.org/x/term v0.24.0 + golang.org/x/text v0.18.0 golang.org/x/time v0.5.0 golang.org/x/tools v0.24.0 google.golang.org/api v0.169.0 @@ -308,7 +308,7 @@ require ( go.opentelemetry.io/otel/sdk v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect - golang.org/x/crypto v0.26.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect golang.org/x/mod v0.20.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index ffe2fd0c4f3bc..4aae5d58fa40e 100644 --- a/go.sum +++ b/go.sum @@ -702,8 +702,8 @@ github.com/pingcap/fn v1.0.0/go.mod h1:u9WZ1ZiOD1RpNhcI42RucFh/lBuzTu6rw88a+oF2Z github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20240904041139-1de8accd5bb7 h1:AHDEjW05jX67LWNOCQtqsirfx1XpGOGI24ey4bPdkB8= -github.com/pingcap/kvproto v0.0.0-20240904041139-1de8accd5bb7/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20240910154453-b242104f8d31 h1:6BY+3T6Hqpw9UZ/D7Om/xB+Xik3NkkYxBV6qCzUdUvU= +github.com/pingcap/kvproto v0.0.0-20240910154453-b242104f8d31/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfUnQGqft0ud+xVFuCdp1XkVL0X1E= @@ -858,10 +858,10 @@ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= -github.com/tikv/client-go/v2 v2.0.8-0.20240821073530-75e3705e58f1 h1:QivCyAQxBOrXWC9X1/yc9U3Hw45usVnicjOg7T/rpgE= -github.com/tikv/client-go/v2 v2.0.8-0.20240821073530-75e3705e58f1/go.mod h1:4HDOAx8OXAJPtqhCZ03IhChXgaFs4B3+vSrPWmiPxjg= -github.com/tikv/pd/client v0.0.0-20240805092608-838ee7983b78 h1:PtW+yTvs9eGTMblulaCHmJ5OtifuE4SJXCACCtkd6ko= -github.com/tikv/pd/client v0.0.0-20240805092608-838ee7983b78/go.mod h1:TxrJRY949Vl14Lmarx6hTNP/HEDYzn4dP0KmjdzQ59w= +github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba h1:dwuuYqPYxU0xcv0bnDgT7I4btQ5c3joBG1HmNOhCTdo= +github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba/go.mod h1:4HDOAx8OXAJPtqhCZ03IhChXgaFs4B3+vSrPWmiPxjg= +github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9 h1:J9LChGMzo95eBrjE03NHITDWgxfPgskH+QrCnlW61/Y= +github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9/go.mod h1:uBHhxAM/SPCMabt483gI/pN/+JXIMKYXohK96s+PwT8= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= @@ -1002,8 +1002,8 @@ golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1215,8 +1215,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1230,8 +1230,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 48fbd26a23146..c2d4a76323646 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -754,6 +754,7 @@ store-limit=0 ttl-refreshed-txn-size=8192 resolve-lock-lite-threshold = 16 copr-req-timeout = "120s" +grpc-keepalive-timeout = 0.2 [tikv-client.async-commit] keys-limit=123 total-key-size-limit=1024 @@ -804,6 +805,7 @@ max_connections = 200 require.Equal(t, uint(6000), conf.TiKVClient.RegionCacheTTL) require.Equal(t, int64(0), conf.TiKVClient.StoreLimit) require.Equal(t, int64(8192), conf.TiKVClient.TTLRefreshedTxnSize) + require.Equal(t, time.Millisecond*200, conf.TiKVClient.GetGrpcKeepAliveTimeout()) require.Equal(t, uint(1000), conf.TokenLimit) require.True(t, conf.EnableTableLock) require.Equal(t, uint64(5), conf.DelayCleanTableLock) @@ -902,6 +904,35 @@ spilled-file-encryption-method = "aes128-ctr" require.NoError(t, f.Sync()) require.NoError(t, conf.Load(configFile)) + conf = NewConfig() + require.Equal(t, time.Second*3, conf.TiKVClient.GetGrpcKeepAliveTimeout()) + err = f.Truncate(0) + require.NoError(t, err) + _, err = f.Seek(0, 0) + require.NoError(t, err) + _, err = f.WriteString(` +[tikv-client] +grpc-keepalive-timeout = 3 +`) + require.NoError(t, err) + require.NoError(t, f.Sync()) + require.NoError(t, conf.Load(configFile)) + require.Equal(t, time.Second*3, conf.TiKVClient.GetGrpcKeepAliveTimeout()) + + err = f.Truncate(0) + require.NoError(t, err) + _, err = f.Seek(0, 0) + require.NoError(t, err) + _, err = f.WriteString(` +[tikv-client] +grpc-keepalive-timeout = 0.01 +`) + require.NoError(t, err) + require.NoError(t, f.Sync()) + require.NoError(t, conf.Load(configFile)) + require.NotNil(t, conf.Valid()) + require.Equal(t, "grpc-keepalive-timeout should be at least 0.05, but got 0.010000", conf.Valid().Error()) + configFile = "config.toml.example" require.NoError(t, conf.Load(configFile)) diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel index 95f2a9375156c..b31ff04a0e137 100644 --- a/pkg/ddl/BUILD.bazel +++ b/pkg/ddl/BUILD.bazel @@ -76,6 +76,7 @@ go_library( "//pkg/ddl/ingest", "//pkg/ddl/label", "//pkg/ddl/logutil", + "//pkg/ddl/notifier", "//pkg/ddl/placement", "//pkg/ddl/resourcegroup", "//pkg/ddl/schemaver", @@ -129,7 +130,6 @@ go_library( "//pkg/sessiontxn", "//pkg/statistics", "//pkg/statistics/handle", - "//pkg/statistics/handle/util", "//pkg/store/driver/txn", "//pkg/store/helper", "//pkg/table", diff --git a/pkg/ddl/add_column.go b/pkg/ddl/add_column.go index 593be2187ecae..f1428f3621445 100644 --- a/pkg/ddl/add_column.go +++ b/pkg/ddl/add_column.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/ddl/logutil" - "github.com/pingcap/tidb/pkg/ddl/util" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/errctx" "github.com/pingcap/tidb/pkg/expression" exprctx "github.com/pingcap/tidb/pkg/expression/context" @@ -42,7 +42,6 @@ import ( field_types "github.com/pingcap/tidb/pkg/parser/types" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" - statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/types" driver "github.com/pingcap/tidb/pkg/types/parser_driver" @@ -133,9 +132,7 @@ func onAddColumn(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, e // Finish this job. job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo) - addColumnEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewAddColumnEvent(tblInfo, []*model.ColumnInfo{columnInfo}), - } + addColumnEvent := notifier.NewAddColumnEvent(tblInfo, []*model.ColumnInfo{columnInfo}) asyncNotifyEvent(jobCtx, addColumnEvent, job) default: err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("column", columnInfo.State) diff --git a/pkg/ddl/cluster.go b/pkg/ddl/cluster.go index b52deef85a6d2..b66894be51d13 100644 --- a/pkg/ddl/cluster.go +++ b/pkg/ddl/cluster.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/ddl/notifier" sess "github.com/pingcap/tidb/pkg/ddl/session" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/infoschema" @@ -38,7 +39,6 @@ import ( "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" - statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/filter" @@ -821,7 +821,7 @@ func (w *worker) onFlashbackCluster(jobCtx *jobContext, t *meta.Meta, job *model case model.StateWriteReorganization: // TODO: Support flashback in unistore. if inFlashbackTest { - asyncNotifyEvent(jobCtx, statsutil.NewFlashbackClusterEvent(), job) + asyncNotifyEvent(jobCtx, notifier.NewFlashbackClusterEvent(), job) job.State = model.JobStateDone job.SchemaState = model.StatePublic return ver, nil @@ -844,7 +844,7 @@ func (w *worker) onFlashbackCluster(jobCtx *jobContext, t *meta.Meta, job *model } } - asyncNotifyEvent(jobCtx, statsutil.NewFlashbackClusterEvent(), job) + asyncNotifyEvent(jobCtx, notifier.NewFlashbackClusterEvent(), job) job.State = model.JobStateDone job.SchemaState = model.StatePublic return updateSchemaVersion(jobCtx, t, job) diff --git a/pkg/ddl/column.go b/pkg/ddl/column.go index ee2e1304ebae5..f575aaf255e91 100644 --- a/pkg/ddl/column.go +++ b/pkg/ddl/column.go @@ -1304,8 +1304,8 @@ func getChangingColumnOriginName(changingColumn *model.ColumnInfo) string { return columnName[:pos] } -func getExpressionIndexOriginName(expressionIdx *model.ColumnInfo) string { - columnName := strings.TrimPrefix(expressionIdx.Name.O, expressionIndexPrefix+"_") +func getExpressionIndexOriginName(originalName pmodel.CIStr) string { + columnName := strings.TrimPrefix(originalName.O, expressionIndexPrefix+"_") var pos int if pos = strings.LastIndex(columnName, "_"); pos == -1 { return columnName diff --git a/pkg/ddl/column_type_change_test.go b/pkg/ddl/column_type_change_test.go index 2452809a72c77..b13337bb4a595 100644 --- a/pkg/ddl/column_type_change_test.go +++ b/pkg/ddl/column_type_change_test.go @@ -260,7 +260,7 @@ func TestRowFormatWithChecksums(t *testing.T) { data, err := h.GetMvccByEncodedKey(encodedKey) require.NoError(t, err) // row value with checksums - expected := []byte{0x80, 0x2, 0x3, 0x0, 0x0, 0x0, 0x1, 0x2, 0x3, 0x1, 0x0, 0x4, 0x0, 0x7, 0x0, 0x1, 0x31, 0x32, 0x33, 0x31, 0x32, 0x33, 0x1, 0xea, 0xb0, 0x41, 0x20} + expected := []byte{0x80, 0x2, 0x3, 0x0, 0x0, 0x0, 0x1, 0x2, 0x3, 0x1, 0x0, 0x4, 0x0, 0x7, 0x0, 0x1, 0x31, 0x32, 0x33, 0x31, 0x32, 0x33, 0x1, 0x2b, 0x9, 0x2d, 0x78} require.Equal(t, expected, data.Info.Writes[0].ShortValue) tk.MustExec("drop table if exists t") } @@ -284,7 +284,7 @@ func TestRowLevelChecksumWithMultiSchemaChange(t *testing.T) { data, err := h.GetMvccByEncodedKey(encodedKey) require.NoError(t, err) // checksum skipped and with a null col vv - expected := []byte{0x80, 0x2, 0x3, 0x0, 0x1, 0x0, 0x1, 0x2, 0x4, 0x3, 0x1, 0x0, 0x4, 0x0, 0x7, 0x0, 0x1, 0x31, 0x32, 0x33, 0x31, 0x32, 0x33, 0x1, 0xa8, 0x88, 0x6f, 0xc8} + expected := []byte{0x80, 0x2, 0x3, 0x0, 0x1, 0x0, 0x1, 0x2, 0x4, 0x3, 0x1, 0x0, 0x4, 0x0, 0x7, 0x0, 0x1, 0x31, 0x32, 0x33, 0x31, 0x32, 0x33, 0x1, 0x69, 0x31, 0x3, 0x90} require.Equal(t, expected, data.Info.Writes[0].ShortValue) tk.MustExec("drop table if exists t") } diff --git a/pkg/ddl/create_table.go b/pkg/ddl/create_table.go index af4dafa0f7198..8dae9acec2504 100644 --- a/pkg/ddl/create_table.go +++ b/pkg/ddl/create_table.go @@ -26,8 +26,8 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/ddl/placement" - "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/infoschema" @@ -41,7 +41,6 @@ import ( field_types "github.com/pingcap/tidb/pkg/parser/types" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" - statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/types" @@ -55,9 +54,9 @@ import ( // DANGER: it is an internal function used by onCreateTable and onCreateTables, for reusing code. Be careful. // 1. it expects the argument of job has been deserialized. // 2. it won't call updateSchemaVersion, FinishTableJob and asyncNotifyEvent. -func createTable(jobCtx *jobContext, t *meta.Meta, job *model.Job, fkCheck bool) (*model.TableInfo, error) { +func createTable(jobCtx *jobContext, t *meta.Meta, job *model.Job, args *model.CreateTableArgs) (*model.TableInfo, error) { schemaID := job.SchemaID - tbInfo := job.Args[0].(*model.TableInfo) + tbInfo, fkCheck := args.TableInfo, args.FKCheck tbInfo.State = model.StateNone err := checkTableNotExists(jobCtx.infoCache, schemaID, tbInfo.Name.L) @@ -157,20 +156,19 @@ func onCreateTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, } }) - // just decode, createTable will use it as Args[0] - tbInfo := &model.TableInfo{} - fkCheck := false - if err := job.DecodeArgs(tbInfo, &fkCheck); err != nil { + args, err := model.GetCreateTableArgs(job) + if err != nil { // Invalid arguments, cancel this job. job.State = model.JobStateCancelled return ver, errors.Trace(err) } + tbInfo := args.TableInfo if len(tbInfo.ForeignKeys) > 0 { - return createTableWithForeignKeys(jobCtx, t, job, tbInfo, fkCheck) + return createTableWithForeignKeys(jobCtx, t, job, args) } - tbInfo, err := createTable(jobCtx, t, job, fkCheck) + tbInfo, err = createTable(jobCtx, t, job, args) if err != nil { return ver, errors.Trace(err) } @@ -182,21 +180,20 @@ func onCreateTable(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, // Finish this job. job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo) - createTableEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewCreateTableEvent(tbInfo), - } + createTableEvent := notifier.NewCreateTableEvent(tbInfo) asyncNotifyEvent(jobCtx, createTableEvent, job) return ver, errors.Trace(err) } -func createTableWithForeignKeys(jobCtx *jobContext, t *meta.Meta, job *model.Job, tbInfo *model.TableInfo, fkCheck bool) (ver int64, err error) { +func createTableWithForeignKeys(jobCtx *jobContext, t *meta.Meta, job *model.Job, args *model.CreateTableArgs) (ver int64, err error) { + tbInfo := args.TableInfo switch tbInfo.State { case model.StateNone, model.StatePublic: // create table in non-public or public state. The function `createTable` will always reset // the `tbInfo.State` with `model.StateNone`, so it's fine to just call the `createTable` with // public state. // when `br` restores table, the state of `tbInfo` will be public. - tbInfo, err = createTable(jobCtx, t, job, fkCheck) + tbInfo, err = createTable(jobCtx, t, job, args) if err != nil { return ver, errors.Trace(err) } @@ -213,9 +210,7 @@ func createTableWithForeignKeys(jobCtx *jobContext, t *meta.Meta, job *model.Job return ver, errors.Trace(err) } job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo) - createTableEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewCreateTableEvent(tbInfo), - } + createTableEvent := notifier.NewCreateTableEvent(tbInfo) asyncNotifyEvent(jobCtx, createTableEvent, job) return ver, nil default: @@ -227,37 +222,38 @@ func createTableWithForeignKeys(jobCtx *jobContext, t *meta.Meta, job *model.Job func onCreateTables(jobCtx *jobContext, t *meta.Meta, job *model.Job) (int64, error) { var ver int64 - var args []*model.TableInfo - fkCheck := false - err := job.DecodeArgs(&args, &fkCheck) + args, err := model.GetBatchCreateTableArgs(job) if err != nil { // Invalid arguments, cancel this job. job.State = model.JobStateCancelled return ver, errors.Trace(err) } + tableInfos := make([]*model.TableInfo, 0, len(args.Tables)) // We don't construct jobs for every table, but only tableInfo // The following loop creates a stub job for every table // // it clones a stub job from the ActionCreateTables job stubJob := job.Clone() stubJob.Args = make([]any, 1) - for i := range args { - stubJob.TableID = args[i].ID - stubJob.Args[0] = args[i] - if args[i].Sequence != nil { - err := createSequenceWithCheck(t, stubJob, args[i]) + for i := range args.Tables { + tblArgs := args.Tables[i] + tableInfo := tblArgs.TableInfo + stubJob.TableID = tableInfo.ID + if tableInfo.Sequence != nil { + err := createSequenceWithCheck(t, stubJob, tableInfo) if err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) } + tableInfos = append(tableInfos, tableInfo) } else { - tbInfo, err := createTable(jobCtx, t, stubJob, fkCheck) + tbInfo, err := createTable(jobCtx, t, stubJob, tblArgs) if err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) } - args[i] = tbInfo + tableInfos = append(tableInfos, tbInfo) } } @@ -268,11 +264,9 @@ func onCreateTables(jobCtx *jobContext, t *meta.Meta, job *model.Job) (int64, er job.State = model.JobStateDone job.SchemaState = model.StatePublic - job.BinlogInfo.SetTableInfos(ver, args) - for i := range args { - createTableEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewCreateTableEvent(args[i]), - } + job.BinlogInfo.SetTableInfos(ver, tableInfos) + for i := range tableInfos { + createTableEvent := notifier.NewCreateTableEvent(tableInfos[i]) asyncNotifyEvent(jobCtx, createTableEvent, job) } @@ -290,14 +284,13 @@ func createTableOrViewWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tb func onCreateView(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) { schemaID := job.SchemaID - tbInfo := &model.TableInfo{} - var orReplace bool - var _placeholder int64 // oldTblInfoID - if err := job.DecodeArgs(tbInfo, &orReplace, &_placeholder); err != nil { + args, err := model.GetCreateTableArgs(job) + if err != nil { // Invalid arguments, cancel this job. job.State = model.JobStateCancelled return ver, errors.Trace(err) } + tbInfo, orReplace := args.TableInfo, args.OnExistReplace tbInfo.State = model.StateNone oldTableID, err := findTableIDByName(jobCtx.infoCache, t, schemaID, tbInfo.Name.L) @@ -681,6 +674,12 @@ func BuildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCh return nil, errors.Trace(err) } + // set default shard row id bits and pre-split regions for table. + if !tbInfo.HasClusteredIndex() && tbInfo.TempTableType == model.TempTableNone { + tbInfo.ShardRowIDBits = ctx.GetSessionVars().ShardRowIDBits + tbInfo.PreSplitRegions = ctx.GetSessionVars().PreSplitRegions + } + if err = handleTableOptions(s.Options, tbInfo); err != nil { return nil, errors.Trace(err) } @@ -806,8 +805,8 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err return dbterror.ErrUnsupportedShardRowIDBits } tbInfo.ShardRowIDBits = op.UintValue - if tbInfo.ShardRowIDBits > shardRowIDBitsMax { - tbInfo.ShardRowIDBits = shardRowIDBitsMax + if tbInfo.ShardRowIDBits > variable.MaxShardRowIDBits { + tbInfo.ShardRowIDBits = variable.MaxShardRowIDBits } tbInfo.MaxShardRowIDBits = tbInfo.ShardRowIDBits case ast.TableOptionPreSplitRegion: diff --git a/pkg/ddl/db_integration_test.go b/pkg/ddl/db_integration_test.go index ae397fc44ea9d..1d8821f4a3bf8 100644 --- a/pkg/ddl/db_integration_test.go +++ b/pkg/ddl/db_integration_test.go @@ -1333,7 +1333,8 @@ func assertAlterWarnExec(tk *testkit.TestKit, t *testing.T, sql string) { } func TestAlterAlgorithm(t *testing.T) { - store := testkit.CreateMockStore(t, mockstore.WithDDLChecker()) + store, dom := testkit.CreateMockStoreAndDomain(t, mockstore.WithDDLChecker()) + ddlChecker := dom.DDL().(*schematracker.Checker) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -1376,6 +1377,20 @@ func TestAlterAlgorithm(t *testing.T) { tk.MustExec("alter table t rename index idx_c2 to idx_c, ALGORITHM=INSTANT") tk.MustExec("alter table t rename index idx_c to idx_c1, ALGORITHM=DEFAULT") + // Test corner case for renameIndexes + tk.MustExec(`create table tscalar(c1 int, col_1_1 int, key col_1(col_1_1))`) + tk.MustExec("alter table tscalar rename index col_1 to col_2") + tk.MustExec("admin check table tscalar") + tk.MustExec("drop table tscalar") + + // Test rename index with scalar function + ddlChecker.Disable() + tk.MustExec(`create table tscalar(id int, col_1 json, KEY idx_1 ((cast(col_1 as char(64) array))))`) + tk.MustExec("alter table tscalar rename index idx_1 to idx_1_1") + tk.MustExec("admin check table tscalar") + tk.MustExec("drop table tscalar") + ddlChecker.Enable() + // partition. assertAlterWarnExec(tk, t, "alter table t ALGORITHM=COPY, truncate partition p1") assertAlterWarnExec(tk, t, "alter table t ALGORITHM=INPLACE, truncate partition p2") diff --git a/pkg/ddl/ddl.go b/pkg/ddl/ddl.go index 61d19e4d544dc..ddad027ad66d5 100644 --- a/pkg/ddl/ddl.go +++ b/pkg/ddl/ddl.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/ddl/ingest" "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/ddl/schemaver" "github.com/pingcap/tidb/pkg/ddl/serverstate" sess "github.com/pingcap/tidb/pkg/ddl/session" @@ -57,7 +58,6 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx/binloginfo" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics/handle" - statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" pumpcli "github.com/pingcap/tidb/pkg/tidb-binlog/pump_client" tidbutil "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/dbterror" @@ -77,8 +77,6 @@ const ( addingDDLJobPrefix = "/tidb/ddl/add_ddl_job_" ddlPrompt = "ddl" - shardRowIDBitsMax = 15 - batchAddingJobs = 100 reorgWorkerCnt = 10 @@ -203,6 +201,7 @@ type JobWrapper struct { // IDAllocated see config of same name in CreateTableConfig. // exported for test. IDAllocated bool + JobArgs model.JobArgs // job submission is run in async, we use this channel to notify the caller. // when fast create table enabled, we might combine multiple jobs into one, and // append the channel to this slice. @@ -220,6 +219,17 @@ func NewJobWrapper(job *model.Job, idAllocated bool) *JobWrapper { } } +// NewJobWrapperWithArgs creates a new JobWrapper with job args. +// TODO: merge with NewJobWrapper later. +func NewJobWrapperWithArgs(job *model.Job, args model.JobArgs, idAllocated bool) *JobWrapper { + return &JobWrapper{ + Job: job, + IDAllocated: idAllocated, + JobArgs: args, + ResultCh: []chan jobSubmitResult{make(chan jobSubmitResult)}, + } +} + // NotifyResult notifies the job submit result. func (t *JobWrapper) NotifyResult(err error) { merged := len(t.ResultCh) > 1 @@ -313,7 +323,7 @@ type ddlCtx struct { schemaVerSyncer schemaver.Syncer serverStateSyncer serverstate.Syncer - ddlEventCh chan<- *statsutil.DDLEvent + ddlEventCh chan<- *notifier.SchemaChangeEvent lease time.Duration // lease is schema lease, default 45s, see config.Lease. binlogCli *pumpcli.PumpsClient // binlogCli is used for Binlog. infoCache *infoschema.InfoCache @@ -555,7 +565,7 @@ func (d *ddl) RegisterStatsHandle(h *handle.Handle) { // asyncNotifyEvent will notify the ddl event to outside world, say statistic handle. When the channel is full, we may // give up notify and log it. -func asyncNotifyEvent(jobCtx *jobContext, e *statsutil.DDLEvent, job *model.Job) { +func asyncNotifyEvent(jobCtx *jobContext, e *notifier.SchemaChangeEvent, job *model.Job) { // skip notify for system databases, system databases are expected to change at // bootstrap and other nodes can also handle the changing in its bootstrap rather // than be notified. diff --git a/pkg/ddl/ddl_history.go b/pkg/ddl/ddl_history.go index 4f4616e9918ac..5c3963ff67cf2 100644 --- a/pkg/ddl/ddl_history.go +++ b/pkg/ddl/ddl_history.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/ddl/logutil" + _ "github.com/pingcap/tidb/pkg/ddl/notifier" // find cycle import sess "github.com/pingcap/tidb/pkg/ddl/session" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/kv" diff --git a/pkg/ddl/ddl_test.go b/pkg/ddl/ddl_test.go index b02681ac6d5ca..47313c49bc5b6 100644 --- a/pkg/ddl/ddl_test.go +++ b/pkg/ddl/ddl_test.go @@ -202,16 +202,6 @@ func TestIgnorableSpec(t *testing.T) { } func TestBuildJobDependence(t *testing.T) { - vers := []model.JobVersion{ - model.JobVersion1, - model.JobVersion2, - } - for _, ver := range vers { - testBuildJobDependence(t, ver) - } -} - -func testBuildJobDependence(t *testing.T, jobVer model.JobVersion) { store := createMockStore(t) defer func() { require.NoError(t, store.Close()) @@ -224,13 +214,7 @@ func testBuildJobDependence(t *testing.T, jobVer model.JobVersion) { job6 := &model.Job{ID: 6, TableID: 1, Type: model.ActionDropTable} job7 := &model.Job{ID: 7, TableID: 2, Type: model.ActionModifyColumn} job9 := &model.Job{ID: 9, SchemaID: 111, Type: model.ActionDropSchema} - job11 := &model.Job{ID: 11, TableID: 2, Type: model.ActionRenameTable, Version: jobVer} - job11.FillArgs(&model.RenameTableArgs{ - OldSchemaID: 111, - NewTableName: pmodel.NewCIStr("new_table_name"), - OldSchemaName: pmodel.NewCIStr("old_db_name"), - }) - + job11 := &model.Job{Version: model.JobVersion1, ID: 11, TableID: 2, Type: model.ActionRenameTable, Args: []any{int64(111), "old db name"}} err := kv.RunInNewTxn(ctx, store, false, func(ctx context.Context, txn kv.Transaction) error { m := meta.NewMeta(txn) require.NoError(t, m.EnQueueDDLJob(job1)) diff --git a/pkg/ddl/executor.go b/pkg/ddl/executor.go index 14707af0d8c24..04de899359857 100644 --- a/pkg/ddl/executor.go +++ b/pkg/ddl/executor.go @@ -1054,8 +1054,8 @@ func (e *executor) createTableWithInfoJob( dbName pmodel.CIStr, tbInfo *model.TableInfo, involvingRef []model.InvolvingSchemaInfo, - onExist OnExist, -) (job *model.Job, err error) { + cfg CreateTableConfig, +) (jobW *JobWrapper, err error) { is := e.infoCache.GetLatest() schema, ok := is.SchemaByName(dbName) if !ok { @@ -1069,7 +1069,7 @@ func (e *executor) createTableWithInfoJob( var oldViewTblID int64 if oldTable, err := is.TableByName(e.ctx, schema.Name, tbInfo.Name); err == nil { err = infoschema.ErrTableExists.GenWithStackByArgs(ast.Ident{Schema: schema.Name, Name: tbInfo.Name}) - switch onExist { + switch cfg.OnExist { case OnExistIgnore: ctx.GetSessionVars().StmtCtx.AppendNote(err) return nil, nil @@ -1094,16 +1094,13 @@ func (e *executor) createTableWithInfoJob( } var actionType model.ActionType - args := []any{tbInfo} switch { case tbInfo.View != nil: actionType = model.ActionCreateView - args = append(args, onExist == OnExistReplace, oldViewTblID) case tbInfo.Sequence != nil: actionType = model.ActionCreateSequence default: actionType = model.ActionCreateTable - args = append(args, ctx.GetSessionVars().ForeignKeyChecks) } var involvingSchemas []model.InvolvingSchemaInfo @@ -1119,18 +1116,24 @@ func (e *executor) createTableWithInfoJob( involvingSchemas = append(involvingSchemas, sharedInvolvingFromTableInfo...) } - job = &model.Job{ + job := &model.Job{ + Version: model.GetJobVerInUse(), SchemaID: schema.ID, SchemaName: schema.Name.L, TableName: tbInfo.Name.L, Type: actionType, BinlogInfo: &model.HistoryInfo{}, - Args: args, CDCWriteSource: ctx.GetSessionVars().CDCWriteSource, InvolvingSchemaInfo: involvingSchemas, SQLMode: ctx.GetSessionVars().SQLMode, } - return job, nil + args := &model.CreateTableArgs{ + TableInfo: tbInfo, + OnExistReplace: cfg.OnExist == OnExistReplace, + OldViewTblID: oldViewTblID, + FKCheck: ctx.GetSessionVars().ForeignKeyChecks, + } + return NewJobWrapperWithArgs(job, args, cfg.IDAllocated), nil } func getSharedInvolvingSchemaInfo(info *model.TableInfo) []model.InvolvingSchemaInfo { @@ -1200,18 +1203,14 @@ func (e *executor) CreateTableWithInfo( ) (err error) { c := GetCreateTableConfig(cs) - job, err := e.createTableWithInfoJob( - ctx, dbName, tbInfo, involvingRef, c.OnExist, - ) + jobW, err := e.createTableWithInfoJob(ctx, dbName, tbInfo, involvingRef, c) if err != nil { return err } - if job == nil { + if jobW == nil { return nil } - jobW := NewJobWrapper(job, c.IDAllocated) - err = e.DoDDLJobWrapper(ctx, jobW) if err != nil { // table exists, but if_not_exists flags is true, so we ignore this error. @@ -1220,7 +1219,7 @@ func (e *executor) CreateTableWithInfo( err = nil } } else { - err = e.createTableWithInfoPost(ctx, tbInfo, job.SchemaID) + err = e.createTableWithInfoPost(ctx, tbInfo, jobW.SchemaID) } return errors.Trace(err) @@ -1239,15 +1238,12 @@ func (e *executor) BatchCreateTableWithInfo(ctx sessionctx.Context, }) c := GetCreateTableConfig(cs) - jobW := NewJobWrapper( - &model.Job{ - BinlogInfo: &model.HistoryInfo{}, - CDCWriteSource: ctx.GetSessionVars().CDCWriteSource, - SQLMode: ctx.GetSessionVars().SQLMode, - }, - c.IDAllocated, - ) - args := make([]*model.TableInfo, 0, len(infos)) + job := &model.Job{ + Version: model.GetJobVerInUse(), + BinlogInfo: &model.HistoryInfo{}, + CDCWriteSource: ctx.GetSessionVars().CDCWriteSource, + SQLMode: ctx.GetSessionVars().SQLMode, + } var err error @@ -1269,43 +1265,41 @@ func (e *executor) BatchCreateTableWithInfo(ctx sessionctx.Context, duplication[info.Name.L] = struct{}{} } + args := &model.BatchCreateTableArgs{ + Tables: make([]*model.CreateTableArgs, 0, len(infos)), + } for _, info := range infos { - job, err := e.createTableWithInfoJob(ctx, dbName, info, nil, c.OnExist) + jobItem, err := e.createTableWithInfoJob(ctx, dbName, info, nil, c) if err != nil { return errors.Trace(err) } - if job == nil { + if jobItem == nil { continue } // if jobW.Type == model.ActionCreateTables, it is initialized // if not, initialize jobW by job.XXXX - if jobW.Type != model.ActionCreateTables { - jobW.Type = model.ActionCreateTables - jobW.SchemaID = job.SchemaID - jobW.SchemaName = job.SchemaName + if job.Type != model.ActionCreateTables { + job.Type = model.ActionCreateTables + job.SchemaID = jobItem.SchemaID + job.SchemaName = jobItem.SchemaName } // append table job args - info, ok := job.Args[0].(*model.TableInfo) - if !ok { - return errors.Trace(fmt.Errorf("except table info")) - } - args = append(args, info) - jobW.InvolvingSchemaInfo = append(jobW.InvolvingSchemaInfo, model.InvolvingSchemaInfo{ + args.Tables = append(args.Tables, jobItem.JobArgs.(*model.CreateTableArgs)) + job.InvolvingSchemaInfo = append(job.InvolvingSchemaInfo, model.InvolvingSchemaInfo{ Database: dbName.L, Table: info.Name.L, }) if sharedInv := getSharedInvolvingSchemaInfo(info); len(sharedInv) > 0 { - jobW.InvolvingSchemaInfo = append(jobW.InvolvingSchemaInfo, sharedInv...) + job.InvolvingSchemaInfo = append(job.InvolvingSchemaInfo, sharedInv...) } } - if len(args) == 0 { + if len(args.Tables) == 0 { return nil } - jobW.Args = append(jobW.Args, args) - jobW.Args = append(jobW.Args, ctx.GetSessionVars().ForeignKeyChecks) + jobW := NewJobWrapperWithArgs(job, args, c.IDAllocated) err = e.DoDDLJobWrapper(ctx, jobW) if err != nil { // table exists, but if_not_exists flags is true, so we ignore this error. @@ -1316,8 +1310,8 @@ func (e *executor) BatchCreateTableWithInfo(ctx sessionctx.Context, return errors.Trace(err) } - for j := range args { - if err = e.createTableWithInfoPost(ctx, args[j], jobW.SchemaID); err != nil { + for _, tblArgs := range args.Tables { + if err = e.createTableWithInfoPost(ctx, tblArgs.TableInfo, jobW.SchemaID); err != nil { return errors.Trace(err) } } @@ -1848,8 +1842,8 @@ func (e *executor) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt for i, opt := range spec.Options { switch opt.Tp { case ast.TableOptionShardRowID: - if opt.UintValue > shardRowIDBitsMax { - opt.UintValue = shardRowIDBitsMax + if opt.UintValue > variable.MaxShardRowIDBits { + opt.UintValue = variable.MaxShardRowIDBits } err = e.ShardRowID(sctx, ident, opt.UintValue) case ast.TableOptionAutoIncrement: @@ -4217,11 +4211,11 @@ func (e *executor) TruncateTable(ctx sessionctx.Context, ti ast.Ident) error { CDCWriteSource: ctx.GetSessionVars().CDCWriteSource, SQLMode: ctx.GetSessionVars().SQLMode, } - job.FillArgs(&model.TruncateTableArgs{ + args := &model.TruncateTableArgs{ FKCheck: fkCheck, OldPartitionIDs: oldPartitionIDs, - }) - err = e.DoDDLJob(ctx, job) + } + err = e.doDDLJob2(ctx, job, args) if err != nil { return errors.Trace(err) } @@ -6263,6 +6257,10 @@ func (e *executor) DoDDLJob(ctx sessionctx.Context, job *model.Job) error { return e.DoDDLJobWrapper(ctx, NewJobWrapper(job, false)) } +func (e *executor) doDDLJob2(ctx sessionctx.Context, job *model.Job, args model.JobArgs) error { + return e.DoDDLJobWrapper(ctx, NewJobWrapperWithArgs(job, args, false)) +} + // DoDDLJobWrapper submit DDL job and wait it finishes. // When fast create is enabled, we might merge multiple jobs into one, so do not // depend on job.ID, use JobID from jobSubmitResult. @@ -6443,13 +6441,6 @@ func (e *executor) DoDDLJobWrapper(ctx sessionctx.Context, jobW *JobWrapper) (re } } -func getTruncateTableNewTableID(job *model.Job) int64 { - if job.Version == model.JobVersion1 { - return job.Args[0].(int64) - } - return job.Args[0].(*model.TruncateTableArgs).NewTableID -} - func getRenameTableUniqueIDs(job *model.Job, schema bool) []int64 { if !schema { return []int64{job.TableID} @@ -6472,7 +6463,7 @@ func HandleLockTablesOnSuccessSubmit(ctx sessionctx.Context, jobW *JobWrapper) { if ok, lockTp := ctx.CheckTableLocked(jobW.TableID); ok { ctx.AddTableLock([]model.TableLockTpInfo{{ SchemaID: jobW.SchemaID, - TableID: getTruncateTableNewTableID(jobW.Job), + TableID: jobW.JobArgs.(*model.TruncateTableArgs).NewTableID, Tp: lockTp, }}) } @@ -6484,7 +6475,7 @@ func HandleLockTablesOnSuccessSubmit(ctx sessionctx.Context, jobW *JobWrapper) { func HandleLockTablesOnFinish(ctx sessionctx.Context, jobW *JobWrapper, ddlErr error) { if jobW.Type == model.ActionTruncateTable { if ddlErr != nil { - newTableID := getTruncateTableNewTableID(jobW.Job) + newTableID := jobW.JobArgs.(*model.TruncateTableArgs).NewTableID ctx.ReleaseTableLockByTableIDs([]int64{newTableID}) return } diff --git a/pkg/ddl/executor_nokit_test.go b/pkg/ddl/executor_nokit_test.go index e1c5fb416b77b..d89426fad0a6e 100644 --- a/pkg/ddl/executor_nokit_test.go +++ b/pkg/ddl/executor_nokit_test.go @@ -69,20 +69,20 @@ func TestBuildQueryStringFromJobs(t *testing.T) { } func TestMergeCreateTableJobsOfSameSchema(t *testing.T) { - job1 := NewJobWrapper(&model.Job{ + job1 := NewJobWrapperWithArgs(&model.Job{ + Version: model.GetJobVerInUse(), SchemaID: 1, Type: model.ActionCreateTable, BinlogInfo: &model.HistoryInfo{}, - Args: []any{&model.TableInfo{Name: pmodel.CIStr{O: "t1", L: "t1"}}, false}, Query: "create table db1.t1 (c1 int, c2 int)", - }, false) - job2 := NewJobWrapper(&model.Job{ + }, &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.CIStr{O: "t1", L: "t1"}}}, false) + job2 := NewJobWrapperWithArgs(&model.Job{ + Version: model.GetJobVerInUse(), SchemaID: 1, Type: model.ActionCreateTable, BinlogInfo: &model.HistoryInfo{}, - Args: []any{&model.TableInfo{Name: pmodel.CIStr{O: "t2", L: "t2"}}, &model.TableInfo{}}, Query: "create table db1.t2 (c1 int, c2 int);", - }, false) + }, &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.CIStr{O: "t2", L: "t2"}}, FKCheck: true}, false) job, err := mergeCreateTableJobsOfSameSchema([]*JobWrapper{job1, job2}) require.NoError(t, err) require.Equal(t, "create table db1.t1 (c1 int, c2 int); create table db1.t2 (c1 int, c2 int);", job.Query) @@ -101,11 +101,11 @@ func TestMergeCreateTableJobs(t *testing.T) { t.Run("non create table are not merged", func(t *testing.T) { jobWs := []*JobWrapper{ - {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr("t1")}, false}}}, + {Job: &model.Job{Version: model.GetJobVerInUse(), SchemaName: "db", Type: model.ActionCreateTable}, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr("t1")}}}, {Job: &model.Job{SchemaName: "db", Type: model.ActionAddColumn}}, - {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr("t2")}, false}}}, + {Job: &model.Job{Version: model.GetJobVerInUse(), SchemaName: "db", Type: model.ActionCreateTable}, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr("t2")}}}, } newWs, err := mergeCreateTableJobs(jobWs) require.NoError(t, err) @@ -122,17 +122,16 @@ func TestMergeCreateTableJobs(t *testing.T) { t.Run("jobs of pre allocated ids are not merged", func(t *testing.T) { jobWs := []*JobWrapper{ - {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr("t1")}, false}}, IDAllocated: true}, - {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr("t2")}, false}}}, + {Job: &model.Job{Version: model.GetJobVerInUse(), SchemaName: "db", Type: model.ActionCreateTable}, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr("t1")}}, IDAllocated: true}, + {Job: &model.Job{Version: model.GetJobVerInUse(), SchemaName: "db", Type: model.ActionCreateTable}, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr("t2")}}}, } newWs, err := mergeCreateTableJobs(jobWs) slices.SortFunc(newWs, func(a, b *JobWrapper) int { - if aName, bName := a.Args[0].(*model.TableInfo).Name.L, b.Args[0].(*model.TableInfo).Name.L; aName != bName { - return strings.Compare(aName, bName) - } - return 0 + argsA := a.JobArgs.(*model.CreateTableArgs) + argsB := b.JobArgs.(*model.CreateTableArgs) + return strings.Compare(argsA.TableInfo.Name.L, argsB.TableInfo.Name.L) }) require.NoError(t, err) require.EqualValues(t, jobWs, newWs) @@ -140,17 +139,16 @@ func TestMergeCreateTableJobs(t *testing.T) { t.Run("jobs of foreign keys are not merged", func(t *testing.T) { jobWs := []*JobWrapper{ - {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{ForeignKeys: []*model.FKInfo{{}}}, false}}}, - {Job: &model.Job{SchemaName: "db", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr("t2")}, false}}}, + {Job: &model.Job{Version: model.GetJobVerInUse(), SchemaName: "db", Type: model.ActionCreateTable}, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{ForeignKeys: []*model.FKInfo{{}}}}}, + {Job: &model.Job{Version: model.GetJobVerInUse(), SchemaName: "db", Type: model.ActionCreateTable}, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr("t2")}}}, } newWs, err := mergeCreateTableJobs(jobWs) slices.SortFunc(newWs, func(a, b *JobWrapper) int { - if aName, bName := a.Args[0].(*model.TableInfo).Name.L, b.Args[0].(*model.TableInfo).Name.L; aName != bName { - return strings.Compare(aName, bName) - } - return 0 + argsA := a.JobArgs.(*model.CreateTableArgs) + argsB := b.JobArgs.(*model.CreateTableArgs) + return strings.Compare(argsA.TableInfo.Name.L, argsB.TableInfo.Name.L) }) require.NoError(t, err) require.EqualValues(t, jobWs, newWs) @@ -158,17 +156,14 @@ func TestMergeCreateTableJobs(t *testing.T) { t.Run("jobs of different schema are not merged", func(t *testing.T) { jobWs := []*JobWrapper{ - {Job: &model.Job{SchemaName: "db1", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr("t1")}, false}}}, - {Job: &model.Job{SchemaName: "db2", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr("t2")}, false}}}, + {Job: &model.Job{Version: model.GetJobVerInUse(), SchemaName: "db1", Type: model.ActionCreateTable}, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr("t1")}}}, + {Job: &model.Job{Version: model.GetJobVerInUse(), SchemaName: "db2", Type: model.ActionCreateTable}, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr("t2")}}}, } newWs, err := mergeCreateTableJobs(jobWs) slices.SortFunc(newWs, func(a, b *JobWrapper) int { - if aName, bName := a.SchemaName, b.SchemaName; aName != bName { - return strings.Compare(aName, bName) - } - return 0 + return strings.Compare(a.SchemaName, b.SchemaName) }) require.NoError(t, err) require.EqualValues(t, jobWs, newWs) @@ -176,61 +171,52 @@ func TestMergeCreateTableJobs(t *testing.T) { t.Run("max batch size 8", func(t *testing.T) { jobWs := make([]*JobWrapper, 0, 100) + jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: "db0", Type: model.ActionAddColumn}, false)) + jobW := NewJobWrapperWithArgs(&model.Job{Version: model.GetJobVerInUse(), SchemaName: "db1", Type: model.ActionCreateTable}, + &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr("t1")}}, true) + jobWs = append(jobWs, jobW) + jobW = NewJobWrapperWithArgs(&model.Job{Version: model.GetJobVerInUse(), SchemaName: "db2", Type: model.ActionCreateTable}, + &model.CreateTableArgs{TableInfo: &model.TableInfo{ForeignKeys: []*model.FKInfo{{}}}}, false) + jobWs = append(jobWs, jobW) for db, cnt := range map[string]int{ - "db0": 9, - "db1": 7, - "db2": 22, + "db3": 9, + "db4": 7, + "db5": 22, } { for i := 0; i < cnt; i++ { tblName := fmt.Sprintf("t%d", i) - jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: db, Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr(tblName)}, false}}, false)) + jobW := NewJobWrapperWithArgs(&model.Job{Version: model.GetJobVerInUse(), SchemaName: db, Type: model.ActionCreateTable}, + &model.CreateTableArgs{TableInfo: &model.TableInfo{Name: pmodel.NewCIStr(tblName)}}, false) + jobWs = append(jobWs, jobW) } } - jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: "dbx", Type: model.ActionAddColumn}, false)) - jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: "dbxx", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{Name: pmodel.NewCIStr("t1")}, false}}, true)) - jobWs = append(jobWs, NewJobWrapper(&model.Job{SchemaName: "dbxxx", Type: model.ActionCreateTable, - Args: []any{&model.TableInfo{ForeignKeys: []*model.FKInfo{{}}}, false}}, false)) newWs, err := mergeCreateTableJobs(jobWs) slices.SortFunc(newWs, func(a, b *JobWrapper) int { - if a.Type != b.Type { - return int(b.Type - a.Type) - } - if aName, bName := a.SchemaName, b.SchemaName; aName != bName { - return strings.Compare(aName, bName) - } - aTableInfo, aOK := a.Args[0].(*model.TableInfo) - bTableInfo, bOK := b.Args[0].(*model.TableInfo) - if aOK && bOK && aTableInfo.Name.L != bTableInfo.Name.L { - return strings.Compare(aTableInfo.Name.L, bTableInfo.Name.L) - } - - return 0 + return strings.Compare(a.SchemaName, b.SchemaName) }) require.NoError(t, err) // 3 non-mergeable + 2 + 1 + 3 require.Len(t, newWs, 9) require.Equal(t, model.ActionAddColumn, newWs[0].Type) require.Equal(t, model.ActionCreateTable, newWs[1].Type) - require.Equal(t, "dbxx", newWs[1].SchemaName) + require.Equal(t, "db1", newWs[1].SchemaName) require.Equal(t, model.ActionCreateTable, newWs[2].Type) - require.Equal(t, "dbxxx", newWs[2].SchemaName) + require.Equal(t, "db2", newWs[2].SchemaName) schemaCnts := make(map[string][]int, 3) for i := 3; i < 9; i++ { require.Equal(t, model.ActionCreateTables, newWs[i].Type) - infos := newWs[i].Args[0].([]*model.TableInfo) - schemaCnts[newWs[i].SchemaName] = append(schemaCnts[newWs[i].SchemaName], len(infos)) - require.Equal(t, len(infos), len(newWs[i].ResultCh)) + args := newWs[i].JobArgs.(*model.BatchCreateTableArgs) + schemaCnts[newWs[i].SchemaName] = append(schemaCnts[newWs[i].SchemaName], len(args.Tables)) + require.Equal(t, len(args.Tables), len(newWs[i].ResultCh)) } for k := range schemaCnts { slices.Sort(schemaCnts[k]) } require.Equal(t, map[string][]int{ - "db0": {4, 5}, - "db1": {7}, - "db2": {7, 7, 8}, + "db3": {4, 5}, + "db4": {7}, + "db5": {7, 7, 8}, }, schemaCnts) }) } diff --git a/pkg/ddl/executor_test.go b/pkg/ddl/executor_test.go index 1193613e760fa..6899262783363 100644 --- a/pkg/ddl/executor_test.go +++ b/pkg/ddl/executor_test.go @@ -270,8 +270,7 @@ func TestHandleLockTable(t *testing.T) { Type: model.ActionTruncateTable, TableID: 1, } - job.FillArgs(&model.TruncateTableArgs{NewTableID: 2}) - jobW := ddl.NewJobWrapper(job, false) + jobW := ddl.NewJobWrapperWithArgs(job, &model.TruncateTableArgs{NewTableID: 2}, false) t.Run("target table not locked", func(t *testing.T) { se.ReleaseAllTableLocks() diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go index 897f788c26542..2f292703bff7d 100644 --- a/pkg/ddl/index.go +++ b/pkg/ddl/index.go @@ -2611,12 +2611,19 @@ func renameIndexes(tblInfo *model.TableInfo, from, to pmodel.CIStr) { idx.Name.L = strings.Replace(idx.Name.L, from.L, to.L, 1) idx.Name.O = strings.Replace(idx.Name.O, from.O, to.O, 1) } + for _, col := range idx.Columns { + originalCol := tblInfo.Columns[col.Offset] + if originalCol.Hidden && getExpressionIndexOriginName(col.Name) == from.O { + col.Name.L = strings.Replace(col.Name.L, from.L, to.L, 1) + col.Name.O = strings.Replace(col.Name.O, from.O, to.O, 1) + } + } } } func renameHiddenColumns(tblInfo *model.TableInfo, from, to pmodel.CIStr) { for _, col := range tblInfo.Columns { - if col.Hidden && getExpressionIndexOriginName(col) == from.O { + if col.Hidden && getExpressionIndexOriginName(col.Name) == from.O { col.Name.L = strings.Replace(col.Name.L, from.L, to.L, 1) col.Name.O = strings.Replace(col.Name.O, from.O, to.O, 1) } diff --git a/pkg/ddl/job_scheduler.go b/pkg/ddl/job_scheduler.go index 0258775d620d5..7d95ad09cdc1f 100644 --- a/pkg/ddl/job_scheduler.go +++ b/pkg/ddl/job_scheduler.go @@ -675,6 +675,11 @@ func insertDDLJobs2Table(ctx context.Context, se *sess.Session, jobWs ...*JobWra var sql bytes.Buffer sql.WriteString(addDDLJobSQL) for i, jobW := range jobWs { + // TODO remove this check when all job type pass args in this way. + if jobW.JobArgs != nil { + jobW.FillArgs(jobW.JobArgs) + } + injectModifyJobArgFailPoint(jobWs) b, err := jobW.Encode(true) if err != nil { return err @@ -683,7 +688,7 @@ func insertDDLJobs2Table(ctx context.Context, se *sess.Session, jobWs ...*JobWra sql.WriteString(",") } fmt.Fprintf(&sql, "(%d, %t, %s, %s, %s, %d, %t)", jobW.ID, jobW.MayNeedReorg(), - strconv.Quote(job2SchemaIDs(jobW.Job)), strconv.Quote(job2TableIDs(jobW.Job)), + strconv.Quote(job2SchemaIDs(jobW)), strconv.Quote(job2TableIDs(jobW)), util.WrapKey2String(b), jobW.Type, jobW.Started()) } se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull) @@ -692,25 +697,25 @@ func insertDDLJobs2Table(ctx context.Context, se *sess.Session, jobWs ...*JobWra return errors.Trace(err) } -func job2SchemaIDs(job *model.Job) string { - return job2UniqueIDs(job, true) +func job2SchemaIDs(jobW *JobWrapper) string { + return job2UniqueIDs(jobW, true) } -func job2TableIDs(job *model.Job) string { - return job2UniqueIDs(job, false) +func job2TableIDs(jobW *JobWrapper) string { + return job2UniqueIDs(jobW, false) } -func job2UniqueIDs(job *model.Job, schema bool) string { - switch job.Type { +func job2UniqueIDs(jobW *JobWrapper, schema bool) string { + switch jobW.Type { case model.ActionExchangeTablePartition, model.ActionRenameTables, model.ActionRenameTable: var ids []int64 - if job.Type == model.ActionRenameTable { - ids = getRenameTableUniqueIDs(job, schema) + if jobW.Type == model.ActionRenameTable { + ids = getRenameTableUniqueIDs(jobW.Job, schema) } else { if schema { - ids = job.CtxVars[0].([]int64) + ids = jobW.CtxVars[0].([]int64) } else { - ids = job.CtxVars[1].([]int64) + ids = jobW.CtxVars[1].([]int64) } } @@ -727,15 +732,15 @@ func job2UniqueIDs(job *model.Job, schema bool) string { return strings.Join(s, ",") case model.ActionTruncateTable: if schema { - return strconv.FormatInt(job.SchemaID, 10) + return strconv.FormatInt(jobW.SchemaID, 10) } - newTableID := getTruncateTableNewTableID(job) - return strconv.FormatInt(job.TableID, 10) + "," + strconv.FormatInt(newTableID, 10) + newTableID := jobW.JobArgs.(*model.TruncateTableArgs).NewTableID + return strconv.FormatInt(jobW.TableID, 10) + "," + strconv.FormatInt(newTableID, 10) } if schema { - return strconv.FormatInt(job.SchemaID, 10) + return strconv.FormatInt(jobW.SchemaID, 10) } - return strconv.FormatInt(job.TableID, 10) + return strconv.FormatInt(jobW.TableID, 10) } func updateDDLJob2Table(se *sess.Session, job *model.Job, updateRawArgs bool) error { diff --git a/pkg/ddl/job_submitter.go b/pkg/ddl/job_submitter.go index b824ef9059ee4..d8908d5a09235 100644 --- a/pkg/ddl/job_submitter.go +++ b/pkg/ddl/job_submitter.go @@ -153,8 +153,8 @@ func mergeCreateTableJobs(jobWs []*JobWrapper) ([]*JobWrapper, error) { continue } // ActionCreateTables doesn't support foreign key now. - tbInfo, ok := jobW.Args[0].(*model.TableInfo) - if !ok || len(tbInfo.ForeignKeys) > 0 { + args := jobW.JobArgs.(*model.CreateTableArgs) + if len(args.TableInfo.ForeignKeys) > 0 { resJobWs = append(resJobWs, jobW) continue } @@ -173,22 +173,13 @@ func mergeCreateTableJobs(jobWs []*JobWrapper) ([]*JobWrapper, error) { start := 0 for _, batchSize := range mathutil.Divide2Batches(total, batchCount) { batch := jobs[start : start+batchSize] - job, err := mergeCreateTableJobsOfSameSchema(batch) + newJobW, err := mergeCreateTableJobsOfSameSchema(batch) if err != nil { return nil, err } start += batchSize logutil.DDLLogger().Info("merge create table jobs", zap.String("schema", schema), zap.Int("total", total), zap.Int("batch_size", batchSize)) - - newJobW := &JobWrapper{ - Job: job, - ResultCh: make([]chan jobSubmitResult, 0, batchSize), - } - // merge the result channels. - for _, j := range batch { - newJobW.ResultCh = append(newJobW.ResultCh, j.ResultCh...) - } resJobWs = append(resJobWs, newJobW) } } @@ -217,16 +208,18 @@ func buildQueryStringFromJobs(jobs []*JobWrapper) string { } // mergeCreateTableJobsOfSameSchema combine CreateTableJobs to BatchCreateTableJob. -func mergeCreateTableJobsOfSameSchema(jobWs []*JobWrapper) (*model.Job, error) { +func mergeCreateTableJobsOfSameSchema(jobWs []*JobWrapper) (*JobWrapper, error) { if len(jobWs) == 0 { return nil, errors.Trace(fmt.Errorf("expect non-empty jobs")) } - var combinedJob *model.Job - - args := make([]*model.TableInfo, 0, len(jobWs)) - involvingSchemaInfo := make([]model.InvolvingSchemaInfo, 0, len(jobWs)) - var foreignKeyChecks bool + var ( + combinedJob *model.Job + args = &model.BatchCreateTableArgs{ + Tables: make([]*model.CreateTableArgs, 0, len(jobWs)), + } + involvingSchemaInfo = make([]model.InvolvingSchemaInfo, 0, len(jobWs)) + ) // if there is any duplicated table name duplication := make(map[string]struct{}) @@ -234,16 +227,11 @@ func mergeCreateTableJobsOfSameSchema(jobWs []*JobWrapper) (*model.Job, error) { if combinedJob == nil { combinedJob = job.Clone() combinedJob.Type = model.ActionCreateTables - combinedJob.Args = combinedJob.Args[:0] - foreignKeyChecks = job.Args[1].(bool) } - // append table job args - info, ok := job.Args[0].(*model.TableInfo) - if !ok { - return nil, errors.Trace(fmt.Errorf("expect model.TableInfo, but got %T", job.Args[0])) - } - args = append(args, info) + jobArgs := job.JobArgs.(*model.CreateTableArgs) + args.Tables = append(args.Tables, jobArgs) + info := jobArgs.TableInfo if _, ok := duplication[info.Name.L]; ok { // return err even if create table if not exists return nil, infoschema.ErrTableExists.FastGenByArgs("can not batch create tables with same name") @@ -258,12 +246,20 @@ func mergeCreateTableJobsOfSameSchema(jobWs []*JobWrapper) (*model.Job, error) { }) } - combinedJob.Args = append(combinedJob.Args, args) - combinedJob.Args = append(combinedJob.Args, foreignKeyChecks) combinedJob.InvolvingSchemaInfo = involvingSchemaInfo combinedJob.Query = buildQueryStringFromJobs(jobWs) - return combinedJob, nil + newJobW := &JobWrapper{ + Job: combinedJob, + JobArgs: args, + ResultCh: make([]chan jobSubmitResult, 0, len(jobWs)), + } + // merge the result channels. + for _, j := range jobWs { + newJobW.ResultCh = append(newJobW.ResultCh, j.ResultCh...) + } + + return newJobW, nil } // addBatchDDLJobs2Table gets global job IDs and puts the DDL jobs in the DDL job table. @@ -383,6 +379,10 @@ func (s *JobSubmitter) addBatchDDLJobs2Queue(jobWs []*JobWrapper) error { } for _, jobW := range jobWs { + // TODO remove this check when all job type pass args in this way. + if jobW.JobArgs != nil { + jobW.FillArgs(jobW.JobArgs) + } job := jobW.Job job.StartTS = txn.StartTS() setJobStateToQueueing(job) @@ -434,7 +434,6 @@ func (s *JobSubmitter) GenGIDAndInsertJobsWithRetry(ctx context.Context, ddlSe * } }) assignGIDsForJobs(jobWs, ids) - injectModifyJobArgFailPoint(jobWs) // job scheduler will start run them after txn commit, we want to make sure // the channel exists before the jobs are submitted. for i, jobW := range jobWs { @@ -494,12 +493,12 @@ func getRequiredGIDCount(jobWs []*JobWrapper) int { } switch jobW.Type { case model.ActionCreateView, model.ActionCreateSequence, model.ActionCreateTable: - info := jobW.Args[0].(*model.TableInfo) - count += idCountForTable(info) + args := jobW.JobArgs.(*model.CreateTableArgs) + count += idCountForTable(args.TableInfo) case model.ActionCreateTables: - infos := jobW.Args[0].([]*model.TableInfo) - for _, info := range infos { - count += idCountForTable(info) + args := jobW.JobArgs.(*model.BatchCreateTableArgs) + for _, tblArgs := range args.Tables { + count += idCountForTable(tblArgs.TableInfo) } case model.ActionCreateSchema, model.ActionCreateResourceGroup: count++ @@ -519,12 +518,7 @@ func getRequiredGIDCount(jobWs []*JobWrapper) int { pInfo := jobW.Args[1].(*model.PartitionInfo) count += len(pInfo.Definitions) case model.ActionTruncateTable: - if jobW.Version == model.JobVersion1 { - partCount := jobW.Args[3].(int) - count += 1 + partCount - } else { - count += 1 + len(jobW.Args[0].(*model.TruncateTableArgs).OldPartitionIDs) - } + count += 1 + len(jobW.JobArgs.(*model.TruncateTableArgs).OldPartitionIDs) } } return count @@ -537,16 +531,16 @@ func assignGIDsForJobs(jobWs []*JobWrapper, ids []int64) { for _, jobW := range jobWs { switch jobW.Type { case model.ActionCreateView, model.ActionCreateSequence, model.ActionCreateTable: - info := jobW.Args[0].(*model.TableInfo) + args := jobW.JobArgs.(*model.CreateTableArgs) if !jobW.IDAllocated { - alloc.assignIDsForTable(info) + alloc.assignIDsForTable(args.TableInfo) } - jobW.TableID = info.ID + jobW.TableID = args.TableInfo.ID case model.ActionCreateTables: if !jobW.IDAllocated { - infos := jobW.Args[0].([]*model.TableInfo) - for _, info := range infos { - alloc.assignIDsForTable(info) + args := jobW.JobArgs.(*model.BatchCreateTableArgs) + for _, tblArgs := range args.Tables { + alloc.assignIDsForTable(tblArgs.TableInfo) } } case model.ActionCreateSchema: @@ -599,23 +593,13 @@ func assignGIDsForJobs(jobWs []*JobWrapper, ids []int64) { pInfo.NewTableID = pInfo.Definitions[0].ID case model.ActionTruncateTable: if !jobW.IDAllocated { - if jobW.Version == model.JobVersion1 { - jobW.Args[0] = alloc.next() - partCount := jobW.Args[3].(int) - partIDs := make([]int64, partCount) - for i := range partIDs { - partIDs[i] = alloc.next() - } - jobW.Args[2] = partIDs - } else { - args := jobW.Args[0].(*model.TruncateTableArgs) - args.NewTableID = alloc.next() - partIDs := make([]int64, len(args.OldPartitionIDs)) - for i := range partIDs { - partIDs[i] = alloc.next() - } - args.NewPartitionIDs = partIDs + args := jobW.JobArgs.(*model.TruncateTableArgs) + args.NewTableID = alloc.next() + partIDs := make([]int64, len(args.OldPartitionIDs)) + for i := range partIDs { + partIDs[i] = alloc.next() } + args.NewPartitionIDs = partIDs } } jobW.ID = alloc.next() diff --git a/pkg/ddl/job_submitter_test.go b/pkg/ddl/job_submitter_test.go index cdcddba8c68a7..0bae829a9d0ca 100644 --- a/pkg/ddl/job_submitter_test.go +++ b/pkg/ddl/job_submitter_test.go @@ -65,11 +65,12 @@ func TestGenIDAndInsertJobsWithRetry(t *testing.T) { jobs := []*ddl.JobWrapper{{ Job: &model.Job{ + Version: model.GetJobVerInUse(), Type: model.ActionCreateTable, SchemaName: "test", TableName: "t1", - Args: []any{&model.TableInfo{}}, }, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{}}, }} initialGID := getGlobalID(ctx, t, store) threads, iterations := 10, 500 @@ -134,24 +135,32 @@ func TestCombinedIDAllocation(t *testing.T) { return info } - genCreateTblJob := func(tp model.ActionType, partitionCnt int) *model.Job { - return &model.Job{ - Version: model.JobVersion1, - Type: tp, - Args: []any{genTblInfo(partitionCnt)}, - } + genCreateTblJobW := func(tp model.ActionType, partitionCnt int, idAllocated bool) *ddl.JobWrapper { + return ddl.NewJobWrapperWithArgs( + &model.Job{ + Version: model.GetJobVerInUse(), + Type: tp, + }, + &model.CreateTableArgs{TableInfo: genTblInfo(partitionCnt)}, + idAllocated, + ) } - genCreateTblsJob := func(partitionCounts ...int) *model.Job { - infos := make([]*model.TableInfo, 0, len(partitionCounts)) - for _, c := range partitionCounts { - infos = append(infos, genTblInfo(c)) + genCreateTblsJobW := func(idAllocated bool, partitionCounts ...int) *ddl.JobWrapper { + args := &model.BatchCreateTableArgs{ + Tables: make([]*model.CreateTableArgs, 0, len(partitionCounts)), } - return &model.Job{ - Version: model.JobVersion1, - Type: model.ActionCreateTables, - Args: []any{infos}, + for _, c := range partitionCounts { + args.Tables = append(args.Tables, &model.CreateTableArgs{TableInfo: genTblInfo(c)}) } + return ddl.NewJobWrapperWithArgs( + &model.Job{ + Version: model.JobVersion1, + Type: model.ActionCreateTables, + }, + args, + idAllocated, + ) } genCreateDBJob := func() *model.Job { @@ -220,50 +229,50 @@ func TestCombinedIDAllocation(t *testing.T) { } } - genTruncTblJob := func(partCnt int) *model.Job { + genTruncTblJob := func(partCnt int, idAllocated bool) *ddl.JobWrapper { j := &model.Job{ Version: model.GetJobVerInUse(), Type: model.ActionTruncateTable, } - j.FillArgs(&model.TruncateTableArgs{OldPartitionIDs: make([]int64, partCnt)}) - return j + args := &model.TruncateTableArgs{OldPartitionIDs: make([]int64, partCnt)} + return ddl.NewJobWrapperWithArgs(j, args, idAllocated) } cases := []idAllocationCase{ { - jobW: ddl.NewJobWrapper(genCreateTblsJob(1, 2, 0), false), + jobW: genCreateTblsJobW(false, 1, 2, 0), requiredIDCount: 1 + 3 + 1 + 2, }, { - jobW: ddl.NewJobWrapper(genCreateTblsJob(3, 4), true), + jobW: genCreateTblsJobW(true, 3, 4), requiredIDCount: 1, }, { - jobW: ddl.NewJobWrapper(genCreateTblJob(model.ActionCreateTable, 3), false), + jobW: genCreateTblJobW(model.ActionCreateTable, 3, false), requiredIDCount: 1 + 1 + 3, }, { - jobW: ddl.NewJobWrapper(genCreateTblJob(model.ActionCreateTable, 0), false), + jobW: genCreateTblJobW(model.ActionCreateTable, 0, false), requiredIDCount: 1 + 1, }, { - jobW: ddl.NewJobWrapper(genCreateTblJob(model.ActionCreateTable, 8), true), + jobW: genCreateTblJobW(model.ActionCreateTable, 8, true), requiredIDCount: 1, }, { - jobW: ddl.NewJobWrapper(genCreateTblJob(model.ActionCreateSequence, 0), false), + jobW: genCreateTblJobW(model.ActionCreateSequence, 0, false), requiredIDCount: 2, }, { - jobW: ddl.NewJobWrapper(genCreateTblJob(model.ActionCreateSequence, 0), true), + jobW: genCreateTblJobW(model.ActionCreateSequence, 0, true), requiredIDCount: 1, }, { - jobW: ddl.NewJobWrapper(genCreateTblJob(model.ActionCreateView, 0), false), + jobW: genCreateTblJobW(model.ActionCreateView, 0, false), requiredIDCount: 2, }, { - jobW: ddl.NewJobWrapper(genCreateTblJob(model.ActionCreateView, 0), true), + jobW: genCreateTblJobW(model.ActionCreateView, 0, true), requiredIDCount: 1, }, { @@ -323,11 +332,11 @@ func TestCombinedIDAllocation(t *testing.T) { requiredIDCount: 1, }, { - jobW: ddl.NewJobWrapper(genTruncTblJob(17), false), + jobW: genTruncTblJob(17, false), requiredIDCount: 19, }, { - jobW: ddl.NewJobWrapper(genTruncTblJob(6), true), + jobW: genTruncTblJob(6, true), requiredIDCount: 1, }, } @@ -402,15 +411,15 @@ func TestCombinedIDAllocation(t *testing.T) { switch j.Type { case model.ActionCreateTable, model.ActionCreateView, model.ActionCreateSequence: require.Greater(t, j.TableID, initialGlobalID) - info := &model.TableInfo{} - require.NoError(t, j.DecodeArgs(info)) - require.Equal(t, j.TableID, info.ID) - checkTableInfo(info) + args, err := model.GetCreateTableArgs(j) + require.NoError(t, err) + require.Equal(t, j.TableID, args.TableInfo.ID) + checkTableInfo(args.TableInfo) case model.ActionCreateTables: - var infos []*model.TableInfo - require.NoError(t, j.DecodeArgs(&infos)) - for _, info := range infos { - checkTableInfo(info) + args, err := model.GetBatchCreateTableArgs(j) + require.NoError(t, err) + for _, tblArgs := range args.Tables { + checkTableInfo(tblArgs.TableInfo) } case model.ActionCreateSchema: require.Greater(t, j.SchemaID, initialGlobalID) @@ -481,11 +490,12 @@ func TestGenIDAndInsertJobsWithRetryQPS(t *testing.T) { payload := strings.Repeat("a", payloadSize) jobs := []*ddl.JobWrapper{{ Job: &model.Job{ + Version: model.GetJobVerInUse(), Type: model.ActionCreateTable, SchemaName: "test", TableName: "t1", - Args: []any{&model.TableInfo{Comment: payload}}, }, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{Comment: payload}}, }} counters := make([]atomic.Int64, thread+1) var wg util.WaitGroupWrapper @@ -541,11 +551,12 @@ func TestGenGIDAndInsertJobsWithRetryOnErr(t *testing.T) { ddlSe := sess.NewSession(tk.Session()) jobs := []*ddl.JobWrapper{{ Job: &model.Job{ + Version: model.GetJobVerInUse(), Type: model.ActionCreateTable, SchemaName: "test", TableName: "t1", - Args: []any{&model.TableInfo{}}, }, + JobArgs: &model.CreateTableArgs{TableInfo: &model.TableInfo{}}, }} submitter := ddl.NewJobSubmitterForTest() // retry for 3 times diff --git a/pkg/ddl/modify_column.go b/pkg/ddl/modify_column.go index cb3edff8b574d..feba8ce322a3d 100644 --- a/pkg/ddl/modify_column.go +++ b/pkg/ddl/modify_column.go @@ -23,8 +23,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/ddl/notifier" sess "github.com/pingcap/tidb/pkg/ddl/session" - ddlutil "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/errctx" "github.com/pingcap/tidb/pkg/expression" exprctx "github.com/pingcap/tidb/pkg/expression/context" @@ -42,7 +42,6 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" - statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/types" @@ -531,9 +530,7 @@ func (w *worker) doModifyColumnTypeWithData( job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo) // Refactor the job args to add the old index ids into delete range table. job.Args = []any{rmIdxIDs, getPartitionIDs(tblInfo)} - modifyColumnEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: ddlutil.NewModifyColumnEvent(tblInfo, []*model.ColumnInfo{changingCol}), - } + modifyColumnEvent := notifier.NewModifyColumnEvent(tblInfo, []*model.ColumnInfo{changingCol}) asyncNotifyEvent(jobCtx, modifyColumnEvent, job) default: err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("column", changingCol.State) diff --git a/pkg/ddl/notifier/BUILD.bazel b/pkg/ddl/notifier/BUILD.bazel new file mode 100644 index 0000000000000..298a7e859f532 --- /dev/null +++ b/pkg/ddl/notifier/BUILD.bazel @@ -0,0 +1,35 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "notifier", + srcs = [ + "publish.go", + "schema_change_notifier.go", + "store.go", + ], + importpath = "github.com/pingcap/tidb/pkg/ddl/notifier", + visibility = ["//visibility:public"], + deps = [ + "//pkg/ddl/session", + "//pkg/meta/model", + "//pkg/util/intest", + ], +) + +go_test( + name = "notifier_test", + timeout = "short", + srcs = [ + "publish_testkit_test.go", + "schema_change_notifier_test.go", + ], + embed = [":notifier"], + flaky = True, + deps = [ + "//pkg/ddl/session", + "//pkg/meta/model", + "//pkg/parser/model", + "//pkg/testkit", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/ddl/notifier/publish.go b/pkg/ddl/notifier/publish.go new file mode 100644 index 0000000000000..ce66f9b0903e3 --- /dev/null +++ b/pkg/ddl/notifier/publish.go @@ -0,0 +1,70 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package notifier + +import ( + "context" + + sess "github.com/pingcap/tidb/pkg/ddl/session" +) + +// PubSchemaChange publishes schema changes to the cluster to notify other +// components. It stages changes in given `se` so they will be visible when `se` +// further commits. When the schema change is not from multi-schema change DDL, +// `multiSchemaChangeSeq` is -1. Otherwise, `multiSchemaChangeSeq` is the sub-job +// index of the multi-schema change DDL. +func PubSchemaChange( + ctx context.Context, + se *sess.Session, + ddlJobID int64, + multiSchemaChangeSeq int64, + event *SchemaChangeEvent, +) error { + return PubSchemeChangeToStore( + ctx, + se, + ddlJobID, + multiSchemaChangeSeq, + event, + DefaultStore, + ) +} + +// PubSchemeChangeToStore is exposed for testing. Caller should use +// PubSchemaChange instead. +func PubSchemeChangeToStore( + ctx context.Context, + se *sess.Session, + ddlJobID int64, + multiSchemaChangeSeq int64, + event *SchemaChangeEvent, + store Store, +) error { + change := &schemaChange{ + ddlJobID: ddlJobID, + multiSchemaChangeSeq: multiSchemaChangeSeq, + event: event, + } + return store.Insert(ctx, se, change) +} + +// schemaChange is the Golang representation of the persistent data. (ddlJobID, +// multiSchemaChangeSeq) should be unique in the cluster. +type schemaChange struct { + ddlJobID int64 + multiSchemaChangeSeq int64 + event *SchemaChangeEvent + processedByFlag uint64 +} diff --git a/pkg/ddl/notifier/publish_testkit_test.go b/pkg/ddl/notifier/publish_testkit_test.go new file mode 100644 index 0000000000000..2f5b535c3d4d0 --- /dev/null +++ b/pkg/ddl/notifier/publish_testkit_test.go @@ -0,0 +1,59 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package notifier_test + +import ( + "context" + "testing" + + "github.com/pingcap/tidb/pkg/ddl/notifier" + sess "github.com/pingcap/tidb/pkg/ddl/session" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/stretchr/testify/require" +) + +func TestPublishToTableStore(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("USE test") + tk.MustExec("DROP TABLE IF EXISTS ddl_notifier") + tk.MustExec(` +CREATE TABLE ddl_notifier ( + ddl_job_id BIGINT, + multi_schema_change_seq BIGINT COMMENT '-1 if the schema change does not belong to a multi-schema change DDL. 0 or positive numbers representing the sub-job index of a multi-schema change DDL', + schema_change JSON COMMENT 'SchemaChange at rest', + processed_by_flag BIGINT UNSIGNED DEFAULT 0 COMMENT 'flag to mark which subscriber has processed the event', + PRIMARY KEY(ddl_job_id, multi_schema_change_seq) +) +`) + + ctx := context.Background() + s := notifier.OpenTableStore("test", "ddl_notifier") + se := sess.NewSession(tk.Session()) + err := notifier.PubSchemeChangeToStore(ctx, se, 1, -1, nil, s) + require.NoError(t, err) + err = notifier.PubSchemeChangeToStore(ctx, se, 2, -1, nil, s) + require.NoError(t, err) + + got, err := s.List(ctx, se, 1) + require.NoError(t, err) + require.Len(t, got, 1) + got, err = s.List(ctx, se, 2) + require.NoError(t, err) + require.Len(t, got, 2) + got, err = s.List(ctx, se, 3) + require.NoError(t, err) + require.Len(t, got, 2) +} diff --git a/pkg/ddl/util/schema_change_notifier.go b/pkg/ddl/notifier/schema_change_notifier.go similarity index 84% rename from pkg/ddl/util/schema_change_notifier.go rename to pkg/ddl/notifier/schema_change_notifier.go index 4962091530ca2..90fdf1c48ce6f 100644 --- a/pkg/ddl/util/schema_change_notifier.go +++ b/pkg/ddl/notifier/schema_change_notifier.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package notifier import ( "fmt" @@ -27,15 +27,14 @@ import ( // check the GetType of SchemaChange and call the corresponding getter function // to retrieve the needed information. type SchemaChangeEvent struct { - // todo: field and method will be added in the next few pr on demand tableInfo *model.TableInfo oldTableInfo *model.TableInfo addedPartInfo *model.PartitionInfo droppedPartInfo *model.PartitionInfo columnInfos []*model.ColumnInfo - // nonPartTableID is used to store the non-partitioned table that is converted to - // a partitioned table in NewAddPartitioningEvent. - nonPartTableID int64 + // oldTableID4Partition is used to store the table ID when a table transitions from being partitioned to non-partitioned, + // or vice versa. + oldTableID4Partition int64 tp model.ActionType } @@ -54,17 +53,23 @@ func (s *SchemaChangeEvent) String() string { if s.oldTableInfo != nil { _, _ = fmt.Fprintf(&sb, ", Old Table ID: %d, Old Table Name: %s", s.oldTableInfo.ID, s.oldTableInfo.Name) } - if s.nonPartTableID != 0 { - _, _ = fmt.Fprintf(&sb, ", Old Table ID for Partition: %d", s.nonPartTableID) + if s.oldTableID4Partition != 0 { + _, _ = fmt.Fprintf(&sb, ", Old Table ID for Partition: %d", s.oldTableID4Partition) } if s.addedPartInfo != nil { for _, partDef := range s.addedPartInfo.Definitions { - _, _ = fmt.Fprintf(&sb, ", Partition Name: %s, Partition ID: %d", partDef.Name, partDef.ID) + if partDef.Name.L != "" { + _, _ = fmt.Fprintf(&sb, ", Partition Name: %s", partDef.Name) + } + _, _ = fmt.Fprintf(&sb, ", Partition ID: %d", partDef.ID) } } if s.droppedPartInfo != nil { for _, partDef := range s.droppedPartInfo.Definitions { - _, _ = fmt.Fprintf(&sb, ", Dropped Partition Name: %s, Dropped Partition ID: %d", partDef.Name, partDef.ID) + if partDef.Name.L != "" { + _, _ = fmt.Fprintf(&sb, ", Dropped Partition Name: %s", partDef.Name) + } + _, _ = fmt.Fprintf(&sb, ", Dropped Partition ID: %d", partDef.ID) } } for _, columnInfo := range s.columnInfos { @@ -323,10 +328,10 @@ func NewAddPartitioningEvent( addedPartInfo *model.PartitionInfo, ) *SchemaChangeEvent { return &SchemaChangeEvent{ - tp: model.ActionAlterTablePartitioning, - nonPartTableID: nonPartTableID, - tableInfo: newGlobalTableInfo, - addedPartInfo: addedPartInfo, + tp: model.ActionAlterTablePartitioning, + oldTableID4Partition: nonPartTableID, + tableInfo: newGlobalTableInfo, + addedPartInfo: addedPartInfo, } } @@ -339,5 +344,39 @@ func (s *SchemaChangeEvent) GetAddPartitioningInfo() ( addedPartInfo *model.PartitionInfo, ) { intest.Assert(s.tp == model.ActionAlterTablePartitioning) - return s.nonPartTableID, s.tableInfo, s.addedPartInfo + return s.oldTableID4Partition, s.tableInfo, s.addedPartInfo +} + +// NewRemovePartitioningEvent creates a schema change event whose type is +// ActionRemovePartitioning. +func NewRemovePartitioningEvent( + oldPartitionedTableID int64, + nonPartitionTableInfo *model.TableInfo, + droppedPartInfo *model.PartitionInfo, +) *SchemaChangeEvent { + return &SchemaChangeEvent{ + tp: model.ActionRemovePartitioning, + oldTableID4Partition: oldPartitionedTableID, + tableInfo: nonPartitionTableInfo, + droppedPartInfo: droppedPartInfo, + } +} + +// GetRemovePartitioningInfo returns the table info and partition info of the SchemaChangeEvent whose type is +// ActionRemovePartitioning. +func (s *SchemaChangeEvent) GetRemovePartitioningInfo() ( + oldPartitionedTableID int64, + newSingleTableInfo *model.TableInfo, + droppedPartInfo *model.PartitionInfo, +) { + intest.Assert(s.tp == model.ActionRemovePartitioning) + return s.oldTableID4Partition, s.tableInfo, s.droppedPartInfo +} + +// NewFlashbackClusterEvent creates a schema change event whose type is +// ActionFlashbackCluster. +func NewFlashbackClusterEvent() *SchemaChangeEvent { + return &SchemaChangeEvent{ + tp: model.ActionFlashbackCluster, + } } diff --git a/pkg/statistics/handle/util/ddl_event_test.go b/pkg/ddl/notifier/schema_change_notifier_test.go similarity index 71% rename from pkg/statistics/handle/util/ddl_event_test.go rename to pkg/ddl/notifier/schema_change_notifier_test.go index ac9c953aa4e1d..511ffa1c115a2 100644 --- a/pkg/statistics/handle/util/ddl_event_test.go +++ b/pkg/ddl/notifier/schema_change_notifier_test.go @@ -1,10 +1,10 @@ -// Copyright 2023 PingCAP, Inc. +// Copyright 2024 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package util +package notifier import ( "testing" @@ -24,14 +24,13 @@ import ( func TestEventString(t *testing.T) { // Create an Event object - e := &DDLEvent{ - tp: model.ActionAddColumn, - schemaID: 1, + e := &SchemaChangeEvent{ + tp: model.ActionAddColumn, tableInfo: &model.TableInfo{ ID: 1, Name: pmodel.NewCIStr("Table1"), }, - partInfo: &model.PartitionInfo{ + addedPartInfo: &model.PartitionInfo{ Definitions: []model.PartitionDefinition{ {ID: 2}, {ID: 3}, @@ -41,7 +40,7 @@ func TestEventString(t *testing.T) { ID: 4, Name: pmodel.NewCIStr("Table2"), }, - oldPartInfo: &model.PartitionInfo{ + droppedPartInfo: &model.PartitionInfo{ Definitions: []model.PartitionDefinition{ {ID: 5}, {ID: 6}, @@ -57,9 +56,8 @@ func TestEventString(t *testing.T) { result := e.String() // Check the result - expected := "(Event Type: add column, Schema ID: 1, Table ID: 1, Table Name: Table1, " + - "Partition IDs: [2 3], Old Table ID: 4, Old Table Name: Table2, " + - "Old Partition IDs: [5 6], Column ID: 7, Column Name: Column1, " + - "Column ID: 8, Column Name: Column2" + expected := "(Event Type: add column, Table ID: 1, Table Name: Table1, Old Table ID: 4, Old Table Name: Table2," + + " Partition ID: 2, Partition ID: 3, Dropped Partition ID: 5, Dropped Partition ID: 6, " + + "Column ID: 7, Column Name: Column1, Column ID: 8, Column Name: Column2)" require.Equal(t, expected, result) } diff --git a/pkg/ddl/notifier/store.go b/pkg/ddl/notifier/store.go new file mode 100644 index 0000000000000..840dc5e06dcd8 --- /dev/null +++ b/pkg/ddl/notifier/store.go @@ -0,0 +1,111 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package notifier + +import ( + "context" + "fmt" + + sess "github.com/pingcap/tidb/pkg/ddl/session" +) + +// Store is the (de)serialization and persistent layer. +type Store interface { + Insert(context.Context, *sess.Session, *schemaChange) error + UpdateProcessed( + ctx context.Context, + se *sess.Session, + ddlJobID int64, + multiSchemaChangeID int, + processedBy uint64, + ) error + Delete(ctx context.Context, se *sess.Session, ddlJobID int64, multiSchemaChangeID int) error + List(ctx context.Context, se *sess.Session, limit int) ([]*schemaChange, error) +} + +// DefaultStore is the system table store. Still WIP now. +var DefaultStore Store + +type tableStore struct { + db string + table string +} + +func (t *tableStore) Insert(ctx context.Context, s *sess.Session, change *schemaChange) error { + // TODO: fill schema_change after we implement JSON serialization. + sql := fmt.Sprintf(` + INSERT INTO %s.%s ( + ddl_job_id, + multi_schema_change_seq, + schema_change, + processed_by_flag + ) VALUES (%d, %d, '%s', 0)`, + t.db, t.table, + change.ddlJobID, change.multiSchemaChangeSeq, "{}", + ) + _, err := s.Execute(ctx, sql, "ddl_notifier") + return err +} + +//revive:disable + +func (t *tableStore) UpdateProcessed(ctx context.Context, se *sess.Session, ddlJobID int64, multiSchemaChangeID int, processedBy uint64) error { + //TODO implement me + panic("implement me") +} + +func (t *tableStore) Delete(ctx context.Context, se *sess.Session, ddlJobID int64, multiSchemaChangeID int) error { + //TODO implement me + panic("implement me") +} + +//revive:enable + +func (t *tableStore) List(ctx context.Context, se *sess.Session, limit int) ([]*schemaChange, error) { + sql := fmt.Sprintf(` + SELECT + ddl_job_id, + multi_schema_change_seq, + schema_change, + processed_by_flag + FROM %s.%s ORDER BY ddl_job_id, multi_schema_change_seq LIMIT %d`, + t.db, t.table, limit) + rows, err := se.Execute(ctx, sql, "ddl_notifier") + if err != nil { + return nil, err + } + ret := make([]*schemaChange, 0, len(rows)) + for _, row := range rows { + ret = append(ret, &schemaChange{ + ddlJobID: row.GetInt64(0), + multiSchemaChangeSeq: row.GetInt64(1), + // TODO: fill schema_change after we implement JSON serialization. + processedByFlag: row.GetUint64(3), + }) + } + return ret, nil +} + +// OpenTableStore opens a store on a created table `db`.`table`. The table should +// be created with the table structure: +// +// ddl_job_id BIGINT, +// multi_schema_change_seq BIGINT COMMENT '-1 if the schema change does not belong to a multi-schema change DDL. 0 or positive numbers representing the sub-job index of a multi-schema change DDL', +// schema_change JSON COMMENT 'SchemaChange at rest', +// processed_by_flag BIGINT UNSIGNED DEFAULT 0 COMMENT 'flag to mark which subscriber has processed the event', +// PRIMARY KEY(ddl_job_id, multi_schema_change_id) +func OpenTableStore(db, table string) Store { + return &tableStore{db: db, table: table} +} diff --git a/pkg/ddl/partition.go b/pkg/ddl/partition.go index dc4e1689e054f..8c619d91c2e40 100644 --- a/pkg/ddl/partition.go +++ b/pkg/ddl/partition.go @@ -29,9 +29,9 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/pkg/ddl/label" "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/ddl/placement" sess "github.com/pingcap/tidb/pkg/ddl/session" - "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/infoschema" @@ -50,7 +50,6 @@ import ( field_types "github.com/pingcap/tidb/pkg/parser/types" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" - statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/tablecodec" @@ -230,9 +229,7 @@ func (w *worker) onAddTablePartition(jobCtx *jobContext, t *meta.Meta, job *mode // Finish this job. job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo) - addPartitionEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewAddPartitionEvent(tblInfo, partInfo), - } + addPartitionEvent := notifier.NewAddPartitionEvent(tblInfo, partInfo) asyncNotifyEvent(jobCtx, addPartitionEvent, job) default: err = dbterror.ErrInvalidDDLState.GenWithStackByArgs("partition", job.SchemaState) @@ -2332,12 +2329,10 @@ func (w *worker) onDropTablePartition(jobCtx *jobContext, t *meta.Meta, job *mod } job.SchemaState = model.StateNone job.FinishTableJob(model.JobStateDone, model.StateNone, ver, tblInfo) - dropPartitionEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewDropPartitionEvent( - tblInfo, - &model.PartitionInfo{Definitions: droppedDefs}, - ), - } + dropPartitionEvent := notifier.NewDropPartitionEvent( + tblInfo, + &model.PartitionInfo{Definitions: droppedDefs}, + ) asyncNotifyEvent(jobCtx, dropPartitionEvent, job) // A background job will be created to delete old partition data. job.Args = []any{physicalTableIDs} @@ -2425,13 +2420,11 @@ func (w *worker) onTruncateTablePartition(jobCtx *jobContext, t *meta.Meta, job // Finish this job. job.FinishTableJob(model.JobStateDone, model.StateNone, ver, tblInfo) - truncatePartitionEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewTruncatePartitionEvent( - tblInfo, - &model.PartitionInfo{Definitions: newPartitions}, - &model.PartitionInfo{Definitions: oldPartitions}, - ), - } + truncatePartitionEvent := notifier.NewTruncatePartitionEvent( + tblInfo, + &model.PartitionInfo{Definitions: newPartitions}, + &model.PartitionInfo{Definitions: oldPartitions}, + ) asyncNotifyEvent(jobCtx, truncatePartitionEvent, job) // A background job will be created to delete old partition data. job.Args = []any{oldIDs} @@ -2565,13 +2558,11 @@ func (w *worker) onTruncateTablePartition(jobCtx *jobContext, t *meta.Meta, job } // Finish this job. job.FinishTableJob(model.JobStateDone, model.StateNone, ver, tblInfo) - truncatePartitionEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewTruncatePartitionEvent( - tblInfo, - &model.PartitionInfo{Definitions: newPartitions}, - &model.PartitionInfo{Definitions: oldPartitions}, - ), - } + truncatePartitionEvent := notifier.NewTruncatePartitionEvent( + tblInfo, + &model.PartitionInfo{Definitions: newPartitions}, + &model.PartitionInfo{Definitions: oldPartitions}, + ) asyncNotifyEvent(jobCtx, truncatePartitionEvent, job) // A background job will be created to delete old partition data. job.Args = []any{oldIDs} @@ -2939,13 +2930,11 @@ func (w *worker) onExchangeTablePartition(jobCtx *jobContext, t *meta.Meta, job } job.FinishTableJob(model.JobStateDone, model.StateNone, ver, pt) - exchangePartitionEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewExchangePartitionEvent( - pt, - &model.PartitionInfo{Definitions: []model.PartitionDefinition{originalPartitionDef}}, - originalNt, - ), - } + exchangePartitionEvent := notifier.NewExchangePartitionEvent( + pt, + &model.PartitionInfo{Definitions: []model.PartitionDefinition{originalPartitionDef}}, + originalNt, + ) asyncNotifyEvent(jobCtx, exchangePartitionEvent, job) return ver, nil } @@ -3477,10 +3466,7 @@ func (w *worker) onReorganizePartition(jobCtx *jobContext, t *meta.Meta, job *mo // Should it actually be synchronous? // Include the old table ID, if changed, which may contain global statistics, // so it can be reused for the new (non)partitioned table. - event, err := newStatsDDLEventForJob( - job.SchemaID, - job.Type, oldTblID, tblInfo, statisticsPartInfo, droppedPartInfo, - ) + event, err := newStatsDDLEventForJob(job.Type, oldTblID, tblInfo, statisticsPartInfo, droppedPartInfo) if err != nil { return ver, errors.Trace(err) } @@ -3495,37 +3481,31 @@ func (w *worker) onReorganizePartition(jobCtx *jobContext, t *meta.Meta, job *mo return ver, errors.Trace(err) } -// newStatsDDLEventForJob creates a statsutil.DDLEvent for a job. +// newStatsDDLEventForJob creates a util.SchemaChangeEvent for a job. // It is used for reorganize partition, add partitioning and remove partitioning. func newStatsDDLEventForJob( - schemaID int64, jobType model.ActionType, oldTblID int64, tblInfo *model.TableInfo, addedPartInfo *model.PartitionInfo, droppedPartInfo *model.PartitionInfo, -) (*statsutil.DDLEvent, error) { - var event *statsutil.DDLEvent +) (*notifier.SchemaChangeEvent, error) { + var event *notifier.SchemaChangeEvent switch jobType { case model.ActionReorganizePartition: - event = &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewReorganizePartitionEvent( - tblInfo, - addedPartInfo, - droppedPartInfo, - ), - } + event = notifier.NewReorganizePartitionEvent( + tblInfo, + addedPartInfo, + droppedPartInfo, + ) case model.ActionAlterTablePartitioning: - event = &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewAddPartitioningEvent( - oldTblID, - tblInfo, - addedPartInfo, - ), - } + event = notifier.NewAddPartitioningEvent( + oldTblID, + tblInfo, + addedPartInfo, + ) case model.ActionRemovePartitioning: - event = statsutil.NewRemovePartitioningEvent( - schemaID, + event = notifier.NewRemovePartitioningEvent( oldTblID, tblInfo, droppedPartInfo, diff --git a/pkg/ddl/restart_test.go b/pkg/ddl/restart_test.go index 47443ce57e35a..028cdc8e116bd 100644 --- a/pkg/ddl/restart_test.go +++ b/pkg/ddl/restart_test.go @@ -59,7 +59,7 @@ func restartWorkers(t *testing.T, store kv.Storage, d *domain.Domain) { } // runInterruptedJob should be called concurrently with restartWorkers -func runInterruptedJob(t *testing.T, store kv.Storage, d ddl.Executor, job *model.Job, doneCh chan error) { +func runInterruptedJob(t *testing.T, store kv.Storage, d ddl.Executor, job *model.Job, args model.JobArgs, doneCh chan error) { var ( history *model.Job err error @@ -68,7 +68,7 @@ func runInterruptedJob(t *testing.T, store kv.Storage, d ddl.Executor, job *mode de := d.(ddl.ExecutorForTest) ctx := testkit.NewTestKit(t, store).Session() ctx.SetValue(sessionctx.QueryString, "skip") - err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapper(job, true)) + err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapperWithArgs(job, args, true)) if errors.Is(err, context.Canceled) { endlessLoopTime := time.Now().Add(time.Minute) for history == nil { @@ -88,9 +88,9 @@ func runInterruptedJob(t *testing.T, store kv.Storage, d ddl.Executor, job *mode doneCh <- err } -func testRunInterruptedJob(t *testing.T, store kv.Storage, d *domain.Domain, job *model.Job) { +func testRunInterruptedJob(t *testing.T, store kv.Storage, d *domain.Domain, job *model.Job, args model.JobArgs) { done := make(chan error, 1) - go runInterruptedJob(t, store, d.DDLExecutor(), job, done) + go runInterruptedJob(t, store, d.DDLExecutor(), job, args, done) ticker := time.NewTicker(d.GetSchemaLease()) defer ticker.Stop() @@ -121,11 +121,11 @@ func TestSchemaResume(t *testing.T) { BinlogInfo: &model.HistoryInfo{}, } job.FillArgs(&model.CreateSchemaArgs{DBInfo: dbInfo}) - testRunInterruptedJob(t, store, dom, job) + testRunInterruptedJob(t, store, dom, job, nil) testCheckSchemaState(t, store, dbInfo, model.StatePublic) job = buildDropSchemaJob(dbInfo) - testRunInterruptedJob(t, store, dom, job) + testRunInterruptedJob(t, store, dom, job, nil) testCheckSchemaState(t, store, dbInfo, model.StateNone) } @@ -139,7 +139,7 @@ func TestStat(t *testing.T) { job := buildDropSchemaJob(dbInfo) done := make(chan error, 1) - go runInterruptedJob(t, store, dom.DDLExecutor(), job, done) + go runInterruptedJob(t, store, dom.DDLExecutor(), job, nil, done) ticker := time.NewTicker(dom.GetSchemaLease() * 1) defer ticker.Stop() @@ -174,15 +174,15 @@ func TestTableResume(t *testing.T) { tblInfo, err := testTableInfo(store, "t1", 3) require.NoError(t, err) job := &model.Job{ + Version: model.GetJobVerInUse(), SchemaID: dbInfo.ID, SchemaName: dbInfo.Name.L, TableID: tblInfo.ID, TableName: tblInfo.Name.L, Type: model.ActionCreateTable, BinlogInfo: &model.HistoryInfo{}, - Args: []any{tblInfo}, } - testRunInterruptedJob(t, store, dom, job) + testRunInterruptedJob(t, store, dom, job, &model.CreateTableArgs{TableInfo: tblInfo}) testCheckTableState(t, store, dbInfo, tblInfo, model.StatePublic) job = &model.Job{ @@ -193,6 +193,6 @@ func TestTableResume(t *testing.T) { Type: model.ActionDropTable, BinlogInfo: &model.HistoryInfo{}, } - testRunInterruptedJob(t, store, dom, job) + testRunInterruptedJob(t, store, dom, job, nil) testCheckTableState(t, store, dbInfo, tblInfo, model.StateNone) } diff --git a/pkg/ddl/schema_test.go b/pkg/ddl/schema_test.go index 46705075b4842..53246265db0fa 100644 --- a/pkg/ddl/schema_test.go +++ b/pkg/ddl/schema_test.go @@ -40,16 +40,17 @@ import ( func testCreateTable(t *testing.T, ctx sessionctx.Context, d ddl.ExecutorForTest, dbInfo *model.DBInfo, tblInfo *model.TableInfo) *model.Job { job := &model.Job{ + Version: model.GetJobVerInUse(), SchemaID: dbInfo.ID, SchemaName: dbInfo.Name.L, TableID: tblInfo.ID, TableName: tblInfo.Name.L, Type: model.ActionCreateTable, BinlogInfo: &model.HistoryInfo{}, - Args: []any{tblInfo}, } + args := &model.CreateTableArgs{TableInfo: tblInfo} ctx.SetValue(sessionctx.QueryString, "skip") - err := d.DoDDLJobWrapper(ctx, ddl.NewJobWrapper(job, true)) + err := d.DoDDLJobWrapper(ctx, ddl.NewJobWrapperWithArgs(job, args, true)) require.NoError(t, err) v := getSchemaVer(t, ctx) @@ -337,8 +338,9 @@ func TestSchemaWaitJob(t *testing.T) { require.NoError(t, err) schemaID := genIDs[0] doDDLJobErr(t, schemaID, 0, "test_schema", "", model.ActionCreateSchema, - testkit.NewTestKit(t, store).Session(), det2, store, func(job *model.Job) { + testkit.NewTestKit(t, store).Session(), det2, store, func(job *model.Job) model.JobArgs { job.FillArgs(&model.CreateSchemaArgs{DBInfo: dbInfo}) + return nil }) } @@ -350,7 +352,7 @@ func doDDLJobErr( ctx sessionctx.Context, d ddl.ExecutorForTest, store kv.Storage, - handler func(job *model.Job), + handler func(job *model.Job) model.JobArgs, ) *model.Job { job := &model.Job{ Version: model.GetJobVerInUse(), @@ -361,10 +363,10 @@ func doDDLJobErr( Type: tp, BinlogInfo: &model.HistoryInfo{}, } - handler(job) + args := handler(job) // TODO: check error detail ctx.SetValue(sessionctx.QueryString, "skip") - require.Error(t, d.DoDDLJobWrapper(ctx, ddl.NewJobWrapper(job, true))) + require.Error(t, d.DoDDLJobWrapper(ctx, ddl.NewJobWrapperWithArgs(job, args, true))) testCheckJobCancelled(t, store, job, nil) return job diff --git a/pkg/ddl/schema_version.go b/pkg/ddl/schema_version.go index 420c65f1c1a2b..7f8f3aa7b95e9 100644 --- a/pkg/ddl/schema_version.go +++ b/pkg/ddl/schema_version.go @@ -31,18 +31,18 @@ import ( // SetSchemaDiffForCreateTables set SchemaDiff for ActionCreateTables. func SetSchemaDiffForCreateTables(diff *model.SchemaDiff, job *model.Job) error { - var tableInfos []*model.TableInfo - err := job.DecodeArgs(&tableInfos) + args, err := model.GetBatchCreateTableArgs(job) if err != nil { return errors.Trace(err) } - diff.AffectedOpts = make([]*model.AffectedOption, len(tableInfos)) - for i := range tableInfos { + diff.AffectedOpts = make([]*model.AffectedOption, len(args.Tables)) + for i := range args.Tables { + tblInfo := args.Tables[i].TableInfo diff.AffectedOpts[i] = &model.AffectedOption{ SchemaID: job.SchemaID, OldSchemaID: job.SchemaID, - TableID: tableInfos[i].ID, - OldTableID: tableInfos[i].ID, + TableID: tblInfo.ID, + OldTableID: tblInfo.ID, } } return nil @@ -75,12 +75,11 @@ func SetSchemaDiffForTruncateTable(diff *model.SchemaDiff, job *model.Job) error // SetSchemaDiffForCreateView set SchemaDiff for ActionCreateView. func SetSchemaDiffForCreateView(diff *model.SchemaDiff, job *model.Job) error { - tbInfo := &model.TableInfo{} - var orReplace bool - var oldTbInfoID int64 - if err := job.DecodeArgs(tbInfo, &orReplace, &oldTbInfoID); err != nil { + args, err := model.GetCreateTableArgs(job) + if err != nil { return errors.Trace(err) } + tbInfo, orReplace, oldTbInfoID := args.TableInfo, args.OnExistReplace, args.OldViewTblID // When the statement is "create or replace view " and we need to drop the old view, // it has two table IDs and should be handled differently. if oldTbInfoID > 0 && orReplace { @@ -249,19 +248,26 @@ func SetSchemaDiffForPartitionModify(diff *model.SchemaDiff, job *model.Job) err } // SetSchemaDiffForCreateTable set SchemaDiff for ActionCreateTable. -func SetSchemaDiffForCreateTable(diff *model.SchemaDiff, job *model.Job) { +func SetSchemaDiffForCreateTable(diff *model.SchemaDiff, job *model.Job) error { diff.TableID = job.TableID - if len(job.Args) > 0 { - tbInfo, _ := job.Args[0].(*model.TableInfo) - // When create table with foreign key, there are two schema status change: - // 1. none -> write-only - // 2. write-only -> public - // In the second status change write-only -> public, infoschema loader should apply drop old table first, then - // apply create new table. So need to set diff.OldTableID here to make sure it. - if tbInfo != nil && tbInfo.State == model.StatePublic && len(tbInfo.ForeignKeys) > 0 { - diff.OldTableID = job.TableID - } + var tbInfo *model.TableInfo + // create table with foreign key will update tableInfo in the job args, so we + // must reuse already decoded ones. + // TODO make DecodeArgs can reuse already decoded args, so we can use GetCreateTableArgs. + if job.Version == model.JobVersion1 { + tbInfo, _ = job.Args[0].(*model.TableInfo) + } else { + tbInfo = job.Args[0].(*model.CreateTableArgs).TableInfo + } + // When create table with foreign key, there are two schema status change: + // 1. none -> write-only + // 2. write-only -> public + // In the second status change write-only -> public, infoschema loader should apply drop old table first, then + // apply create new table. So need to set diff.OldTableID here to make sure it. + if tbInfo.State == model.StatePublic && len(tbInfo.ForeignKeys) > 0 { + diff.OldTableID = job.TableID } + return nil } // SetSchemaDiffForRecoverSchema set SchemaDiff for ActionRecoverSchema. @@ -355,7 +361,7 @@ func updateSchemaVersion(jobCtx *jobContext, t *meta.Meta, job *model.Job, multi case model.ActionRemovePartitioning, model.ActionAlterTablePartitioning: err = SetSchemaDiffForPartitionModify(diff, job) case model.ActionCreateTable: - SetSchemaDiffForCreateTable(diff, job) + err = SetSchemaDiffForCreateTable(diff, job) case model.ActionRecoverSchema: err = SetSchemaDiffForRecoverSchema(diff, job) case model.ActionFlashbackCluster: diff --git a/pkg/ddl/schematracker/BUILD.bazel b/pkg/ddl/schematracker/BUILD.bazel index de05d2d58abcb..209b1af2b649e 100644 --- a/pkg/ddl/schematracker/BUILD.bazel +++ b/pkg/ddl/schematracker/BUILD.bazel @@ -47,7 +47,7 @@ go_test( ], embed = [":schematracker"], flaky = True, - shard_count = 15, + shard_count = 16, deps = [ "//pkg/executor", "//pkg/infoschema", diff --git a/pkg/ddl/schematracker/checker.go b/pkg/ddl/schematracker/checker.go index 4742209f46799..c2112a12c3174 100644 --- a/pkg/ddl/schematracker/checker.go +++ b/pkg/ddl/schematracker/checker.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "regexp" "strings" "sync/atomic" @@ -164,8 +165,24 @@ func (d *Checker) checkTableInfo(ctx sessionctx.Context, dbName, tableName pmode s1 := removeClusteredIndexComment(result.String()) s2 := removeClusteredIndexComment(result2.String()) + // Remove shard_row_id_bits and pre_split_regions comments. + if ctx.GetSessionVars().ShardRowIDBits != 0 || ctx.GetSessionVars().PreSplitRegions != 0 { + removeShardPreSplitComment := func(s string) string { + pattern := ` \/\*T! SHARD_ROW_ID_BITS=.*?\*\/` + re := regexp.MustCompile(pattern) + ret := re.ReplaceAllString(s, "") + pattern = ` \/\*T! PRE_SPLIT_REGIONS=.*?\*\/` + re = regexp.MustCompile(pattern) + ret = re.ReplaceAllString(ret, "") + return ret + } + + s1 = removeShardPreSplitComment(s1) + s2 = removeShardPreSplitComment(s2) + } + if s1 != s2 { - errStr := fmt.Sprintf("%s != %s", s1, s2) + errStr := fmt.Sprintf("%s\n!=\n%s", s1, s2) panic(errStr) } } @@ -223,7 +240,7 @@ func (*Checker) RecoverSchema(_ sessionctx.Context, _ *ddl.RecoverSchemaInfo) (e // CreateTable implements the DDL interface. func (d *Checker) CreateTable(ctx sessionctx.Context, stmt *ast.CreateTableStmt) error { err := d.realExecutor.CreateTable(ctx, stmt) - if err != nil { + if err != nil || d.closed.Load() { return err } @@ -332,7 +349,7 @@ func (d *Checker) DropIndex(ctx sessionctx.Context, stmt *ast.DropIndexStmt) err // AlterTable implements the DDL interface. func (d *Checker) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast.AlterTableStmt) error { err := d.realExecutor.AlterTable(ctx, sctx, stmt) - if err != nil { + if err != nil || d.closed.Load() { return err } diff --git a/pkg/ddl/schematracker/dm_tracker_test.go b/pkg/ddl/schematracker/dm_tracker_test.go index 89b47bc8b21a5..9ab0e28f90a67 100644 --- a/pkg/ddl/schematracker/dm_tracker_test.go +++ b/pkg/ddl/schematracker/dm_tracker_test.go @@ -219,6 +219,24 @@ func TestIndexLength(t *testing.T) { checkShowCreateTable(t, tblInfo, expected) } +func TestCreateTableWithIndex(t *testing.T) { + // See issue 56045 + sql := "create table test.t(col_1 json, KEY idx_1 ((cast(col_1 as char(64) array))))" + tracker := schematracker.NewSchemaTracker(2) + tracker.CreateTestDB(nil) + execCreate(t, tracker, sql) + + sql = "alter table test.t rename index idx_1 to idx_1_1" + execAlter(t, tracker, sql) + + tblInfo := mustTableByName(t, tracker, "test", "t") + expected := "CREATE TABLE `t` (\n" + + " `col_1` json DEFAULT NULL,\n" + + " KEY `idx_1_1` ((cast(`col_1` as char(64) array)))\n" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" + checkShowCreateTable(t, tblInfo, expected) +} + func TestIssue5092(t *testing.T) { // copy TestIssue5092 in db_integration_test.go sql := "create table test.t (a int)" diff --git a/pkg/ddl/sequence.go b/pkg/ddl/sequence.go index 22c5092110b66..76fe7c60320e5 100644 --- a/pkg/ddl/sequence.go +++ b/pkg/ddl/sequence.go @@ -29,15 +29,16 @@ import ( func onCreateSequence(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver int64, _ error) { schemaID := job.SchemaID - tbInfo := &model.TableInfo{} - if err := job.DecodeArgs(tbInfo); err != nil { + args, err := model.GetCreateTableArgs(job) + if err != nil { // Invalid arguments, cancel this job. job.State = model.JobStateCancelled return ver, errors.Trace(err) } + tbInfo := args.TableInfo tbInfo.State = model.StateNone - err := checkTableNotExists(jobCtx.infoCache, schemaID, tbInfo.Name.L) + err = checkTableNotExists(jobCtx.infoCache, schemaID, tbInfo.Name.L) if err != nil { if infoschema.ErrDatabaseNotExists.Equal(err) || infoschema.ErrTableExists.Equal(err) { job.State = model.JobStateCancelled diff --git a/pkg/ddl/table.go b/pkg/ddl/table.go index a8f70823e3f0b..a53946e7d7bf9 100644 --- a/pkg/ddl/table.go +++ b/pkg/ddl/table.go @@ -26,9 +26,9 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/ddl/label" "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/ddl/placement" sess "github.com/pingcap/tidb/pkg/ddl/session" - "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/meta" @@ -39,7 +39,6 @@ import ( pmodel "github.com/pingcap/tidb/pkg/parser/model" field_types "github.com/pingcap/tidb/pkg/parser/types" "github.com/pingcap/tidb/pkg/sessionctx/variable" - statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/tablecodec" @@ -124,9 +123,7 @@ func onDropTableOrView(jobCtx *jobContext, t *meta.Meta, job *model.Job) (ver in startKey := tablecodec.EncodeTablePrefix(job.TableID) job.Args = append(job.Args, startKey, oldIDs, ruleIDs) if !tblInfo.IsSequence() && !tblInfo.IsView() { - dropTableEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewDropTableEvent(tblInfo), - } + dropTableEvent := notifier.NewDropTableEvent(tblInfo) asyncNotifyEvent(jobCtx, dropTableEvent, job) } default: @@ -573,9 +570,7 @@ func (w *worker) onTruncateTable(jobCtx *jobContext, t *meta.Meta, job *model.Jo return ver, errors.Trace(err) } job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tblInfo) - truncateTableEvent := &statsutil.DDLEvent{ - SchemaChangeEvent: util.NewTruncateTableEvent(tblInfo, oldTblInfo), - } + truncateTableEvent := notifier.NewTruncateTableEvent(tblInfo, oldTblInfo) asyncNotifyEvent(jobCtx, truncateTableEvent, job) // see truncateTableByReassignPartitionIDs for why they might change. args.OldPartitionIDs = oldPartitionIDs diff --git a/pkg/ddl/table_test.go b/pkg/ddl/table_test.go index fc12715c7c9fa..e3006a3ade29f 100644 --- a/pkg/ddl/table_test.go +++ b/pkg/ddl/table_test.go @@ -170,9 +170,9 @@ func testTruncateTable(t *testing.T, ctx sessionctx.Context, store kv.Storage, d Type: model.ActionTruncateTable, BinlogInfo: &model.HistoryInfo{}, } - job.FillArgs(&model.TruncateTableArgs{NewTableID: newTableID}) + args := &model.TruncateTableArgs{NewTableID: newTableID} ctx.SetValue(sessionctx.QueryString, "skip") - err = d.DoDDLJobWrapper(ctx, ddl.NewJobWrapper(job, true)) + err = d.DoDDLJobWrapper(ctx, ddl.NewJobWrapperWithArgs(job, args, true)) require.NoError(t, err) v := getSchemaVer(t, ctx) @@ -228,8 +228,8 @@ func TestTable(t *testing.T) { newTblInfo, err := testTableInfo(store, "t", 3) require.NoError(t, err) doDDLJobErr(t, dbInfo.ID, newTblInfo.ID, dbInfo.Name.L, newTblInfo.Name.L, model.ActionCreateTable, - ctx, de, store, func(job *model.Job) { - job.Args = []any{newTblInfo} + ctx, de, store, func(job *model.Job) model.JobArgs { + return &model.CreateTableArgs{TableInfo: newTblInfo} }) ctx = testkit.NewTestKit(t, store).Session() @@ -302,16 +302,17 @@ func TestCreateView(t *testing.T) { newTblInfo0, err := testTableInfo(store, "v", 3) require.NoError(t, err) job = &model.Job{ + Version: model.GetJobVerInUse(), SchemaID: dbInfo.ID, SchemaName: dbInfo.Name.L, TableID: tblInfo.ID, TableName: tblInfo.Name.L, Type: model.ActionCreateView, BinlogInfo: &model.HistoryInfo{}, - Args: []any{newTblInfo0}, } + args := &model.CreateTableArgs{TableInfo: newTblInfo0} ctx.SetValue(sessionctx.QueryString, "skip") - err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapper(job, true)) + err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapperWithArgs(job, args, true)) require.NoError(t, err) v := getSchemaVer(t, ctx) @@ -325,16 +326,17 @@ func TestCreateView(t *testing.T) { newTblInfo1, err := testTableInfo(store, "v", 3) require.NoError(t, err) job = &model.Job{ + Version: model.GetJobVerInUse(), SchemaID: dbInfo.ID, SchemaName: dbInfo.Name.L, TableID: tblInfo.ID, TableName: tblInfo.Name.L, Type: model.ActionCreateView, BinlogInfo: &model.HistoryInfo{}, - Args: []any{newTblInfo1, true, newTblInfo0.ID}, } + args = &model.CreateTableArgs{TableInfo: newTblInfo1, OnExistReplace: true, OldViewTblID: newTblInfo0.ID} ctx.SetValue(sessionctx.QueryString, "skip") - err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapper(job, true)) + err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapperWithArgs(job, args, true)) require.NoError(t, err) v = getSchemaVer(t, ctx) @@ -348,16 +350,17 @@ func TestCreateView(t *testing.T) { newTblInfo2, err := testTableInfo(store, "v", 3) require.NoError(t, err) job = &model.Job{ + Version: model.GetJobVerInUse(), SchemaID: dbInfo.ID, SchemaName: dbInfo.Name.L, TableID: tblInfo.ID, TableName: tblInfo.Name.L, Type: model.ActionCreateView, BinlogInfo: &model.HistoryInfo{}, - Args: []any{newTblInfo2, true, newTblInfo0.ID}, } + args = &model.CreateTableArgs{TableInfo: newTblInfo2, OnExistReplace: true, OldViewTblID: newTblInfo0.ID} ctx.SetValue(sessionctx.QueryString, "skip") - err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapper(job, true)) + err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapperWithArgs(job, args, true)) // The non-existing table id in job args will not be considered anymore. require.NoError(t, err) } @@ -484,28 +487,26 @@ func TestCreateTables(t *testing.T) { ctx := testkit.NewTestKit(t, store).Session() - var infos []*model.TableInfo genIDs, err := genGlobalIDs(store, 3) require.NoError(t, err) - infos = append(infos, &model.TableInfo{ - ID: genIDs[0], - Name: pmodel.NewCIStr("s1"), - }) - infos = append(infos, &model.TableInfo{ - ID: genIDs[1], - Name: pmodel.NewCIStr("s2"), - }) - infos = append(infos, &model.TableInfo{ - ID: genIDs[2], - Name: pmodel.NewCIStr("s3"), - }) + args := &model.BatchCreateTableArgs{ + Tables: make([]*model.CreateTableArgs, 0, 3), + } + for i := 0; i < 3; i++ { + args.Tables = append(args.Tables, &model.CreateTableArgs{ + TableInfo: &model.TableInfo{ + ID: genIDs[i], + Name: pmodel.NewCIStr(fmt.Sprintf("s%d", i+1)), + }, + }) + } job := &model.Job{ + Version: model.GetJobVerInUse(), SchemaID: dbInfo.ID, Type: model.ActionCreateTables, BinlogInfo: &model.HistoryInfo{}, - Args: []any{infos}, InvolvingSchemaInfo: []model.InvolvingSchemaInfo{ {Database: "test_table", Table: "s1"}, {Database: "test_table", Table: "s2"}, @@ -520,7 +521,7 @@ func TestCreateTables(t *testing.T) { *errP = errors.New("mock get job by ID failed") }) }) - err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapper(job, true)) + err = de.DoDDLJobWrapper(ctx, ddl.NewJobWrapperWithArgs(job, args, true)) require.NoError(t, err) testGetTable(t, domain, genIDs[0]) diff --git a/pkg/ddl/util/BUILD.bazel b/pkg/ddl/util/BUILD.bazel index 277f63044c813..9ca3879436e23 100644 --- a/pkg/ddl/util/BUILD.bazel +++ b/pkg/ddl/util/BUILD.bazel @@ -5,7 +5,6 @@ go_library( srcs = [ "dead_table_lock_checker.go", "mock.go", - "schema_change_notifier.go", "util.go", "watcher.go", ], @@ -22,7 +21,6 @@ go_library( "//pkg/sessionctx/variable", "//pkg/table/tables", "//pkg/util/chunk", - "//pkg/util/intest", "//pkg/util/mock", "//pkg/util/sqlexec", "@com_github_pingcap_errors//:errors", diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go index d059b7c734968..8ddfd0ef3d7ce 100644 --- a/pkg/domain/domain.go +++ b/pkg/domain/domain.go @@ -299,6 +299,15 @@ func (do *Domain) loadInfoSchema(startTS uint64, isSnapshot bool) (infoschema.In schemaTs = 0 } + var oldIsV2 bool + enableV2 := variable.SchemaCacheSize.Load() > 0 + currentSchemaVersion := int64(0) + if oldInfoSchema := do.infoCache.GetLatest(); oldInfoSchema != nil { + currentSchemaVersion = oldInfoSchema.SchemaMetaVersion() + oldIsV2, _ = infoschema.IsV2(oldInfoSchema) + } + useV2, isV1V2Switch := shouldUseV2(enableV2, oldIsV2, isSnapshot) + if is := do.infoCache.GetByVersion(neededSchemaVersion); is != nil { isV2, raw := infoschema.IsV2(is) if isV2 { @@ -314,17 +323,10 @@ func (do *Domain) loadInfoSchema(startTS uint64, isSnapshot bool) (infoschema.In // the insert method check if schemaTs is zero do.infoCache.Insert(is, schemaTs) - return is, true, 0, nil, nil - } - - var oldIsV2 bool - enableV2 := variable.SchemaCacheSize.Load() > 0 - currentSchemaVersion := int64(0) - if oldInfoSchema := do.infoCache.GetLatest(); oldInfoSchema != nil { - currentSchemaVersion = oldInfoSchema.SchemaMetaVersion() - oldIsV2, _ = infoschema.IsV2(oldInfoSchema) + if !isV1V2Switch { + return is, true, 0, nil, nil + } } - useV2, isV1V2Switch := shouldUseV2(enableV2, oldIsV2, isSnapshot) // TODO: tryLoadSchemaDiffs has potential risks of failure. And it becomes worse in history reading cases. // It is only kept because there is no alternative diff/partial loading solution. @@ -371,7 +373,17 @@ func (do *Domain) loadInfoSchema(startTS uint64, isSnapshot bool) (infoschema.In } infoschema_metrics.LoadSchemaDurationLoadAll.Observe(time.Since(startTime).Seconds()) - builder := infoschema.NewBuilder(do, do.sysFacHack, do.infoCache.Data, useV2) + data := do.infoCache.Data + if isSnapshot { + // Use a NewData() to avoid adding the snapshot schema to the infoschema history. + // Why? imagine that the current schema version is [103 104 105 ...] + // Then a snapshot read require infoschem version 53, and it's added + // Now the history becomes [53, ... 103, 104, 105 ...] + // Then if a query ask for version 74, we'll mistakenly use 53! + // Not adding snapshot schema to history can avoid such cases. + data = infoschema.NewData() + } + builder := infoschema.NewBuilder(do, do.sysFacHack, data, useV2) err = builder.InitWithDBInfos(schemas, policies, resourceGroups, neededSchemaVersion) if err != nil { return nil, false, currentSchemaVersion, nil, err @@ -388,6 +400,7 @@ func (do *Domain) loadInfoSchema(startTS uint64, isSnapshot bool) (infoschema.In // Reset the whole info cache to avoid co-existing of both v1 and v2, causing the memory usage doubled. fn := do.infoCache.Upsert(is, schemaTs) do.deferFn.add(fn, time.Now().Add(10*time.Minute)) + logutil.BgLogger().Info("infoschema v1/v2 switch") } else { do.infoCache.Insert(is, schemaTs) } diff --git a/pkg/errno/errcode.go b/pkg/errno/errcode.go index 5d56f339273ff..ed109587548cf 100644 --- a/pkg/errno/errcode.go +++ b/pkg/errno/errcode.go @@ -1084,6 +1084,7 @@ const ( ErrBRJobNotFound = 8174 ErrMemoryExceedForQuery = 8175 ErrMemoryExceedForInstance = 8176 + ErrDeleteNotFoundColumn = 8177 // Error codes used by TiDB ddl package ErrUnsupportedDDLOperation = 8200 diff --git a/pkg/errno/errname.go b/pkg/errno/errname.go index aadcbb3dfcbca..868d3329ba136 100644 --- a/pkg/errno/errname.go +++ b/pkg/errno/errname.go @@ -1077,6 +1077,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrLoadDataPreCheckFailed: mysql.Message("PreCheck failed: %s", nil), ErrMemoryExceedForQuery: mysql.Message("Your query has been cancelled due to exceeding the allowed memory limit for a single SQL query. Please try narrowing your query scope or increase the tidb_mem_quota_query limit and try again.[conn=%d]", nil), ErrMemoryExceedForInstance: mysql.Message("Your query has been cancelled due to exceeding the allowed memory limit for the tidb-server instance and this query is currently using the most memory. Please try narrowing your query scope or increase the tidb_server_memory_limit and try again.[conn=%d]", nil), + ErrDeleteNotFoundColumn: mysql.Message("Delete can not find column %s for table %s", nil), ErrHTTPServiceError: mysql.Message("HTTP request failed with status %s", nil), ErrWarnOptimizerHintInvalidInteger: mysql.Message("integer value is out of range in '%s'", nil), diff --git a/pkg/executor/delete.go b/pkg/executor/delete.go index 23c1b77244279..29218e0e6051f 100644 --- a/pkg/executor/delete.go +++ b/pkg/executor/delete.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta/model" plannercore "github.com/pingcap/tidb/pkg/planner/core" - "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/sessiontxn" @@ -62,16 +61,16 @@ func (e *DeleteExec) Next(ctx context.Context, req *chunk.Chunk) error { return e.deleteSingleTableByChunk(ctx) } -func (e *DeleteExec) deleteOneRow(tbl table.Table, handleCols util.HandleCols, isExtraHandle bool, row []types.Datum) error { +func (e *DeleteExec) deleteOneRow(tbl table.Table, colInfo *plannercore.TblColPosInfo, isExtraHandle bool, row []types.Datum) error { end := len(row) if isExtraHandle { end-- } - handle, err := handleCols.BuildHandleByDatums(row) + handle, err := colInfo.HandleCols.BuildHandleByDatums(row) if err != nil { return err } - err = e.removeRow(e.Ctx(), tbl, handle, row[:end]) + err = e.removeRow(e.Ctx(), tbl, handle, row[:end], colInfo) if err != nil { return err } @@ -79,19 +78,10 @@ func (e *DeleteExec) deleteOneRow(tbl table.Table, handleCols util.HandleCols, i } func (e *DeleteExec) deleteSingleTableByChunk(ctx context.Context) error { - var ( - tbl table.Table - isExtrahandle bool - handleCols util.HandleCols - rowCount int - ) - for _, info := range e.tblColPosInfos { - tbl = e.tblID2Table[info.TblID] - handleCols = info.HandleCols - if !tbl.Meta().IsCommonHandle { - isExtrahandle = handleCols.IsInt() && handleCols.GetCol(0).ID == model.ExtraHandleID - } - } + colPosInfo := &e.tblColPosInfos[0] + tbl := e.tblID2Table[colPosInfo.TblID] + handleCols := colPosInfo.HandleCols + isExtraHandle := !tbl.Meta().IsCommonHandle && handleCols.IsInt() && handleCols.GetCol(0).ID == model.ExtraHandleID batchDMLSize := e.Ctx().GetSessionVars().DMLBatchSize // If tidb_batch_delete is ON and not in a transaction, we could use BatchDelete mode. @@ -101,6 +91,7 @@ func (e *DeleteExec) deleteSingleTableByChunk(ctx context.Context) error { datumRow := make([]types.Datum, 0, len(fields)) chk := exec.TryNewCacheChunk(e.Children(0)) columns := e.Children(0).Schema().Columns + rowCount := 0 if len(columns) != len(fields) { logutil.BgLogger().Error("schema columns and fields mismatch", zap.Int("len(columns)", len(columns)), @@ -137,7 +128,7 @@ func (e *DeleteExec) deleteSingleTableByChunk(ctx context.Context) error { datumRow = append(datumRow, datum) } - err = e.deleteOneRow(tbl, handleCols, isExtrahandle, datumRow) + err = e.deleteOneRow(tbl, colPosInfo, isExtraHandle, datumRow) if err != nil { return err } @@ -167,12 +158,13 @@ func (e *DeleteExec) doBatchDelete(ctx context.Context) error { func (e *DeleteExec) composeTblRowMap(tblRowMap tableRowMapType, colPosInfos []plannercore.TblColPosInfo, joinedRow []types.Datum) error { // iterate all the joined tables, and got the corresponding rows in joinedRow. var totalMemDelta int64 - for _, info := range colPosInfos { + for i := range colPosInfos { + info := colPosInfos[i] if unmatchedOuterRow(info, joinedRow) { continue } if tblRowMap[info.TblID] == nil { - tblRowMap[info.TblID] = kv.NewMemAwareHandleMap[[]types.Datum]() + tblRowMap[info.TblID] = kv.NewMemAwareHandleMap[handleInfoPair]() } handle, err := info.HandleCols.BuildHandleByDatums(joinedRow) if err != nil { @@ -181,10 +173,10 @@ func (e *DeleteExec) composeTblRowMap(tblRowMap tableRowMapType, colPosInfos []p // tblRowMap[info.TblID][handle] hold the row datas binding to this table and this handle. row, exist := tblRowMap[info.TblID].Get(handle) if !exist { - row = make([]types.Datum, info.End-info.Start) + row = handleInfoPair{handleVal: make([]types.Datum, info.End-info.Start), posInfo: &info} } for i, d := range joinedRow[info.Start:info.End] { - d.Copy(&row[i]) + d.Copy(&row.handleVal[i]) } memDelta := tblRowMap[info.TblID].Set(handle, row) if !exist { @@ -231,32 +223,30 @@ func (e *DeleteExec) deleteMultiTablesByChunk(ctx context.Context) error { } } } - return e.removeRowsInTblRowMap(tblRowMap) } func (e *DeleteExec) removeRowsInTblRowMap(tblRowMap tableRowMapType) error { for id, rowMap := range tblRowMap { var err error - rowMap.Range(func(h kv.Handle, val []types.Datum) bool { - err = e.removeRow(e.Ctx(), e.tblID2Table[id], h, val) + rowMap.Range(func(h kv.Handle, val handleInfoPair) bool { + err = e.removeRow(e.Ctx(), e.tblID2Table[id], h, val.handleVal, val.posInfo) return err == nil }) if err != nil { return err } } - return nil } -func (e *DeleteExec) removeRow(ctx sessionctx.Context, t table.Table, h kv.Handle, data []types.Datum) error { +func (e *DeleteExec) removeRow(ctx sessionctx.Context, t table.Table, h kv.Handle, data []types.Datum, posInfo *plannercore.TblColPosInfo) error { txn, err := e.Ctx().Txn(true) if err != nil { return err } - err = t.RemoveRecord(ctx.GetTableCtx(), txn, h, data) + err = t.RemoveRecord(ctx.GetTableCtx(), txn, h, data, posInfo.IndexesForDelete) if err != nil { return err } @@ -323,7 +313,12 @@ func (e *DeleteExec) HasFKCascades() bool { return len(e.fkCascades) > 0 } +type handleInfoPair struct { + handleVal []types.Datum + posInfo *plannercore.TblColPosInfo +} + // tableRowMapType is a map for unique (Table, Row) pair. key is the tableID. // the key in map[int64]Row is the joined table handle, which represent a unique reference row. // the value in map[int64]Row is the deleting row. -type tableRowMapType map[int64]*kv.MemAwareHandleMap[[]types.Datum] +type tableRowMapType map[int64]*kv.MemAwareHandleMap[handleInfoPair] diff --git a/pkg/executor/detach_integration_test.go b/pkg/executor/detach_integration_test.go index 9af9e05a73741..12922f1caba2e 100644 --- a/pkg/executor/detach_integration_test.go +++ b/pkg/executor/detach_integration_test.go @@ -355,17 +355,24 @@ func TestDetachProjection(t *testing.T) { require.NoError(t, rs.Close()) // Projection with user variable is allowed + // Also test NOW() function will return right value, see issue: https://github.com/pingcap/tidb/issues/56051 tk.MustExec("set @a = 1") tk.MustExec("set @b = 10") - tk.MustHavePlan("select a + b + @a + getvar('b') from t where a > 100 and a < 200", "Projection") - rs, err = tk.Exec("select a + b + @a + getvar('b') from t where a > ? and a < ?", 100, 200) + tk.MustExec("set @@timestamp=360000") + tk.MustHavePlan( + "select a + b + @a + getvar('b'), UNIX_TIMESTAMP(NOW()) from t where a > 100 and a < 200", + "Projection", + ) + rs, err = tk.Exec( + "select a + b + @a + getvar('b'), UNIX_TIMESTAMP(NOW()) from t where a > ? and a < ?", + 100, 200, + ) require.NoError(t, err) drs, ok, err = rs.(sqlexec.DetachableRecordSet).TryDetach() require.NoError(t, err) require.True(t, ok) - // set user variable to another value to test the expression should not change after detaching - tk.MustExec("set @a=100") - tk.MustExec("set @b=1000") + // set user variable and current time to another value to test the expression should not change after detaching + tk.MustExec("set @a=100,@b=1000,@@timestamp=0") chk = drs.NewChunk(nil) expectedSelect = 101 for { @@ -377,6 +384,7 @@ func TestDetachProjection(t *testing.T) { } for i := 0; i < chk.NumRows(); i++ { require.Equal(t, float64(2*expectedSelect+11), chk.GetRow(i).GetFloat64(0)) + require.Equal(t, int64(360000), chk.GetRow(i).GetInt64(1)) expectedSelect++ } } diff --git a/pkg/executor/importer/importer_testkit_test.go b/pkg/executor/importer/importer_testkit_test.go index 4f516b9a33585..df13450cb0a27 100644 --- a/pkg/executor/importer/importer_testkit_test.go +++ b/pkg/executor/importer/importer_testkit_test.go @@ -353,7 +353,7 @@ func TestProcessChunkWith(t *testing.T) { require.Len(t, progress.GetColSize(), 3) checksumMap := checksum.GetInnerChecksums() require.Len(t, checksumMap, 1) - require.Equal(t, verify.MakeKVChecksum(111, 3, 14231358899564314836), *checksumMap[verify.DataKVGroupID]) + require.Equal(t, verify.MakeKVChecksum(111, 3, 13867387642099248025), *checksumMap[verify.DataKVGroupID]) }) } diff --git a/pkg/executor/infoschema_cluster_table_test.go b/pkg/executor/infoschema_cluster_table_test.go index cb63dfed7d6a6..8549d16a4b453 100644 --- a/pkg/executor/infoschema_cluster_table_test.go +++ b/pkg/executor/infoschema_cluster_table_test.go @@ -397,7 +397,7 @@ func TestTableStorageStats(t *testing.T) { "test 2", )) rows := tk.MustQuery("select TABLE_NAME from information_schema.TABLE_STORAGE_STATS where TABLE_SCHEMA = 'mysql';").Rows() - result := 54 + result := 55 require.Len(t, rows, result) // More tests about the privileges. diff --git a/pkg/executor/infoschema_reader_test.go b/pkg/executor/infoschema_reader_test.go index 364ffd1fc2a93..45cfb2b04238c 100644 --- a/pkg/executor/infoschema_reader_test.go +++ b/pkg/executor/infoschema_reader_test.go @@ -293,16 +293,16 @@ func TestForAnalyzeStatus(t *testing.T) { " `TABLE_NAME` varchar(64) DEFAULT NULL,\n" + " `PARTITION_NAME` varchar(64) DEFAULT NULL,\n" + " `JOB_INFO` longtext DEFAULT NULL,\n" + - " `PROCESSED_ROWS` bigint(64) unsigned DEFAULT NULL,\n" + + " `PROCESSED_ROWS` bigint(21) unsigned DEFAULT NULL,\n" + " `START_TIME` datetime DEFAULT NULL,\n" + " `END_TIME` datetime DEFAULT NULL,\n" + " `STATE` varchar(64) DEFAULT NULL,\n" + " `FAIL_REASON` longtext DEFAULT NULL,\n" + " `INSTANCE` varchar(512) DEFAULT NULL,\n" + - " `PROCESS_ID` bigint(64) unsigned DEFAULT NULL,\n" + + " `PROCESS_ID` bigint(21) unsigned DEFAULT NULL,\n" + " `REMAINING_SECONDS` varchar(512) DEFAULT NULL,\n" + " `PROGRESS` double(22,6) DEFAULT NULL,\n" + - " `ESTIMATED_TOTAL_ROWS` bigint(64) unsigned DEFAULT NULL\n" + + " `ESTIMATED_TOTAL_ROWS` bigint(21) unsigned DEFAULT NULL\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" tk.MustQuery("show create table information_schema.analyze_status").Check(testkit.Rows("ANALYZE_STATUS " + analyzeStatusTable)) tk.MustExec("delete from mysql.analyze_jobs") @@ -608,7 +608,7 @@ func TestColumnTable(t *testing.T) { testkit.RowsWithSep("|", "test|tbl1|col_2")) tk.MustQuery(`select count(*) from information_schema.columns;`).Check( - testkit.RowsWithSep("|", "4937")) + testkit.RowsWithSep("|", "4944")) } func TestIndexUsageTable(t *testing.T) { @@ -655,7 +655,7 @@ func TestIndexUsageTable(t *testing.T) { testkit.RowsWithSep("|", "test|idt2|idx_4")) tk.MustQuery(`select count(*) from information_schema.tidb_index_usage;`).Check( - testkit.RowsWithSep("|", "72")) + testkit.RowsWithSep("|", "73")) tk.MustQuery(`select TABLE_SCHEMA, TABLE_NAME, INDEX_NAME from information_schema.tidb_index_usage where TABLE_SCHEMA = 'test1';`).Check(testkit.Rows()) @@ -806,7 +806,7 @@ func TestReferencedTableSchemaWithForeignKey(t *testing.T) { tk.MustExec("create table test.t1(id int primary key);") tk.MustExec("create table test2.t2(i int, id int, foreign key (id) references test.t1(id));") - tk.MustQuery(`SELECT column_name, referenced_column_name, referenced_table_name, table_schema, referenced_table_schema + tk.MustQuery(`SELECT column_name, referenced_column_name, referenced_table_name, table_schema, referenced_table_schema FROM information_schema.key_column_usage WHERE table_name = 't2' AND table_schema = 'test2';`).Check(testkit.Rows( "id id t1 test2 test")) @@ -858,22 +858,22 @@ func TestInfoSchemaDDLJobs(t *testing.T) { tk2 := testkit.NewTestKit(t, store) tk2.MustQuery(`SELECT JOB_ID, JOB_TYPE, SCHEMA_STATE, SCHEMA_ID, TABLE_ID, table_name, STATE FROM information_schema.ddl_jobs WHERE table_name = "t1";`).Check(testkit.RowsWithSep("|", - "125|add index /* txn-merge */|public|118|123|t1|synced", - "124|create table|public|118|123|t1|synced", - "111|add index /* txn-merge */|public|104|109|t1|synced", - "110|create table|public|104|109|t1|synced", + "127|add index /* txn-merge */|public|120|125|t1|synced", + "126|create table|public|120|125|t1|synced", + "113|add index /* txn-merge */|public|106|111|t1|synced", + "112|create table|public|106|111|t1|synced", )) tk2.MustQuery(`SELECT JOB_ID, JOB_TYPE, SCHEMA_STATE, SCHEMA_ID, TABLE_ID, table_name, STATE FROM information_schema.ddl_jobs WHERE db_name = "d1" and JOB_TYPE LIKE "add index%%";`).Check(testkit.RowsWithSep("|", - "131|add index /* txn-merge */|public|118|129|t3|synced", - "128|add index /* txn-merge */|public|118|126|t2|synced", - "125|add index /* txn-merge */|public|118|123|t1|synced", - "122|add index /* txn-merge */|public|118|120|t0|synced", + "133|add index /* txn-merge */|public|120|131|t3|synced", + "130|add index /* txn-merge */|public|120|128|t2|synced", + "127|add index /* txn-merge */|public|120|125|t1|synced", + "124|add index /* txn-merge */|public|120|122|t0|synced", )) tk2.MustQuery(`SELECT JOB_ID, JOB_TYPE, SCHEMA_STATE, SCHEMA_ID, TABLE_ID, table_name, STATE FROM information_schema.ddl_jobs WHERE db_name = "d0" and table_name = "t3";`).Check(testkit.RowsWithSep("|", - "117|add index /* txn-merge */|public|104|115|t3|synced", - "116|create table|public|104|115|t3|synced", + "119|add index /* txn-merge */|public|106|117|t3|synced", + "118|create table|public|106|117|t3|synced", )) tk2.MustQuery(`SELECT JOB_ID, JOB_TYPE, SCHEMA_STATE, SCHEMA_ID, TABLE_ID, table_name, STATE FROM information_schema.ddl_jobs WHERE state = "running";`).Check(testkit.Rows()) @@ -884,15 +884,15 @@ func TestInfoSchemaDDLJobs(t *testing.T) { if job.SchemaState == model.StateWriteOnly && loaded.CompareAndSwap(false, true) { tk2.MustQuery(`SELECT JOB_ID, JOB_TYPE, SCHEMA_STATE, SCHEMA_ID, TABLE_ID, table_name, STATE FROM information_schema.ddl_jobs WHERE table_name = "t0" and state = "running";`).Check(testkit.RowsWithSep("|", - "132|add index /* txn-merge */|write only|104|106|t0|running", + "134|add index /* txn-merge */|write only|106|108|t0|running", )) tk2.MustQuery(`SELECT JOB_ID, JOB_TYPE, SCHEMA_STATE, SCHEMA_ID, TABLE_ID, table_name, STATE FROM information_schema.ddl_jobs WHERE db_name = "d0" and state = "running";`).Check(testkit.RowsWithSep("|", - "132|add index /* txn-merge */|write only|104|106|t0|running", + "134|add index /* txn-merge */|write only|106|108|t0|running", )) tk2.MustQuery(`SELECT JOB_ID, JOB_TYPE, SCHEMA_STATE, SCHEMA_ID, TABLE_ID, table_name, STATE FROM information_schema.ddl_jobs WHERE state = "running";`).Check(testkit.RowsWithSep("|", - "132|add index /* txn-merge */|write only|104|106|t0|running", + "134|add index /* txn-merge */|write only|106|108|t0|running", )) } }) @@ -908,8 +908,8 @@ func TestInfoSchemaDDLJobs(t *testing.T) { tk.MustExec("create table test2.t1(id int)") tk.MustQuery(`SELECT JOB_ID, JOB_TYPE, SCHEMA_STATE, SCHEMA_ID, TABLE_ID, table_name, STATE FROM information_schema.ddl_jobs WHERE db_name = "test2" and table_name = "t1"`).Check(testkit.RowsWithSep("|", - "141|create table|public|138|140|t1|synced", - "136|create table|public|133|135|t1|synced", + "143|create table|public|140|142|t1|synced", + "138|create table|public|135|137|t1|synced", )) // Test explain output, since the output may change in future. diff --git a/pkg/executor/internal/querywatch/query_watch.go b/pkg/executor/internal/querywatch/query_watch.go index 98847a2b101df..3d1c861462461 100644 --- a/pkg/executor/internal/querywatch/query_watch.go +++ b/pkg/executor/internal/querywatch/query_watch.go @@ -122,7 +122,7 @@ func fromQueryWatchOptionList(ctx context.Context, sctx, newSctx sessionctx.Cont optionList []*ast.QueryWatchOption) (*runaway.QuarantineRecord, error) { record := &runaway.QuarantineRecord{ Source: runaway.ManualSource, - StartTime: time.Now(), + StartTime: time.Now().UTC(), EndTime: runaway.NullTime, } for _, op := range optionList { diff --git a/pkg/executor/internal/querywatch/query_watch_test.go b/pkg/executor/internal/querywatch/query_watch_test.go index 72381f52c523e..fa461083270e5 100644 --- a/pkg/executor/internal/querywatch/query_watch_test.go +++ b/pkg/executor/internal/querywatch/query_watch_test.go @@ -29,6 +29,10 @@ import ( ) func TestQueryWatch(t *testing.T) { + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC", `return(true)`)) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC")) + }() store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) if variable.SchemaCacheSize.Load() != 0 { @@ -117,7 +121,7 @@ func TestQueryWatch(t *testing.T) { require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/store/copr/sleepCoprRequest", fmt.Sprintf("return(%d)", 60))) err = tk.QueryToErr("select /*+ resource_group(rg1) */ * from t3") require.ErrorContains(t, err, "[executor:8253]Query execution was interrupted, identified as runaway query") - tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, match_type from mysql.tidb_runaway_queries", nil, + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, match_type from mysql.tidb_runaway_queries", nil, testkit.Rows( "rg1 select /*+ resource_group(rg1) */ * from t3 watch", "rg1 select /*+ resource_group(rg1) */ * from t3 identify", diff --git a/pkg/executor/mem_reader.go b/pkg/executor/mem_reader.go index 83abf8b26de68..a86da267c92af 100644 --- a/pkg/executor/mem_reader.go +++ b/pkg/executor/mem_reader.go @@ -39,7 +39,6 @@ import ( ) type memReader interface { - getMemRows(ctx context.Context) ([][]types.Datum, error) getMemRowsHandle() ([]kv.Handle, error) } @@ -65,6 +64,10 @@ type memIndexReader struct { physTblIDIdx int partitionIDMap map[int64]struct{} compareExec + + buf [16]byte + decodeBuff [][]byte + resultRows []types.Datum } func buildMemIndexReader(ctx context.Context, us *UnionScanExec, idxReader *IndexReaderExecutor) *memIndexReader { @@ -90,6 +93,7 @@ func buildMemIndexReader(ctx context.Context, us *UnionScanExec, idxReader *Inde compareExec: us.compareExec, physTblIDIdx: us.physTblIDIdx, partitionIDMap: us.partitionIDMap, + resultRows: make([]types.Datum, 0, len(outputOffset)), } } @@ -163,6 +167,7 @@ func (m *memIndexReader) getMemRows(ctx context.Context) ([][]types.Datum, error return err } m.addedRows = append(m.addedRows, data) + m.resultRows = make([]types.Datum, 0, len(data)) return nil }) @@ -189,7 +194,15 @@ func (m *memIndexReader) decodeIndexKeyValue(key, value []byte, tps []*types.Fie if mysql.HasUnsignedFlag(tps[len(m.index.Columns)].GetFlag()) { hdStatus = tablecodec.HandleIsUnsigned } - values, err := tablecodec.DecodeIndexKV(key, value, len(m.index.Columns), hdStatus, colInfos) + + colsLen := len(m.index.Columns) + if m.decodeBuff == nil { + m.decodeBuff = make([][]byte, colsLen, colsLen+len(colInfos)) + } else { + m.decodeBuff = m.decodeBuff[: colsLen : colsLen+len(colInfos)] + } + buf := m.buf[:0] + values, err := tablecodec.DecodeIndexKVEx(key, value, colsLen, hdStatus, colInfos, buf, m.decodeBuff) if err != nil { return nil, errors.Trace(err) } @@ -199,7 +212,7 @@ func (m *memIndexReader) decodeIndexKeyValue(key, value []byte, tps []*types.Fie physTblIDColumnIdx = m.outputOffset[m.physTblIDIdx] } - ds := make([]types.Datum, 0, len(m.outputOffset)) + ds := m.resultRows[:0] for i, offset := range m.outputOffset { // The `value` slice doesn't contain the value of `physTblID`, it fills by `tablecodec.DecodeKeyHead` function. // For example, the schema is `[a, b, physTblID, c]`, `value` is `[v_a, v_b, v_c]`, `outputOffset` is `[0, 1, 2, 3]` @@ -676,6 +689,7 @@ type memIndexLookUpReader struct { table table.Table conditions []expression.Expression retFieldTypes []*types.FieldType + schema *expression.Schema idxReader *memIndexReader @@ -704,6 +718,7 @@ func buildMemIndexLookUpReader(ctx context.Context, us *UnionScanExec, idxLookUp outputOffset: outputOffset, cacheTable: us.cacheTable, partitionIDMap: us.partitionIDMap, + resultRows: make([]types.Datum, 0, len(outputOffset)), } return &memIndexLookUpReader{ @@ -713,6 +728,7 @@ func buildMemIndexLookUpReader(ctx context.Context, us *UnionScanExec, idxLookUp table: idxLookUpReader.table, conditions: us.conditions, retFieldTypes: exec.RetTypes(us), + schema: us.Schema(), idxReader: memIdxReader, partitionMode: idxLookUpReader.partitionTableMode, @@ -726,17 +742,6 @@ func buildMemIndexLookUpReader(ctx context.Context, us *UnionScanExec, idxLookUp } func (m *memIndexLookUpReader) getMemRowsIter(ctx context.Context) (memRowsIter, error) { - data, err := m.getMemRows(ctx) - if err != nil { - return nil, errors.Trace(err) - } - return &defaultRowsIter{data: data}, nil -} - -func (m *memIndexLookUpReader) getMemRows(ctx context.Context) ([][]types.Datum, error) { - r, ctx := tracing.StartRegionEx(ctx, "memIndexLookUpReader.getMemRows") - defer r.End() - kvRanges := [][]kv.KeyRange{m.idxReader.kvRanges} tbls := []table.Table{m.table} if m.partitionMode { @@ -763,13 +768,14 @@ func (m *memIndexLookUpReader) getMemRows(ctx context.Context) ([][]types.Datum, tblKVRanges = append(tblKVRanges, ranges...) } if numHandles == 0 { - return nil, nil + return &defaultRowsIter{}, nil } if m.desc { slices.Reverse(tblKVRanges) } + cd := NewRowDecoder(m.ctx, m.schema, m.table.Meta()) colIDs, pkColIDs, rd := getColIDAndPkColIDs(m.ctx, m.table, m.columns) memTblReader := &memTableReader{ ctx: m.ctx, @@ -784,13 +790,14 @@ func (m *memIndexLookUpReader) getMemRows(ctx context.Context) ([][]types.Datum, buffer: allocBuf{ handleBytes: make([]byte, 0, 16), rd: rd, + cd: cd, }, cacheTable: m.cacheTable, keepOrder: m.keepOrder, compareExec: m.compareExec, } - return memTblReader.getMemRows(ctx) + return memTblReader.getMemRowsIter(ctx) } func (*memIndexLookUpReader) getMemRowsHandle() ([]kv.Handle, error) { @@ -849,6 +856,7 @@ func buildMemIndexMergeReader(ctx context.Context, us *UnionScanExec, indexMerge retFieldTypes: exec.RetTypes(us), outputOffset: outputOffset, partitionIDMap: indexMergeReader.partitionIDMap, + resultRows: make([]types.Datum, 0, len(outputOffset)), }) } } diff --git a/pkg/executor/partition_table_test.go b/pkg/executor/partition_table_test.go index 7dbe10f818478..2d6440ed0a0c9 100644 --- a/pkg/executor/partition_table_test.go +++ b/pkg/executor/partition_table_test.go @@ -2412,25 +2412,51 @@ func TestGlobalIndexWithSelectLock(t *testing.T) { tk1 := testkit.NewTestKit(t, store) tk1.MustExec("set tidb_enable_global_index = true") tk1.MustExec("use test") - tk1.MustExec("create table t(a int, b int, unique index(b) global, primary key(a)) partition by hash(a) partitions 5;") - tk1.MustExec("insert into t values (1,1),(2,2),(3,3),(4,4),(5,5);") - tk1.MustExec("begin") - tk1.MustExec("select * from t use index(b) where b = 2 order by b limit 1 for update;") + tk1.MustExec("create table t(" + + " a int, " + + " b int, " + + " c int, " + + " unique index(b) global, " + + " unique index(c) global, " + + " primary key(a)) " + + "partition by hash(a) partitions 5") + + tk1.MustExec("insert into t values (1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5)") + + cases := []struct { + sql string + plan string + }{ + {"select * from t use index(b) where b > 1 order by b limit 1 for update", "IndexLookUp"}, + {"select b from t use index(b) where b > 1 for update", "IndexReader"}, + {"select * from t use index(b) where b = 2 for update", "Point_Get"}, + {"select * from t use index(b) where b in (2, 3) for update", "Batch_Point_Get"}, + {"select /*+ USE_INDEX_MERGE(t, b, c) */ * from t where b = 2 or c = 3 for update", "IndexMerge"}, + } + + for _, c := range cases { + tk1.MustExec("begin") + tk1.MustHavePlan(c.sql, c.plan) + tk1.MustExec(c.sql) + + tk2 := testkit.NewTestKit(t, store) + tk2.MustExec("use test") + + ch := make(chan int, 10) + go func() { + // Check the key is locked. + tk2.MustExec("update t set b = 6 where b = 2") + ch <- 1 + }() - tk2 := testkit.NewTestKit(t, store) - tk2.MustExec("use test") + time.Sleep(50 * time.Millisecond) + ch <- 0 + tk1.MustExec("commit") - ch := make(chan int, 10) - go func() { - // Check the key is locked. - tk2.MustExec("update t set b = 6 where b = 2") - ch <- 1 - }() - - time.Sleep(50 * time.Millisecond) - ch <- 0 - tk1.MustExec("commit") + require.Equal(t, <-ch, 0) + require.Equal(t, <-ch, 1) - require.Equal(t, <-ch, 0) - require.Equal(t, <-ch, 1) + require.Equal(t, tk1.MustQuery("select * from t where b = 6").Rows(), testkit.Rows("2 6 2")) + tk1.MustExec("update t set b = 2 where b = 6") + } } diff --git a/pkg/executor/slow_query_test.go b/pkg/executor/slow_query_test.go index e63f95f0f2cd3..c4a755c5464ab 100644 --- a/pkg/executor/slow_query_test.go +++ b/pkg/executor/slow_query_test.go @@ -147,8 +147,8 @@ func TestParseSlowLogFile(t *testing.T) { # Request_unit_read: 2.158 # Request_unit_write: 2.123 # Time_queued_by_rc: 0.05 -# Tidb_cpu_usage: 0.01 -# Tikv_cpu_usage: 0.021 +# Tidb_cpu_time: 0.01 +# Tikv_cpu_time: 0.021 # Plan_digest: 60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4 # Prev_stmt: update t set i = 1; use test; diff --git a/pkg/executor/test/ddl/BUILD.bazel b/pkg/executor/test/ddl/BUILD.bazel index cd755519d7edf..0b1818981dd53 100644 --- a/pkg/executor/test/ddl/BUILD.bazel +++ b/pkg/executor/test/ddl/BUILD.bazel @@ -8,7 +8,7 @@ go_test( "main_test.go", ], flaky = True, - shard_count = 19, + shard_count = 20, deps = [ "//pkg/config", "//pkg/ddl/schematracker", diff --git a/pkg/executor/test/ddl/ddl_test.go b/pkg/executor/test/ddl/ddl_test.go index b0e6cc8a110b6..cab1926b536d8 100644 --- a/pkg/executor/test/ddl/ddl_test.go +++ b/pkg/executor/test/ddl/ddl_test.go @@ -1106,3 +1106,41 @@ func TestRenameMultiTables(t *testing.T) { tk.MustExec("drop database rename2") tk.MustExec("drop database rename3") } + +func TestDefShardTables(t *testing.T) { + store := testkit.CreateMockStore(t, mockstore.WithDDLChecker()) + + tk := testkit.NewTestKit(t, store) + + tk.MustExec("set @@session.tidb_enable_clustered_index = off") + tk.MustExec("set @@session.tidb_shard_row_id_bits = 4") + tk.MustExec("set @@session.tidb_pre_split_regions = 4") + tk.MustExec("use test") + tk.MustExec("create table t (i int primary key)") + result := tk.MustQuery("show create table t") + createSQL := result.Rows()[0][1] + expected := "CREATE TABLE `t` (\n `i` int(11) NOT NULL,\n PRIMARY KEY (`i`) /*T![clustered_index] NONCLUSTERED */\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T! SHARD_ROW_ID_BITS=4 PRE_SPLIT_REGIONS=4 */" + require.Equal(t, expected, createSQL) + + // test for manual setup shard_row_id_bits and pre_split_regions + tk.MustExec("create table t0 (i int primary key) /*T! SHARD_ROW_ID_BITS=2 PRE_SPLIT_REGIONS=2 */") + result = tk.MustQuery("show create table t0") + createSQL = result.Rows()[0][1] + expected = "CREATE TABLE `t0` (\n `i` int(11) NOT NULL,\n PRIMARY KEY (`i`) /*T![clustered_index] NONCLUSTERED */\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T! SHARD_ROW_ID_BITS=2 PRE_SPLIT_REGIONS=2 */" + require.Equal(t, expected, createSQL) + + // test for clustered index table + tk.MustExec("set @@session.tidb_enable_clustered_index = on") + tk.MustExec("create table t1 (i int primary key)") + result = tk.MustQuery("show create table t1") + createSQL = result.Rows()[0][1] + expected = "CREATE TABLE `t1` (\n `i` int(11) NOT NULL,\n PRIMARY KEY (`i`) /*T![clustered_index] CLUSTERED */\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" + require.Equal(t, expected, createSQL) + + // test for global temporary table + tk.MustExec("create global temporary table tengine (id int) engine = 'innodb' on commit delete rows") + result = tk.MustQuery("show create table tengine") + createSQL = result.Rows()[0][1] + expected = "CREATE GLOBAL TEMPORARY TABLE `tengine` (\n `id` int(11) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ON COMMIT DELETE ROWS" + require.Equal(t, expected, createSQL) +} diff --git a/pkg/executor/test/seqtest/seq_executor_test.go b/pkg/executor/test/seqtest/seq_executor_test.go index 14f68aaa0abd3..784bd8ad3c7c8 100644 --- a/pkg/executor/test/seqtest/seq_executor_test.go +++ b/pkg/executor/test/seqtest/seq_executor_test.go @@ -760,6 +760,8 @@ func checkGoroutineExists(keyword string) bool { } func TestAdminShowNextID(t *testing.T) { + step := int64(10) + autoid.SetStep(step) store := testkit.CreateMockStore(t) HelperTestAdminShowNextID(t, store, `admin show `) @@ -771,10 +773,6 @@ func HelperTestAdminShowNextID(t *testing.T, store kv.Storage, str string) { defer func() { require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/meta/autoid/mockAutoIDChange")) }() - step := int64(10) - autoIDStep := autoid.GetStep() - autoid.SetStep(step) - defer autoid.SetStep(autoIDStep) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("drop table if exists t,tt") @@ -787,7 +785,7 @@ func HelperTestAdminShowNextID(t *testing.T, store kv.Storage, str string) { r = tk.MustQuery(str + " t next_row_id") r.Check(testkit.Rows("test t _tidb_rowid 11 _TIDB_ROWID")) // Row ID is original + step. - for i := 0; i < int(step); i++ { + for i := 0; i < int(10); i++ { tk.MustExec("insert into t values(10000, 1)") } r = tk.MustQuery(str + " t next_row_id") diff --git a/pkg/executor/test/tiflashtest/BUILD.bazel b/pkg/executor/test/tiflashtest/BUILD.bazel index 6f1da72d25ce1..7713977d4ca95 100644 --- a/pkg/executor/test/tiflashtest/BUILD.bazel +++ b/pkg/executor/test/tiflashtest/BUILD.bazel @@ -9,7 +9,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 43, + shard_count = 44, deps = [ "//pkg/config", "//pkg/domain", diff --git a/pkg/executor/test/tiflashtest/tiflash_test.go b/pkg/executor/test/tiflashtest/tiflash_test.go index 288bb7d46a9dd..e19bcea99a403 100644 --- a/pkg/executor/test/tiflashtest/tiflash_test.go +++ b/pkg/executor/test/tiflashtest/tiflash_test.go @@ -20,6 +20,7 @@ import ( "math/rand" "strings" "sync" + "sync/atomic" "testing" "time" @@ -2032,3 +2033,87 @@ func TestMppAggShouldAlignFinalMode(t *testing.T) { err = failpoint.Disable("github.com/pingcap/tidb/pkg/expression/aggregation/show-agg-mode") require.Nil(t, err) } + +func TestMppTableReaderCacheForSingleSQL(t *testing.T) { + store := testkit.CreateMockStore(t, withMockTiFlash(1)) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t(a int, b int, primary key(a))") + tk.MustExec("alter table t set tiflash replica 1") + tb := external.GetTableByName(t, tk, "test", "t") + err := domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true) + require.NoError(t, err) + + tk.MustExec("create table t2(a int, b int) partition by hash(b) partitions 4") + tk.MustExec("alter table t2 set tiflash replica 1") + tb = external.GetTableByName(t, tk, "test", "t2") + err = domain.GetDomain(tk.Session()).DDLExecutor().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true) + require.NoError(t, err) + tk.MustExec("insert into t values(1, 1)") + tk.MustExec("insert into t values(2, 2)") + tk.MustExec("insert into t values(3, 3)") + tk.MustExec("insert into t values(4, 4)") + tk.MustExec("insert into t values(5, 5)") + + tk.MustExec("insert into t2 values(1, 1)") + tk.MustExec("insert into t2 values(2, 2)") + tk.MustExec("insert into t2 values(3, 3)") + tk.MustExec("insert into t2 values(4, 4)") + tk.MustExec("insert into t2 values(5, 5)") + + tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") + tk.MustExec("set @@session.tidb_allow_mpp=ON") + tk.MustExec("set @@session.tidb_enforce_mpp=ON") + tk.MustExec("set @@session.tidb_max_chunk_size=32") + + // Test TableReader cache for single SQL. + type testCase struct { + sql string + expectHitNum int32 + expectMissNum int32 + } + + testCases := []testCase{ + // Non-Partition + // Cache hit + {"select * from t", 0, 1}, + {"select * from t union select * from t", 1, 1}, + {"select * from t union select * from t t1 union select * from t t2", 2, 1}, + {"select * from t where b <= 3 union select * from t where b > 3", 1, 1}, // both full range + {"select * from t where a <= 3 union select * from t where a <= 3", 1, 1}, // same range + {"select * from t t1 join t t2 on t1.b=t2.b", 1, 1}, + + // Cache miss + {"select * from t union all select * from t", 0, 2}, // different mpp task root + {"select * from t where a <= 3 union select * from t where a > 3", 0, 2}, // different range + + // Partition + // Cache hit + {"select * from t2 union select * from t2", 1, 1}, + {"select * from t2 where b = 1 union select * from t2 where b = 5", 1, 1}, // same partition, full range + {"select * from t2 where b = 1 and a < 3 union select * from t2 where b = 5 and a < 3", 1, 1}, // same partition, same range + {"select * from t2 t1 join t2 t2 on t1.b=t2.b", 1, 1}, + {"select * from t2 t1 join t2 t2 on t1.b=t2.b where t1.a = 2 and t2.a = 2", 1, 1}, + + // Cache miss + {"select * from t2 union select * from t2 where b = 1", 0, 2}, // different partition + {"select * from t2 where b = 2 union select * from t2 where b = 1", 0, 2}, // different partition + } + + var hitNum, missNum atomic.Int32 + hitFunc := func() { + hitNum.Add(1) + } + missFunc := func() { + missNum.Add(1) + } + failpoint.EnableCall("github.com/pingcap/tidb/pkg/planner/core/mppTaskGeneratorTableReaderCacheHit", hitFunc) + failpoint.EnableCall("github.com/pingcap/tidb/pkg/planner/core/mppTaskGeneratorTableReaderCacheMiss", missFunc) + for _, tc := range testCases { + hitNum.Store(0) + missNum.Store(0) + tk.MustQuery(tc.sql) + require.Equal(t, tc.expectHitNum, hitNum.Load()) + require.Equal(t, tc.expectMissNum, missNum.Load()) + } +} diff --git a/pkg/expression/BUILD.bazel b/pkg/expression/BUILD.bazel index bfb562f84bd2c..503c4f9839503 100644 --- a/pkg/expression/BUILD.bazel +++ b/pkg/expression/BUILD.bazel @@ -81,6 +81,7 @@ go_library( "//pkg/infoschema/context", "//pkg/kv", "//pkg/meta/model", + "//pkg/param", "//pkg/parser", "//pkg/parser/ast", "//pkg/parser/auth", @@ -122,6 +123,7 @@ go_library( "//pkg/util/size", "//pkg/util/sqlexec", "//pkg/util/stringutil", + "//pkg/util/timeutil", "//pkg/util/vitess", "//pkg/util/zeropool", "@com_github_gogo_protobuf//proto", diff --git a/pkg/expression/builtin_time.go b/pkg/expression/builtin_time.go index e2c58fd32699c..d044b554fa2b7 100644 --- a/pkg/expression/builtin_time.go +++ b/pkg/expression/builtin_time.go @@ -2524,7 +2524,7 @@ func evalNowWithFsp(ctx EvalContext, fsp int) (types.Time, bool, error) { return types.ZeroTime, true, err } - err = result.ConvertTimeZone(time.Local, location(ctx)) + err = result.ConvertTimeZone(nowTs.Location(), location(ctx)) if err != nil { return types.ZeroTime, true, err } diff --git a/pkg/expression/helper.go b/pkg/expression/helper.go index 154d599142d95..b95571b6e7831 100644 --- a/pkg/expression/helper.go +++ b/pkg/expression/helper.go @@ -17,6 +17,7 @@ package expression import ( "math" "strings" + "sync" "time" "github.com/pingcap/failpoint" @@ -25,6 +26,10 @@ import ( "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/types" driver "github.com/pingcap/tidb/pkg/types/parser_driver" + "github.com/pingcap/tidb/pkg/util/intest" + "github.com/pingcap/tidb/pkg/util/logutil" + "github.com/pingcap/tidb/pkg/util/timeutil" + "go.uber.org/zap" ) func boolToInt64(v bool) int64 { @@ -73,7 +78,7 @@ func getTimeCurrentTimeStamp(ctx EvalContext, tp byte, fsp int) (t types.Time, e } value.SetCoreTime(types.FromGoTime(defaultTime.Truncate(time.Duration(math.Pow10(9-fsp)) * time.Nanosecond))) if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime || tp == mysql.TypeDate { - err = value.ConvertTimeZone(time.Local, ctx.Location()) + err = value.ConvertTimeZone(defaultTime.Location(), ctx.Location()) if err != nil { return value, err } @@ -159,9 +164,47 @@ func GetTimeValue(ctx BuildContext, v any, tp byte, fsp int, explicitTz *time.Lo return d, nil } +// randomNowLocationForTest is only used for test +var randomNowLocationForTest *time.Location +var randomNowLocationForTestOnce sync.Once + +func pickRandomLocationForTest() *time.Location { + randomNowLocationForTestOnce.Do(func() { + names := []string{ + "", + "UTC", + "Asia/Shanghai", + "America/Los_Angeles", + "Asia/Tokyo", + "Europe/Berlin", + } + name := names[int(time.Now().UnixMilli())%len(names)] + loc := time.Local + if name != "" { + var err error + loc, err = timeutil.LoadLocation(name) + terror.MustNil(err) + } + randomNowLocationForTest = loc + logutil.BgLogger().Info( + "set random timezone for getStmtTimestamp", + zap.String("timezone", loc.String()), + ) + }) + return randomNowLocationForTest +} + // if timestamp session variable set, use session variable as current time, otherwise use cached time // during one sql statement, the "current_time" should be the same -func getStmtTimestamp(ctx EvalContext) (time.Time, error) { +func getStmtTimestamp(ctx EvalContext) (now time.Time, err error) { + if intest.InTest { + // When in a test, return the now with random location to make sure all outside code will + // respect the location of return value `now` instead of having a strong assumption what its location is. + defer func() { + now = now.In(pickRandomLocationForTest()) + }() + } + failpoint.Inject("injectNow", func(val failpoint.Value) { v := time.Unix(int64(val.(int)), 0) failpoint.Return(v, nil) diff --git a/pkg/expression/integration_test/integration_test.go b/pkg/expression/integration_test/integration_test.go index a1cd13635bb9d..7c5c83fc2d988 100644 --- a/pkg/expression/integration_test/integration_test.go +++ b/pkg/expression/integration_test/integration_test.go @@ -1116,18 +1116,18 @@ func TestTiDBEncodeKey(t *testing.T) { err := tk.QueryToErr("select tidb_encode_record_key('test', 't1', 0);") require.ErrorContains(t, err, "doesn't exist") tk.MustQuery("select tidb_encode_record_key('test', 't', 1);"). - Check(testkit.Rows("7480000000000000685f728000000000000001")) + Check(testkit.Rows("74800000000000006a5f728000000000000001")) tk.MustExec("alter table t add index i(b);") err = tk.QueryToErr("select tidb_encode_index_key('test', 't', 'i1', 1);") require.ErrorContains(t, err, "index not found") tk.MustQuery("select tidb_encode_index_key('test', 't', 'i', 1, 1);"). - Check(testkit.Rows("7480000000000000685f698000000000000001038000000000000001038000000000000001")) + Check(testkit.Rows("74800000000000006a5f698000000000000001038000000000000001038000000000000001")) tk.MustExec("create table t1 (a int primary key, b int) partition by hash(a) partitions 4;") tk.MustExec("insert into t1 values (1, 1);") - tk.MustQuery("select tidb_encode_record_key('test', 't1(p1)', 1);").Check(testkit.Rows("74800000000000006d5f728000000000000001")) - rs := tk.MustQuery("select tidb_mvcc_info('74800000000000006d5f728000000000000001');") + tk.MustQuery("select tidb_encode_record_key('test', 't1(p1)', 1);").Check(testkit.Rows("74800000000000006f5f728000000000000001")) + rs := tk.MustQuery("select tidb_mvcc_info('74800000000000006f5f728000000000000001');") mvccInfo := rs.Rows()[0][0].(string) require.NotEqual(t, mvccInfo, `{"info":{}}`) @@ -1136,14 +1136,14 @@ func TestTiDBEncodeKey(t *testing.T) { tk2 := testkit.NewTestKit(t, store) err = tk2.Session().Auth(&auth.UserIdentity{Username: "alice", Hostname: "localhost"}, nil, nil, nil) require.NoError(t, err) - err = tk2.QueryToErr("select tidb_mvcc_info('74800000000000006d5f728000000000000001');") + err = tk2.QueryToErr("select tidb_mvcc_info('74800000000000006f5f728000000000000001');") require.ErrorContains(t, err, "Access denied") err = tk2.QueryToErr("select tidb_encode_record_key('test', 't1(p1)', 1);") require.ErrorContains(t, err, "SELECT command denied") err = tk2.QueryToErr("select tidb_encode_index_key('test', 't', 'i1', 1);") require.ErrorContains(t, err, "SELECT command denied") tk.MustExec("grant select on test.t1 to 'alice'@'%';") - tk2.MustQuery("select tidb_encode_record_key('test', 't1(p1)', 1);").Check(testkit.Rows("74800000000000006d5f728000000000000001")) + tk2.MustQuery("select tidb_encode_record_key('test', 't1(p1)', 1);").Check(testkit.Rows("74800000000000006f5f728000000000000001")) } func TestIssue9710(t *testing.T) { diff --git a/pkg/expression/util.go b/pkg/expression/util.go index 95e0c90fde533..4e39f8f24da66 100644 --- a/pkg/expression/util.go +++ b/pkg/expression/util.go @@ -17,6 +17,8 @@ package expression import ( "bytes" "context" + "encoding/binary" + "fmt" "math" "strconv" "strings" @@ -27,6 +29,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/expression/contextopt" "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/param" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/charset" "github.com/pingcap/tidb/pkg/parser/mysql" @@ -36,6 +39,7 @@ import ( driver "github.com/pingcap/tidb/pkg/types/parser_driver" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/collate" + "github.com/pingcap/tidb/pkg/util/hack" "github.com/pingcap/tidb/pkg/util/intset" "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" @@ -1920,3 +1924,241 @@ func ExprHasSetVarOrSleep(expr Expression) bool { } return false } + +// ExecBinaryParam parse execute binary param arguments to datum slice. +func ExecBinaryParam(typectx types.Context, binaryParams []param.BinaryParam) (params []Expression, err error) { + var ( + tmp any + ) + + params = make([]Expression, len(binaryParams)) + args := make([]types.Datum, len(binaryParams)) + for i := 0; i < len(args); i++ { + tp := binaryParams[i].Tp + isUnsigned := binaryParams[i].IsUnsigned + + switch tp { + case mysql.TypeNull: + var nilDatum types.Datum + nilDatum.SetNull() + args[i] = nilDatum + continue + + case mysql.TypeTiny: + if isUnsigned { + args[i] = types.NewUintDatum(uint64(binaryParams[i].Val[0])) + } else { + args[i] = types.NewIntDatum(int64(int8(binaryParams[i].Val[0]))) + } + continue + + case mysql.TypeShort, mysql.TypeYear: + valU16 := binary.LittleEndian.Uint16(binaryParams[i].Val) + if isUnsigned { + args[i] = types.NewUintDatum(uint64(valU16)) + } else { + args[i] = types.NewIntDatum(int64(int16(valU16))) + } + continue + + case mysql.TypeInt24, mysql.TypeLong: + valU32 := binary.LittleEndian.Uint32(binaryParams[i].Val) + if isUnsigned { + args[i] = types.NewUintDatum(uint64(valU32)) + } else { + args[i] = types.NewIntDatum(int64(int32(valU32))) + } + continue + + case mysql.TypeLonglong: + valU64 := binary.LittleEndian.Uint64(binaryParams[i].Val) + if isUnsigned { + args[i] = types.NewUintDatum(valU64) + } else { + args[i] = types.NewIntDatum(int64(valU64)) + } + continue + + case mysql.TypeFloat: + args[i] = types.NewFloat32Datum(math.Float32frombits(binary.LittleEndian.Uint32(binaryParams[i].Val))) + continue + + case mysql.TypeDouble: + args[i] = types.NewFloat64Datum(math.Float64frombits(binary.LittleEndian.Uint64(binaryParams[i].Val))) + continue + + case mysql.TypeDate, mysql.TypeTimestamp, mysql.TypeDatetime: + switch len(binaryParams[i].Val) { + case 0: + tmp = types.ZeroDatetimeStr + case 4: + _, tmp = binaryDate(0, binaryParams[i].Val) + case 7: + _, tmp = binaryDateTime(0, binaryParams[i].Val) + case 11: + _, tmp = binaryTimestamp(0, binaryParams[i].Val) + case 13: + _, tmp = binaryTimestampWithTZ(0, binaryParams[i].Val) + default: + err = mysql.ErrMalformPacket + return + } + // TODO: generate the time datum directly + var parseTime func(types.Context, string) (types.Time, error) + switch tp { + case mysql.TypeDate: + parseTime = types.ParseDate + case mysql.TypeDatetime: + parseTime = types.ParseDatetime + case mysql.TypeTimestamp: + // To be compatible with MySQL, even the type of parameter is + // TypeTimestamp, the return type should also be `Datetime`. + parseTime = types.ParseDatetime + } + var time types.Time + time, err = parseTime(typectx, tmp.(string)) + err = typectx.HandleTruncate(err) + if err != nil { + return + } + args[i] = types.NewDatum(time) + continue + + case mysql.TypeDuration: + fsp := 0 + switch len(binaryParams[i].Val) { + case 0: + tmp = "0" + case 8: + isNegative := binaryParams[i].Val[0] + if isNegative > 1 { + err = mysql.ErrMalformPacket + return + } + _, tmp = binaryDuration(1, binaryParams[i].Val, isNegative) + case 12: + isNegative := binaryParams[i].Val[0] + if isNegative > 1 { + err = mysql.ErrMalformPacket + return + } + _, tmp = binaryDurationWithMS(1, binaryParams[i].Val, isNegative) + fsp = types.MaxFsp + default: + err = mysql.ErrMalformPacket + return + } + // TODO: generate the duration datum directly + var dur types.Duration + dur, _, err = types.ParseDuration(typectx, tmp.(string), fsp) + err = typectx.HandleTruncate(err) + if err != nil { + return + } + args[i] = types.NewDatum(dur) + continue + case mysql.TypeNewDecimal: + if binaryParams[i].IsNull { + args[i] = types.NewDecimalDatum(nil) + } else { + var dec types.MyDecimal + err = typectx.HandleTruncate(dec.FromString(binaryParams[i].Val)) + if err != nil { + return nil, err + } + args[i] = types.NewDecimalDatum(&dec) + } + continue + case mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: + if binaryParams[i].IsNull { + args[i] = types.NewBytesDatum(nil) + } else { + args[i] = types.NewBytesDatum(binaryParams[i].Val) + } + continue + case mysql.TypeUnspecified, mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, + mysql.TypeEnum, mysql.TypeSet, mysql.TypeGeometry, mysql.TypeBit: + if !binaryParams[i].IsNull { + tmp = string(hack.String(binaryParams[i].Val)) + } else { + tmp = nil + } + args[i] = types.NewDatum(tmp) + continue + default: + err = param.ErrUnknownFieldType.GenWithStack("stmt unknown field type %d", tp) + return + } + } + + for i := range params { + ft := new(types.FieldType) + types.InferParamTypeFromUnderlyingValue(args[i].GetValue(), ft) + params[i] = &Constant{Value: args[i], RetType: ft} + } + return +} + +func binaryDate(pos int, paramValues []byte) (int, string) { + year := binary.LittleEndian.Uint16(paramValues[pos : pos+2]) + pos += 2 + month := paramValues[pos] + pos++ + day := paramValues[pos] + pos++ + return pos, fmt.Sprintf("%04d-%02d-%02d", year, month, day) +} + +func binaryDateTime(pos int, paramValues []byte) (int, string) { + pos, date := binaryDate(pos, paramValues) + hour := paramValues[pos] + pos++ + minute := paramValues[pos] + pos++ + second := paramValues[pos] + pos++ + return pos, fmt.Sprintf("%s %02d:%02d:%02d", date, hour, minute, second) +} + +func binaryTimestamp(pos int, paramValues []byte) (int, string) { + pos, dateTime := binaryDateTime(pos, paramValues) + microSecond := binary.LittleEndian.Uint32(paramValues[pos : pos+4]) + pos += 4 + return pos, fmt.Sprintf("%s.%06d", dateTime, microSecond) +} + +func binaryTimestampWithTZ(pos int, paramValues []byte) (int, string) { + pos, timestamp := binaryTimestamp(pos, paramValues) + tzShiftInMin := int16(binary.LittleEndian.Uint16(paramValues[pos : pos+2])) + tzShiftHour := tzShiftInMin / 60 + tzShiftAbsMin := tzShiftInMin % 60 + if tzShiftAbsMin < 0 { + tzShiftAbsMin = -tzShiftAbsMin + } + pos += 2 + return pos, fmt.Sprintf("%s%+02d:%02d", timestamp, tzShiftHour, tzShiftAbsMin) +} + +func binaryDuration(pos int, paramValues []byte, isNegative uint8) (int, string) { + sign := "" + if isNegative == 1 { + sign = "-" + } + days := binary.LittleEndian.Uint32(paramValues[pos : pos+4]) + pos += 4 + hours := paramValues[pos] + pos++ + minutes := paramValues[pos] + pos++ + seconds := paramValues[pos] + pos++ + return pos, fmt.Sprintf("%s%d %02d:%02d:%02d", sign, days, hours, minutes, seconds) +} + +func binaryDurationWithMS(pos int, paramValues []byte, + isNegative uint8) (int, string) { + pos, dur := binaryDuration(pos, paramValues, isNegative) + microSecond := binary.LittleEndian.Uint32(paramValues[pos : pos+4]) + pos += 4 + return pos, fmt.Sprintf("%s.%06d", dur, microSecond) +} diff --git a/pkg/infoschema/internal/testkit.go b/pkg/infoschema/internal/testkit.go index a8a18875172a9..c68efa9752126 100644 --- a/pkg/infoschema/internal/testkit.go +++ b/pkg/infoschema/internal/testkit.go @@ -98,8 +98,8 @@ select * from t_slim; # Resource_group: rg1 # Request_unit_read: 96.66703066666668 # Request_unit_write: 3182.424414062492 -# Tidb_cpu_usage: 0.01 -# Tikv_cpu_usage: 0.021 +# Tidb_cpu_time: 0.01 +# Tikv_cpu_time: 0.021 INSERT INTO ...; `) require.NoError(t, f.Close()) diff --git a/pkg/infoschema/perfschema/tables.go b/pkg/infoschema/perfschema/tables.go index f43ca58c96bd5..bfb804f9bb673 100644 --- a/pkg/infoschema/perfschema/tables.go +++ b/pkg/infoschema/perfschema/tables.go @@ -207,6 +207,11 @@ func (vt *perfSchemaTable) Indices() []table.Index { return vt.indices } +// DeletableIndices implements table.Table DeletableIndices interface. +func (vt *perfSchemaTable) DeletableIndices() []table.Index { + return nil +} + // GetPartitionedTable implements table.Table GetPartitionedTable interface. func (vt *perfSchemaTable) GetPartitionedTable() table.PartitionedTable { return nil diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go index a1e95c174a84a..f66a3813eef1f 100644 --- a/pkg/infoschema/tables.go +++ b/pkg/infoschema/tables.go @@ -466,7 +466,7 @@ var columnsCols = []columnInfo{ {name: "TABLE_SCHEMA", tp: mysql.TypeVarchar, size: 64}, {name: "TABLE_NAME", tp: mysql.TypeVarchar, size: 64}, {name: "COLUMN_NAME", tp: mysql.TypeVarchar, size: 64}, - {name: "ORDINAL_POSITION", tp: mysql.TypeLonglong, size: 64}, + {name: "ORDINAL_POSITION", tp: mysql.TypeLonglong, size: 21}, {name: "COLUMN_DEFAULT", tp: mysql.TypeBlob, size: 196606}, {name: "IS_NULLABLE", tp: mysql.TypeVarchar, size: 3}, {name: "DATA_TYPE", tp: mysql.TypeVarchar, size: 64}, @@ -858,8 +858,8 @@ var tableProcesslistCols = []columnInfo{ {name: "RESOURCE_GROUP", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag, deflt: ""}, {name: "SESSION_ALIAS", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, deflt: ""}, {name: "ROWS_AFFECTED", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, - {name: "TIDB_CPU", tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag, deflt: 0}, - {name: "TIKV_CPU", tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag, deflt: 0}, + {name: "TIDB_CPU", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag, deflt: 0}, + {name: "TIKV_CPU", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag, deflt: 0}, } var tableTiDBIndexesCols = []columnInfo{ @@ -1032,16 +1032,16 @@ var tableAnalyzeStatusCols = []columnInfo{ {name: "TABLE_NAME", tp: mysql.TypeVarchar, size: 64}, {name: "PARTITION_NAME", tp: mysql.TypeVarchar, size: 64}, {name: "JOB_INFO", tp: mysql.TypeLongBlob, size: types.UnspecifiedLength}, - {name: "PROCESSED_ROWS", tp: mysql.TypeLonglong, size: 64, flag: mysql.UnsignedFlag}, + {name: "PROCESSED_ROWS", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "START_TIME", tp: mysql.TypeDatetime}, {name: "END_TIME", tp: mysql.TypeDatetime}, {name: "STATE", tp: mysql.TypeVarchar, size: 64}, {name: "FAIL_REASON", tp: mysql.TypeLongBlob, size: types.UnspecifiedLength}, {name: "INSTANCE", tp: mysql.TypeVarchar, size: 512}, - {name: "PROCESS_ID", tp: mysql.TypeLonglong, size: 64, flag: mysql.UnsignedFlag}, + {name: "PROCESS_ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "REMAINING_SECONDS", tp: mysql.TypeVarchar, size: 512}, {name: "PROGRESS", tp: mysql.TypeDouble, size: 22, decimal: 6}, - {name: "ESTIMATED_TOTAL_ROWS", tp: mysql.TypeLonglong, size: 64, flag: mysql.UnsignedFlag}, + {name: "ESTIMATED_TOTAL_ROWS", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, } // TableTiKVRegionStatusCols is TiKV region status mem table columns. @@ -1197,7 +1197,7 @@ var tableTableTiFlashReplicaCols = []columnInfo{ {name: "TABLE_SCHEMA", tp: mysql.TypeVarchar, size: 64}, {name: "TABLE_NAME", tp: mysql.TypeVarchar, size: 64}, {name: "TABLE_ID", tp: mysql.TypeLonglong, size: 21}, - {name: "REPLICA_COUNT", tp: mysql.TypeLonglong, size: 64}, + {name: "REPLICA_COUNT", tp: mysql.TypeLonglong, size: 21}, {name: "LOCATION_LABELS", tp: mysql.TypeVarchar, size: 64}, {name: "AVAILABLE", tp: mysql.TypeTiny, size: 1}, {name: "PROGRESS", tp: mysql.TypeDouble, size: 22}, @@ -1409,8 +1409,8 @@ var tableStorageStatsCols = []columnInfo{ {name: "PEER_COUNT", tp: mysql.TypeLonglong, size: 21}, {name: "REGION_COUNT", tp: mysql.TypeLonglong, size: 21, comment: "The region count of single replica of the table"}, {name: "EMPTY_REGION_COUNT", tp: mysql.TypeLonglong, size: 21, comment: "The region count of single replica of the table"}, - {name: "TABLE_SIZE", tp: mysql.TypeLonglong, size: 64, comment: "The disk usage(MB) of single replica of the table, if the table size is empty or less than 1MB, it would show 1MB "}, - {name: "TABLE_KEYS", tp: mysql.TypeLonglong, size: 64, comment: "The count of keys of single replica of the table"}, + {name: "TABLE_SIZE", tp: mysql.TypeLonglong, size: 21, comment: "The disk usage(MB) of single replica of the table, if the table size is empty or less than 1MB, it would show 1MB "}, + {name: "TABLE_KEYS", tp: mysql.TypeLonglong, size: 21, comment: "The count of keys of single replica of the table"}, } var tableTableTiFlashTablesCols = []columnInfo{ @@ -1421,54 +1421,54 @@ var tableTableTiFlashTablesCols = []columnInfo{ {name: "TIDB_DATABASE", tp: mysql.TypeVarchar, size: 64}, {name: "TIDB_TABLE", tp: mysql.TypeVarchar, size: 64}, {name: "TABLE_ID", tp: mysql.TypeLonglong, size: 21}, - {name: "IS_TOMBSTONE", tp: mysql.TypeLonglong, size: 64}, - {name: "SEGMENT_COUNT", tp: mysql.TypeLonglong, size: 64}, - {name: "TOTAL_ROWS", tp: mysql.TypeLonglong, size: 64}, - {name: "TOTAL_SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "TOTAL_DELETE_RANGES", tp: mysql.TypeLonglong, size: 64}, + {name: "IS_TOMBSTONE", tp: mysql.TypeLonglong, size: 21}, + {name: "SEGMENT_COUNT", tp: mysql.TypeLonglong, size: 21}, + {name: "TOTAL_ROWS", tp: mysql.TypeLonglong, size: 21}, + {name: "TOTAL_SIZE", tp: mysql.TypeLonglong, size: 21}, + {name: "TOTAL_DELETE_RANGES", tp: mysql.TypeLonglong, size: 21}, {name: "DELTA_RATE_ROWS", tp: mysql.TypeDouble, size: 64}, {name: "DELTA_RATE_SEGMENTS", tp: mysql.TypeDouble, size: 64}, {name: "DELTA_PLACED_RATE", tp: mysql.TypeDouble, size: 64}, - {name: "DELTA_CACHE_SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_CACHE_SIZE", tp: mysql.TypeLonglong, size: 21}, {name: "DELTA_CACHE_RATE", tp: mysql.TypeDouble, size: 64}, {name: "DELTA_CACHE_WASTED_RATE", tp: mysql.TypeDouble, size: 64}, - {name: "DELTA_INDEX_SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_INDEX_SIZE", tp: mysql.TypeLonglong, size: 21}, {name: "AVG_SEGMENT_ROWS", tp: mysql.TypeDouble, size: 64}, {name: "AVG_SEGMENT_SIZE", tp: mysql.TypeDouble, size: 64}, - {name: "DELTA_COUNT", tp: mysql.TypeLonglong, size: 64}, - {name: "TOTAL_DELTA_ROWS", tp: mysql.TypeLonglong, size: 64}, - {name: "TOTAL_DELTA_SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_COUNT", tp: mysql.TypeLonglong, size: 21}, + {name: "TOTAL_DELTA_ROWS", tp: mysql.TypeLonglong, size: 21}, + {name: "TOTAL_DELTA_SIZE", tp: mysql.TypeLonglong, size: 21}, {name: "AVG_DELTA_ROWS", tp: mysql.TypeDouble, size: 64}, {name: "AVG_DELTA_SIZE", tp: mysql.TypeDouble, size: 64}, {name: "AVG_DELTA_DELETE_RANGES", tp: mysql.TypeDouble, size: 64}, - {name: "STABLE_COUNT", tp: mysql.TypeLonglong, size: 64}, - {name: "TOTAL_STABLE_ROWS", tp: mysql.TypeLonglong, size: 64}, - {name: "TOTAL_STABLE_SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "TOTAL_STABLE_SIZE_ON_DISK", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_COUNT", tp: mysql.TypeLonglong, size: 21}, + {name: "TOTAL_STABLE_ROWS", tp: mysql.TypeLonglong, size: 21}, + {name: "TOTAL_STABLE_SIZE", tp: mysql.TypeLonglong, size: 21}, + {name: "TOTAL_STABLE_SIZE_ON_DISK", tp: mysql.TypeLonglong, size: 21}, {name: "AVG_STABLE_ROWS", tp: mysql.TypeDouble, size: 64}, {name: "AVG_STABLE_SIZE", tp: mysql.TypeDouble, size: 64}, - {name: "TOTAL_PACK_COUNT_IN_DELTA", tp: mysql.TypeLonglong, size: 64}, - {name: "MAX_PACK_COUNT_IN_DELTA", tp: mysql.TypeLonglong, size: 64}, + {name: "TOTAL_PACK_COUNT_IN_DELTA", tp: mysql.TypeLonglong, size: 21}, + {name: "MAX_PACK_COUNT_IN_DELTA", tp: mysql.TypeLonglong, size: 21}, {name: "AVG_PACK_COUNT_IN_DELTA", tp: mysql.TypeDouble, size: 64}, {name: "AVG_PACK_ROWS_IN_DELTA", tp: mysql.TypeDouble, size: 64}, {name: "AVG_PACK_SIZE_IN_DELTA", tp: mysql.TypeDouble, size: 64}, - {name: "TOTAL_PACK_COUNT_IN_STABLE", tp: mysql.TypeLonglong, size: 64}, + {name: "TOTAL_PACK_COUNT_IN_STABLE", tp: mysql.TypeLonglong, size: 21}, {name: "AVG_PACK_COUNT_IN_STABLE", tp: mysql.TypeDouble, size: 64}, {name: "AVG_PACK_ROWS_IN_STABLE", tp: mysql.TypeDouble, size: 64}, {name: "AVG_PACK_SIZE_IN_STABLE", tp: mysql.TypeDouble, size: 64}, - {name: "STORAGE_STABLE_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_STABLE_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 21}, {name: "STORAGE_STABLE_OLDEST_SNAPSHOT_LIFETIME", tp: mysql.TypeDouble, size: 64}, - {name: "STORAGE_STABLE_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_STABLE_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 21}, {name: "STORAGE_STABLE_OLDEST_SNAPSHOT_TRACING_ID", tp: mysql.TypeVarchar, size: 128}, - {name: "STORAGE_DELTA_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_DELTA_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 21}, {name: "STORAGE_DELTA_OLDEST_SNAPSHOT_LIFETIME", tp: mysql.TypeDouble, size: 64}, - {name: "STORAGE_DELTA_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_DELTA_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 21}, {name: "STORAGE_DELTA_OLDEST_SNAPSHOT_TRACING_ID", tp: mysql.TypeVarchar, size: 128}, - {name: "STORAGE_META_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_META_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 21}, {name: "STORAGE_META_OLDEST_SNAPSHOT_LIFETIME", tp: mysql.TypeDouble, size: 64}, - {name: "STORAGE_META_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_META_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 21}, {name: "STORAGE_META_OLDEST_SNAPSHOT_TRACING_ID", tp: mysql.TypeVarchar, size: 128}, - {name: "BACKGROUND_TASKS_LENGTH", tp: mysql.TypeLonglong, size: 64}, + {name: "BACKGROUND_TASKS_LENGTH", tp: mysql.TypeLonglong, size: 21}, {name: "TIFLASH_INSTANCE", tp: mysql.TypeVarchar, size: 64}, } @@ -1480,61 +1480,61 @@ var tableTableTiFlashSegmentsCols = []columnInfo{ {name: "TIDB_DATABASE", tp: mysql.TypeVarchar, size: 64}, {name: "TIDB_TABLE", tp: mysql.TypeVarchar, size: 64}, {name: "TABLE_ID", tp: mysql.TypeLonglong, size: 21}, - {name: "IS_TOMBSTONE", tp: mysql.TypeLonglong, size: 64}, - {name: "SEGMENT_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "IS_TOMBSTONE", tp: mysql.TypeLonglong, size: 21}, + {name: "SEGMENT_ID", tp: mysql.TypeLonglong, size: 21}, {name: "RANGE", tp: mysql.TypeVarchar, size: 64}, - {name: "EPOCH", tp: mysql.TypeLonglong, size: 64}, - {name: "ROWS", tp: mysql.TypeLonglong, size: 64}, - {name: "SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "EPOCH", tp: mysql.TypeLonglong, size: 21}, + {name: "ROWS", tp: mysql.TypeLonglong, size: 21}, + {name: "SIZE", tp: mysql.TypeLonglong, size: 21}, {name: "DELTA_RATE", tp: mysql.TypeDouble, size: 64}, - {name: "DELTA_MEMTABLE_ROWS", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_MEMTABLE_SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_MEMTABLE_COLUMN_FILES", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_MEMTABLE_DELETE_RANGES", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_PERSISTED_PAGE_ID", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_PERSISTED_ROWS", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_PERSISTED_SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_PERSISTED_COLUMN_FILES", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_PERSISTED_DELETE_RANGES", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_CACHE_SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_INDEX_SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_PAGE_ID", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_ROWS", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_DMFILES", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_DMFILES_ID_0", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_DMFILES_ROWS", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_DMFILES_SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_DMFILES_SIZE_ON_DISK", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_DMFILES_PACKS", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_MEMTABLE_ROWS", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_MEMTABLE_SIZE", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_MEMTABLE_COLUMN_FILES", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_MEMTABLE_DELETE_RANGES", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_PERSISTED_PAGE_ID", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_PERSISTED_ROWS", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_PERSISTED_SIZE", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_PERSISTED_COLUMN_FILES", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_PERSISTED_DELETE_RANGES", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_CACHE_SIZE", tp: mysql.TypeLonglong, size: 21}, + {name: "DELTA_INDEX_SIZE", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_PAGE_ID", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_ROWS", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_SIZE", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_DMFILES", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_DMFILES_ID_0", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_DMFILES_ROWS", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_DMFILES_SIZE", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_DMFILES_SIZE_ON_DISK", tp: mysql.TypeLonglong, size: 21}, + {name: "STABLE_DMFILES_PACKS", tp: mysql.TypeLonglong, size: 21}, {name: "TIFLASH_INSTANCE", tp: mysql.TypeVarchar, size: 64}, } var tableClientErrorsSummaryGlobalCols = []columnInfo{ - {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, {name: "ERROR_MESSAGE", tp: mysql.TypeVarchar, size: 1024, flag: mysql.NotNullFlag}, - {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, - {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, + {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, {name: "FIRST_SEEN", tp: mysql.TypeTimestamp, size: 26}, {name: "LAST_SEEN", tp: mysql.TypeTimestamp, size: 26}, } var tableClientErrorsSummaryByUserCols = []columnInfo{ {name: "USER", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag}, - {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, {name: "ERROR_MESSAGE", tp: mysql.TypeVarchar, size: 1024, flag: mysql.NotNullFlag}, - {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, - {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, + {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, {name: "FIRST_SEEN", tp: mysql.TypeTimestamp, size: 26}, {name: "LAST_SEEN", tp: mysql.TypeTimestamp, size: 26}, } var tableClientErrorsSummaryByHostCols = []columnInfo{ {name: "HOST", tp: mysql.TypeVarchar, size: 255, flag: mysql.NotNullFlag}, - {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, {name: "ERROR_MESSAGE", tp: mysql.TypeVarchar, size: 1024, flag: mysql.NotNullFlag}, - {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, - {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, + {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, {name: "FIRST_SEEN", tp: mysql.TypeTimestamp, size: 26}, {name: "LAST_SEEN", tp: mysql.TypeTimestamp, size: 26}, } @@ -1546,8 +1546,8 @@ var tableTiDBTrxCols = []columnInfo{ {name: txninfo.CurrentSQLDigestTextStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "The normalized sql the transaction are currently running"}, {name: txninfo.StateStr, tp: mysql.TypeEnum, size: 16, enumElems: txninfo.TxnRunningStateStrs, comment: "Current running state of the transaction"}, {name: txninfo.WaitingStartTimeStr, tp: mysql.TypeTimestamp, decimal: 6, size: 26, comment: "Current lock waiting's start time"}, - {name: txninfo.MemBufferKeysStr, tp: mysql.TypeLonglong, size: 64, comment: "How many entries are in MemDB"}, - {name: txninfo.MemBufferBytesStr, tp: mysql.TypeLonglong, size: 64, comment: "MemDB used memory"}, + {name: txninfo.MemBufferKeysStr, tp: mysql.TypeLonglong, size: 21, comment: "How many entries are in MemDB"}, + {name: txninfo.MemBufferBytesStr, tp: mysql.TypeLonglong, size: 21, comment: "MemDB used memory"}, {name: txninfo.SessionIDStr, tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag, comment: "Which session this transaction belongs to"}, {name: txninfo.UserStr, tp: mysql.TypeVarchar, size: 16, comment: "The user who open this session"}, {name: txninfo.DBStr, tp: mysql.TypeVarchar, size: 64, comment: "The schema this transaction works on"}, @@ -1580,7 +1580,7 @@ var tableDataLockWaitsCols = []columnInfo{ var tableStatementsSummaryEvictedCols = []columnInfo{ {name: "BEGIN_TIME", tp: mysql.TypeTimestamp, size: 26}, {name: "END_TIME", tp: mysql.TypeTimestamp, size: 26}, - {name: "EVICTED_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "EVICTED_COUNT", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, } var tableAttributesCols = []columnInfo{ @@ -1596,7 +1596,7 @@ var tableTrxSummaryCols = []columnInfo{ } var tablePlacementPoliciesCols = []columnInfo{ - {name: "POLICY_ID", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "POLICY_ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, {name: "CATALOG_NAME", tp: mysql.TypeVarchar, size: 512, flag: mysql.NotNullFlag}, {name: "POLICY_NAME", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag}, // Catalog wide policy {name: "PRIMARY_REGION", tp: mysql.TypeVarchar, size: 1024}, @@ -1606,8 +1606,8 @@ var tablePlacementPoliciesCols = []columnInfo{ {name: "FOLLOWER_CONSTRAINTS", tp: mysql.TypeVarchar, size: 1024}, {name: "LEARNER_CONSTRAINTS", tp: mysql.TypeVarchar, size: 1024}, {name: "SCHEDULE", tp: mysql.TypeVarchar, size: 20}, // EVEN or MAJORITY_IN_PRIMARY - {name: "FOLLOWERS", tp: mysql.TypeLonglong, size: 64}, - {name: "LEARNERS", tp: mysql.TypeLonglong, size: 64}, + {name: "FOLLOWERS", tp: mysql.TypeLonglong, size: 21}, + {name: "LEARNERS", tp: mysql.TypeLonglong, size: 21}, } var tableVariablesInfoCols = []columnInfo{ @@ -1615,8 +1615,8 @@ var tableVariablesInfoCols = []columnInfo{ {name: "VARIABLE_SCOPE", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag}, {name: "DEFAULT_VALUE", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag}, {name: "CURRENT_VALUE", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag}, - {name: "MIN_VALUE", tp: mysql.TypeLonglong, size: 64}, - {name: "MAX_VALUE", tp: mysql.TypeLonglong, size: 64, flag: mysql.UnsignedFlag}, + {name: "MIN_VALUE", tp: mysql.TypeLonglong, size: 21}, + {name: "MAX_VALUE", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "POSSIBLE_VALUES", tp: mysql.TypeVarchar, size: 256}, {name: "IS_NOOP", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag}, } @@ -1666,7 +1666,7 @@ var tableResourceGroupsCols = []columnInfo{ } var tableRunawayWatchListCols = []columnInfo{ - {name: "ID", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag}, {name: "RESOURCE_GROUP_NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag}, {name: "START_TIME", tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag}, {name: "END_TIME", tp: mysql.TypeVarchar, size: 32}, @@ -2422,6 +2422,11 @@ func (it *infoschemaTable) Indices() []table.Index { return nil } +// DeletableIndices implements table.Table DeletableIndices interface. +func (it *infoschemaTable) DeletableIndices() []table.Index { + return nil +} + // WritableConstraint implements table.Table WritableConstraint interface. func (it *infoschemaTable) WritableConstraint() []*table.Constraint { return nil @@ -2443,7 +2448,7 @@ func (it *infoschemaTable) AddRecord(ctx table.MutateContext, txn kv.Transaction } // RemoveRecord implements table.Table RemoveRecord interface. -func (it *infoschemaTable) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum) error { +func (it *infoschemaTable) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum, opts ...table.RemoveRecordOption) error { return table.ErrUnsupportedOp } @@ -2515,6 +2520,11 @@ func (vt *VirtualTable) Indices() []table.Index { return nil } +// DeletableIndices implements table.Table DeletableIndices interface. +func (vt *VirtualTable) DeletableIndices() []table.Index { + return nil +} + // WritableConstraint implements table.Table WritableConstraint interface. func (vt *VirtualTable) WritableConstraint() []*table.Constraint { return nil @@ -2536,7 +2546,7 @@ func (vt *VirtualTable) AddRecord(ctx table.MutateContext, txn kv.Transaction, r } // RemoveRecord implements table.Table RemoveRecord interface. -func (vt *VirtualTable) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum) error { +func (vt *VirtualTable) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum, opts ...table.RemoveRecordOption) error { return table.ErrUnsupportedOp } diff --git a/pkg/infoschema/test/clustertablestest/tables_test.go b/pkg/infoschema/test/clustertablestest/tables_test.go index 8576d21fad91f..ac80fac0867fb 100644 --- a/pkg/infoschema/test/clustertablestest/tables_test.go +++ b/pkg/infoschema/test/clustertablestest/tables_test.go @@ -170,8 +170,8 @@ func TestInfoSchemaFieldValue(t *testing.T) { " `RESOURCE_GROUP` varchar(32) NOT NULL DEFAULT '',\n" + " `SESSION_ALIAS` varchar(64) NOT NULL DEFAULT '',\n" + " `ROWS_AFFECTED` bigint(21) unsigned DEFAULT NULL,\n" + - " `TIDB_CPU` double NOT NULL DEFAULT '0',\n" + - " `TIKV_CPU` double NOT NULL DEFAULT '0'\n" + + " `TIDB_CPU` bigint(21) NOT NULL DEFAULT '0',\n" + + " `TIKV_CPU` bigint(21) NOT NULL DEFAULT '0'\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")) tk.MustQuery("show create table information_schema.cluster_log").Check( testkit.Rows("" + diff --git a/pkg/kv/mpp.go b/pkg/kv/mpp.go index 14ee3b95765c2..aa41b8622764b 100644 --- a/pkg/kv/mpp.go +++ b/pkg/kv/mpp.go @@ -237,6 +237,29 @@ type MPPBuildTasksRequest struct { PartitionIDAndRanges []PartitionIDAndRanges } +// ToString returns a string representation of MPPBuildTasksRequest. Used for CacheKey. +func (req *MPPBuildTasksRequest) ToString() string { + sb := strings.Builder{} + if req.KeyRanges != nil { // Non-partiton + for i, keyRange := range req.KeyRanges { + sb.WriteString("range_id" + strconv.Itoa(i)) + sb.WriteString(keyRange.StartKey.String()) + sb.WriteString(keyRange.EndKey.String()) + } + return sb.String() + } + // Partition + for _, partitionIDAndRange := range req.PartitionIDAndRanges { + sb.WriteString("partition_id" + strconv.Itoa(int(partitionIDAndRange.ID))) + for i, keyRange := range partitionIDAndRange.KeyRanges { + sb.WriteString("range_id" + strconv.Itoa(i)) + sb.WriteString(keyRange.StartKey.String()) + sb.WriteString(keyRange.EndKey.String()) + } + } + return sb.String() +} + // ExchangeCompressionMode means the compress method used in exchange operator type ExchangeCompressionMode int diff --git a/pkg/lightning/common/retry.go b/pkg/lightning/common/retry.go index a2fa52041c287..ed1f7d4510ffb 100644 --- a/pkg/lightning/common/retry.go +++ b/pkg/lightning/common/retry.go @@ -123,7 +123,7 @@ func isSingleRetryableError(err error) bool { case *mysql.MySQLError: switch nerr.Number { // ErrLockDeadlock can retry to commit while meet deadlock - case tmysql.ErrUnknown, tmysql.ErrLockDeadlock, tmysql.ErrWriteConflict, tmysql.ErrWriteConflictInTiDB, + case tmysql.ErrUnknown, tmysql.ErrLockDeadlock, tmysql.ErrLockWaitTimeout, tmysql.ErrWriteConflict, tmysql.ErrWriteConflictInTiDB, tmysql.ErrPDServerTimeout, tmysql.ErrTiKVServerTimeout, tmysql.ErrTiKVServerBusy, tmysql.ErrResolveLockTimeout, tmysql.ErrRegionUnavailable, tmysql.ErrInfoSchemaExpired, tmysql.ErrInfoSchemaChanged, tmysql.ErrTxnRetryable: return true diff --git a/pkg/meta/model/job.go b/pkg/meta/model/job.go index 637324a34d01b..bec4012419941 100644 --- a/pkg/meta/model/job.go +++ b/pkg/meta/model/job.go @@ -515,6 +515,7 @@ func (job *Job) Encode(updateRawArgs bool) ([]byte, error) { } else { var arg any if len(job.Args) > 0 { + intest.Assert(len(job.Args) == 1, "Job.Args should have only one element") arg = job.Args[0] } job.RawArgs, err = json.Marshal(arg) @@ -542,6 +543,7 @@ func (job *Job) Decode(b []byte) error { // DecodeArgs decodes serialized job arguments from job.RawArgs into the given // variables, and also save the result in job.Args. It's for JobVersion1. +// TODO make it un-exported after we finish the migration to JobVersion2. func (job *Job) DecodeArgs(args ...any) error { intest.Assert(job.Version == JobVersion1, "Job.DecodeArgs is only used for JobVersion1") var rawArgs []json.RawMessage diff --git a/pkg/meta/model/job_args.go b/pkg/meta/model/job_args.go index 135fbd5780f60..9aabd96e7d767 100644 --- a/pkg/meta/model/job_args.go +++ b/pkg/meta/model/job_args.go @@ -27,6 +27,7 @@ import ( func getOrDecodeArgsV2[T JobArgs](job *Job) (T, error) { intest.Assert(job.Version == JobVersion2, "job version is not v2") if len(job.Args) > 0 { + intest.Assert(len(job.Args) == 1, "job args length is not 1") return job.Args[0].(T), nil } var v T @@ -184,6 +185,100 @@ func GetModifySchemaArgs(job *Job) (*ModifySchemaArgs, error) { return getOrDecodeArgsV2[*ModifySchemaArgs](job) } +// CreateTableArgs is the arguments for create table/view/sequence job. +type CreateTableArgs struct { + TableInfo *TableInfo `json:"table_info,omitempty"` + // below 2 are used for create view. + OnExistReplace bool `json:"on_exist_replace,omitempty"` + OldViewTblID int64 `json:"old_view_tbl_id,omitempty"` + // used for create table. + FKCheck bool `json:"fk_check,omitempty"` +} + +func (a *CreateTableArgs) fillJob(job *Job) { + if job.Version == JobVersion1 { + switch job.Type { + case ActionCreateTable: + job.Args = []any{a.TableInfo, a.FKCheck} + case ActionCreateView: + job.Args = []any{a.TableInfo, a.OnExistReplace, a.OldViewTblID} + case ActionCreateSequence: + job.Args = []any{a.TableInfo} + } + return + } + job.Args = []any{a} +} + +// GetCreateTableArgs gets the create-table args. +func GetCreateTableArgs(job *Job) (*CreateTableArgs, error) { + if job.Version == JobVersion1 { + var ( + tableInfo = &TableInfo{} + onExistReplace bool + oldViewTblID int64 + fkCheck bool + ) + switch job.Type { + case ActionCreateTable: + if err := job.DecodeArgs(tableInfo, &fkCheck); err != nil { + return nil, errors.Trace(err) + } + case ActionCreateView: + if err := job.DecodeArgs(tableInfo, &onExistReplace, &oldViewTblID); err != nil { + return nil, errors.Trace(err) + } + case ActionCreateSequence: + if err := job.DecodeArgs(tableInfo); err != nil { + return nil, errors.Trace(err) + } + } + return &CreateTableArgs{ + TableInfo: tableInfo, + OnExistReplace: onExistReplace, + OldViewTblID: oldViewTblID, + FKCheck: fkCheck, + }, nil + } + return getOrDecodeArgsV2[*CreateTableArgs](job) +} + +// BatchCreateTableArgs is the arguments for batch create table job. +type BatchCreateTableArgs struct { + Tables []*CreateTableArgs `json:"tables,omitempty"` +} + +func (a *BatchCreateTableArgs) fillJob(job *Job) { + if job.Version == JobVersion1 { + infos := make([]*TableInfo, 0, len(a.Tables)) + for _, info := range a.Tables { + infos = append(infos, info.TableInfo) + } + job.Args = []any{infos, a.Tables[0].FKCheck} + return + } + job.Args = []any{a} +} + +// GetBatchCreateTableArgs gets the batch create-table args. +func GetBatchCreateTableArgs(job *Job) (*BatchCreateTableArgs, error) { + if job.Version == JobVersion1 { + var ( + tableInfos []*TableInfo + fkCheck bool + ) + if err := job.DecodeArgs(&tableInfos, &fkCheck); err != nil { + return nil, errors.Trace(err) + } + args := &BatchCreateTableArgs{Tables: make([]*CreateTableArgs, 0, len(tableInfos))} + for _, info := range tableInfos { + args.Tables = append(args.Tables, &CreateTableArgs{TableInfo: info, FKCheck: fkCheck}) + } + return args, nil + } + return getOrDecodeArgsV2[*BatchCreateTableArgs](job) +} + // TruncateTableArgs is the arguments for truncate table job. type TruncateTableArgs struct { FKCheck bool `json:"fk_check,omitempty"` diff --git a/pkg/meta/model/job_args_test.go b/pkg/meta/model/job_args_test.go index d8f5a00586546..14ca86a9a66dc 100644 --- a/pkg/meta/model/job_args_test.go +++ b/pkg/meta/model/job_args_test.go @@ -131,6 +131,75 @@ func TestModifySchemaArgs(t *testing.T) { } } +func TestCreateTableArgs(t *testing.T) { + t.Run("create table", func(t *testing.T) { + inArgs := &CreateTableArgs{ + TableInfo: &TableInfo{ID: 100}, + FKCheck: true, + } + for _, v := range []JobVersion{JobVersion1, JobVersion2} { + j2 := &Job{} + require.NoError(t, j2.Decode(getJobBytes(t, inArgs, v, ActionCreateTable))) + args, err := GetCreateTableArgs(j2) + require.NoError(t, err) + require.EqualValues(t, inArgs.TableInfo, args.TableInfo) + require.EqualValues(t, inArgs.FKCheck, args.FKCheck) + } + }) + t.Run("create view", func(t *testing.T) { + inArgs := &CreateTableArgs{ + TableInfo: &TableInfo{ID: 122}, + OnExistReplace: true, + OldViewTblID: 123, + } + for _, v := range []JobVersion{JobVersion1, JobVersion2} { + j2 := &Job{} + require.NoError(t, j2.Decode(getJobBytes(t, inArgs, v, ActionCreateView))) + args, err := GetCreateTableArgs(j2) + require.NoError(t, err) + require.EqualValues(t, inArgs.TableInfo, args.TableInfo) + require.EqualValues(t, inArgs.OnExistReplace, args.OnExistReplace) + require.EqualValues(t, inArgs.OldViewTblID, args.OldViewTblID) + } + }) + t.Run("create sequence", func(t *testing.T) { + inArgs := &CreateTableArgs{ + TableInfo: &TableInfo{ID: 22}, + } + for _, v := range []JobVersion{JobVersion1, JobVersion2} { + j2 := &Job{} + require.NoError(t, j2.Decode(getJobBytes(t, inArgs, v, ActionCreateSequence))) + args, err := GetCreateTableArgs(j2) + require.NoError(t, err) + require.EqualValues(t, inArgs.TableInfo, args.TableInfo) + } + }) +} + +func TestBatchCreateTableArgs(t *testing.T) { + inArgs := &BatchCreateTableArgs{ + Tables: []*CreateTableArgs{ + {TableInfo: &TableInfo{ID: 100}, FKCheck: true}, + {TableInfo: &TableInfo{ID: 101}, FKCheck: false}, + }, + } + // in job version 1, we only save one FKCheck value for all tables. + j2 := &Job{} + require.NoError(t, j2.Decode(getJobBytes(t, inArgs, JobVersion1, ActionCreateTables))) + args, err := GetBatchCreateTableArgs(j2) + require.NoError(t, err) + for i := 0; i < len(inArgs.Tables); i++ { + require.EqualValues(t, inArgs.Tables[i].TableInfo, args.Tables[i].TableInfo) + require.EqualValues(t, true, args.Tables[i].FKCheck) + } + + j2 = &Job{} + require.NoError(t, j2.Decode(getJobBytes(t, inArgs, JobVersion2, ActionCreateTables))) + args, err = GetBatchCreateTableArgs(j2) + require.NoError(t, err) + require.EqualValues(t, inArgs.Tables, args.Tables) +} + func TestTruncateTableArgs(t *testing.T) { inArgs := &TruncateTableArgs{ NewTableID: 1, diff --git a/pkg/metrics/grafana/tidb_runtime.json b/pkg/metrics/grafana/tidb_runtime.json index ef5a4b8bca8ca..6e1ecb40f7f3f 100644 --- a/pkg/metrics/grafana/tidb_runtime.json +++ b/pkg/metrics/grafana/tidb_runtime.json @@ -527,7 +527,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB process Go garbage collection STW pause duration", + "description": "TiDB process Go garbage collection STW pause duration. (Deprecated in Go v1.22.0 and above)", "editable": true, "error": false, "fill": 1, @@ -1574,6 +1574,1129 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "(Available in Go v1.22.0 and above).\n\n stopping: the time from deciding to stop the world until all Ps are stopped\n\nSTW: the time from deciding to stop the world until the world is started again\n\nstopping time is a subset of the total STW time, as it only covers the period when all processors are being stopped.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 43 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:38" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.999,sum(rate(go_sched_pauses_total_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".999 GC STW", + "refId": "A", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_sched_pauses_total_gc_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s]) / rate(go_sched_pauses_total_gc_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg GC STW", + "refId": "B", + "step": 40 + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.999,sum(rate(go_sched_pauses_total_other_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".999 non-GC STW", + "refId": "C" + }, + { + "exemplar": true, + "expr": "rate(go_sched_pauses_total_other_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])/rate(go_sched_pauses_total_other_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg non-GC STW", + "refId": "D", + "step": 40 + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.999,sum(rate(go_sched_pauses_stopping_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".999 GC stopping", + "refId": "E", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_sched_pauses_stopping_gc_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s]) / rate(go_sched_pauses_stopping_gc_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg GC stopping", + "refId": "F", + "step": 40 + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.999,sum(rate(go_sched_pauses_stopping_other_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".999 non-GC stopping", + "refId": "G" + }, + { + "exemplar": true, + "expr": "rate(go_sched_pauses_stopping_other_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])/rate(go_sched_pauses_stopping_other_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg non-GC stopping", + "refId": "H", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC STW Latency(>= go1.22.0)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "This panel shows the proportion of CPU time spent on various activities, including user code execution, garbage collection, and memory scavenging. It helps in understanding how CPU resources are utilized by the Go runtime in TiDB processes.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 50 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "scopedVars": { + "instance": { + "selected": false, + "text": "127.0.0.1:10080", + "value": "127.0.0.1:10080" + } + }, + "seriesOverrides": [ + { + "$$hashKey": "object:532", + "alias": "gc_mark_assist", + "bars": true, + "lines": false, + "stack": true + }, + { + "$$hashKey": "object:834", + "alias": "gc_mark_dedicated", + "bars": true, + "lines": false, + "stack": true + }, + { + "$$hashKey": "object:851", + "alias": "gc_mark_idle", + "bars": true, + "lines": false, + "stack": true + }, + { + "$$hashKey": "object:868", + "alias": "gc_pause", + "bars": true, + "lines": false, + "stack": true + }, + { + "$$hashKey": "object:1147", + "alias": "user_total", + "bars": true, + "lines": false, + "stack": true + }, + { + "$$hashKey": "object:1164", + "alias": "scavenge_total", + "bars": true, + "lines": false, + "stack": true + }, + { + "$$hashKey": "object:1181", + "alias": "idle_total", + "bars": true, + "lines": false, + "stack": true + }, + { + "$$hashKey": "object:1366", + "alias": "gc_total", + "bars": true, + "lines": false, + "stack": true + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(go_cpu_classes_user_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "user_total", + "refId": "A", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_gc_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "gc_total", + "refId": "B", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_idle_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "idle_total", + "refId": "C", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_scavenge_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "scavenge_total", + "refId": "D", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_gc_mark_dedicated_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "gc_mark_dedicated", + "refId": "E", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_gc_mark_assist_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "gc_mark_assist", + "refId": "F", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_gc_mark_idle_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "gc_mark_idle", + "refId": "G", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_gc_pause_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "gc_pause", + "refId": "H", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_scavenge_assist_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "scavenge_assist", + "refId": "I", + "step": 40 + }, + { + "exemplar": true, + "expr": "rate(go_cpu_classes_scavenge_background_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])/rate(go_cpu_classes_total_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "scavenge_background", + "refId": "J", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "estimated portion of CPU time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1028", + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1029", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "datasource": "${DS_TEST-CLUSTER}", + "description": "auto gc: completed GC cycles generated by the Go runtime\nforced gc: completed GC cycles forced by the application\ntotal gc: all completed GC cycles\ngc limiter enabled: the number of times the GC CPU limiter has been enabled over the past", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 85, + "links": [], + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.17", + "scopedVars": { + "instance": { + "selected": true, + "text": "127.0.0.1:10080", + "value": "127.0.0.1:10080" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "increase(go_gc_cycles_automatic_gc_cycles_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "auto gc-{{instance}}", + "refId": "A", + "step": 40 + }, + { + "exemplar": true, + "expr": "increase(go_gc_cycles_forced_gc_cycles_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "forced gc-{{instance}}", + "refId": "B", + "step": 40 + }, + { + "exemplar": true, + "expr": "increase(go_gc_cycles_total_gc_cycles_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total gc-{{instance}}", + "refId": "C", + "step": 40 + }, + { + "exemplar": true, + "expr": "changes(go_gc_limiter_last_enabled_gc_cycle{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "gc limiter enabled-{{instance}}", + "refId": "D", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "golang GC", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Approximate cumulative time goroutines have spent blocked on a sync.Mutex or sync.RWMutex.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:38" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "irate(go_sync_mutex_wait_total_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "mutex_wait_seconds-{{instance}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "sync mutex wait", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Value of GOGC and GOMEMLIMIT", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 87, + "links": [], + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.17", + "scopedVars": { + "instance": { + "selected": true, + "text": "127.0.0.1:10080", + "value": "127.0.0.1:10080" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "go_gc_gogc_percent{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "GOGC-{{instance}}", + "refId": "A", + "step": 40 + }, + { + "exemplar": true, + "expr": "go_gc_gomemlimit_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "GOMEMLIMIT-{{instance}}", + "refId": "B", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "GOGC & GOMEMLIMIT", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "alloc-objects-total" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "none" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 107, + "links": [], + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.17", + "repeatDirection": "h", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(go_gc_heap_allocs_by_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".999 alloc-by-size", + "refId": "A" + }, + { + "exemplar": true, + "expr": "increase(go_gc_heap_allocs_by_size_bytes_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])/increase(go_gc_heap_allocs_by_size_bytes_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "avg alloc-by-size", + "refId": "B" + }, + { + "exemplar": true, + "expr": "increase(go_gc_heap_allocs_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "alloc-bytes-total", + "refId": "C" + }, + { + "exemplar": true, + "expr": "increase(go_gc_heap_allocs_objects_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "alloc-objects-total", + "refId": "D" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "heap alloc", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "free-objects-total" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "none" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 108, + "links": [], + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.17", + "repeatDirection": "h", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(go_gc_heap_frees_by_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".999 free-by-size", + "refId": "A" + }, + { + "exemplar": true, + "expr": "increase(go_gc_heap_frees_by_size_bytes_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])/increase(go_gc_heap_frees_by_size_bytes_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "hide": false, + "interval": "", + "legendFormat": "avg free-by-size", + "refId": "B" + }, + { + "exemplar": true, + "expr": "increase(go_gc_heap_frees_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "free-bytes-total", + "refId": "C" + }, + { + "exemplar": true, + "expr": "increase(go_gc_heap_frees_objects_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "free-objects-total", + "refId": "D" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "heap free", + "type": "timeseries" } ], "repeat": "instance", diff --git a/pkg/param/BUILD.bazel b/pkg/param/BUILD.bazel index 1b2204b5e7f45..5434815a2d029 100644 --- a/pkg/param/BUILD.bazel +++ b/pkg/param/BUILD.bazel @@ -7,10 +7,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/errno", - "//pkg/expression", - "//pkg/parser/mysql", - "//pkg/types", "//pkg/util/dbterror", - "//pkg/util/hack", ], ) diff --git a/pkg/param/binary_params.go b/pkg/param/binary_params.go index 3cd7b31973a83..0a3d5197bb49a 100644 --- a/pkg/param/binary_params.go +++ b/pkg/param/binary_params.go @@ -15,19 +15,12 @@ package param import ( - "encoding/binary" - "fmt" - "math" - "github.com/pingcap/tidb/pkg/errno" - "github.com/pingcap/tidb/pkg/expression" - "github.com/pingcap/tidb/pkg/parser/mysql" - "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/hack" ) -var errUnknownFieldType = dbterror.ClassServer.NewStd(errno.ErrUnknownFieldType) +// ErrUnknownFieldType is returned when datatype of the binary param is unknown +var ErrUnknownFieldType = dbterror.ClassServer.NewStd(errno.ErrUnknownFieldType) // BinaryParam stores the information decoded from the binary protocol // It can be further parsed into `expression.Expression` through the `ExecArgs` function in this package @@ -37,241 +30,3 @@ type BinaryParam struct { IsNull bool Val []byte } - -// ExecArgs parse execute arguments to datum slice. -func ExecArgs(typectx types.Context, binaryParams []BinaryParam) (params []expression.Expression, err error) { - var ( - tmp any - ) - - params = make([]expression.Expression, len(binaryParams)) - args := make([]types.Datum, len(binaryParams)) - for i := 0; i < len(args); i++ { - tp := binaryParams[i].Tp - isUnsigned := binaryParams[i].IsUnsigned - - switch tp { - case mysql.TypeNull: - var nilDatum types.Datum - nilDatum.SetNull() - args[i] = nilDatum - continue - - case mysql.TypeTiny: - if isUnsigned { - args[i] = types.NewUintDatum(uint64(binaryParams[i].Val[0])) - } else { - args[i] = types.NewIntDatum(int64(int8(binaryParams[i].Val[0]))) - } - continue - - case mysql.TypeShort, mysql.TypeYear: - valU16 := binary.LittleEndian.Uint16(binaryParams[i].Val) - if isUnsigned { - args[i] = types.NewUintDatum(uint64(valU16)) - } else { - args[i] = types.NewIntDatum(int64(int16(valU16))) - } - continue - - case mysql.TypeInt24, mysql.TypeLong: - valU32 := binary.LittleEndian.Uint32(binaryParams[i].Val) - if isUnsigned { - args[i] = types.NewUintDatum(uint64(valU32)) - } else { - args[i] = types.NewIntDatum(int64(int32(valU32))) - } - continue - - case mysql.TypeLonglong: - valU64 := binary.LittleEndian.Uint64(binaryParams[i].Val) - if isUnsigned { - args[i] = types.NewUintDatum(valU64) - } else { - args[i] = types.NewIntDatum(int64(valU64)) - } - continue - - case mysql.TypeFloat: - args[i] = types.NewFloat32Datum(math.Float32frombits(binary.LittleEndian.Uint32(binaryParams[i].Val))) - continue - - case mysql.TypeDouble: - args[i] = types.NewFloat64Datum(math.Float64frombits(binary.LittleEndian.Uint64(binaryParams[i].Val))) - continue - - case mysql.TypeDate, mysql.TypeTimestamp, mysql.TypeDatetime: - switch len(binaryParams[i].Val) { - case 0: - tmp = types.ZeroDatetimeStr - case 4: - _, tmp = binaryDate(0, binaryParams[i].Val) - case 7: - _, tmp = binaryDateTime(0, binaryParams[i].Val) - case 11: - _, tmp = binaryTimestamp(0, binaryParams[i].Val) - case 13: - _, tmp = binaryTimestampWithTZ(0, binaryParams[i].Val) - default: - err = mysql.ErrMalformPacket - return - } - // TODO: generate the time datum directly - var parseTime func(types.Context, string) (types.Time, error) - switch tp { - case mysql.TypeDate: - parseTime = types.ParseDate - case mysql.TypeDatetime: - parseTime = types.ParseDatetime - case mysql.TypeTimestamp: - // To be compatible with MySQL, even the type of parameter is - // TypeTimestamp, the return type should also be `Datetime`. - parseTime = types.ParseDatetime - } - var time types.Time - time, err = parseTime(typectx, tmp.(string)) - err = typectx.HandleTruncate(err) - if err != nil { - return - } - args[i] = types.NewDatum(time) - continue - - case mysql.TypeDuration: - fsp := 0 - switch len(binaryParams[i].Val) { - case 0: - tmp = "0" - case 8: - isNegative := binaryParams[i].Val[0] - if isNegative > 1 { - err = mysql.ErrMalformPacket - return - } - _, tmp = binaryDuration(1, binaryParams[i].Val, isNegative) - case 12: - isNegative := binaryParams[i].Val[0] - if isNegative > 1 { - err = mysql.ErrMalformPacket - return - } - _, tmp = binaryDurationWithMS(1, binaryParams[i].Val, isNegative) - fsp = types.MaxFsp - default: - err = mysql.ErrMalformPacket - return - } - // TODO: generate the duration datum directly - var dur types.Duration - dur, _, err = types.ParseDuration(typectx, tmp.(string), fsp) - err = typectx.HandleTruncate(err) - if err != nil { - return - } - args[i] = types.NewDatum(dur) - continue - case mysql.TypeNewDecimal: - if binaryParams[i].IsNull { - args[i] = types.NewDecimalDatum(nil) - } else { - var dec types.MyDecimal - err = typectx.HandleTruncate(dec.FromString(binaryParams[i].Val)) - if err != nil { - return nil, err - } - args[i] = types.NewDecimalDatum(&dec) - } - continue - case mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: - if binaryParams[i].IsNull { - args[i] = types.NewBytesDatum(nil) - } else { - args[i] = types.NewBytesDatum(binaryParams[i].Val) - } - continue - case mysql.TypeUnspecified, mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, - mysql.TypeEnum, mysql.TypeSet, mysql.TypeGeometry, mysql.TypeBit: - if !binaryParams[i].IsNull { - tmp = string(hack.String(binaryParams[i].Val)) - } else { - tmp = nil - } - args[i] = types.NewDatum(tmp) - continue - default: - err = errUnknownFieldType.GenWithStack("stmt unknown field type %d", tp) - return - } - } - - for i := range params { - ft := new(types.FieldType) - types.InferParamTypeFromUnderlyingValue(args[i].GetValue(), ft) - params[i] = &expression.Constant{Value: args[i], RetType: ft} - } - return -} - -func binaryDate(pos int, paramValues []byte) (int, string) { - year := binary.LittleEndian.Uint16(paramValues[pos : pos+2]) - pos += 2 - month := paramValues[pos] - pos++ - day := paramValues[pos] - pos++ - return pos, fmt.Sprintf("%04d-%02d-%02d", year, month, day) -} - -func binaryDateTime(pos int, paramValues []byte) (int, string) { - pos, date := binaryDate(pos, paramValues) - hour := paramValues[pos] - pos++ - minute := paramValues[pos] - pos++ - second := paramValues[pos] - pos++ - return pos, fmt.Sprintf("%s %02d:%02d:%02d", date, hour, minute, second) -} - -func binaryTimestamp(pos int, paramValues []byte) (int, string) { - pos, dateTime := binaryDateTime(pos, paramValues) - microSecond := binary.LittleEndian.Uint32(paramValues[pos : pos+4]) - pos += 4 - return pos, fmt.Sprintf("%s.%06d", dateTime, microSecond) -} - -func binaryTimestampWithTZ(pos int, paramValues []byte) (int, string) { - pos, timestamp := binaryTimestamp(pos, paramValues) - tzShiftInMin := int16(binary.LittleEndian.Uint16(paramValues[pos : pos+2])) - tzShiftHour := tzShiftInMin / 60 - tzShiftAbsMin := tzShiftInMin % 60 - if tzShiftAbsMin < 0 { - tzShiftAbsMin = -tzShiftAbsMin - } - pos += 2 - return pos, fmt.Sprintf("%s%+02d:%02d", timestamp, tzShiftHour, tzShiftAbsMin) -} - -func binaryDuration(pos int, paramValues []byte, isNegative uint8) (int, string) { - sign := "" - if isNegative == 1 { - sign = "-" - } - days := binary.LittleEndian.Uint32(paramValues[pos : pos+4]) - pos += 4 - hours := paramValues[pos] - pos++ - minutes := paramValues[pos] - pos++ - seconds := paramValues[pos] - pos++ - return pos, fmt.Sprintf("%s%d %02d:%02d:%02d", sign, days, hours, minutes, seconds) -} - -func binaryDurationWithMS(pos int, paramValues []byte, - isNegative uint8) (int, string) { - pos, dur := binaryDuration(pos, paramValues, isNegative) - microSecond := binary.LittleEndian.Uint32(paramValues[pos : pos+4]) - pos += 4 - return pos, fmt.Sprintf("%s.%06d", dur, microSecond) -} diff --git a/pkg/planner/cascades/memo/BUILD.bazel b/pkg/planner/cascades/memo/BUILD.bazel new file mode 100644 index 0000000000000..88670682bca8a --- /dev/null +++ b/pkg/planner/cascades/memo/BUILD.bazel @@ -0,0 +1,39 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "memo", + srcs = [ + "group.go", + "group_expr.go", + "group_id_generator.go", + "memo.go", + ], + importpath = "github.com/pingcap/tidb/pkg/planner/cascades/memo", + visibility = ["//visibility:public"], + deps = [ + "//pkg/planner/cascades/base", + "//pkg/planner/core/base", + "//pkg/planner/pattern", + "//pkg/planner/property", + "//pkg/sessionctx", + "//pkg/util/intest", + ], +) + +go_test( + name = "memo_test", + timeout = "short", + srcs = [ + "group_and_expr_test.go", + "group_id_generator_test.go", + ], + embed = [":memo"], + flaky = True, + shard_count = 3, + deps = [ + "//pkg/expression", + "//pkg/planner/cascades/base", + "//pkg/planner/core/operator/logicalop", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/planner/cascades/memo/group.go b/pkg/planner/cascades/memo/group.go new file mode 100644 index 0000000000000..c5a387a1fc861 --- /dev/null +++ b/pkg/planner/cascades/memo/group.go @@ -0,0 +1,115 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memo + +import ( + "container/list" + + "github.com/pingcap/tidb/pkg/planner/cascades/base" + "github.com/pingcap/tidb/pkg/planner/pattern" + "github.com/pingcap/tidb/pkg/planner/property" +) + +var _ base.HashEquals = &Group{} + +// Group is basic infra to store all the logically equivalent expressions +// for one logical operator in current context. +type Group struct { + // groupID indicates the uniqueness of this group, also for encoding. + groupID GroupID + + // logicalExpressions indicates the logical equiv classes for this group. + logicalExpressions *list.List + + // operand2FirstExpr is used to locate to the first same type logical expression + // in list above instead of traverse them all. + operand2FirstExpr map[pattern.Operand]*list.Element + + // hash2GroupExpr is used to de-duplication in the list. + hash2GroupExpr map[uint64]*list.Element + + // logicalProp indicates the logical property. + logicalProp *property.LogicalProperty + + // explored indicates whether this group has been explored. + explored bool +} + +// ******************************************* start of HashEqual methods ******************************************* + +// Hash64 implements the HashEquals.<0th> interface. +func (g *Group) Hash64(h base.Hasher) { + h.HashUint64(uint64(g.groupID)) +} + +// Equals implements the HashEquals.<1st> interface. +func (g *Group) Equals(other any) bool { + if other == nil { + return false + } + switch x := other.(type) { + case *Group: + return g.groupID == x.groupID + case Group: + return g.groupID == x.groupID + default: + return false + } +} + +// ******************************************* end of HashEqual methods ******************************************* + +// Exists checks whether a Group expression existed in a Group. +func (g *Group) Exists(hash64u uint64) bool { + _, ok := g.hash2GroupExpr[hash64u] + return ok +} + +// Insert adds a GroupExpression to the Group. +func (g *Group) Insert(e *GroupExpression) bool { + if e == nil { + return false + } + // GroupExpressions hash should be initialized within Init(xxx) method. + hash64 := e.Sum64() + if g.Exists(hash64) { + return false + } + operand := pattern.GetOperand(e.logicalPlan) + var newEquiv *list.Element + mark, ok := g.operand2FirstExpr[operand] + if ok { + // cluster same operands together. + newEquiv = g.logicalExpressions.InsertAfter(e, mark) + } else { + // otherwise, put it at the end. + newEquiv = g.logicalExpressions.PushBack(e) + g.operand2FirstExpr[operand] = newEquiv + } + g.hash2GroupExpr[hash64] = newEquiv + e.group = g + return true +} + +// NewGroup creates a new Group with given logical prop. +func NewGroup(prop *property.LogicalProperty) *Group { + g := &Group{ + logicalExpressions: list.New(), + hash2GroupExpr: make(map[uint64]*list.Element), + operand2FirstExpr: make(map[pattern.Operand]*list.Element), + logicalProp: prop, + } + return g +} diff --git a/pkg/planner/cascades/memo/group_and_expr_test.go b/pkg/planner/cascades/memo/group_and_expr_test.go new file mode 100644 index 0000000000000..4fd379964cbda --- /dev/null +++ b/pkg/planner/cascades/memo/group_and_expr_test.go @@ -0,0 +1,75 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memo + +import ( + "testing" + + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/planner/cascades/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" + "github.com/stretchr/testify/require" +) + +func TestGroupHashEquals(t *testing.T) { + hasher1 := base.NewHashEqualer() + hasher2 := base.NewHashEqualer() + a := Group{groupID: 1} + b := Group{groupID: 1} + a.Hash64(hasher1) + b.Hash64(hasher2) + require.Equal(t, hasher1.Sum64(), hasher2.Sum64()) + require.True(t, a.Equals(b)) + require.True(t, a.Equals(&b)) + + // change the id. + b.groupID = 2 + hasher2.Reset() + b.Hash64(hasher2) + require.NotEqual(t, hasher1.Sum64(), hasher2.Sum64()) + require.False(t, a.Equals(b)) + require.False(t, a.Equals(&b)) +} + +func TestGroupExpressionHashEquals(t *testing.T) { + hasher1 := base.NewHashEqualer() + hasher2 := base.NewHashEqualer() + child1 := &Group{groupID: 1} + child2 := &Group{groupID: 2} + a := GroupExpression{ + group: &Group{groupID: 3}, + inputs: []*Group{child1, child2}, + logicalPlan: &logicalop.LogicalProjection{Exprs: []expression.Expression{expression.NewOne()}}, + } + b := GroupExpression{ + // root group should change the hash. + group: &Group{groupID: 4}, + inputs: []*Group{child1, child2}, + logicalPlan: &logicalop.LogicalProjection{Exprs: []expression.Expression{expression.NewOne()}}, + } + a.Hash64(hasher1) + b.Hash64(hasher2) + require.Equal(t, hasher1.Sum64(), hasher2.Sum64()) + require.True(t, a.Equals(b)) + require.True(t, a.Equals(&b)) + + // change the children order, like join commutative. + b.inputs = []*Group{child2, child1} + hasher2.Reset() + b.Hash64(hasher2) + require.NotEqual(t, hasher1.Sum64(), hasher2.Sum64()) + require.False(t, a.Equals(b)) + require.False(t, a.Equals(&b)) +} diff --git a/pkg/planner/cascades/memo/group_expr.go b/pkg/planner/cascades/memo/group_expr.go new file mode 100644 index 0000000000000..97ca867ad1fef --- /dev/null +++ b/pkg/planner/cascades/memo/group_expr.go @@ -0,0 +1,106 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memo + +import ( + base2 "github.com/pingcap/tidb/pkg/planner/cascades/base" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/pattern" + "github.com/pingcap/tidb/pkg/util/intest" +) + +// GroupExpression is a single expression from the equivalent list classes inside a group. +// it is a node in the expression tree, while it takes groups as inputs. This kind of loose +// coupling between Group and GroupExpression is the key to the success of the memory compact +// of representing a forest. +type GroupExpression struct { + // group is the Group that this GroupExpression belongs to. + group *Group + + // inputs stores the Groups that this GroupExpression based on. + inputs []*Group + + // logicalPlan is internal logical expression stands for this groupExpr. + logicalPlan base.LogicalPlan + + // hash64 is the unique fingerprint of the GroupExpression. + hash64 uint64 +} + +// Sum64 returns the cached hash64 of the GroupExpression. +func (e *GroupExpression) Sum64() uint64 { + intest.Assert(e.hash64 != 0, "hash64 should not be 0") + return e.hash64 +} + +// Hash64 implements the Hash64 interface. +func (e *GroupExpression) Hash64(h base2.Hasher) { + // logical plan hash. + e.logicalPlan.Hash64(h) + // children group hash. + for _, child := range e.inputs { + child.Hash64(h) + } +} + +// Equals implements the Equals interface. +func (e *GroupExpression) Equals(other any) bool { + if other == nil { + return false + } + var e2 *GroupExpression + switch x := other.(type) { + case *GroupExpression: + e2 = x + case GroupExpression: + e2 = &x + default: + return false + } + if len(e.inputs) != len(e2.inputs) { + return false + } + if pattern.GetOperand(e.logicalPlan) != pattern.GetOperand(e2.logicalPlan) { + return false + } + // current logical operator meta cmp, logical plan don't care logicalPlan's children. + // when we convert logicalPlan to GroupExpression, we will set children to nil. + if !e.logicalPlan.Equals(e2.logicalPlan) { + return false + } + // if one of the children is different, then the two GroupExpressions are different. + for i, one := range e.inputs { + if !one.Equals(e2.inputs[i]) { + return false + } + } + return true +} + +// NewGroupExpression creates a new GroupExpression with the given logical plan and children. +func NewGroupExpression(lp base.LogicalPlan, inputs []*Group) *GroupExpression { + return &GroupExpression{ + group: nil, + inputs: inputs, + logicalPlan: lp, + hash64: 0, + } +} + +// Init initializes the GroupExpression with the given group and hasher. +func (e *GroupExpression) Init(h base2.Hasher) { + e.Hash64(h) + e.hash64 = h.Sum64() +} diff --git a/pkg/planner/cascades/memo/group_id_generator.go b/pkg/planner/cascades/memo/group_id_generator.go new file mode 100644 index 0000000000000..32aa90bfefc1c --- /dev/null +++ b/pkg/planner/cascades/memo/group_id_generator.go @@ -0,0 +1,30 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memo + +// GroupID is the unique id for a group. +type GroupID uint64 + +// GroupIDGenerator is used to generate group id. +type GroupIDGenerator struct { + id uint64 +} + +// NextGroupID generates the next group id. +// It is not thread-safe, since memo optimizing is also in one thread. +func (gi *GroupIDGenerator) NextGroupID() GroupID { + gi.id++ + return GroupID(gi.id) +} diff --git a/pkg/planner/cascades/memo/group_id_generator_test.go b/pkg/planner/cascades/memo/group_id_generator_test.go new file mode 100644 index 0000000000000..fb2a644ca3a8b --- /dev/null +++ b/pkg/planner/cascades/memo/group_id_generator_test.go @@ -0,0 +1,48 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memo + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGroupIDGenerator_NextGroupID(t *testing.T) { + g := GroupIDGenerator{} + got := g.NextGroupID() + require.Equal(t, GroupID(1), got) + got = g.NextGroupID() + require.Equal(t, GroupID(2), got) + got = g.NextGroupID() + require.Equal(t, GroupID(3), got) + + // adjust the id. + g.id = 100 + got = g.NextGroupID() + require.Equal(t, GroupID(101), got) + got = g.NextGroupID() + require.Equal(t, GroupID(102), got) + got = g.NextGroupID() + require.Equal(t, GroupID(103), got) + + g.id = math.MaxUint64 + got = g.NextGroupID() + // rewire to 0. + require.Equal(t, GroupID(0), got) + got = g.NextGroupID() + require.Equal(t, GroupID(1), got) +} diff --git a/pkg/planner/cascades/memo/memo.go b/pkg/planner/cascades/memo/memo.go new file mode 100644 index 0000000000000..89758ee0e63e2 --- /dev/null +++ b/pkg/planner/cascades/memo/memo.go @@ -0,0 +1,107 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memo + +import ( + "container/list" + "sync" + + base2 "github.com/pingcap/tidb/pkg/planner/cascades/base" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/util/intest" +) + +// Memo is the main structure of the memo package. +type Memo struct { + // ctx is the context of the memo. + sCtx sessionctx.Context + + // groupIDGen is the incremental group id for internal usage. + groupIDGen GroupIDGenerator + + // rootGroup is the root group of the memo. + rootGroup *Group + + // groups is the list of all groups in the memo. + groups *list.List + + // groupID2Group is the map from group id to group. + groupID2Group map[GroupID]*list.Element + + // hash2GroupExpr is the map from hash to group expression. + hash2GroupExpr map[uint64]*list.Element + + // hasherPool is the pool of hasher. + hasherPool *sync.Pool +} + +// NewMemo creates a new memo. +func NewMemo(ctx sessionctx.Context) *Memo { + return &Memo{ + sCtx: ctx, + groupIDGen: GroupIDGenerator{id: 0}, + groups: list.New(), + groupID2Group: make(map[GroupID]*list.Element), + hasherPool: &sync.Pool{New: func() any { return base2.NewHashEqualer() }}, + } +} + +// CopyIn copies a logical plan into the memo with format as GroupExpression. +func (m *Memo) CopyIn(target *Group, lp base.LogicalPlan) (*GroupExpression, bool) { + // Group the children first. + childGroups := make([]*Group, 0, len(lp.Children())) + for _, child := range lp.Children() { + // todo: child.getGroupExpression.GetGroup directly + groupExpr, ok := m.CopyIn(nil, child) + group := groupExpr.group + intest.Assert(ok) + intest.Assert(group != nil) + intest.Assert(group != target) + childGroups = append(childGroups, group) + } + + hasher := m.hasherPool.Get().(base2.Hasher) + hasher.Reset() + groupExpr := NewGroupExpression(lp, childGroups) + groupExpr.Init(hasher) + m.hasherPool.Put(hasher) + + ok := m.insertGroupExpression(groupExpr, target) + // todo: new group need to derive the logical property. + return groupExpr, ok +} + +// @bool indicates whether the groupExpr is inserted to a new group. +func (m *Memo) insertGroupExpression(groupExpr *GroupExpression, target *Group) bool { + // for group merge, here groupExpr is the new groupExpr with undetermined belonged group. + // we need to use groupExpr hash to find whether there is same groupExpr existed before. + // if existed and the existed groupExpr.Group is not same with target, we should merge them up. + // todo: merge group + if target == nil { + target = m.NewGroup() + m.groups.PushBack(target) + m.groupID2Group[target.groupID] = m.groups.Back() + } + target.Insert(groupExpr) + return true +} + +// NewGroup creates a new group. +func (m *Memo) NewGroup() *Group { + group := NewGroup(nil) + group.groupID = m.groupIDGen.NextGroupID() + return group +} diff --git a/pkg/planner/core/fragment.go b/pkg/planner/core/fragment.go index 014dd3187c95c..5fa55c7d44436 100644 --- a/pkg/planner/core/fragment.go +++ b/pkg/planner/core/fragment.go @@ -23,6 +23,7 @@ import ( "unsafe" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/distsql" "github.com/pingcap/tidb/pkg/expression" @@ -107,21 +108,23 @@ type mppTaskGenerator struct { // For MPPGather under UnionScan, need keyRange to scan MemBuffer. KVRanges []kv.KeyRange - nodeInfo map[string]bool + nodeInfo map[string]bool + tableReaderCache map[string][]kv.MPPTaskMeta // cache for table reader } // GenerateRootMPPTasks generate all mpp tasks and return root ones. func GenerateRootMPPTasks(ctx sessionctx.Context, startTs uint64, mppGatherID uint64, mppQueryID kv.MPPQueryID, sender *PhysicalExchangeSender, is infoschema.InfoSchema) ([]*Fragment, []kv.KeyRange, map[string]bool, error) { g := &mppTaskGenerator{ - ctx: ctx, - gatherID: mppGatherID, - startTS: startTs, - mppQueryID: mppQueryID, - is: is, - cache: make(map[int]tasksAndFrags), - KVRanges: make([]kv.KeyRange, 0), - nodeInfo: make(map[string]bool), + ctx: ctx, + gatherID: mppGatherID, + startTS: startTs, + mppQueryID: mppQueryID, + is: is, + cache: make(map[int]tasksAndFrags), + KVRanges: make([]kv.KeyRange, 0), + nodeInfo: make(map[string]bool), + tableReaderCache: make(map[string][]kv.MPPTaskMeta), } frags, err := g.generateMPPTasks(sender) if err != nil { @@ -587,9 +590,18 @@ func (e *mppTaskGenerator) constructMPPTasksImpl(ctx context.Context, ts *Physic ttl = time.Duration(0) } tiflashReplicaRead := e.ctx.GetSessionVars().TiFlashReplicaRead - metas, err := e.ctx.GetMPPClient().ConstructMPPTasks(ctx, req, ttl, dispatchPolicy, tiflashReplicaRead, e.ctx.GetSessionVars().StmtCtx.AppendWarning) - if err != nil { - return nil, errors.Trace(err) + + var metas []kv.MPPTaskMeta + if val := req.ToString(); e.tableReaderCache[val] != nil { + metas = e.tableReaderCache[val] + failpoint.InjectCall("mppTaskGeneratorTableReaderCacheHit") + } else { + metas, err = e.ctx.GetMPPClient().ConstructMPPTasks(ctx, req, ttl, dispatchPolicy, tiflashReplicaRead, e.ctx.GetSessionVars().StmtCtx.AppendWarning) + if err != nil { + return nil, errors.Trace(err) + } + e.tableReaderCache[val] = metas + failpoint.InjectCall("mppTaskGeneratorTableReaderCacheMiss") } mppVersion := e.ctx.GetSessionVars().ChooseMppVersion() diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index ac76eb9fe37ea..d48cee0af1fca 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -5183,6 +5183,8 @@ type TblColPosInfo struct { Start, End int // HandleOrdinal represents the ordinal of the handle column. HandleCols util.HandleCols + + IndexesForDelete table.IndexesLayout } // MemoryUsage return the memory usage of TblColPosInfo @@ -5230,35 +5232,112 @@ func (c TblColPosInfoSlice) FindTblIdx(colOrdinal int) (int, bool) { return rangeBehindOrdinal - 1, true } -// buildColumns2Handle builds columns to handle mapping. -func buildColumns2Handle( +// buildColumns2HandleWithWrtiableColumns builds columns to handle mapping. +// This func is called by Update and can only see writable columns. +func buildColumns2HandleWithWrtiableColumns( names []*types.FieldName, tblID2Handle map[int64][]util.HandleCols, tblID2Table map[int64]table.Table, - onlyWritableCol bool, ) (TblColPosInfoSlice, error) { var cols2Handles TblColPosInfoSlice for tblID, handleCols := range tblID2Handle { tbl := tblID2Table[tblID] - var tblLen int - if onlyWritableCol { - tblLen = len(tbl.WritableCols()) - } else { - tblLen = len(tbl.Cols()) - } + tblLen := len(tbl.WritableCols()) for _, handleCol := range handleCols { offset, err := getTableOffset(names, names[handleCol.GetCol(0).Index]) if err != nil { return nil, err } end := offset + tblLen - cols2Handles = append(cols2Handles, TblColPosInfo{tblID, offset, end, handleCol}) + cols2Handles = append(cols2Handles, TblColPosInfo{TblID: tblID, Start: offset, End: end, HandleCols: handleCol}) } } sort.Sort(cols2Handles) return cols2Handles, nil } +// buildColPositionInfoForDelete builds columns to handle mapping for delete. +// We'll have two kinds of columns seen by DELETE: +// 1. The columns that are public. They are the columns that not affected by any DDL. +// 2. The columns that are not public. They are the columns that are affected by DDL. +// But we need them because the non-public indexes may rely on them. +// +// The two kind of columns forms the whole columns of the table. Public part first. +// This function records the following things: +// 1. The position of the columns used by indexes in the DELETE's select's output. +// 2. The row id's position in the output. +func buildColPositionInfoForDelete( + names []*types.FieldName, + tblID2Handle map[int64][]util.HandleCols, + tblID2Table map[int64]table.Table, +) (TblColPosInfoSlice, error) { + var cols2PosInfos TblColPosInfoSlice + for tblID, handleCols := range tblID2Handle { + tbl := tblID2Table[tblID] + deletableIdxs := tbl.DeletableIndices() + deletableCols := tbl.DeletableCols() + tblInfo := tbl.Meta() + + for _, handleCol := range handleCols { + curColPosInfo, err := buildSingleTableColPosInfoForDelete(names, handleCol, deletableIdxs, deletableCols, tblInfo) + if err != nil { + return nil, err + } + cols2PosInfos = append(cols2PosInfos, curColPosInfo) + } + } + sort.Sort(cols2PosInfos) + return cols2PosInfos, nil +} + +// buildSingleTableColPosInfoForDelete builds columns mapping for delete. +func buildSingleTableColPosInfoForDelete( + names []*types.FieldName, + handleCol util.HandleCols, + deletableIdxs []table.Index, + deletableCols []*table.Column, + tblInfo *model.TableInfo, +) (TblColPosInfo, error) { + // Columns can be seen by DELETE are the deletable columns. + tblLen := len(deletableCols) + offset, err := getTableOffset(names, names[handleCol.GetCol(0).Index]) + if err != nil { + return TblColPosInfo{}, err + } + end := offset + tblLen + + // Index only records its columns' offsets in the deletableCols(the whole columns). + // So we need to first change the offsets to the real position of SELECT's output. + offsetMap := make(map[int]int, len(deletableCols)) + // For multi-delete case, the offset is recorded for the mixed row. We need to use its position in single table's row layout. + // e.g. The multi-delete case is [t1.a, t1.b, t2.a, t2.b] and There's a index [t2.b]. + // The idxCols' original offset is [3]. And we should use [1] instead. + for i, name := range names[offset:end] { + found := false + for j, col := range deletableCols { + if col.Name.L == name.ColName.L { + offsetMap[j] = i + found = true + break + } + } + if !found { + return TblColPosInfo{}, plannererrors.ErrDeleteNotFoundColumn.GenWithStackByArgs(name.ColName.O, tblInfo.Name.O) + } + } + indexColMap := make(map[int64]table.IndexRowLayoutOption, len(deletableIdxs)) + for _, idx := range deletableIdxs { + idxCols := idx.Meta().Columns + colPos := make([]int, 0, len(idxCols)) + for _, col := range idxCols { + colPos = append(colPos, offsetMap[col.Offset]) + } + indexColMap[idx.Meta().ID] = colPos + } + + return TblColPosInfo{tblInfo.ID, offset, end, handleCol, indexColMap}, nil +} + func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) (base.Plan, error) { b.pushSelectOffset(0) b.pushTableHints(update.TableHints, 0) @@ -5375,7 +5454,7 @@ func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) ( for id := range tblID2Handle { tblID2table[id], _ = b.is.TableByID(ctx, id) } - updt.TblColPosInfos, err = buildColumns2Handle(updt.OutputNames(), tblID2Handle, tblID2table, true) + updt.TblColPosInfos, err = buildColumns2HandleWithWrtiableColumns(updt.OutputNames(), tblID2Handle, tblID2table) if err != nil { return nil, err } @@ -5850,7 +5929,7 @@ func (b *PlanBuilder) buildDelete(ctx context.Context, ds *ast.DeleteStmt) (base for id := range tblID2Handle { tblID2table[id], _ = b.is.TableByID(ctx, id) } - del.TblColPosInfos, err = buildColumns2Handle(del.names, tblID2Handle, tblID2table, false) + del.TblColPosInfos, err = buildColPositionInfoForDelete(del.names, tblID2Handle, tblID2table) if err != nil { return nil, err } diff --git a/pkg/planner/core/operator/logicalop/logical_projection.go b/pkg/planner/core/operator/logicalop/logical_projection.go index dc3d9a6b990d0..15bc95a0f9448 100644 --- a/pkg/planner/core/operator/logicalop/logical_projection.go +++ b/pkg/planner/core/operator/logicalop/logical_projection.go @@ -60,18 +60,10 @@ func (p LogicalProjection) Init(ctx base.PlanContext, qbOffset int) *LogicalProj // Hash64 implements the base.Hash64.<0th> interface. func (p *LogicalProjection) Hash64(h base2.Hasher) { - // todo: LogicalSchemaProducer should implement HashEquals interface, otherwise, its self elements - // like schema and names are lost. - p.LogicalSchemaProducer.Hash64(h) - // todo: if we change the logicalProjection's Expr definition as:Exprs []memo.ScalarOperator[any], - // we should use like below: - // for _, one := range p.Exprs { - // one.Hash64(one) - // } - // otherwise, we would use the belowing code. - //for _, one := range p.Exprs { - // one.Hash64(h) - //} + h.HashInt(len(p.Exprs)) + for _, one := range p.Exprs { + one.Hash64(h) + } h.HashBool(p.CalculateNoDelay) h.HashBool(p.Proj4Expand) } @@ -81,24 +73,24 @@ func (p *LogicalProjection) Equals(other any) bool { if other == nil { return false } - proj, ok := other.(*LogicalProjection) - if !ok { + var p2 *LogicalProjection + switch x := other.(type) { + case *LogicalProjection: + p2 = x + case LogicalProjection: + p2 = &x + default: return false } - // todo: LogicalSchemaProducer should implement HashEquals interface, otherwise, its self elements - // like schema and names are lost. - if !p.LogicalSchemaProducer.Equals(&proj.LogicalSchemaProducer) { + if len(p.Exprs) != len(p2.Exprs) { return false } - //for i, one := range p.Exprs { - // if !one.(memo.ScalarOperator[any]).Equals(other.Exprs[i]) { - // return false - // } - //} - if p.CalculateNoDelay != proj.CalculateNoDelay { - return false + for i, one := range p.Exprs { + if !one.Equals(p2.Exprs[i]) { + return false + } } - return p.Proj4Expand == proj.Proj4Expand + return p.CalculateNoDelay == p2.CalculateNoDelay && p.Proj4Expand == p2.Proj4Expand } // *************************** start implementation of Plan interface ********************************** diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index e7d4228dc5ea9..c6005381e6b0b 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -4470,6 +4470,11 @@ func (b *PlanBuilder) buildImportInto(ctx context.Context, ld *ast.ImportIntoStm } tnW := b.resolveCtx.GetTableName(ld.Table) + if tnW.TableInfo.TempTableType != model.TempTableNone { + return nil, errors.Errorf("IMPORT INTO does not support temporary table") + } else if tnW.TableInfo.TableCacheStatusType != model.TableCacheStatusDisable { + return nil, errors.Errorf("IMPORT INTO does not support cached table") + } p := ImportInto{ Path: ld.Path, Format: ld.Format, diff --git a/pkg/planner/core/point_get_plan.go b/pkg/planner/core/point_get_plan.go index 2825a4dde1508..a87be087c7ef4 100644 --- a/pkg/planner/core/point_get_plan.go +++ b/pkg/planner/core/point_get_plan.go @@ -1943,7 +1943,7 @@ func buildPointUpdatePlan(ctx base.PlanContext, pointPlan base.PhysicalPlan, dbN if orderedList == nil { return nil } - handleCols := buildHandleCols(ctx, tbl, pointPlan.Schema()) + handleCols := buildHandleCols(ctx, dbName, tbl, pointPlan) updatePlan := Update{ SelectPlan: pointPlan, OrderedList: orderedList, @@ -2064,27 +2064,25 @@ func buildPointDeletePlan(ctx base.PlanContext, pointPlan base.PhysicalPlan, dbN if checkFastPlanPrivilege(ctx, dbName, tbl.Name.L, mysql.SelectPriv, mysql.DeletePriv) != nil { return nil } - handleCols := buildHandleCols(ctx, tbl, pointPlan.Schema()) - delPlan := Delete{ - SelectPlan: pointPlan, - TblColPosInfos: TblColPosInfoSlice{ - TblColPosInfo{ - TblID: tbl.ID, - Start: 0, - End: pointPlan.Schema().Len(), - HandleCols: handleCols, - }, - }, - }.Init(ctx) + handleCols := buildHandleCols(ctx, dbName, tbl, pointPlan) var err error is := ctx.GetInfoSchema().(infoschema.InfoSchema) t, _ := is.TableByID(context.Background(), tbl.ID) - if t != nil { - tblID2Table := map[int64]table.Table{tbl.ID: t} - err = delPlan.buildOnDeleteFKTriggers(ctx, is, tblID2Table) - if err != nil { - return nil - } + intest.Assert(t != nil, "The point get executor is accessing a table without meta info.") + idxs := t.DeletableIndices() + deletableCols := t.DeletableCols() + colPosInfo, err := buildSingleTableColPosInfoForDelete(pointPlan.OutputNames(), handleCols, idxs, deletableCols, tbl) + if err != nil { + return nil + } + delPlan := Delete{ + SelectPlan: pointPlan, + TblColPosInfos: []TblColPosInfo{colPosInfo}, + }.Init(ctx) + tblID2Table := map[int64]table.Table{tbl.ID: t} + err = delPlan.buildOnDeleteFKTriggers(ctx, is, tblID2Table) + if err != nil { + return nil } return delPlan } @@ -2113,7 +2111,8 @@ func colInfoToColumn(col *model.ColumnInfo, idx int) *expression.Column { } } -func buildHandleCols(ctx base.PlanContext, tbl *model.TableInfo, schema *expression.Schema) util.HandleCols { +func buildHandleCols(ctx base.PlanContext, dbName string, tbl *model.TableInfo, pointget base.PhysicalPlan) util.HandleCols { + schema := pointget.Schema() // fields len is 0 for update and delete. if tbl.PKIsHandle { for i, col := range tbl.Columns { @@ -2130,6 +2129,18 @@ func buildHandleCols(ctx base.PlanContext, tbl *model.TableInfo, schema *express handleCol := colInfoToColumn(model.NewExtraHandleColInfo(), schema.Len()) schema.Append(handleCol) + newOutputNames := pointget.OutputNames().Shallow() + tableAliasName := tbl.Name + if schema.Len() > 0 { + tableAliasName = pointget.OutputNames()[0].TblName + } + newOutputNames = append(newOutputNames, &types.FieldName{ + DBName: pmodel.NewCIStr(dbName), + TblName: tableAliasName, + OrigTblName: tbl.Name, + ColName: model.ExtraHandleName, + }) + pointget.SetOutputNames(newOutputNames) return util.NewIntHandleCols(handleCol) } diff --git a/pkg/planner/core/rule_column_pruning.go b/pkg/planner/core/rule_column_pruning.go index 8692a854994b0..1cf25ac1bde89 100644 --- a/pkg/planner/core/rule_column_pruning.go +++ b/pkg/planner/core/rule_column_pruning.go @@ -18,7 +18,6 @@ import ( "context" "slices" - "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/mysql" @@ -41,27 +40,29 @@ func (*ColumnPruner) Optimize(_ context.Context, lp base.LogicalPlan, opt *optim if err != nil { return nil, planChanged, err } - intest.AssertNoError(noZeroColumnLayOut(lp), "After column pruning, some operator got zero row output. Please fix it.") + intest.AssertFunc(func() bool { + return noZeroColumnLayOut(lp) + }, "After column pruning, some operator got zero row output. Please fix it.") return lp, planChanged, nil } -func noZeroColumnLayOut(p base.LogicalPlan) error { +func noZeroColumnLayOut(p base.LogicalPlan) bool { for _, child := range p.Children() { - if err := noZeroColumnLayOut(child); err != nil { - return err + if success := noZeroColumnLayOut(child); !success { + return false } } if p.Schema().Len() == 0 { // The p don't hold its schema. So we don't need check itself. if len(p.Children()) > 0 && p.Schema() == p.Children()[0].Schema() { - return nil + return true } _, ok := p.(*logicalop.LogicalTableDual) if !ok { - return errors.Errorf("Operator %s has zero row output", p.ExplainID().String()) + return false } } - return nil + return true } func pruneByItems(p base.LogicalPlan, old []*util.ByItems, opt *optimizetrace.LogicalOptimizeOp) (byItems []*util.ByItems, diff --git a/pkg/resourcegroup/runaway/BUILD.bazel b/pkg/resourcegroup/runaway/BUILD.bazel index 9dcbbe78c898e..269fea00c3937 100644 --- a/pkg/resourcegroup/runaway/BUILD.bazel +++ b/pkg/resourcegroup/runaway/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "runaway", @@ -38,3 +38,12 @@ go_library( "@org_uber_go_zap//:zap", ], ) + +go_test( + name = "runaway_test", + timeout = "short", + srcs = ["record_test.go"], + embed = [":runaway"], + flaky = True, + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/pkg/resourcegroup/runaway/checker.go b/pkg/resourcegroup/runaway/checker.go index 4592f2a338405..e397a411972a4 100644 --- a/pkg/resourcegroup/runaway/checker.go +++ b/pkg/resourcegroup/runaway/checker.go @@ -300,7 +300,7 @@ func (r *Checker) markRunaway(matchType string, action rmpb.RunawayAction, switc } actionStr = strings.ToLower(actionStr) metrics.RunawayCheckerCounter.WithLabelValues(r.resourceGroupName, matchType, actionStr).Inc() - r.manager.markRunaway(r.resourceGroupName, r.originalSQL, r.planDigest, actionStr, matchType, now) + r.manager.markRunaway(r, actionStr, matchType, now) } func (r *Checker) getSettingConvictIdentifier() string { diff --git a/pkg/resourcegroup/runaway/manager.go b/pkg/resourcegroup/runaway/manager.go index 831c3db285197..5b9dee83c8231 100644 --- a/pkg/resourcegroup/runaway/manager.go +++ b/pkg/resourcegroup/runaway/manager.go @@ -43,7 +43,7 @@ const ( maxWatchListCap = 10000 maxWatchRecordChannelSize = 1024 - runawayRecordFlushInterval = time.Second + runawayRecordFlushInterval = 30 * time.Second runawayRecordGCInterval = time.Hour * 24 runawayRecordExpiredDuration = time.Hour * 24 * 7 @@ -155,17 +155,19 @@ func (rm *Manager) RunawayRecordFlushLoop() { quarantineRecordCh := rm.quarantineRecordChan() staleQuarantineRecordCh := rm.staleQuarantineRecordChan() flushThreshold := flushThreshold() - records := make([]*Record, 0, flushThreshold) + // recordMap is used to deduplicate records. + recordMap = make(map[recordKey]*Record, flushThreshold) flushRunawayRecords := func() { - if len(records) == 0 { + if len(recordMap) == 0 { return } - sql, params := genRunawayQueriesStmt(records) + sql, params := genRunawayQueriesStmt(recordMap) if _, err := ExecRCRestrictedSQL(rm.sysSessionPool, sql, params); err != nil { - logutil.BgLogger().Error("flush runaway records failed", zap.Error(err), zap.Int("count", len(records))) + logutil.BgLogger().Error("flush runaway records failed", zap.Error(err), zap.Int("count", len(recordMap))) } - records = records[:0] + // reset the map. + recordMap = make(map[recordKey]*Record, flushThreshold) } for { @@ -176,11 +178,21 @@ func (rm *Manager) RunawayRecordFlushLoop() { flushRunawayRecords() fired = true case r := <-recordCh: - records = append(records, r) + key := recordKey{ + ResourceGroupName: r.ResourceGroupName, + SQLDigest: r.SQLDigest, + PlanDigest: r.PlanDigest, + Match: r.Match, + } + if _, exists := recordMap[key]; exists { + recordMap[key].Repeats++ + } else { + recordMap[key] = r + } failpoint.Inject("FastRunawayGC", func() { flushRunawayRecords() }) - if len(records) >= flushThreshold { + if len(recordMap) >= flushThreshold { flushRunawayRecords() } else if fired { fired = false @@ -321,7 +333,7 @@ func (rm *Manager) getWatchFromWatchList(key string) *QuarantineRecord { return nil } -func (rm *Manager) markRunaway(resourceGroupName, originalSQL, planDigest, action, matchType string, now *time.Time) { +func (rm *Manager) markRunaway(checker *Checker, action, matchType string, now *time.Time) { source := rm.serverID if !rm.syncerInitialized.Load() { rm.logOnce.Do(func() { @@ -331,13 +343,16 @@ func (rm *Manager) markRunaway(resourceGroupName, originalSQL, planDigest, actio } select { case rm.runawayQueriesChan <- &Record{ - ResourceGroupName: resourceGroupName, - Time: *now, + ResourceGroupName: checker.resourceGroupName, + StartTime: *now, Match: matchType, Action: action, - SQLText: originalSQL, - PlanDigest: planDigest, + SampleText: checker.originalSQL, + SQLDigest: checker.sqlDigest, + PlanDigest: checker.planDigest, Source: source, + // default value for Repeats + Repeats: 1, }: default: // TODO: add warning for discard flush records diff --git a/pkg/resourcegroup/runaway/record.go b/pkg/resourcegroup/runaway/record.go index d8762417e22c1..7e9ce1cab51f2 100644 --- a/pkg/resourcegroup/runaway/record.go +++ b/pkg/resourcegroup/runaway/record.go @@ -17,6 +17,7 @@ package runaway import ( "context" "fmt" + "hash/fnv" "strings" "time" @@ -52,36 +53,62 @@ var NullTime time.Time // Record is used to save records which will be inserted into mysql.tidb_runaway_queries. type Record struct { ResourceGroupName string - Time time.Time + StartTime time.Time Match string Action string - SQLText string + SampleText string + SQLDigest string PlanDigest string Source string + // Repeats is used to avoid inserting the same record multiple times. + // It records the number of times after flushing the record(10s) to the table or len(map) exceeds the threshold(1024). + // We only consider `resource_group_name`, `sql_digest`, `plan_digest` and `match_type` when comparing records. + // default value is 1. + Repeats int +} + +// recordMap is used to save records which will be inserted into `mysql.tidb_runaway_queries` by function `flushRunawayRecords`. +var recordMap map[recordKey]*Record + +// recordKey represents the composite key for record key in `tidb_runaway_queries`. +type recordKey struct { + ResourceGroupName string + SQLDigest string + PlanDigest string + Match string +} + +// Hash generates a hash for the recordKey. +// Because `tidb_runaway_queries` is informational and not performance-critical, +// we can lose some accuracy for other component's performance. +func (k recordKey) Hash() uint64 { + h := fnv.New64a() + h.Write([]byte(k.ResourceGroupName)) + h.Write([]byte(k.SQLDigest)) + h.Write([]byte(k.PlanDigest)) + h.Write([]byte(k.Match)) + return h.Sum64() } // genRunawayQueriesStmt generates statement with given RunawayRecords. -func genRunawayQueriesStmt(records []*Record) (string, []any) { +func genRunawayQueriesStmt(recordMap map[recordKey]*Record) (string, []any) { var builder strings.Builder - params := make([]any, 0, len(records)*7) - builder.WriteString("insert into mysql.tidb_runaway_queries VALUES ") - for count, r := range records { - if count > 0 { + params := make([]any, 0, len(recordMap)*9) + builder.WriteString("INSERT INTO mysql.tidb_runaway_queries " + + "(resource_group_name, start_time, match_type, action, sample_sql, sql_digest, plan_digest, tidb_server, repeats) VALUES ") + firstRecord := true + for _, r := range recordMap { + if !firstRecord { builder.WriteByte(',') } - builder.WriteString("(%?, %?, %?, %?, %?, %?, %?)") - params = append(params, r.ResourceGroupName) - params = append(params, r.Time) - params = append(params, r.Match) - params = append(params, r.Action) - params = append(params, r.SQLText) - params = append(params, r.PlanDigest) - params = append(params, r.Source) + firstRecord = false + builder.WriteString("(%?, %?, %?, %?, %?, %?, %?, %?, %?)") + params = append(params, r.ResourceGroupName, r.StartTime, r.Match, r.Action, r.SampleText, r.SQLDigest, r.PlanDigest, r.Source, r.Repeats) } return builder.String(), params } -// QuarantineRecord is used to save records which will be insert into mysql.tidb_runaway_watch. +// QuarantineRecord is used to save records which will be inserted into mysql.tidb_runaway_watch. type QuarantineRecord struct { ID int64 ResourceGroupName string @@ -183,7 +210,7 @@ func (r *QuarantineRecord) genDeletionStmt() (string, []any) { func (rm *Manager) deleteExpiredRows(expiredDuration time.Duration) { const ( tableName = "tidb_runaway_queries" - colName = "time" + colName = "start_time" ) var systemSchemaCIStr = model.NewCIStr("mysql") diff --git a/pkg/resourcegroup/runaway/record_test.go b/pkg/resourcegroup/runaway/record_test.go new file mode 100644 index 0000000000000..8e51f97e4e24b --- /dev/null +++ b/pkg/resourcegroup/runaway/record_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runaway + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRecordKey(t *testing.T) { + // Initialize test data + key1 := recordKey{ + ResourceGroupName: "group1", + SQLDigest: "digest1", + PlanDigest: "plan1", + } + // key2 is identical to key1 + key2 := recordKey{ + ResourceGroupName: "group1", + SQLDigest: "digest1", + PlanDigest: "plan1", + } + key3 := recordKey{ + ResourceGroupName: "group2", + } + // Test Hash method + hash1 := key1.Hash() + hash2 := key2.Hash() + hash3 := key3.Hash() + + assert.Equal(t, hash1, hash2, "Hashes should be equal for identical keys") + assert.NotEqual(t, hash1, hash3, "Hashes should not be equal for different keys") + + // Test MapKey method + recordMap = make(map[recordKey]*Record) + record1 := &Record{ + ResourceGroupName: "group1", + SQLDigest: "digest1", + PlanDigest: "plan1", + } + // put key1 into recordMap + recordMap[key1] = record1 + assert.Len(t, recordMap, 1, "recordMap should have 1 element") + assert.Equal(t, "group1", recordMap[key1].ResourceGroupName, "Repeats should not be updated") + assert.Equal(t, 0, recordMap[key1].Repeats, "Repeats should be incremented") + // key2 is identical to key1, so we can use key2 to get the record + assert.NotNil(t, recordMap[key1], "key1 should exist in recordMap") + assert.NotNil(t, recordMap[key2], "key2 should exist in recordMap") + assert.Nil(t, recordMap[key3], "key3 should not exist in recordMap") + + // put key2 into recordMap and update Repeats + record2 := &Record{ + ResourceGroupName: "group1", + Repeats: 1, + } + recordMap[key2] = record2 + assert.Len(t, recordMap, 1, "recordMap should have 1 element") + assert.Equal(t, 1, recordMap[key1].Repeats, "Repeats should be updated") + // change ResourceGroupName of key2 will not affect key1 + key2.ResourceGroupName = "group2" + record3 := &Record{ + ResourceGroupName: "group2", + } + recordMap[key2] = record3 + assert.Len(t, recordMap, 2, "recordMap should have 1 element") + assert.Equal(t, "group1", recordMap[key1].ResourceGroupName, "Repeats should not be updated") + assert.Equal(t, "group2", recordMap[key2].ResourceGroupName, "ResourceGroupName should be updated") +} diff --git a/pkg/resourcegroup/tests/BUILD.bazel b/pkg/resourcegroup/tests/BUILD.bazel index 9ef1e1a79bbd2..ebc61d5cba148 100644 --- a/pkg/resourcegroup/tests/BUILD.bazel +++ b/pkg/resourcegroup/tests/BUILD.bazel @@ -6,7 +6,7 @@ go_test( srcs = ["resource_group_test.go"], flaky = True, race = "on", - shard_count = 6, + shard_count = 7, deps = [ "//pkg/ddl/resourcegroup", "//pkg/domain", diff --git a/pkg/resourcegroup/tests/resource_group_test.go b/pkg/resourcegroup/tests/resource_group_test.go index 4b7e57dd7d9ac..532bc3cc7bb00 100644 --- a/pkg/resourcegroup/tests/resource_group_test.go +++ b/pkg/resourcegroup/tests/resource_group_test.go @@ -322,9 +322,9 @@ func TestResourceGroupRunaway(t *testing.T) { tryInterval := time.Millisecond * 100 maxWaitDuration := time.Second * 5 - tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, match_type from mysql.tidb_runaway_queries", nil, + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, match_type from mysql.tidb_runaway_queries", nil, testkit.Rows("rg1 select /*+ resource_group(rg1) */ * from t identify"), maxWaitDuration, tryInterval) - tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, time from mysql.tidb_runaway_queries", nil, + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, start_time from mysql.tidb_runaway_queries", nil, nil, maxWaitDuration, tryInterval) tk.MustExec("alter resource group rg1 RU_PER_SEC=1000 QUERY_LIMIT=(EXEC_ELAPSED='100ms' ACTION=COOLDOWN)") tk.MustQuery("select /*+ resource_group(rg1) */ * from t").Check(testkit.Rows("1")) @@ -335,7 +335,7 @@ func TestResourceGroupRunaway(t *testing.T) { err = tk.QueryToErr("select /*+ resource_group(rg2) */ * from t") require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query") tk.MustGetErrCode("select /*+ resource_group(rg2) */ * from t", mysql.ErrResourceGroupQueryRunawayQuarantine) - tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, match_type from mysql.tidb_runaway_queries", nil, + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, match_type from mysql.tidb_runaway_queries", nil, testkit.Rows("rg2 select /*+ resource_group(rg2) */ * from t identify", "rg2 select /*+ resource_group(rg2) */ * from t watch"), maxWaitDuration, tryInterval) tk.MustQuery("select SQL_NO_CACHE resource_group_name, watch_text from mysql.tidb_runaway_watch"). @@ -345,7 +345,7 @@ func TestResourceGroupRunaway(t *testing.T) { err = tk.QueryToErr("select /*+ resource_group(rg2) */ * from t") require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query") - tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, time from mysql.tidb_runaway_queries", nil, + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, start_time from mysql.tidb_runaway_queries", nil, nil, maxWaitDuration, tryInterval) tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, watch_text, end_time from mysql.tidb_runaway_watch", nil, nil, maxWaitDuration, tryInterval) @@ -407,9 +407,61 @@ func TestResourceGroupRunawayExceedTiDBSide(t *testing.T) { tryInterval := time.Millisecond * 100 maxWaitDuration := time.Second * 5 - tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, match_type from mysql.tidb_runaway_queries", nil, + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, match_type from mysql.tidb_runaway_queries", nil, testkit.Rows("rg1 select /*+ resource_group(rg1) */ sleep(0.5) from t identify"), maxWaitDuration, tryInterval) - tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, original_sql, time from mysql.tidb_runaway_queries", nil, + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, start_time from mysql.tidb_runaway_queries", nil, + nil, maxWaitDuration, tryInterval) +} + +func TestResourceGroupRunawayFlood(t *testing.T) { + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC", `return(true)`)) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC")) + }() + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost"}, nil, nil, nil)) + + tk.MustExec("use test") + tk.MustExec("create table t(a int)") + tk.MustExec("insert into t values(1)") + tk.MustExec("set global tidb_enable_resource_control='on'") + tk.MustExec("create resource group rg1 RU_PER_SEC=1000 QUERY_LIMIT=(EXEC_ELAPSED='50ms' ACTION=KILL)") + tk.MustQuery("select /*+ resource_group(rg1) */ * from t").Check(testkit.Rows("1")) + require.Eventually(t, func() bool { + return dom.RunawayManager().IsSyncerInitialized() + }, 20*time.Second, 300*time.Millisecond) + + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/store/copr/sleepCoprRequest", fmt.Sprintf("return(%d)", 60))) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/store/copr/sleepCoprRequest")) + }() + err := tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.1) from t") + require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query") + + tryInterval := time.Millisecond * 100 + maxWaitDuration := time.Second * 5 + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, repeats, match_type from mysql.tidb_runaway_queries", nil, + testkit.Rows("rg1 select /*+ resource_group(rg1) */ sleep(0.1) from t 1 identify"), maxWaitDuration, tryInterval) + // wait for the runaway watch to be cleaned up + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, repeats from mysql.tidb_runaway_queries", nil, + nil, maxWaitDuration, tryInterval) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC")) + + // check thrice to make sure the runaway query be regarded as a repeated query. + err = tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.2) from t") + require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query") + err = tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.3) from t") + require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query") + // using FastRunawayGC to trigger flush + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/resourcegroup/runaway/FastRunawayGC", `return(true)`)) + err = tk.QueryToErr("select /*+ resource_group(rg1) */ sleep(0.4) from t") + require.ErrorContains(t, err, "Query execution was interrupted, identified as runaway query") + // only have one runaway query + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, repeats, match_type from mysql.tidb_runaway_queries", nil, + testkit.Rows("rg1 select /*+ resource_group(rg1) */ sleep(0.2) from t 3 identify"), maxWaitDuration, tryInterval) + // wait for the runaway watch to be cleaned up + tk.EventuallyMustQueryAndCheck("select SQL_NO_CACHE resource_group_name, sample_sql, repeats from mysql.tidb_runaway_queries", nil, nil, maxWaitDuration, tryInterval) } diff --git a/pkg/server/conn_stmt_params_test.go b/pkg/server/conn_stmt_params_test.go index 33d54f2d85c1e..66fa3754069ba 100644 --- a/pkg/server/conn_stmt_params_test.go +++ b/pkg/server/conn_stmt_params_test.go @@ -45,7 +45,7 @@ func decodeAndParse(typectx types.Context, args []expression.Expression, boundPa return err } - parsedArgs, err := param.ExecArgs(typectx, binParams) + parsedArgs, err := expression.ExecBinaryParam(typectx, binParams) if err != nil { return err } diff --git a/pkg/server/extension.go b/pkg/server/extension.go index e6d0f1a32bc62..11022a83a82a1 100644 --- a/pkg/server/extension.go +++ b/pkg/server/extension.go @@ -17,6 +17,7 @@ package server import ( "fmt" + "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/extension" "github.com/pingcap/tidb/pkg/param" "github.com/pingcap/tidb/pkg/parser" @@ -92,7 +93,7 @@ func (cc *clientConn) onExtensionStmtEnd(node any, stmtCtxValid bool, err error, // eliminate one of them by storing the parsed result. typectx := ctx.GetSessionVars().StmtCtx.TypeCtx() typectx = types.NewContext(typectx.Flags(), typectx.Location(), contextutil.IgnoreWarn) - params, _ := param.ExecArgs(typectx, args) + params, _ := expression.ExecBinaryParam(typectx, args) info.executeStmt = &ast.ExecuteStmt{ PrepStmt: prepared, BinaryArgs: params, diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index 37321950f8e96..ec74727a89189 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -621,14 +621,16 @@ const ( // CreateRunawayTable stores the query which is identified as runaway or quarantined because of in watch list. CreateRunawayTable = `CREATE TABLE IF NOT EXISTS mysql.tidb_runaway_queries ( resource_group_name varchar(32) not null, - time TIMESTAMP NOT NULL, + start_time TIMESTAMP NOT NULL, + repeats int default 1, match_type varchar(12) NOT NULL, action varchar(12) NOT NULL, - original_sql TEXT NOT NULL, - plan_digest TEXT NOT NULL, + sample_sql TEXT NOT NULL, + sql_digest varchar(64) NOT NULL, + plan_digest varchar(64) NOT NULL, tidb_server varchar(512), INDEX plan_index(plan_digest(64)) COMMENT "accelerate the speed when select runaway query", - INDEX time_index(time) COMMENT "accelerate the speed when querying with active watch" + INDEX time_index(start_time) COMMENT "accelerate the speed when querying with active watch" ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;` // CreateRunawayWatchTable stores the condition which is used to check whether query should be quarantined. @@ -692,6 +694,15 @@ const ( KEY (created_by), KEY (status));` + // CreatePITRIDMap is a table that records the id map from upstream to downstream for PITR. + CreatePITRIDMap = `CREATE TABLE IF NOT EXISTS mysql.tidb_pitr_id_map ( + restored_ts BIGINT NOT NULL, + upstream_cluster_id BIGINT NOT NULL, + segment_id BIGINT NOT NULL, + id_map BLOB(524288) NOT NULL, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (restored_ts, upstream_cluster_id, segment_id));` + // DropMySQLIndexUsageTable removes the table `mysql.schema_index_usage` DropMySQLIndexUsageTable = "DROP TABLE IF EXISTS mysql.schema_index_usage" @@ -1113,13 +1124,20 @@ const ( // version211 add column `summary` to `mysql.tidb_background_subtask_history`. version211 = 211 - // version212 add column `switch_group_name` to `mysql.tidb_runaway_watch` and `mysql.tidb_runaway_watch_done`. + // version212 changed a lots of runaway related table. + // 1. switchGroup: add column `switch_group_name` to `mysql.tidb_runaway_watch` and `mysql.tidb_runaway_watch_done`. + // 2. modify column `plan_digest` type, modify column `time` to `start_time, + // modify column `original_sql` to `sample_sql` to `mysql.tidb_runaway_queries`. version212 = 212 + + // version 213 + // create `mysql.tidb_pitr_id_map` table + version213 = 213 ) // currentBootstrapVersion is defined as a variable, so we can modify its value for testing. // please make sure this is the largest version -var currentBootstrapVersion int64 = version212 +var currentBootstrapVersion int64 = version213 // DDL owner key's expired time is ManagerSessionTTL seconds, we should wait the time and give more time to have a chance to finish it. var internalSQLTimeout = owner.ManagerSessionTTL + 15 @@ -1287,6 +1305,7 @@ var ( upgradeToVer210, upgradeToVer211, upgradeToVer212, + upgradeToVer213, } ) @@ -3090,8 +3109,36 @@ func upgradeToVer212(s sessiontypes.Session, ver int64) { if ver >= version212 { return } + // need to ensure curVersion has the column before rename. + // version169 created `tidb_runaway_queries` table + // version172 created `tidb_runaway_watch` and `tidb_runaway_watch_done` tables + if ver < version172 { + return + } + // version212 changed a lots of runaway related table. + // 1. switchGroup: add column `switch_group_name` to `mysql.tidb_runaway_watch` and `mysql.tidb_runaway_watch_done`. doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch ADD COLUMN `switch_group_name` VARCHAR(32) DEFAULT '' AFTER `action`;", infoschema.ErrColumnExists) doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_watch_done ADD COLUMN `switch_group_name` VARCHAR(32) DEFAULT '' AFTER `action`;", infoschema.ErrColumnExists) + // 2. modify column `plan_digest` type, modify column `time` to `start_time, + // modify column `original_sql` to `sample_sql` and unique union key to `mysql.tidb_runaway_queries`. + // add column `sql_digest`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries ADD COLUMN `sql_digest` varchar(64) DEFAULT '' AFTER `original_sql`;", infoschema.ErrColumnExists) + // add column `repeats`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries ADD COLUMN `repeats` int DEFAULT 1 AFTER `time`;", infoschema.ErrColumnExists) + // rename column name from `time` to `start_time`, will auto rebuild the index. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `time` TO `start_time`") + // rename column `original_sql` to `sample_sql`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `original_sql` TO `sample_sql`") + // modify column type of `plan_digest`. + doReentrantDDL(s, "ALTER TABLE mysql.tidb_runaway_queries MODIFY COLUMN `plan_digest` varchar(64) DEFAULT '';", infoschema.ErrColumnExists) +} + +func upgradeToVer213(s sessiontypes.Session, ver int64) { + if ver >= version213 { + return + } + + mustExecute(s, CreatePITRIDMap) } // initGlobalVariableIfNotExists initialize a global variable with specific val if it does not exist. @@ -3238,6 +3285,8 @@ func doDDLWorks(s sessiontypes.Session) { mustExecute(s, CreateDistFrameworkMeta) // create request_unit_by_group mustExecute(s, CreateRequestUnitByGroupTable) + // create tidb_pitr_id_map + mustExecute(s, CreatePITRIDMap) // create `sys` schema mustExecute(s, CreateSysSchema) // create `sys.schema_unused_indexes` view diff --git a/pkg/session/bootstrap_test.go b/pkg/session/bootstrap_test.go index 0987adb382ddd..c0150461388e0 100644 --- a/pkg/session/bootstrap_test.go +++ b/pkg/session/bootstrap_test.go @@ -270,6 +270,12 @@ func revertVersionAndVariables(t *testing.T, se sessiontypes.Session, ver int) { // for version <= version195, tidb_enable_dist_task should be disabled before upgrade MustExec(t, se, "update mysql.global_variables set variable_value='off' where variable_name='tidb_enable_dist_task'") } + if ver < version212 && ver >= version172 { + // for version < version212, revert column changes related to function `upgradeToVer212`. + // related tables created after version172. + MustExec(t, se, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `start_time` TO `time`") + MustExec(t, se, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `sample_sql` TO `original_sql`") + } } // TestUpgrade tests upgrading @@ -312,6 +318,7 @@ func TestUpgrade(t *testing.T) { MustExec(t, se1, fmt.Sprintf(`delete from mysql.global_variables where VARIABLE_NAME="%s"`, variable.TiDBDistSQLScanConcurrency)) MustExec(t, se1, `commit`) unsetStoreBootstrapped(store.UUID()) + revertVersionAndVariables(t, se1, 0) // Make sure the version is downgraded. r = MustExecToRecodeSet(t, se1, `SELECT VARIABLE_VALUE from mysql.TiDB where VARIABLE_NAME="tidb_server_version"`) req = r.NewChunk(nil) @@ -2412,3 +2419,35 @@ func TestTiDBHistoryTableConsistent(t *testing.T) { dom.Close() } + +func TestTiDBUpgradeToVer212(t *testing.T) { + store, dom := CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + + // bootstrap as version198, version 199~208 is reserved for v8.1.x bugfix patch. + ver198 := version198 + seV198 := CreateSessionAndSetID(t, store) + txn, err := store.Begin() + require.NoError(t, err) + m := meta.NewMeta(txn) + err = m.FinishBootstrap(int64(ver198)) + require.NoError(t, err) + revertVersionAndVariables(t, seV198, ver198) + // simulate a real ver198 where mysql.tidb_runaway_queries` doesn't have `start_time`/`sample_sql` columns yet. + MustExec(t, seV198, "select original_sql, time from mysql.tidb_runaway_queries") + err = txn.Commit(context.Background()) + require.NoError(t, err) + unsetStoreBootstrapped(store.UUID()) + + // upgrade to ver212 + dom.Close() + domCurVer, err := BootstrapSession(store) + require.NoError(t, err) + defer domCurVer.Close() + seCurVer := CreateSessionAndSetID(t, store) + ver, err := getBootstrapVersion(seCurVer) + require.NoError(t, err) + require.Equal(t, currentBootstrapVersion, ver) + // the columns are changed automatically + MustExec(t, seCurVer, "select sample_sql, start_time, plan_digest from mysql.tidb_runaway_queries") +} diff --git a/pkg/session/bootstraptest/bootstrap_upgrade_test.go b/pkg/session/bootstraptest/bootstrap_upgrade_test.go index 29107c9739675..dd1e5a2aea677 100644 --- a/pkg/session/bootstraptest/bootstrap_upgrade_test.go +++ b/pkg/session/bootstraptest/bootstrap_upgrade_test.go @@ -103,6 +103,12 @@ func revertVersionAndVariables(t *testing.T, se sessiontypes.Session, ver int) { // for version <= version195, tidb_enable_dist_task should be disabled before upgrade session.MustExec(t, se, "update mysql.global_variables set variable_value='off' where variable_name='tidb_enable_dist_task'") } + if ver < 212 && ver >= 172 { + // for version < version212, revert column changes related to function `upgradeToVer212`. + // related tables created after version172. + session.MustExec(t, se, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `start_time` TO `time`") + session.MustExec(t, se, "ALTER TABLE mysql.tidb_runaway_queries RENAME COLUMN `sample_sql` TO `original_sql`") + } } func TestUpgradeVersion66(t *testing.T) { @@ -618,7 +624,8 @@ func TestUpgradeVersionForResumeJob(t *testing.T) { idxFinishTS = runJob.BinlogInfo.FinishedTS } else { // The second add index op. - if strings.Contains(runJob.TableName, "upgrade_tbl") { + // notice: upgrade `tidb_runaway_queries` table will happened in `upgradeToVer212` function which is before the second add index op. + if strings.Contains(runJob.TableName, "upgrade_tbl") || strings.Contains(runJob.TableName, "tidb_runaway_queries") { require.Greater(t, runJob.BinlogInfo.FinishedTS, idxFinishTS) } else { // The upgrade DDL ops. These jobs' finishedTS must less than add index ops. diff --git a/pkg/session/session.go b/pkg/session/session.go index f12e2c1d53f95..5c5ccf36c749a 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -2022,7 +2022,7 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex } if execStmt, ok := stmtNode.(*ast.ExecuteStmt); ok { if binParam, ok := execStmt.BinaryArgs.([]param.BinaryParam); ok { - args, err := param.ExecArgs(s.GetSessionVars().StmtCtx.TypeCtx(), binParam) + args, err := expression.ExecBinaryParam(s.GetSessionVars().StmtCtx.TypeCtx(), binParam) if err != nil { return nil, err } diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index fce1e39177bba..c214de45631be 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -837,6 +837,12 @@ type SessionVars struct { // status stands for the session status. e.g. in transaction or not, auto commit is on or off, and so on. status atomic.Uint32 + // ShardRowIDBits is the number of shard bits for user table row ID. + ShardRowIDBits uint64 + + // PreSplitRegions is the number of regions that should be pre-split for the table. + PreSplitRegions uint64 + // ClientCapability is client's capability. ClientCapability uint32 @@ -3325,9 +3331,9 @@ const ( // SlowLogWaitRUDuration is the total duration for kv requests to wait available request-units. SlowLogWaitRUDuration = "Time_queued_by_rc" // SlowLogTidbCPUUsageDuration is the total tidb cpu usages. - SlowLogTidbCPUUsageDuration = "Tidb_cpu_usage" + SlowLogTidbCPUUsageDuration = "Tidb_cpu_time" // SlowLogTikvCPUUsageDuration is the total tikv cpu usages. - SlowLogTikvCPUUsageDuration = "Tikv_cpu_usage" + SlowLogTikvCPUUsageDuration = "Tikv_cpu_time" ) // GenerateBinaryPlan decides whether we should record binary plan in slow log and stmt summary. diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index a2b623206579a..730f223037764 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -2060,6 +2060,14 @@ var defaultSysVars = []*SysVar{ s.AllowBatchCop = int(TidbOptInt64(val, DefTiDBAllowBatchCop)) return nil }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBShardRowIDBits, Value: strconv.Itoa(DefShardRowIDBits), Type: TypeInt, MinValue: 0, MaxValue: MaxShardRowIDBits, SetSession: func(s *SessionVars, val string) error { + s.ShardRowIDBits = TidbOptUint64(val, DefShardRowIDBits) + return nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBPreSplitRegions, Value: strconv.Itoa(DefPreSplitRegions), Type: TypeInt, MinValue: 0, MaxValue: MaxPreSplitRegions, SetSession: func(s *SessionVars, val string) error { + s.PreSplitRegions = TidbOptUint64(val, DefPreSplitRegions) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBInitChunkSize, Value: strconv.Itoa(DefInitChunkSize), Type: TypeUnsigned, MinValue: 1, MaxValue: initChunkSizeUpperBound, SetSession: func(s *SessionVars, val string) error { s.InitChunkSize = tidbOptPositiveInt32(val, DefInitChunkSize) return nil diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index 9998376cf3647..267c920827587 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -387,6 +387,14 @@ const ( // The default value is 0 TiDBAllowBatchCop = "tidb_allow_batch_cop" + // TiDBShardRowIDBits means all the tables created in the current session will be sharded. + // The default value is 0 + TiDBShardRowIDBits = "tidb_shard_row_id_bits" + + // TiDBPreSplitRegions means all the tables created in the current session will be pre-splited. + // The default value is 0 + TiDBPreSplitRegions = "tidb_pre_split_regions" + // TiDBAllowMPPExecution means if we should use mpp way to execute query or not. // Default value is `true`, means to be determined by the optimizer. // Value set to `false` means never use mpp. @@ -1207,6 +1215,12 @@ const ( // MaxConfigurableConcurrency is the maximum number of "threads" (goroutines) that can be specified // for any type of configuration item that has concurrent workers. MaxConfigurableConcurrency = 256 + + // MaxShardRowIDBits is the maximum number of bits that can be used for row-id sharding. + MaxShardRowIDBits = 15 + + // MaxPreSplitRegions is the maximum number of regions that can be pre-split. + MaxPreSplitRegions = 15 ) // Default TiDB system variable values. @@ -1279,6 +1293,8 @@ const ( DefTiDBEnableOuterJoinReorder = true DefTiDBEnableNAAJ = true DefTiDBAllowBatchCop = 1 + DefShardRowIDBits = 0 + DefPreSplitRegions = 0 DefBlockEncryptionMode = "aes-128-ecb" DefTiDBAllowMPPExecution = true DefTiDBAllowTiFlashCop = false diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/BUILD.bazel b/pkg/statistics/handle/autoanalyze/priorityqueue/BUILD.bazel index 927b1f0b50522..16ba16edb0787 100644 --- a/pkg/statistics/handle/autoanalyze/priorityqueue/BUILD.bazel +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "priorityqueue", srcs = [ + "analysis_job_factory.go", "calculator.go", "dynamic_partitioned_table_analysis_job.go", "interval.go", @@ -14,13 +15,18 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/priorityqueue", visibility = ["//visibility:public"], deps = [ + "//pkg/meta/model", "//pkg/sessionctx", "//pkg/sessionctx/sysproctrack", "//pkg/sessionctx/variable", + "//pkg/statistics", "//pkg/statistics/handle/autoanalyze/exec", "//pkg/statistics/handle/logutil", "//pkg/statistics/handle/types", "//pkg/statistics/handle/util", + "//pkg/util/intest", + "//pkg/util/timeutil", + "@com_github_tikv_client_go_v2//oracle", "@org_uber_go_zap//:zap", ], ) @@ -29,6 +35,7 @@ go_test( name = "priorityqueue_test", timeout = "short", srcs = [ + "analysis_job_factory_test.go", "calculator_test.go", "dynamic_partitioned_table_analysis_job_test.go", "interval_test.go", @@ -39,15 +46,18 @@ go_test( "static_partitioned_table_analysis_job_test.go", ], flaky = True, - shard_count = 22, + shard_count = 28, deps = [ ":priorityqueue", + "//pkg/meta/model", "//pkg/parser/model", "//pkg/session", "//pkg/sessionctx", + "//pkg/statistics", "//pkg/testkit", "//pkg/testkit/testsetup", "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//oracle", "@org_uber_go_goleak//:goleak", ], ) diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/analysis_job_factory.go b/pkg/statistics/handle/autoanalyze/priorityqueue/analysis_job_factory.go new file mode 100644 index 0000000000000..8e0dbaaf21837 --- /dev/null +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/analysis_job_factory.go @@ -0,0 +1,382 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package priorityqueue + +import ( + "time" + + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/statistics" + statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types" + "github.com/pingcap/tidb/pkg/util/intest" + "github.com/pingcap/tidb/pkg/util/timeutil" + "github.com/tikv/client-go/v2/oracle" +) + +const ( + // unanalyzedTableDefaultChangePercentage is the default change percentage of unanalyzed table. + unanalyzedTableDefaultChangePercentage = 1 + // unanalyzedTableDefaultLastUpdateDuration is the default last update duration of unanalyzed table. + unanalyzedTableDefaultLastUpdateDuration = -30 * time.Minute +) + +// AnalysisJobFactory is responsible for creating different types of analysis jobs. +// NOTE: This struct is not thread-safe. +type AnalysisJobFactory struct { + sctx sessionctx.Context + autoAnalyzeRatio float64 + // The current TSO. + currentTs uint64 +} + +// NewAnalysisJobFactory creates a new AnalysisJobFactory. +func NewAnalysisJobFactory(sctx sessionctx.Context, autoAnalyzeRatio float64, currentTs uint64) *AnalysisJobFactory { + return &AnalysisJobFactory{ + sctx: sctx, + autoAnalyzeRatio: autoAnalyzeRatio, + currentTs: currentTs, + } +} + +// CreateNonPartitionedTableAnalysisJob creates a job for non-partitioned tables. +func (f *AnalysisJobFactory) CreateNonPartitionedTableAnalysisJob( + tableSchema string, + tblInfo *model.TableInfo, + tblStats *statistics.Table, +) AnalysisJob { + if !tblStats.IsEligibleForAnalysis() { + return nil + } + + tableStatsVer := f.sctx.GetSessionVars().AnalyzeVersion + statistics.CheckAnalyzeVerOnTable(tblStats, &tableStatsVer) + + changePercentage := f.CalculateChangePercentage(tblStats) + tableSize := f.CalculateTableSize(tblStats) + lastAnalysisDuration := f.GetTableLastAnalyzeDuration(tblStats) + indexes := f.CheckIndexesNeedAnalyze(tblInfo, tblStats) + + // No need to analyze. + // We perform a separate check because users may set the auto analyze ratio to 0, + // yet still wish to analyze newly added indexes and tables that have not been analyzed. + if changePercentage == 0 && len(indexes) == 0 { + return nil + } + + return NewNonPartitionedTableAnalysisJob( + tableSchema, + tblInfo.Name.O, + tblInfo.ID, + indexes, + tableStatsVer, + changePercentage, + tableSize, + lastAnalysisDuration, + ) +} + +// CreateStaticPartitionAnalysisJob creates a job for static partitions. +func (f *AnalysisJobFactory) CreateStaticPartitionAnalysisJob( + tableSchema string, + globalTblInfo *model.TableInfo, + partitionID int64, + partitionName string, + partitionStats *statistics.Table, +) AnalysisJob { + if !partitionStats.IsEligibleForAnalysis() { + return nil + } + + tableStatsVer := f.sctx.GetSessionVars().AnalyzeVersion + statistics.CheckAnalyzeVerOnTable(partitionStats, &tableStatsVer) + + changePercentage := f.CalculateChangePercentage(partitionStats) + tableSize := f.CalculateTableSize(partitionStats) + lastAnalysisDuration := f.GetTableLastAnalyzeDuration(partitionStats) + indexes := f.CheckIndexesNeedAnalyze(globalTblInfo, partitionStats) + + // No need to analyze. + // We perform a separate check because users may set the auto analyze ratio to 0, + // yet still wish to analyze newly added indexes and tables that have not been analyzed. + if changePercentage == 0 && len(indexes) == 0 { + return nil + } + + return NewStaticPartitionTableAnalysisJob( + tableSchema, + globalTblInfo.Name.O, + globalTblInfo.ID, + partitionName, + partitionID, + indexes, + tableStatsVer, + changePercentage, + tableSize, + lastAnalysisDuration, + ) +} + +// CreateDynamicPartitionedTableAnalysisJob creates a job for dynamic partitioned tables. +func (f *AnalysisJobFactory) CreateDynamicPartitionedTableAnalysisJob( + tableSchema string, + globalTblInfo *model.TableInfo, + globalTblStats *statistics.Table, + partitionStats map[PartitionIDAndName]*statistics.Table, +) AnalysisJob { + if !globalTblStats.IsEligibleForAnalysis() { + return nil + } + + // TODO: figure out how to check the table stats version correctly for partitioned tables. + tableStatsVer := f.sctx.GetSessionVars().AnalyzeVersion + statistics.CheckAnalyzeVerOnTable(globalTblStats, &tableStatsVer) + + avgChange, avgSize, minLastAnalyzeDuration, partitionNames := f.CalculateIndicatorsForPartitions(globalTblStats, partitionStats) + partitionIndexes := f.CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable(globalTblInfo, partitionStats) + + // No need to analyze. + // We perform a separate check because users may set the auto analyze ratio to 0, + // yet still wish to analyze newly added indexes and tables that have not been analyzed. + if len(partitionNames) == 0 && len(partitionIndexes) == 0 { + return nil + } + + return NewDynamicPartitionedTableAnalysisJob( + tableSchema, + globalTblInfo.Name.O, + globalTblInfo.ID, + partitionNames, + partitionIndexes, + tableStatsVer, + avgChange, + avgSize, + minLastAnalyzeDuration, + ) +} + +// CalculateChangePercentage calculates the change percentage of the table +// based on the change count and the analysis count. +func (f *AnalysisJobFactory) CalculateChangePercentage(tblStats *statistics.Table) float64 { + if !tblStats.IsAnalyzed() { + return unanalyzedTableDefaultChangePercentage + } + + // Auto analyze based on the change percentage is disabled. + // However, this check should not affect the analysis of indexes, + // as index analysis is still needed for query performance. + if f.autoAnalyzeRatio == 0 { + return 0 + } + + tblCnt := float64(tblStats.RealtimeCount) + if histCnt := tblStats.GetAnalyzeRowCount(); histCnt > 0 { + tblCnt = histCnt + } + res := float64(tblStats.ModifyCount) / tblCnt + if res > f.autoAnalyzeRatio { + return res + } + + return 0 +} + +// CalculateTableSize calculates the size of the table. +func (*AnalysisJobFactory) CalculateTableSize(tblStats *statistics.Table) float64 { + tblCnt := float64(tblStats.RealtimeCount) + colCnt := float64(tblStats.ColAndIdxExistenceMap.ColNum()) + intest.Assert(colCnt != 0, "Column count should not be 0") + + return tblCnt * colCnt +} + +// GetTableLastAnalyzeDuration gets the last analyze duration of the table. +func (f *AnalysisJobFactory) GetTableLastAnalyzeDuration(tblStats *statistics.Table) time.Duration { + lastTime := f.FindLastAnalyzeTime(tblStats) + currentTime := oracle.GetTimeFromTS(f.currentTs) + + // Calculate the duration since last analyze. + return currentTime.Sub(lastTime) +} + +// FindLastAnalyzeTime finds the last analyze time of the table. +// It uses `LastUpdateVersion` to find the last analyze time. +// The `LastUpdateVersion` is the version of the transaction that updates the statistics. +// It always not null(default 0), so we can use it to find the last analyze time. +func (f *AnalysisJobFactory) FindLastAnalyzeTime(tblStats *statistics.Table) time.Time { + if !tblStats.IsAnalyzed() { + phy := oracle.GetTimeFromTS(f.currentTs) + return phy.Add(unanalyzedTableDefaultLastUpdateDuration) + } + return oracle.GetTimeFromTS(tblStats.LastAnalyzeVersion) +} + +// CheckIndexesNeedAnalyze checks if the indexes need to be analyzed. +func (*AnalysisJobFactory) CheckIndexesNeedAnalyze(tblInfo *model.TableInfo, tblStats *statistics.Table) []string { + // If table is not analyzed, we need to analyze whole table. + // So we don't need to check indexes. + if !tblStats.IsAnalyzed() { + return nil + } + + indexes := make([]string, 0, len(tblInfo.Indices)) + // Check if missing index stats. + for _, idx := range tblInfo.Indices { + if idxStats := tblStats.GetIdx(idx.ID); idxStats == nil && !tblStats.ColAndIdxExistenceMap.HasAnalyzed(idx.ID, true) && idx.State == model.StatePublic { + indexes = append(indexes, idx.Name.O) + } + } + + return indexes +} + +// CalculateIndicatorsForPartitions calculates the average change percentage, +// average size and average last analyze duration for the partitions that meet the threshold. +// Change percentage is the ratio of the number of modified rows to the total number of rows. +// Size is the product of the number of rows and the number of columns. +// Last analyze duration is the duration since the last analyze. +func (f *AnalysisJobFactory) CalculateIndicatorsForPartitions( + globalStats *statistics.Table, + partitionStats map[PartitionIDAndName]*statistics.Table, +) ( + avgChange float64, + avgSize float64, + avgLastAnalyzeDuration time.Duration, + partitionNames []string, +) { + totalChangePercent := 0.0 + totalSize := 0.0 + count := 0.0 + partitionNames = make([]string, 0, len(partitionStats)) + cols := float64(globalStats.ColAndIdxExistenceMap.ColNum()) + intest.Assert(cols != 0, "Column count should not be 0") + totalLastAnalyzeDuration := time.Duration(0) + + for pIDAndName, tblStats := range partitionStats { + // Skip partition analysis if it doesn't meet the threshold, stats are not yet loaded, + // or the auto analyze ratio is set to 0 by the user. + changePercent := f.CalculateChangePercentage(tblStats) + if changePercent == 0 { + continue + } + + totalChangePercent += changePercent + // size = count * cols + totalSize += float64(tblStats.RealtimeCount) * cols + lastAnalyzeDuration := f.GetTableLastAnalyzeDuration(tblStats) + totalLastAnalyzeDuration += lastAnalyzeDuration + partitionNames = append(partitionNames, pIDAndName.Name) + count++ + } + if len(partitionNames) == 0 { + return 0, 0, 0, partitionNames + } + + avgChange = totalChangePercent / count + avgSize = totalSize / count + avgLastAnalyzeDuration = totalLastAnalyzeDuration / time.Duration(count) + + return avgChange, avgSize, avgLastAnalyzeDuration, partitionNames +} + +// CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable checks if the indexes of the partitioned table need to be analyzed. +// It returns a map from index name to the names of the partitions that need to be analyzed. +// NOTE: This is only for newly added indexes. +func (*AnalysisJobFactory) CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable( + tblInfo *model.TableInfo, + partitionStats map[PartitionIDAndName]*statistics.Table, +) map[string][]string { + partitionIndexes := make(map[string][]string, len(tblInfo.Indices)) + + for _, idx := range tblInfo.Indices { + // No need to analyze the index if it's not public. + if idx.State != model.StatePublic { + continue + } + + // Find all the partitions that need to analyze this index. + names := make([]string, 0, len(partitionStats)) + for pIDAndName, tblStats := range partitionStats { + if idxStats := tblStats.GetIdx(idx.ID); idxStats == nil && !tblStats.ColAndIdxExistenceMap.HasAnalyzed(idx.ID, true) { + names = append(names, pIDAndName.Name) + } + } + + if len(names) > 0 { + partitionIndexes[idx.Name.O] = names + } + } + + return partitionIndexes +} + +// PartitionIDAndName is a struct that contains the ID and name of a partition. +// Exported for testing purposes. Do not use it in other packages. +type PartitionIDAndName struct { + Name string + ID int64 +} + +// NewPartitionIDAndName creates a new PartitionIDAndName. +func NewPartitionIDAndName(name string, id int64) PartitionIDAndName { + return PartitionIDAndName{ + Name: name, + ID: id, + } +} + +// GetPartitionStats gets the partition stats. +func GetPartitionStats( + statsHandle statstypes.StatsHandle, + tblInfo *model.TableInfo, + defs []model.PartitionDefinition, +) map[PartitionIDAndName]*statistics.Table { + partitionStats := make(map[PartitionIDAndName]*statistics.Table, len(defs)) + + for _, def := range defs { + stats := statsHandle.GetPartitionStatsForAutoAnalyze(tblInfo, def.ID) + // Ignore the partition if it's not ready to analyze. + if !stats.IsEligibleForAnalysis() { + continue + } + d := NewPartitionIDAndName(def.Name.O, def.ID) + partitionStats[d] = stats + } + + return partitionStats +} + +// AutoAnalysisTimeWindow is a struct that contains the start and end time of the auto analyze time window. +type AutoAnalysisTimeWindow struct { + start time.Time + end time.Time +} + +// NewAutoAnalysisTimeWindow creates a new AutoAnalysisTimeWindow. +func NewAutoAnalysisTimeWindow(start, end time.Time) AutoAnalysisTimeWindow { + return AutoAnalysisTimeWindow{ + start: start, + end: end, + } +} + +// IsWithinTimeWindow checks if the current time is within the time window. +// If the auto analyze time window is not set or the current time is not in the window, return false. +func (a AutoAnalysisTimeWindow) IsWithinTimeWindow(currentTime time.Time) bool { + if a.start.IsZero() || a.end.IsZero() { + return false + } + return timeutil.WithinDayTimePeriod(a.start, a.end, currentTime) +} diff --git a/pkg/statistics/handle/autoanalyze/priorityqueue/analysis_job_factory_test.go b/pkg/statistics/handle/autoanalyze/priorityqueue/analysis_job_factory_test.go new file mode 100644 index 0000000000000..cce1fe5f3e63a --- /dev/null +++ b/pkg/statistics/handle/autoanalyze/priorityqueue/analysis_job_factory_test.go @@ -0,0 +1,479 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package priorityqueue_test + +import ( + "sort" + "testing" + "time" + + "github.com/pingcap/tidb/pkg/meta/model" + pmodel "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/statistics" + "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/priorityqueue" + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/oracle" +) + +func TestCalculateChangePercentage(t *testing.T) { + tests := []struct { + name string + tblStats *statistics.Table + autoAnalyzeRatio float64 + want float64 + }{ + { + name: "Unanalyzed table", + tblStats: &statistics.Table{ + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, nil, nil), + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 0), + }, + autoAnalyzeRatio: 0.5, + want: 1, + }, + { + name: "Analyzed table with change percentage above threshold", + tblStats: &statistics.Table{ + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, 100, 60, nil, nil), + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(1, 1), + LastAnalyzeVersion: 1, + }, + autoAnalyzeRatio: 0.5, + want: 0.6, + }, + { + name: "Analyzed table with change percentage below threshold", + tblStats: &statistics.Table{ + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, 100, 40, nil, nil), + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(1, 1), + LastAnalyzeVersion: 1, + }, + autoAnalyzeRatio: 0.5, + want: 0, + }, + { + name: "Auto analyze ratio set to 0", + tblStats: &statistics.Table{ + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, 100, 60, nil, nil), + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(1, 1), + LastAnalyzeVersion: 1, + }, + autoAnalyzeRatio: 0, + want: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := priorityqueue.NewAnalysisJobFactory(nil, tt.autoAnalyzeRatio, 0) + got := factory.CalculateChangePercentage(tt.tblStats) + require.InDelta(t, tt.want, got, 0.001) + }) + } +} + +func TestGetTableLastAnalyzeDuration(t *testing.T) { + tests := []struct { + name string + tblStats *statistics.Table + currentTs uint64 + wantDuration time.Duration + }{ + { + name: "Analyzed table", + tblStats: &statistics.Table{ + LastAnalyzeVersion: oracle.GoTimeToTS(time.Now().Add(-24 * time.Hour)), + }, + currentTs: oracle.GoTimeToTS(time.Now()), + wantDuration: 24 * time.Hour, + }, + { + name: "Unanalyzed table", + tblStats: &statistics.Table{ + HistColl: statistics.HistColl{}, + }, + currentTs: oracle.GoTimeToTS(time.Now()), + wantDuration: 30 * time.Minute, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := priorityqueue.NewAnalysisJobFactory(nil, 0, tt.currentTs) + got := factory.GetTableLastAnalyzeDuration(tt.tblStats) + require.InDelta(t, tt.wantDuration, got, float64(time.Second)) + }) + } +} + +func TestCheckIndexesNeedAnalyze(t *testing.T) { + analyzedMap := statistics.NewColAndIndexExistenceMap(1, 0) + analyzedMap.InsertCol(1, nil, true) + analyzedMap.InsertIndex(1, nil, false) + tests := []struct { + name string + tblInfo *model.TableInfo + tblStats *statistics.Table + want []string + }{ + { + name: "Test Table not analyzed", + tblInfo: &model.TableInfo{ + Indices: []*model.IndexInfo{ + { + ID: 1, + Name: pmodel.NewCIStr("index1"), + State: model.StatePublic, + }, + }, + }, + tblStats: &statistics.Table{ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 0)}, + want: nil, + }, + { + name: "Test Index not analyzed", + tblInfo: &model.TableInfo{ + Indices: []*model.IndexInfo{ + { + ID: 1, + Name: pmodel.NewCIStr("index1"), + State: model.StatePublic, + }, + }, + }, + tblStats: &statistics.Table{ + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, 0, 0, map[int64]*statistics.Column{ + 1: { + StatsVer: 2, + }, + }, nil), + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: 1, + }, + want: []string{"index1"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := priorityqueue.NewAnalysisJobFactory(nil, 0, 0) + got := factory.CheckIndexesNeedAnalyze(tt.tblInfo, tt.tblStats) + require.Equal(t, tt.want, got) + }) + } +} + +func TestCalculateIndicatorsForPartitions(t *testing.T) { + // 2024-01-01 10:00:00 + currentTime := time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC) + currentTs := oracle.GoTimeToTS(currentTime) + // 2023-12-31 10:00:00 + lastUpdateTime := time.Date(2023, 12, 31, 10, 0, 0, 0, time.UTC) + lastUpdateTs := oracle.GoTimeToTS(lastUpdateTime) + unanalyzedMap := statistics.NewColAndIndexExistenceMap(0, 0) + analyzedMap := statistics.NewColAndIndexExistenceMap(2, 1) + analyzedMap.InsertCol(1, nil, true) + analyzedMap.InsertCol(2, nil, true) + analyzedMap.InsertIndex(1, nil, true) + tests := []struct { + name string + globalStats *statistics.Table + partitionStats map[priorityqueue.PartitionIDAndName]*statistics.Table + defs []model.PartitionDefinition + autoAnalyzeRatio float64 + currentTs uint64 + wantAvgChangePercentage float64 + wantAvgSize float64 + wantAvgLastAnalyzeDuration time.Duration + wantPartitions []string + }{ + { + name: "Test Table not analyzed", + globalStats: &statistics.Table{ + ColAndIdxExistenceMap: analyzedMap, + }, + partitionStats: map[priorityqueue.PartitionIDAndName]*statistics.Table{ + priorityqueue.NewPartitionIDAndName("p0", 1): { + HistColl: statistics.HistColl{ + Pseudo: false, + RealtimeCount: statistics.AutoAnalyzeMinCnt + 1, + }, + ColAndIdxExistenceMap: unanalyzedMap, + }, + priorityqueue.NewPartitionIDAndName("p1", 2): { + HistColl: statistics.HistColl{ + Pseudo: false, + RealtimeCount: statistics.AutoAnalyzeMinCnt + 1, + }, + ColAndIdxExistenceMap: unanalyzedMap, + }, + }, + defs: []model.PartitionDefinition{ + { + ID: 1, + Name: pmodel.NewCIStr("p0"), + }, + { + ID: 2, + Name: pmodel.NewCIStr("p1"), + }, + }, + autoAnalyzeRatio: 0.5, + currentTs: currentTs, + wantAvgChangePercentage: 1, + wantAvgSize: 2002, + wantAvgLastAnalyzeDuration: 1800 * time.Second, + wantPartitions: []string{"p0", "p1"}, + }, + { + name: "Test Table analyzed and only one partition meets the threshold", + globalStats: &statistics.Table{ + ColAndIdxExistenceMap: analyzedMap, + }, + partitionStats: map[priorityqueue.PartitionIDAndName]*statistics.Table{ + priorityqueue.NewPartitionIDAndName("p0", 1): { + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, (statistics.AutoAnalyzeMinCnt+1)*2, map[int64]*statistics.Column{ + 1: { + StatsVer: 2, + Histogram: statistics.Histogram{ + LastUpdateVersion: lastUpdateTs, + }, + }, + 2: { + StatsVer: 2, + Histogram: statistics.Histogram{ + LastUpdateVersion: lastUpdateTs, + }, + }, + }, nil), + Version: currentTs, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: lastUpdateTs, + }, + priorityqueue.NewPartitionIDAndName("p1", 2): { + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, map[int64]*statistics.Column{ + 1: { + StatsVer: 2, + Histogram: statistics.Histogram{ + LastUpdateVersion: lastUpdateTs, + }, + }, + 2: { + StatsVer: 2, + Histogram: statistics.Histogram{ + LastUpdateVersion: lastUpdateTs, + }, + }, + }, nil), + Version: currentTs, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: lastUpdateTs, + }, + }, + defs: []model.PartitionDefinition{ + { + ID: 1, + Name: pmodel.NewCIStr("p0"), + }, + { + ID: 2, + Name: pmodel.NewCIStr("p1"), + }, + }, + autoAnalyzeRatio: 0.5, + currentTs: currentTs, + wantAvgChangePercentage: 2, + wantAvgSize: 2002, + wantAvgLastAnalyzeDuration: 24 * time.Hour, + wantPartitions: []string{"p0"}, + }, + { + name: "No partition meets the threshold", + globalStats: &statistics.Table{ + ColAndIdxExistenceMap: analyzedMap, + }, + partitionStats: map[priorityqueue.PartitionIDAndName]*statistics.Table{ + priorityqueue.NewPartitionIDAndName("p0", 1): { + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, map[int64]*statistics.Column{ + 1: { + StatsVer: 2, + Histogram: statistics.Histogram{ + LastUpdateVersion: lastUpdateTs, + }, + }, + 2: { + StatsVer: 2, + Histogram: statistics.Histogram{ + LastUpdateVersion: lastUpdateTs, + }, + }, + }, nil), + Version: currentTs, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: lastUpdateTs, + }, + priorityqueue.NewPartitionIDAndName("p1", 2): { + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, map[int64]*statistics.Column{ + 1: { + StatsVer: 2, + Histogram: statistics.Histogram{ + LastUpdateVersion: lastUpdateTs, + }, + }, + 2: { + StatsVer: 2, + Histogram: statistics.Histogram{ + LastUpdateVersion: lastUpdateTs, + }, + }, + }, nil), + Version: currentTs, + ColAndIdxExistenceMap: analyzedMap, + LastAnalyzeVersion: lastUpdateTs, + }, + }, + defs: []model.PartitionDefinition{ + { + ID: 1, + Name: pmodel.NewCIStr("p0"), + }, + { + ID: 2, + Name: pmodel.NewCIStr("p1"), + }, + }, + autoAnalyzeRatio: 0.5, + currentTs: currentTs, + wantAvgChangePercentage: 0, + wantAvgSize: 0, + wantAvgLastAnalyzeDuration: 0, + wantPartitions: []string{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := priorityqueue.NewAnalysisJobFactory(nil, tt.autoAnalyzeRatio, tt.currentTs) + gotAvgChangePercentage, + gotAvgSize, + gotAvgLastAnalyzeDuration, + gotPartitions := + factory.CalculateIndicatorsForPartitions( + tt.globalStats, + tt.partitionStats, + ) + require.Equal(t, tt.wantAvgChangePercentage, gotAvgChangePercentage) + require.Equal(t, tt.wantAvgSize, gotAvgSize) + require.Equal(t, tt.wantAvgLastAnalyzeDuration, gotAvgLastAnalyzeDuration) + // Sort the partitions. + sort.Strings(tt.wantPartitions) + sort.Strings(gotPartitions) + require.Equal(t, tt.wantPartitions, gotPartitions) + }) + } +} + +func TestCheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable(t *testing.T) { + tblInfo := model.TableInfo{ + Indices: []*model.IndexInfo{ + { + ID: 1, + Name: pmodel.NewCIStr("index1"), + State: model.StatePublic, + }, + { + ID: 2, + Name: pmodel.NewCIStr("index2"), + State: model.StatePublic, + }, + }, + Columns: []*model.ColumnInfo{ + { + ID: 1, + }, + { + ID: 2, + }, + }, + } + partitionStats := map[priorityqueue.PartitionIDAndName]*statistics.Table{ + priorityqueue.NewPartitionIDAndName("p0", 1): { + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, nil, map[int64]*statistics.Index{}), + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 0), + }, + priorityqueue.NewPartitionIDAndName("p1", 2): { + HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, nil, map[int64]*statistics.Index{ + 2: { + StatsVer: 2, + }, + }), + ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 1), + }, + } + + factory := priorityqueue.NewAnalysisJobFactory(nil, 0, 0) + partitionIndexes := factory.CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable(&tblInfo, partitionStats) + expected := map[string][]string{"index1": {"p0", "p1"}, "index2": {"p0"}} + require.Equal(t, len(expected), len(partitionIndexes)) + + for k, v := range expected { + sort.Strings(v) + if val, ok := partitionIndexes[k]; ok { + sort.Strings(val) + require.Equal(t, v, val) + } else { + require.Fail(t, "key not found in partitionIndexes: "+k) + } + } +} + +func TestAutoAnalysisTimeWindow(t *testing.T) { + tests := []struct { + name string + start time.Time + end time.Time + current time.Time + wantWithin bool + }{ + { + name: "Within time window", + start: time.Date(2024, 1, 1, 1, 0, 0, 0, time.UTC), + end: time.Date(2024, 1, 1, 5, 0, 0, 0, time.UTC), + current: time.Date(2024, 1, 1, 3, 0, 0, 0, time.UTC), + wantWithin: true, + }, + { + name: "Outside time window", + start: time.Date(2024, 1, 1, 1, 0, 0, 0, time.UTC), + end: time.Date(2024, 1, 1, 5, 0, 0, 0, time.UTC), + current: time.Date(2024, 1, 1, 6, 0, 0, 0, time.UTC), + wantWithin: false, + }, + { + name: "Empty time window", + start: time.Time{}, + end: time.Time{}, + current: time.Now(), + wantWithin: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + window := priorityqueue.NewAutoAnalysisTimeWindow(tt.start, tt.end) + got := window.IsWithinTimeWindow(tt.current) + require.Equal(t, tt.wantWithin, got) + }) + } +} diff --git a/pkg/statistics/handle/autoanalyze/refresher/BUILD.bazel b/pkg/statistics/handle/autoanalyze/refresher/BUILD.bazel index 1274b67db2a4c..97c8f62b369f7 100644 --- a/pkg/statistics/handle/autoanalyze/refresher/BUILD.bazel +++ b/pkg/statistics/handle/autoanalyze/refresher/BUILD.bazel @@ -14,7 +14,6 @@ go_library( "//pkg/sessionctx", "//pkg/sessionctx/sysproctrack", "//pkg/sessionctx/variable", - "//pkg/statistics", "//pkg/statistics/handle/autoanalyze/exec", "//pkg/statistics/handle/autoanalyze/priorityqueue", "//pkg/statistics/handle/lockstats", @@ -23,8 +22,6 @@ go_library( "//pkg/statistics/handle/util", "//pkg/util", "//pkg/util/intest", - "//pkg/util/timeutil", - "@com_github_tikv_client_go_v2//oracle", "@org_uber_go_zap//:zap", ], ) @@ -38,10 +35,9 @@ go_test( "worker_test.go", ], flaky = True, - shard_count = 15, + shard_count = 9, deps = [ ":refresher", - "//pkg/meta/model", "//pkg/parser/model", "//pkg/sessionctx", "//pkg/sessionctx/sysproctrack", @@ -51,7 +47,6 @@ go_test( "//pkg/testkit", "//pkg/testkit/testsetup", "@com_github_stretchr_testify//require", - "@com_github_tikv_client_go_v2//oracle", "@org_uber_go_goleak//:goleak", ], ) diff --git a/pkg/statistics/handle/autoanalyze/refresher/refresher.go b/pkg/statistics/handle/autoanalyze/refresher/refresher.go index 4372ab0254bcd..5a991cb354924 100644 --- a/pkg/statistics/handle/autoanalyze/refresher/refresher.go +++ b/pkg/statistics/handle/autoanalyze/refresher/refresher.go @@ -23,7 +23,6 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/sysproctrack" "github.com/pingcap/tidb/pkg/sessionctx/variable" - "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/exec" "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/priorityqueue" "github.com/pingcap/tidb/pkg/statistics/handle/lockstats" @@ -32,25 +31,16 @@ import ( statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/intest" - "github.com/pingcap/tidb/pkg/util/timeutil" - "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" ) -const ( - // unanalyzedTableDefaultChangePercentage is the default change percentage of unanalyzed table. - unanalyzedTableDefaultChangePercentage = 1 - // unanalyzedTableDefaultLastUpdateDuration is the default last update duration of unanalyzed table. - unanalyzedTableDefaultLastUpdateDuration = -30 * time.Minute -) - // Refresher provides methods to refresh stats info. // NOTE: Refresher is not thread-safe. type Refresher struct { statsHandle statstypes.StatsHandle sysProcTracker sysproctrack.Tracker // This will be refreshed every time we rebuild the priority queue. - autoAnalysisTimeWindow + autoAnalysisTimeWindow priorityqueue.AutoAnalysisTimeWindow // Jobs is the priority queue of analysis jobs. // Exported for testing purposes. @@ -84,7 +74,7 @@ func (r *Refresher) UpdateConcurrency() { // AnalyzeHighestPriorityTables picks tables with the highest priority and analyzes them. func (r *Refresher) AnalyzeHighestPriorityTables() bool { - if !r.autoAnalysisTimeWindow.isWithinTimeWindow(time.Now()) { + if !r.autoAnalysisTimeWindow.IsWithinTimeWindow(time.Now()) { return false } @@ -182,11 +172,8 @@ func (r *Refresher) RebuildTableAnalysisJobQueue() error { } // We will check it again when we try to execute the job. // So store the time window for later use. - r.autoAnalysisTimeWindow = autoAnalysisTimeWindow{ - start: start, - end: end, - } - if !r.autoAnalysisTimeWindow.isWithinTimeWindow(time.Now()) { + r.autoAnalysisTimeWindow = priorityqueue.NewAutoAnalysisTimeWindow(start, end) + if !r.autoAnalysisTimeWindow.IsWithinTimeWindow(time.Now()) { return nil } calculator := priorityqueue.NewPriorityCalculator() @@ -204,11 +191,13 @@ func (r *Refresher) RebuildTableAnalysisJobQueue() error { return err } + jobFactory := priorityqueue.NewAnalysisJobFactory(sctx, autoAnalyzeRatio, currentTs) + dbs := is.AllSchemaNames() for _, db := range dbs { // Sometimes the tables are too many. Auto-analyze will take too much time on it. // so we need to check the available time. - if !r.autoAnalysisTimeWindow.isWithinTimeWindow(time.Now()) { + if !r.autoAnalysisTimeWindow.IsWithinTimeWindow(time.Now()) { return nil } // Ignore the memory and system database. @@ -231,37 +220,13 @@ func (r *Refresher) RebuildTableAnalysisJobQueue() error { continue } pi := tblInfo.GetPartitionInfo() - pushJobFunc := func(job priorityqueue.AnalysisJob) { - if job == nil { - return - } - // Calculate the weight of the job. - weight := calculator.CalculateWeight(job) - // We apply a penalty to larger tables, which can potentially result in a negative weight. - // To prevent this, we filter out any negative weights. Under normal circumstances, table sizes should not be negative. - if weight <= 0 { - statslogutil.SingletonStatsSamplerLogger().Warn( - "Table gets a negative weight", - zap.Float64("weight", weight), - zap.Stringer("job", job), - ) - } - job.SetWeight(weight) - // Push the job onto the queue. - r.Jobs.Push(job) - } - // No partitions, analyze the whole table. if pi == nil { - job := CreateTableAnalysisJob( - sctx, + job := jobFactory.CreateNonPartitionedTableAnalysisJob( db.O, tblInfo, r.statsHandle.GetTableStatsForAutoAnalyze(tblInfo), - autoAnalyzeRatio, - currentTs, ) - pushJobFunc(job) - // Skip the rest of the loop. + r.pushJob(job, calculator) continue } @@ -272,33 +237,27 @@ func (r *Refresher) RebuildTableAnalysisJobQueue() error { partitionDefs = append(partitionDefs, def) } } - partitionStats := getPartitionStats(r.statsHandle, tblInfo, partitionDefs) + partitionStats := priorityqueue.GetPartitionStats(r.statsHandle, tblInfo, partitionDefs) // If the prune mode is static, we need to analyze every partition as a separate table. if pruneMode == variable.Static { for pIDAndName, stats := range partitionStats { - job := CreateStaticPartitionAnalysisJob( - sctx, + job := jobFactory.CreateStaticPartitionAnalysisJob( db.O, tblInfo, pIDAndName.ID, pIDAndName.Name, stats, - autoAnalyzeRatio, - currentTs, ) - pushJobFunc(job) + r.pushJob(job, calculator) } } else { - job := createTableAnalysisJobForPartitions( - sctx, + job := jobFactory.CreateDynamicPartitionedTableAnalysisJob( db.O, tblInfo, r.statsHandle.GetPartitionStatsForAutoAnalyze(tblInfo, tblInfo.ID), partitionStats, - autoAnalyzeRatio, - currentTs, ) - pushJobFunc(job) + r.pushJob(job, calculator) } } } @@ -313,6 +272,25 @@ func (r *Refresher) RebuildTableAnalysisJobQueue() error { return nil } +func (r *Refresher) pushJob(job priorityqueue.AnalysisJob, calculator *priorityqueue.PriorityCalculator) { + if job == nil { + return + } + // We apply a penalty to larger tables, which can potentially result in a negative weight. + // To prevent this, we filter out any negative weights. Under normal circumstances, table sizes should not be negative. + weight := calculator.CalculateWeight(job) + if weight <= 0 { + statslogutil.SingletonStatsSamplerLogger().Warn( + "Table gets a negative weight", + zap.Float64("weight", weight), + zap.Stringer("job", job), + ) + } + job.SetWeight(weight) + // Push the job onto the queue. + r.Jobs.Push(job) +} + // WaitAutoAnalyzeFinishedForTest waits for the auto analyze job to be finished. // Only used in the test. func (r *Refresher) WaitAutoAnalyzeFinishedForTest() { @@ -330,314 +308,6 @@ func (r *Refresher) Close() { r.worker.Stop() } -// CreateTableAnalysisJob creates a TableAnalysisJob for the physical table. -func CreateTableAnalysisJob( - sctx sessionctx.Context, - tableSchema string, - tblInfo *model.TableInfo, - tblStats *statistics.Table, - autoAnalyzeRatio float64, - currentTs uint64, -) priorityqueue.AnalysisJob { - if !tblStats.IsEligibleForAnalysis() { - return nil - } - - tableStatsVer := sctx.GetSessionVars().AnalyzeVersion - statistics.CheckAnalyzeVerOnTable(tblStats, &tableStatsVer) - - changePercentage := CalculateChangePercentage(tblStats, autoAnalyzeRatio) - tableSize := calculateTableSize(tblInfo, tblStats) - lastAnalysisDuration := GetTableLastAnalyzeDuration(tblStats, currentTs) - indexes := CheckIndexesNeedAnalyze(tblInfo, tblStats) - - // No need to analyze. - // We perform a separate check because users may set the auto analyze ratio to 0, - // yet still wish to analyze newly added indexes and tables that have not been analyzed. - if changePercentage == 0 && len(indexes) == 0 { - return nil - } - - job := priorityqueue.NewNonPartitionedTableAnalysisJob( - tableSchema, - tblInfo.Name.O, - tblInfo.ID, - indexes, - tableStatsVer, - changePercentage, - tableSize, - lastAnalysisDuration, - ) - - return job -} - -// CreateStaticPartitionAnalysisJob creates a TableAnalysisJob for the static partition. -func CreateStaticPartitionAnalysisJob( - sctx sessionctx.Context, - tableSchema string, - globalTblInfo *model.TableInfo, - partitionID int64, - partitionName string, - partitionStats *statistics.Table, - autoAnalyzeRatio float64, - currentTs uint64, -) priorityqueue.AnalysisJob { - if !partitionStats.IsEligibleForAnalysis() { - return nil - } - - tableStatsVer := sctx.GetSessionVars().AnalyzeVersion - statistics.CheckAnalyzeVerOnTable(partitionStats, &tableStatsVer) - - changePercentage := CalculateChangePercentage(partitionStats, autoAnalyzeRatio) - tableSize := calculateTableSize(globalTblInfo, partitionStats) - lastAnalysisDuration := GetTableLastAnalyzeDuration(partitionStats, currentTs) - indexes := CheckIndexesNeedAnalyze(globalTblInfo, partitionStats) - - // No need to analyze. - // We perform a separate check because users may set the auto analyze ratio to 0, - // yet still wish to analyze newly added indexes and tables that have not been analyzed. - if changePercentage == 0 && len(indexes) == 0 { - return nil - } - - job := priorityqueue.NewStaticPartitionTableAnalysisJob( - tableSchema, - globalTblInfo.Name.O, - globalTblInfo.ID, - partitionName, - partitionID, - indexes, - tableStatsVer, - changePercentage, - tableSize, - lastAnalysisDuration, - ) - - return job -} - -// CalculateChangePercentage calculates the change percentage of the table -// based on the change count and the analysis count. -func CalculateChangePercentage( - tblStats *statistics.Table, - autoAnalyzeRatio float64, -) float64 { - if !tblStats.IsAnalyzed() { - return unanalyzedTableDefaultChangePercentage - } - - // Auto analyze based on the change percentage is disabled. - // However, this check should not affect the analysis of indexes, - // as index analysis is still needed for query performance. - if autoAnalyzeRatio == 0 { - return 0 - } - - tblCnt := float64(tblStats.RealtimeCount) - if histCnt := tblStats.GetAnalyzeRowCount(); histCnt > 0 { - tblCnt = histCnt - } - res := float64(tblStats.ModifyCount) / tblCnt - if res > autoAnalyzeRatio { - return res - } - - return 0 -} - -func calculateTableSize( - tblInfo *model.TableInfo, - tblStats *statistics.Table, -) float64 { - tblCnt := float64(tblStats.RealtimeCount) - // TODO: Ignore unanalyzable columns. - colCnt := float64(len(tblInfo.Columns)) - - return tblCnt * colCnt -} - -// GetTableLastAnalyzeDuration gets the duration since the last analysis of the table. -func GetTableLastAnalyzeDuration( - tblStats *statistics.Table, - currentTs uint64, -) time.Duration { - lastTime := findLastAnalyzeTime(tblStats, currentTs) - currentTime := oracle.GetTimeFromTS(currentTs) - - // Calculate the duration since last analyze. - return currentTime.Sub(lastTime) -} - -// findLastAnalyzeTime finds the last analyze time of the table. -// It uses `LastUpdateVersion` to find the last analyze time. -// The `LastUpdateVersion` is the version of the transaction that updates the statistics. -// It always not null(default 0), so we can use it to find the last analyze time. -func findLastAnalyzeTime( - tblStats *statistics.Table, - currentTs uint64, -) time.Time { - // Table is not analyzed, compose a fake version. - if !tblStats.IsAnalyzed() { - phy := oracle.GetTimeFromTS(currentTs) - return phy.Add(unanalyzedTableDefaultLastUpdateDuration) - } - return oracle.GetTimeFromTS(tblStats.LastAnalyzeVersion) -} - -// CheckIndexesNeedAnalyze checks if the indexes of the table need to be analyzed. -func CheckIndexesNeedAnalyze( - tblInfo *model.TableInfo, - tblStats *statistics.Table, -) []string { - // If table is not analyzed, we need to analyze whole table. - // So we don't need to check indexes. - if !tblStats.IsAnalyzed() { - return nil - } - - indexes := make([]string, 0, len(tblInfo.Indices)) - // Check if missing index stats. - for _, idx := range tblInfo.Indices { - if idxStats := tblStats.GetIdx(idx.ID); idxStats == nil && !tblStats.ColAndIdxExistenceMap.HasAnalyzed(idx.ID, true) && idx.State == model.StatePublic { - indexes = append(indexes, idx.Name.O) - } - } - - return indexes -} - -func createTableAnalysisJobForPartitions( - sctx sessionctx.Context, - tableSchema string, - tblInfo *model.TableInfo, - tblStats *statistics.Table, - partitionStats map[PartitionIDAndName]*statistics.Table, - autoAnalyzeRatio float64, - currentTs uint64, -) priorityqueue.AnalysisJob { - if !tblStats.IsEligibleForAnalysis() { - return nil - } - - // TODO: figure out how to check the table stats version correctly for partitioned tables. - tableStatsVer := sctx.GetSessionVars().AnalyzeVersion - statistics.CheckAnalyzeVerOnTable(tblStats, &tableStatsVer) - - averageChangePercentage, avgSize, minLastAnalyzeDuration, partitionNames := CalculateIndicatorsForPartitions( - tblInfo, - partitionStats, - autoAnalyzeRatio, - currentTs, - ) - partitionIndexes := CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable( - tblInfo, - partitionStats, - ) - // No need to analyze. - // We perform a separate check because users may set the auto analyze ratio to 0, - // yet still wish to analyze newly added indexes and tables that have not been analyzed. - if len(partitionNames) == 0 && len(partitionIndexes) == 0 { - return nil - } - - job := priorityqueue.NewDynamicPartitionedTableAnalysisJob( - tableSchema, - tblInfo.Name.O, - tblInfo.ID, - partitionNames, - partitionIndexes, - tableStatsVer, - averageChangePercentage, - avgSize, - minLastAnalyzeDuration, - ) - - return job -} - -// CalculateIndicatorsForPartitions calculates the average change percentage, -// average size and average last analyze duration for the partitions that meet the threshold. -// Change percentage is the ratio of the number of modified rows to the total number of rows. -// Size is the product of the number of rows and the number of columns. -// Last analyze duration is the duration since the last analyze. -func CalculateIndicatorsForPartitions( - tblInfo *model.TableInfo, - partitionStats map[PartitionIDAndName]*statistics.Table, - autoAnalyzeRatio float64, - currentTs uint64, -) ( - avgChange float64, - avgSize float64, - avgLastAnalyzeDuration time.Duration, - partitionNames []string, -) { - totalChangePercent := 0.0 - totalSize := 0.0 - count := 0.0 - partitionNames = make([]string, 0, len(partitionStats)) - cols := float64(len(tblInfo.Columns)) - totalLastAnalyzeDuration := time.Duration(0) - - for pIDAndName, tblStats := range partitionStats { - changePercent := CalculateChangePercentage(tblStats, autoAnalyzeRatio) - // Skip partition analysis if it doesn't meet the threshold, stats are not yet loaded, - // or the auto analyze ratio is set to 0 by the user. - if changePercent == 0 { - continue - } - - totalChangePercent += changePercent - // size = count * cols - totalSize += float64(tblStats.RealtimeCount) * cols - lastAnalyzeDuration := GetTableLastAnalyzeDuration(tblStats, currentTs) - totalLastAnalyzeDuration += lastAnalyzeDuration - partitionNames = append(partitionNames, pIDAndName.Name) - count++ - } - if len(partitionNames) == 0 { - return 0, 0, 0, partitionNames - } - - avgChange = totalChangePercent / count - avgSize = totalSize / count - avgLastAnalyzeDuration = totalLastAnalyzeDuration / time.Duration(count) - - return avgChange, avgSize, avgLastAnalyzeDuration, partitionNames -} - -// CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable checks if the indexes of the partitioned table need to be analyzed. -// It returns a map from index name to the names of the partitions that need to be analyzed. -// NOTE: This is only for newly added indexes. -func CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable( - tblInfo *model.TableInfo, - partitionStats map[PartitionIDAndName]*statistics.Table, -) map[string][]string { - partitionIndexes := make(map[string][]string, len(tblInfo.Indices)) - - for _, idx := range tblInfo.Indices { - // No need to analyze the index if it's not public. - if idx.State != model.StatePublic { - continue - } - - // Find all the partitions that need to analyze this index. - names := make([]string, 0, len(partitionStats)) - for pIDAndName, tblStats := range partitionStats { - if idxStats := tblStats.GetIdx(idx.ID); idxStats == nil && !tblStats.ColAndIdxExistenceMap.HasAnalyzed(idx.ID, true) { - names = append(names, pIDAndName.Name) - } - } - - if len(names) > 0 { - partitionIndexes[idx.Name.O] = names - } - } - - return partitionIndexes -} - func getStartTs(sctx sessionctx.Context) (uint64, error) { txn, err := sctx.Txn(true) if err != nil { @@ -645,48 +315,3 @@ func getStartTs(sctx sessionctx.Context) (uint64, error) { } return txn.StartTS(), nil } - -// PartitionIDAndName is a struct that contains the ID and name of a partition. -// Exported for testing purposes. Do not use it in other packages. -type PartitionIDAndName struct { - Name string - ID int64 -} - -func getPartitionStats( - statsHandle statstypes.StatsHandle, - tblInfo *model.TableInfo, - defs []model.PartitionDefinition, -) map[PartitionIDAndName]*statistics.Table { - partitionStats := make(map[PartitionIDAndName]*statistics.Table, len(defs)) - - for _, def := range defs { - stats := statsHandle.GetPartitionStatsForAutoAnalyze(tblInfo, def.ID) - // Ignore the partition if it's not ready to analyze. - if !stats.IsEligibleForAnalysis() { - continue - } - d := PartitionIDAndName{ - ID: def.ID, - Name: def.Name.O, - } - partitionStats[d] = stats - } - - return partitionStats -} - -// autoAnalysisTimeWindow is a struct that contains the start and end time of the auto analyze time window. -type autoAnalysisTimeWindow struct { - start time.Time - end time.Time -} - -// isWithinTimeWindow checks if the current time is within the time window. -// If the auto analyze time window is not set or the current time is not in the window, return false. -func (a autoAnalysisTimeWindow) isWithinTimeWindow(currentTime time.Time) bool { - if a.start == (time.Time{}) || a.end == (time.Time{}) { - return false - } - return timeutil.WithinDayTimePeriod(a.start, a.end, currentTime) -} diff --git a/pkg/statistics/handle/autoanalyze/refresher/refresher_test.go b/pkg/statistics/handle/autoanalyze/refresher/refresher_test.go index 4ac0971081b3c..0ebbe5f8e8a19 100644 --- a/pkg/statistics/handle/autoanalyze/refresher/refresher_test.go +++ b/pkg/statistics/handle/autoanalyze/refresher/refresher_test.go @@ -17,18 +17,15 @@ package refresher_test import ( "context" "math" - "sort" "testing" "time" - "github.com/pingcap/tidb/pkg/meta/model" pmodel "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/priorityqueue" "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze/refresher" "github.com/pingcap/tidb/pkg/testkit" "github.com/stretchr/testify/require" - "github.com/tikv/client-go/v2/oracle" ) func TestSkipAnalyzeTableWhenAutoAnalyzeRatioIsZero(t *testing.T) { @@ -440,492 +437,3 @@ func TestRebuildTableAnalysisJobQueue(t *testing.T) { require.Equal(t, float64(6*2), indicators.TableSize) require.GreaterOrEqual(t, indicators.LastAnalysisDuration, time.Duration(0)) } - -func TestCalculateChangePercentage(t *testing.T) { - unanalyzedColumns := map[int64]*statistics.Column{ - 1: {}, - 2: {}, - } - unanalyzedIndices := map[int64]*statistics.Index{ - 1: {}, - 2: {}, - } - analyzedColumns := map[int64]*statistics.Column{ - 1: { - StatsVer: 2, - }, - 2: { - StatsVer: 2, - }, - } - analyzedIndices := map[int64]*statistics.Index{ - 1: { - StatsVer: 2, - }, - 2: { - StatsVer: 2, - }, - } - bothUnanalyzedMap := statistics.NewColAndIndexExistenceMap(0, 0) - bothAnalyzedMap := statistics.NewColAndIndexExistenceMap(2, 2) - bothAnalyzedMap.InsertCol(1, nil, true) - bothAnalyzedMap.InsertCol(2, nil, true) - bothAnalyzedMap.InsertIndex(1, nil, true) - bothAnalyzedMap.InsertIndex(2, nil, true) - tests := []struct { - name string - tblStats *statistics.Table - autoAnalyzeRatio float64 - want float64 - }{ - { - name: "Test Table not analyzed", - tblStats: &statistics.Table{ - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, unanalyzedColumns, unanalyzedIndices), - ColAndIdxExistenceMap: bothUnanalyzedMap, - }, - autoAnalyzeRatio: 0.5, - want: 1, - }, - { - name: "Based on change percentage", - tblStats: &statistics.Table{ - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, (statistics.AutoAnalyzeMinCnt+1)*2, analyzedColumns, analyzedIndices), - ColAndIdxExistenceMap: bothAnalyzedMap, - LastAnalyzeVersion: 1, - }, - autoAnalyzeRatio: 0.5, - want: 2, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := refresher.CalculateChangePercentage(tt.tblStats, tt.autoAnalyzeRatio) - require.Equal(t, tt.want, got) - }) - } -} - -func TestGetTableLastAnalyzeDuration(t *testing.T) { - // 2023-12-31 10:00:00 - lastUpdateTime := time.Date(2023, 12, 31, 10, 0, 0, 0, time.UTC) - lastUpdateTs := oracle.GoTimeToTS(lastUpdateTime) - tblStats := &statistics.Table{ - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, 0, 0, map[int64]*statistics.Column{ - 1: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - }, nil), - LastAnalyzeVersion: lastUpdateTs, - } - // 2024-01-01 10:00:00 - currentTime := time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC) - currentTs := oracle.GoTimeToTS(currentTime) - want := 24 * time.Hour - - got := refresher.GetTableLastAnalyzeDuration(tblStats, currentTs) - require.Equal(t, want, got) -} - -func TestGetTableLastAnalyzeDurationForUnanalyzedTable(t *testing.T) { - tblStats := &statistics.Table{ - HistColl: statistics.HistColl{}, - } - // 2024-01-01 10:00:00 - currentTime := time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC) - currentTs := oracle.GoTimeToTS(currentTime) - want := 1800 * time.Second - - got := refresher.GetTableLastAnalyzeDuration(tblStats, currentTs) - require.Equal(t, want, got) -} - -func TestCheckIndexesNeedAnalyze(t *testing.T) { - analyzedMap := statistics.NewColAndIndexExistenceMap(1, 0) - analyzedMap.InsertCol(1, nil, true) - analyzedMap.InsertIndex(1, nil, false) - tests := []struct { - name string - tblInfo *model.TableInfo - tblStats *statistics.Table - want []string - }{ - { - name: "Test Table not analyzed", - tblInfo: &model.TableInfo{ - Indices: []*model.IndexInfo{ - { - ID: 1, - Name: pmodel.NewCIStr("index1"), - State: model.StatePublic, - }, - }, - }, - tblStats: &statistics.Table{ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 0)}, - want: nil, - }, - { - name: "Test Index not analyzed", - tblInfo: &model.TableInfo{ - Indices: []*model.IndexInfo{ - { - ID: 1, - Name: pmodel.NewCIStr("index1"), - State: model.StatePublic, - }, - }, - }, - tblStats: &statistics.Table{ - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, 0, 0, map[int64]*statistics.Column{ - 1: { - StatsVer: 2, - }, - }, map[int64]*statistics.Index{}), - ColAndIdxExistenceMap: analyzedMap, - LastAnalyzeVersion: 1, - }, - want: []string{"index1"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := refresher.CheckIndexesNeedAnalyze(tt.tblInfo, tt.tblStats) - require.Equal(t, tt.want, got) - }) - } -} - -func TestCalculateIndicatorsForPartitions(t *testing.T) { - // 2024-01-01 10:00:00 - currentTime := time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC) - currentTs := oracle.GoTimeToTS(currentTime) - // 2023-12-31 10:00:00 - lastUpdateTime := time.Date(2023, 12, 31, 10, 0, 0, 0, time.UTC) - lastUpdateTs := oracle.GoTimeToTS(lastUpdateTime) - unanalyzedMap := statistics.NewColAndIndexExistenceMap(0, 0) - analyzedMap := statistics.NewColAndIndexExistenceMap(2, 1) - analyzedMap.InsertCol(1, nil, true) - analyzedMap.InsertCol(2, nil, true) - analyzedMap.InsertIndex(1, nil, true) - tests := []struct { - name string - tblInfo *model.TableInfo - partitionStats map[refresher.PartitionIDAndName]*statistics.Table - defs []model.PartitionDefinition - autoAnalyzeRatio float64 - currentTs uint64 - wantAvgChangePercentage float64 - wantAvgSize float64 - wantAvgLastAnalyzeDuration time.Duration - wantPartitions []string - }{ - { - name: "Test Table not analyzed", - tblInfo: &model.TableInfo{ - Indices: []*model.IndexInfo{ - { - ID: 1, - Name: pmodel.NewCIStr("index1"), - State: model.StatePublic, - }, - }, - Columns: []*model.ColumnInfo{ - { - ID: 1, - }, - { - ID: 2, - }, - }, - }, - partitionStats: map[refresher.PartitionIDAndName]*statistics.Table{ - { - ID: 1, - Name: "p0", - }: { - HistColl: statistics.HistColl{ - Pseudo: false, - RealtimeCount: statistics.AutoAnalyzeMinCnt + 1, - }, - ColAndIdxExistenceMap: unanalyzedMap, - }, - { - ID: 2, - Name: "p1", - }: { - HistColl: statistics.HistColl{ - Pseudo: false, - RealtimeCount: statistics.AutoAnalyzeMinCnt + 1, - }, - ColAndIdxExistenceMap: unanalyzedMap, - }, - }, - defs: []model.PartitionDefinition{ - { - ID: 1, - Name: pmodel.NewCIStr("p0"), - }, - { - ID: 2, - Name: pmodel.NewCIStr("p1"), - }, - }, - autoAnalyzeRatio: 0.5, - currentTs: currentTs, - wantAvgChangePercentage: 1, - wantAvgSize: 2002, - wantAvgLastAnalyzeDuration: 1800 * time.Second, - wantPartitions: []string{"p0", "p1"}, - }, - { - name: "Test Table analyzed and only one partition meets the threshold", - tblInfo: &model.TableInfo{ - Indices: []*model.IndexInfo{ - { - ID: 1, - Name: pmodel.NewCIStr("index1"), - State: model.StatePublic, - }, - }, - Columns: []*model.ColumnInfo{ - { - ID: 1, - }, - { - ID: 2, - }, - }, - }, - partitionStats: map[refresher.PartitionIDAndName]*statistics.Table{ - { - ID: 1, - Name: "p0", - }: { - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, (statistics.AutoAnalyzeMinCnt+1)*2, map[int64]*statistics.Column{ - 1: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - 2: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - }, nil), - Version: currentTs, - ColAndIdxExistenceMap: analyzedMap, - LastAnalyzeVersion: lastUpdateTs, - }, - { - ID: 2, - Name: "p1", - }: { - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, map[int64]*statistics.Column{ - 1: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - 2: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - }, nil), - Version: currentTs, - ColAndIdxExistenceMap: analyzedMap, - LastAnalyzeVersion: lastUpdateTs, - }, - }, - defs: []model.PartitionDefinition{ - { - ID: 1, - Name: pmodel.NewCIStr("p0"), - }, - { - ID: 2, - Name: pmodel.NewCIStr("p1"), - }, - }, - autoAnalyzeRatio: 0.5, - currentTs: currentTs, - wantAvgChangePercentage: 2, - wantAvgSize: 2002, - wantAvgLastAnalyzeDuration: 24 * time.Hour, - wantPartitions: []string{"p0"}, - }, - { - name: "No partition meets the threshold", - tblInfo: &model.TableInfo{ - Indices: []*model.IndexInfo{ - { - ID: 1, - Name: pmodel.NewCIStr("index1"), - State: model.StatePublic, - }, - }, - Columns: []*model.ColumnInfo{ - { - ID: 1, - }, - { - ID: 2, - }, - }, - }, - partitionStats: map[refresher.PartitionIDAndName]*statistics.Table{ - { - ID: 1, - Name: "p0", - }: { - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, map[int64]*statistics.Column{ - 1: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - 2: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - }, nil), - Version: currentTs, - ColAndIdxExistenceMap: analyzedMap, - LastAnalyzeVersion: lastUpdateTs, - }, - { - ID: 2, - Name: "p1", - }: { - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, map[int64]*statistics.Column{ - 1: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - 2: { - StatsVer: 2, - Histogram: statistics.Histogram{ - LastUpdateVersion: lastUpdateTs, - }, - }, - }, nil), - Version: currentTs, - ColAndIdxExistenceMap: analyzedMap, - LastAnalyzeVersion: lastUpdateTs, - }, - }, - defs: []model.PartitionDefinition{ - { - ID: 1, - Name: pmodel.NewCIStr("p0"), - }, - { - ID: 2, - Name: pmodel.NewCIStr("p1"), - }, - }, - autoAnalyzeRatio: 0.5, - currentTs: currentTs, - wantAvgChangePercentage: 0, - wantAvgSize: 0, - wantAvgLastAnalyzeDuration: 0, - wantPartitions: []string{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotAvgChangePercentage, - gotAvgSize, - gotAvgLastAnalyzeDuration, - gotPartitions := - refresher.CalculateIndicatorsForPartitions( - tt.tblInfo, - tt.partitionStats, - tt.autoAnalyzeRatio, - tt.currentTs, - ) - require.Equal(t, tt.wantAvgChangePercentage, gotAvgChangePercentage) - require.Equal(t, tt.wantAvgSize, gotAvgSize) - require.Equal(t, tt.wantAvgLastAnalyzeDuration, gotAvgLastAnalyzeDuration) - // Sort the partitions. - sort.Strings(tt.wantPartitions) - sort.Strings(gotPartitions) - require.Equal(t, tt.wantPartitions, gotPartitions) - }) - } -} - -func TestCheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable(t *testing.T) { - tblInfo := model.TableInfo{ - Indices: []*model.IndexInfo{ - { - ID: 1, - Name: pmodel.NewCIStr("index1"), - State: model.StatePublic, - }, - { - ID: 2, - Name: pmodel.NewCIStr("index2"), - State: model.StatePublic, - }, - }, - Columns: []*model.ColumnInfo{ - { - ID: 1, - }, - { - ID: 2, - }, - }, - } - partitionStats := map[refresher.PartitionIDAndName]*statistics.Table{ - { - ID: 1, - Name: "p0", - }: { - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, nil, map[int64]*statistics.Index{}), - ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 0), - }, - { - ID: 2, - Name: "p1", - }: { - HistColl: *statistics.NewHistCollWithColsAndIdxs(0, false, statistics.AutoAnalyzeMinCnt+1, 0, nil, map[int64]*statistics.Index{ - 2: { - StatsVer: 2, - }, - }), - ColAndIdxExistenceMap: statistics.NewColAndIndexExistenceMap(0, 1), - }, - } - - partitionIndexes := refresher.CheckNewlyAddedIndexesNeedAnalyzeForPartitionedTable(&tblInfo, partitionStats) - expected := map[string][]string{"index1": {"p0", "p1"}, "index2": {"p0"}} - require.Equal(t, len(expected), len(partitionIndexes)) - - for k, v := range expected { - sort.Strings(v) - if val, ok := partitionIndexes[k]; ok { - sort.Strings(val) - require.Equal(t, v, val) - } else { - require.Fail(t, "key not found in partitionIndexes: "+k) - } - } -} diff --git a/pkg/statistics/handle/bootstrap.go b/pkg/statistics/handle/bootstrap.go index 89fe1c019ee2e..7671c4a2877fa 100644 --- a/pkg/statistics/handle/bootstrap.go +++ b/pkg/statistics/handle/bootstrap.go @@ -570,6 +570,12 @@ func (h *Handle) initStatsFMSketch(cache statstypes.StatsCache) error { func (*Handle) initStatsBuckets4Chunk(cache statstypes.StatsCache, iter *chunk.Iterator4Chunk) { var table *statistics.Table + unspecifiedLengthTp := types.NewFieldType(mysql.TypeBlob) + var ( + hasErr bool + failedTableID int64 + failedHistID int64 + ) for row := iter.Begin(); row != iter.End(); row = iter.Next() { tableID, isIndex, histID := row.GetInt64(0), row.GetInt64(1), row.GetInt64(2) if table == nil || table.PhysicalID != tableID { @@ -604,16 +610,35 @@ func (*Handle) initStatsBuckets4Chunk(cache statstypes.StatsCache, iter *chunk.I hist = &column.Histogram d := types.NewBytesDatum(row.GetBytes(5)) var err error - lower, err = d.ConvertTo(statistics.UTCWithAllowInvalidDateCtx, &column.Info.FieldType) + if column.Info.FieldType.EvalType() == types.ETString && column.Info.FieldType.GetType() != mysql.TypeEnum && column.Info.FieldType.GetType() != mysql.TypeSet { + // For new collation data, when storing the bounds of the histogram, we store the collate key instead of the + // original value. + // But there's additional conversion logic for new collation data, and the collate key might be longer than + // the FieldType.flen. + // If we use the original FieldType here, there might be errors like "Invalid utf8mb4 character string" + // or "Data too long". + // So we change it to TypeBlob to bypass those logics here. + lower, err = d.ConvertTo(statistics.UTCWithAllowInvalidDateCtx, unspecifiedLengthTp) + } else { + lower, err = d.ConvertTo(statistics.UTCWithAllowInvalidDateCtx, &column.Info.FieldType) + } if err != nil { - logutil.BgLogger().Debug("decode bucket lower bound failed", zap.Error(err)) + hasErr = true + failedTableID = tableID + failedHistID = histID table.DelCol(histID) continue } d = types.NewBytesDatum(row.GetBytes(6)) - upper, err = d.ConvertTo(statistics.UTCWithAllowInvalidDateCtx, &column.Info.FieldType) + if column.Info.FieldType.EvalType() == types.ETString && column.Info.FieldType.GetType() != mysql.TypeEnum && column.Info.FieldType.GetType() != mysql.TypeSet { + upper, err = d.ConvertTo(statistics.UTCWithAllowInvalidDateCtx, unspecifiedLengthTp) + } else { + upper, err = d.ConvertTo(statistics.UTCWithAllowInvalidDateCtx, &column.Info.FieldType) + } if err != nil { - logutil.BgLogger().Debug("decode bucket upper bound failed", zap.Error(err)) + hasErr = true + failedTableID = tableID + failedHistID = histID table.DelCol(histID) continue } @@ -623,6 +648,9 @@ func (*Handle) initStatsBuckets4Chunk(cache statstypes.StatsCache, iter *chunk.I if table != nil { cache.Put(table.PhysicalID, table) // put this table in the cache because all statstics of the table have been read. } + if hasErr { + logutil.BgLogger().Error("failed to convert datum for at least one histogram bucket", zap.Int64("table ID", failedTableID), zap.Int64("column ID", failedHistID)) + } } func (h *Handle) initStatsBuckets(cache statstypes.StatsCache, totalMemory uint64) error { diff --git a/pkg/statistics/handle/ddl/BUILD.bazel b/pkg/statistics/handle/ddl/BUILD.bazel index f3ca9919e5df3..268b15ea3e435 100644 --- a/pkg/statistics/handle/ddl/BUILD.bazel +++ b/pkg/statistics/handle/ddl/BUILD.bazel @@ -12,7 +12,7 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/statistics/handle/ddl", visibility = ["//visibility:public"], deps = [ - "//pkg/ddl/util", + "//pkg/ddl/notifier", "//pkg/infoschema", "//pkg/meta/model", "//pkg/sessionctx", @@ -22,6 +22,7 @@ go_library( "//pkg/statistics/handle/storage", "//pkg/statistics/handle/types", "//pkg/statistics/handle/util", + "//pkg/util/intest", "@com_github_pingcap_errors//:errors", "@org_uber_go_zap//:zap", ], @@ -32,14 +33,17 @@ go_test( timeout = "short", srcs = ["ddl_test.go"], flaky = True, - shard_count = 18, + shard_count = 19, deps = [ + ":ddl", + "//pkg/ddl/notifier", "//pkg/meta/model", "//pkg/parser/model", "//pkg/planner/cardinality", - "//pkg/statistics/handle/util", + "//pkg/statistics/handle/storage", "//pkg/testkit", "//pkg/types", + "//pkg/util", "//pkg/util/mock", "@com_github_stretchr_testify//require", ], diff --git a/pkg/statistics/handle/ddl/ddl.go b/pkg/statistics/handle/ddl/ddl.go index fc82f61e90fb6..bbe5e0868a097 100644 --- a/pkg/statistics/handle/ddl/ddl.go +++ b/pkg/statistics/handle/ddl/ddl.go @@ -16,6 +16,7 @@ package ddl import ( "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" @@ -24,11 +25,12 @@ import ( "github.com/pingcap/tidb/pkg/statistics/handle/storage" "github.com/pingcap/tidb/pkg/statistics/handle/types" "github.com/pingcap/tidb/pkg/statistics/handle/util" + "github.com/pingcap/tidb/pkg/util/intest" "go.uber.org/zap" ) type ddlHandlerImpl struct { - ddlEventCh chan *util.DDLEvent + ddlEventCh chan *notifier.SchemaChangeEvent statsWriter types.StatsReadWriter statsHandler types.StatsHandle globalStatsHandler types.StatsGlobal @@ -41,7 +43,7 @@ func NewDDLHandler( globalStatsHandler types.StatsGlobal, ) types.DDL { return &ddlHandlerImpl{ - ddlEventCh: make(chan *util.DDLEvent, 1000), + ddlEventCh: make(chan *notifier.SchemaChangeEvent, 1000), statsWriter: statsWriter, statsHandler: statsHandler, globalStatsHandler: globalStatsHandler, @@ -49,65 +51,10 @@ func NewDDLHandler( } // HandleDDLEvent begins to process a ddl task. -func (h *ddlHandlerImpl) HandleDDLEvent(t *util.DDLEvent) error { - sctx, err := h.statsHandler.SPool().Get() - if err != nil { - return err - } - defer h.statsHandler.SPool().Put(sctx) - - // ActionFlashbackCluster will not create any new stats info - // and it's SchemaID alwayws equals to 0, so skip check it. - if t.GetType() != model.ActionFlashbackCluster && t.SchemaChangeEvent == nil { - if isSysDB, err := t.IsMemOrSysDB(sctx.(sessionctx.Context)); err != nil { - return err - } else if isSysDB { - // EXCHANGE PARTITION EVENT NOTES: - // 1. When a partition is exchanged with a system table, we need to adjust the global statistics - // based on the count delta and modify count delta. However, due to the involvement of the system table, - // a complete update of the global statistics is not feasible. Therefore, we bypass the statistics update - // for the table in this scenario. Despite this, the table id still changes, so the statistics for the - // system table will still be visible. - // 2. If the system table is a partitioned table, we will update the global statistics for the partitioned table. - // It is rare to exchange a partition from a system table, so we can ignore this case. In this case, - // the system table will have statistics, but this is not a significant issue. - logutil.StatsLogger().Info("Skip handle system database ddl event", zap.Stringer("event", t)) - return nil - } - } - if t.SchemaChangeEvent == nil { - // when SchemaChangeEvent is set, it will be printed in the default branch of - // below switch. - logutil.StatsLogger().Info("Handle ddl event", zap.Stringer("event", t)) - } - - switch t.GetType() { - case model.ActionRemovePartitioning: - // Change id for global stats, since the data has not changed! - // Note: This operation will update all tables related to statistics with the new ID. - oldTblID, - newSingleTableInfo, - droppedPartInfo := t.GetRemovePartitioningInfo() - if err := h.statsWriter.ChangeGlobalStatsID(oldTblID, newSingleTableInfo.ID); err != nil { - return err - } - - // Remove partition stats. - for _, def := range droppedPartInfo.Definitions { - if err := h.statsWriter.UpdateStatsMetaVersionForGC(def.ID); err != nil { - return err - } - } - case model.ActionFlashbackCluster: - return h.statsWriter.UpdateStatsVersion() - default: - logutil.StatsLogger().Info("Handle schema change event", zap.Stringer("event", t.SchemaChangeEvent)) - } - - e := t.SchemaChangeEvent - switch e.GetType() { +func (h *ddlHandlerImpl) HandleDDLEvent(s *notifier.SchemaChangeEvent) error { + switch s.GetType() { case model.ActionCreateTable: - newTableInfo := e.GetCreateTableInfo() + newTableInfo := s.GetCreateTableInfo() ids, err := h.getTableIDs(newTableInfo) if err != nil { return err @@ -118,7 +65,7 @@ func (h *ddlHandlerImpl) HandleDDLEvent(t *util.DDLEvent) error { } } case model.ActionTruncateTable: - newTableInfo, droppedTableInfo := e.GetTruncateTableInfo() + newTableInfo, droppedTableInfo := s.GetTruncateTableInfo() ids, err := h.getTableIDs(newTableInfo) if err != nil { return err @@ -140,7 +87,7 @@ func (h *ddlHandlerImpl) HandleDDLEvent(t *util.DDLEvent) error { } } case model.ActionDropTable: - droppedTableInfo := e.GetDropTableInfo() + droppedTableInfo := s.GetDropTableInfo() ids, err := h.getTableIDs(droppedTableInfo) if err != nil { return err @@ -151,7 +98,7 @@ func (h *ddlHandlerImpl) HandleDDLEvent(t *util.DDLEvent) error { } } case model.ActionAddColumn: - newTableInfo, newColumnInfo := e.GetAddColumnInfo() + newTableInfo, newColumnInfo := s.GetAddColumnInfo() ids, err := h.getTableIDs(newTableInfo) if err != nil { return err @@ -162,7 +109,7 @@ func (h *ddlHandlerImpl) HandleDDLEvent(t *util.DDLEvent) error { } } case model.ActionModifyColumn: - newTableInfo, modifiedColumnInfo := e.GetModifyColumnInfo() + newTableInfo, modifiedColumnInfo := s.GetModifyColumnInfo() ids, err := h.getTableIDs(newTableInfo) if err != nil { return err @@ -173,30 +120,40 @@ func (h *ddlHandlerImpl) HandleDDLEvent(t *util.DDLEvent) error { } } case model.ActionAddTablePartition: - globalTableInfo, addedPartitionInfo := e.GetAddPartitionInfo() + globalTableInfo, addedPartitionInfo := s.GetAddPartitionInfo() for _, def := range addedPartitionInfo.Definitions { if err := h.statsWriter.InsertTableStats2KV(globalTableInfo, def.ID); err != nil { return err } } case model.ActionTruncateTablePartition: - if err := h.onTruncatePartitions(e); err != nil { + if err := h.onTruncatePartitions(s); err != nil { return err } case model.ActionDropTablePartition: - if err := h.onDropPartitions(e); err != nil { + if err := h.onDropPartitions(s); err != nil { return err } + // EXCHANGE PARTITION EVENT NOTES: + // 1. When a partition is exchanged with a system table, we need to adjust the global statistics + // based on the count delta and modify count delta. However, due to the involvement of the system table, + // a complete update of the global statistics is not feasible. Therefore, we bypass the statistics update + // for the table in this scenario. Despite this, the table id still changes, so the statistics for the + // system table will still be visible. + // 2. If the system table is a partitioned table, we will update the global statistics for the partitioned table. + // It is rare to exchange a partition from a system table, so we can ignore this case. In this case, + // the system table will have statistics, but this is not a significant issue. + // So we decided to completely ignore the system table event. case model.ActionExchangeTablePartition: - if err := h.onExchangeAPartition(e); err != nil { + if err := h.onExchangeAPartition(s); err != nil { return err } case model.ActionReorganizePartition: - if err := h.onReorganizePartitions(e); err != nil { + if err := h.onReorganizePartitions(s); err != nil { return err } case model.ActionAlterTablePartitioning: - oldSingleTableID, globalTableInfo, addedPartInfo := e.GetAddPartitioningInfo() + oldSingleTableID, globalTableInfo, addedPartInfo := s.GetAddPartitioningInfo() // Add new partition stats. for _, def := range addedPartInfo.Definitions { if err := h.statsWriter.InsertTableStats2KV(globalTableInfo, def.ID); err != nil { @@ -206,10 +163,38 @@ func (h *ddlHandlerImpl) HandleDDLEvent(t *util.DDLEvent) error { // Change id for global stats, since the data has not changed! // Note: This operation will update all tables related to statistics with the new ID. return h.statsWriter.ChangeGlobalStatsID(oldSingleTableID, globalTableInfo.ID) + case model.ActionRemovePartitioning: + // Change id for global stats, since the data has not changed! + // Note: This operation will update all tables related to statistics with the new ID. + oldTblID, newSingleTableInfo, droppedPartInfo := s.GetRemovePartitioningInfo() + if err := h.statsWriter.ChangeGlobalStatsID(oldTblID, newSingleTableInfo.ID); err != nil { + return err + } + + // Remove partition stats. + for _, def := range droppedPartInfo.Definitions { + if err := h.statsWriter.UpdateStatsMetaVersionForGC(def.ID); err != nil { + return err + } + } + case model.ActionFlashbackCluster: + return h.statsWriter.UpdateStatsVersion() + default: + intest.Assert(false) + logutil.StatsLogger().Error("Unhandled schema change event", zap.Stringer("type", s)) } return nil } +// UpdateStatsWithCountDeltaAndModifyCountDeltaForTest updates the global stats with the given count delta and modify count delta. +func UpdateStatsWithCountDeltaAndModifyCountDeltaForTest( + sctx sessionctx.Context, + tableID int64, + countDelta, modifyCountDelta int64, +) error { + return updateStatsWithCountDeltaAndModifyCountDelta(sctx, tableID, countDelta, modifyCountDelta) +} + // updateStatsWithCountDeltaAndModifyCountDelta updates // the global stats with the given count delta and modify count delta. // Only used by some special DDLs, such as exchange partition. @@ -250,7 +235,7 @@ func updateStatsWithCountDeltaAndModifyCountDelta( } // Because count can not be negative, so we need to get the current and calculate the delta. - count, modifyCount, isNull, err := storage.StatsMetaCountAndModifyCount(sctx, tableID) + count, modifyCount, isNull, err := storage.StatsMetaCountAndModifyCountForUpdate(sctx, tableID) if err != nil { return err } @@ -303,6 +288,6 @@ func (h *ddlHandlerImpl) getTableIDs(tblInfo *model.TableInfo) (ids []int64, err } // DDLEventCh returns ddl events channel in handle. -func (h *ddlHandlerImpl) DDLEventCh() chan *util.DDLEvent { +func (h *ddlHandlerImpl) DDLEventCh() chan *notifier.SchemaChangeEvent { return h.ddlEventCh } diff --git a/pkg/statistics/handle/ddl/ddl_test.go b/pkg/statistics/handle/ddl/ddl_test.go index c85d0cfb97095..135d19637a834 100644 --- a/pkg/statistics/handle/ddl/ddl_test.go +++ b/pkg/statistics/handle/ddl/ddl_test.go @@ -19,12 +19,15 @@ import ( "fmt" "testing" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/meta/model" pmodel "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/planner/cardinality" - "github.com/pingcap/tidb/pkg/statistics/handle/util" + "github.com/pingcap/tidb/pkg/statistics/handle/ddl" + "github.com/pingcap/tidb/pkg/statistics/handle/storage" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/mock" "github.com/stretchr/testify/require" ) @@ -1302,15 +1305,38 @@ func TestAddPartitioning(t *testing.T) { ) } -func findEvent(eventCh <-chan *util.DDLEvent, eventType model.ActionType) *util.DDLEvent { +func findEvent(eventCh <-chan *notifier.SchemaChangeEvent, eventType model.ActionType) *notifier.SchemaChangeEvent { // Find the target event. for { event := <-eventCh - if event.SchemaChangeEvent.GetType() == eventType { - return event - } if event.GetType() == eventType { return event } } } + +func TestExchangePartition(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + + tk.MustExec("use test") + tk.MustExec("create table t (c1 int)") + is := dom.InfoSchema() + tbl, err := is.TableByName(context.Background(), pmodel.NewCIStr("test"), pmodel.NewCIStr("t")) + require.NoError(t, err) + var wg util.WaitGroupWrapper + for i := 0; i < 20; i++ { + tk1 := testkit.NewTestKit(t, store) + wg.Run(func() { + tk1.MustExec("begin") + ddl.UpdateStatsWithCountDeltaAndModifyCountDeltaForTest(tk1.Session(), tbl.Meta().ID, 10, 10) + tk1.MustExec("commit") + }) + } + wg.Wait() + count, modifyCount, isNull, err := storage.StatsMetaCountAndModifyCount(tk.Session(), tbl.Meta().ID) + require.NoError(t, err) + require.False(t, isNull) + require.Equal(t, int64(200), count) + require.Equal(t, int64(200), modifyCount) +} diff --git a/pkg/statistics/handle/ddl/drop_partition.go b/pkg/statistics/handle/ddl/drop_partition.go index 96b40355273a9..f0591000260ec 100644 --- a/pkg/statistics/handle/ddl/drop_partition.go +++ b/pkg/statistics/handle/ddl/drop_partition.go @@ -16,7 +16,7 @@ package ddl import ( "github.com/pingcap/errors" - ddlutil "github.com/pingcap/tidb/pkg/ddl/util" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics/handle/lockstats" @@ -24,7 +24,7 @@ import ( "github.com/pingcap/tidb/pkg/statistics/handle/util" ) -func (h *ddlHandlerImpl) onDropPartitions(t *ddlutil.SchemaChangeEvent) error { +func (h *ddlHandlerImpl) onDropPartitions(t *notifier.SchemaChangeEvent) error { globalTableInfo, droppedPartitionInfo := t.GetDropPartitionInfo() // Note: Put all the operations in a transaction. if err := util.CallWithSCtx(h.statsHandler.SPool(), func(sctx sessionctx.Context) error { diff --git a/pkg/statistics/handle/ddl/exchange_partition.go b/pkg/statistics/handle/ddl/exchange_partition.go index 972d1515ed8e6..96f0348020063 100644 --- a/pkg/statistics/handle/ddl/exchange_partition.go +++ b/pkg/statistics/handle/ddl/exchange_partition.go @@ -16,7 +16,7 @@ package ddl import ( "github.com/pingcap/errors" - ddlutil "github.com/pingcap/tidb/pkg/ddl/util" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/sessionctx" @@ -26,7 +26,7 @@ import ( "go.uber.org/zap" ) -func (h *ddlHandlerImpl) onExchangeAPartition(t *ddlutil.SchemaChangeEvent) error { +func (h *ddlHandlerImpl) onExchangeAPartition(t *notifier.SchemaChangeEvent) error { globalTableInfo, originalPartInfo, originalTableInfo := t.GetExchangePartitionInfo() // Note: Put all the operations in a transaction. diff --git a/pkg/statistics/handle/ddl/reorganize_partition.go b/pkg/statistics/handle/ddl/reorganize_partition.go index 266983d4a3f4b..4d039fc0e3277 100644 --- a/pkg/statistics/handle/ddl/reorganize_partition.go +++ b/pkg/statistics/handle/ddl/reorganize_partition.go @@ -15,10 +15,10 @@ package ddl import ( - ddlutil "github.com/pingcap/tidb/pkg/ddl/util" + "github.com/pingcap/tidb/pkg/ddl/notifier" ) -func (h *ddlHandlerImpl) onReorganizePartitions(t *ddlutil.SchemaChangeEvent) error { +func (h *ddlHandlerImpl) onReorganizePartitions(t *notifier.SchemaChangeEvent) error { globalTableInfo, addedPartInfo, droppedPartitionInfo := t.GetReorganizePartitionInfo() diff --git a/pkg/statistics/handle/ddl/truncate_partition.go b/pkg/statistics/handle/ddl/truncate_partition.go index e6f8fc5b0bec3..ec0e854def7c6 100644 --- a/pkg/statistics/handle/ddl/truncate_partition.go +++ b/pkg/statistics/handle/ddl/truncate_partition.go @@ -16,7 +16,7 @@ package ddl import ( "github.com/pingcap/errors" - ddlutil "github.com/pingcap/tidb/pkg/ddl/util" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/sessionctx" @@ -28,7 +28,7 @@ import ( "go.uber.org/zap" ) -func (h *ddlHandlerImpl) onTruncatePartitions(t *ddlutil.SchemaChangeEvent) error { +func (h *ddlHandlerImpl) onTruncatePartitions(t *notifier.SchemaChangeEvent) error { globalTableInfo, addedPartInfo, droppedPartInfo := t.GetTruncatePartitionInfo() // First, add the new stats meta record for the new partitions. for _, def := range addedPartInfo.Definitions { diff --git a/pkg/statistics/handle/storage/dump_test.go b/pkg/statistics/handle/storage/dump_test.go index 08470b647b9df..805202c566f85 100644 --- a/pkg/statistics/handle/storage/dump_test.go +++ b/pkg/statistics/handle/storage/dump_test.go @@ -15,12 +15,14 @@ package storage_test import ( + "cmp" "context" "encoding/json" "errors" "fmt" "math" "runtime" + "slices" "strings" "testing" @@ -600,6 +602,10 @@ func TestJSONTableToBlocks(t *testing.T) { dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true) require.NoError(t, err) + // the slice is generated from a map loop, which is randomly + slices.SortFunc(dumpJSONTable.PredicateColumns, func(a, b *handleutil.JSONPredicateColumn) int { + return cmp.Compare(a.ID, b.ID) + }) jsOrigin, _ := json.Marshal(dumpJSONTable) blockSize := 30 @@ -608,6 +614,10 @@ func TestJSONTableToBlocks(t *testing.T) { dumpJSONBlocks, err := storage.JSONTableToBlocks(js, blockSize) require.NoError(t, err) jsConverted, err := storage.BlocksToJSONTable(dumpJSONBlocks) + // the slice is generated from a map loop, which is randomly + slices.SortFunc(jsConverted.PredicateColumns, func(a, b *handleutil.JSONPredicateColumn) int { + return cmp.Compare(a.ID, b.ID) + }) require.NoError(t, err) jsonStr, err := json.Marshal(jsConverted) require.NoError(t, err) diff --git a/pkg/statistics/handle/storage/read.go b/pkg/statistics/handle/storage/read.go index e101e85e3f03e..456e8154d64ec 100644 --- a/pkg/statistics/handle/storage/read.go +++ b/pkg/statistics/handle/storage/read.go @@ -43,7 +43,20 @@ import ( // StatsMetaCountAndModifyCount reads count and modify_count for the given table from mysql.stats_meta. func StatsMetaCountAndModifyCount(sctx sessionctx.Context, tableID int64) (count, modifyCount int64, isNull bool, err error) { - rows, _, err := util.ExecRows(sctx, "select count, modify_count from mysql.stats_meta where table_id = %?", tableID) + return statsMetaCountAndModifyCount(sctx, tableID, false) +} + +// StatsMetaCountAndModifyCountForUpdate reads count and modify_count for the given table from mysql.stats_meta with lock. +func StatsMetaCountAndModifyCountForUpdate(sctx sessionctx.Context, tableID int64) (count, modifyCount int64, isNull bool, err error) { + return statsMetaCountAndModifyCount(sctx, tableID, true) +} + +func statsMetaCountAndModifyCount(sctx sessionctx.Context, tableID int64, forUpdate bool) (count, modifyCount int64, isNull bool, err error) { + sql := "select count, modify_count from mysql.stats_meta where table_id = %?" + if forUpdate { + sql += " for update" + } + rows, _, err := util.ExecRows(sctx, sql, tableID) if err != nil { return 0, 0, false, err } diff --git a/pkg/statistics/handle/types/BUILD.bazel b/pkg/statistics/handle/types/BUILD.bazel index 997bc1fcd8349..373eb6271edcd 100644 --- a/pkg/statistics/handle/types/BUILD.bazel +++ b/pkg/statistics/handle/types/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/statistics/handle/types", visibility = ["//visibility:public"], deps = [ + "//pkg/ddl/notifier", "//pkg/infoschema", "//pkg/meta/model", "//pkg/parser/ast", diff --git a/pkg/statistics/handle/types/interfaces.go b/pkg/statistics/handle/types/interfaces.go index a98f8b7aba78e..75de80fc3973e 100644 --- a/pkg/statistics/handle/types/interfaces.go +++ b/pkg/statistics/handle/types/interfaces.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/pingcap/tidb/pkg/ddl/notifier" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/ast" @@ -458,9 +459,9 @@ type StatsGlobal interface { // DDL is used to handle ddl events. type DDL interface { // HandleDDLEvent handles ddl events. - HandleDDLEvent(event *statsutil.DDLEvent) error + HandleDDLEvent(changeEvent *notifier.SchemaChangeEvent) error // DDLEventCh returns ddl events channel in handle. - DDLEventCh() chan *statsutil.DDLEvent + DDLEventCh() chan *notifier.SchemaChangeEvent } // StatsHandle is used to manage TiDB Statistics. diff --git a/pkg/statistics/handle/util/BUILD.bazel b/pkg/statistics/handle/util/BUILD.bazel index e46e9dd7366cc..1d6a46b63c17b 100644 --- a/pkg/statistics/handle/util/BUILD.bazel +++ b/pkg/statistics/handle/util/BUILD.bazel @@ -1,10 +1,9 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "util", srcs = [ "auto_analyze_proc_id_generator.go", - "ddl_event.go", "lease_getter.go", "pool.go", "table_info.go", @@ -13,10 +12,8 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/statistics/handle/util", visibility = ["//visibility:public"], deps = [ - "//pkg/ddl/util", "//pkg/infoschema", "//pkg/kv", - "//pkg/meta/model", "//pkg/parser/terror", "//pkg/planner/core/resolve", "//pkg/sessionctx", @@ -36,16 +33,3 @@ go_library( "@org_uber_go_atomic//:atomic", ], ) - -go_test( - name = "util_test", - timeout = "short", - srcs = ["ddl_event_test.go"], - embed = [":util"], - flaky = True, - deps = [ - "//pkg/meta/model", - "//pkg/parser/model", - "@com_github_stretchr_testify//require", - ], -) diff --git a/pkg/statistics/handle/util/ddl_event.go b/pkg/statistics/handle/util/ddl_event.go deleted file mode 100644 index cea3601cc9511..0000000000000 --- a/pkg/statistics/handle/util/ddl_event.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "fmt" - - ddlutil "github.com/pingcap/tidb/pkg/ddl/util" - "github.com/pingcap/tidb/pkg/infoschema" - "github.com/pingcap/tidb/pkg/meta/model" - "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/util" - "github.com/pingcap/tidb/pkg/util/intest" -) - -// DDLEvent contains the information of a ddl event that is used to update stats. -type DDLEvent struct { - // todo: replace DDLEvent by SchemaChangeEvent gradually - SchemaChangeEvent *ddlutil.SchemaChangeEvent - // For different ddl types, the following fields are used. - // They have different meanings for different ddl types. - // Please do **not** use these fields directly, use the corresponding - // NewXXXEvent functions instead. - tableInfo *model.TableInfo - partInfo *model.PartitionInfo - oldTableInfo *model.TableInfo - oldPartInfo *model.PartitionInfo - columnInfos []*model.ColumnInfo - - // schemaID is the ID of the schema that the table belongs to. - // Used to filter out the system or memory tables. - schemaID int64 - // This value is used to store the table ID during a transition. - // It applies when a table structure is being changed from partitioned to non-partitioned, or vice versa. - oldTableID int64 - tp model.ActionType -} - -// IsMemOrSysDB checks whether the table is in the memory or system database. -func (e *DDLEvent) IsMemOrSysDB(sctx sessionctx.Context) (bool, error) { - intest.Assert(e.schemaID != 0, "schemaID should not be 0, please set it when creating the event") - is := sctx.GetDomainInfoSchema().(infoschema.InfoSchema) - schema, ok := is.SchemaByID(e.schemaID) - if !ok { - return false, fmt.Errorf("schema not found for table %s", e.tableInfo.Name) - } - return util.IsMemOrSysDB(schema.Name.L), nil -} - -// NewRemovePartitioningEvent creates a new ddl event that converts a partitioned table to a single table. -// For example, `alter table t remove partitioning`. -func NewRemovePartitioningEvent( - schemaID int64, - oldPartitionedTableID int64, - newSingleTableInfo *model.TableInfo, - droppedPartInfo *model.PartitionInfo, -) *DDLEvent { - return &DDLEvent{ - tp: model.ActionRemovePartitioning, - schemaID: schemaID, - oldTableID: oldPartitionedTableID, - tableInfo: newSingleTableInfo, - oldPartInfo: droppedPartInfo, - } -} - -// GetRemovePartitioningInfo gets the table info of the table that is converted to a single table. -func (e *DDLEvent) GetRemovePartitioningInfo() ( - oldPartitionedTableID int64, - newSingleTableInfo *model.TableInfo, - droppedPartInfo *model.PartitionInfo, -) { - return e.oldTableID, e.tableInfo, e.oldPartInfo -} - -// NewFlashbackClusterEvent creates a new ddl event that flashes back the cluster. -func NewFlashbackClusterEvent() *DDLEvent { - return &DDLEvent{ - tp: model.ActionFlashbackCluster, - } -} - -// GetType returns the type of the ddl event. -func (e *DDLEvent) GetType() model.ActionType { - return e.tp -} - -// String implements fmt.Stringer interface. -func (e *DDLEvent) String() string { - ret := fmt.Sprintf("(Event Type: %s", e.tp) - if e.schemaID != 0 { - ret += fmt.Sprintf(", Schema ID: %d", e.schemaID) - } - if e.tableInfo != nil { - ret += fmt.Sprintf(", Table ID: %d, Table Name: %s", e.tableInfo.ID, e.tableInfo.Name) - } - if e.partInfo != nil { - ids := make([]int64, 0, len(e.partInfo.Definitions)) - for _, def := range e.partInfo.Definitions { - ids = append(ids, def.ID) - } - ret += fmt.Sprintf(", Partition IDs: %v", ids) - } - if e.oldTableInfo != nil { - ret += fmt.Sprintf(", Old Table ID: %d, Old Table Name: %s", e.oldTableInfo.ID, e.oldTableInfo.Name) - } - if e.oldPartInfo != nil { - ids := make([]int64, 0, len(e.oldPartInfo.Definitions)) - for _, def := range e.oldPartInfo.Definitions { - ids = append(ids, def.ID) - } - ret += fmt.Sprintf(", Old Partition IDs: %v", ids) - } - for _, columnInfo := range e.columnInfos { - ret += fmt.Sprintf(", Column ID: %d, Column Name: %s", columnInfo.ID, columnInfo.Name) - } - - return ret -} diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index e7f6510c63ba8..c59cddca90473 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -148,6 +148,11 @@ func (m *ColAndIdxExistenceMap) IsEmpty() bool { return len(m.colInfoMap)+len(m.idxInfoMap) == 0 } +// ColNum returns the number of columns in the map. +func (m *ColAndIdxExistenceMap) ColNum() int { + return len(m.colInfoMap) +} + // Clone deeply copies the map. func (m *ColAndIdxExistenceMap) Clone() *ColAndIdxExistenceMap { mm := NewColAndIndexExistenceMap(len(m.colInfoMap), len(m.idxInfoMap)) diff --git a/pkg/table/table.go b/pkg/table/table.go index 28a02dbdffd32..d48fb475964be 100644 --- a/pkg/table/table.go +++ b/pkg/table/table.go @@ -214,6 +214,47 @@ type UpdateRecordOption interface { applyUpdateRecordOpt(*UpdateRecordOpt) } +// RemoveRecordOpt contains the options will be used when removing a record. +type RemoveRecordOpt struct { + indexesLayoutOffset IndexesLayout +} + +// HasIndexesLayout returns whether the RemoveRecordOpt has indexes layout. +func (opt *RemoveRecordOpt) HasIndexesLayout() bool { + return opt.indexesLayoutOffset != nil +} + +// GetIndexLayout returns the IndexRowLayoutOption for the specified index. +func (opt *RemoveRecordOpt) GetIndexLayout(indexID int64) IndexRowLayoutOption { + return opt.indexesLayoutOffset[indexID] +} + +// NewRemoveRecordOpt creates a new RemoveRecordOpt with options. +func NewRemoveRecordOpt(opts ...RemoveRecordOption) *RemoveRecordOpt { + opt := &RemoveRecordOpt{} + for _, o := range opts { + o.applyRemoveRecordOpt(opt) + } + return opt +} + +// RemoveRecordOption is defined for the RemoveRecord() method of the Table interface. +type RemoveRecordOption interface { + applyRemoveRecordOpt(*RemoveRecordOpt) +} + +// IndexRowLayoutOption is the option for index row layout. +// It is used to specify the order of the index columns in the row. +type IndexRowLayoutOption []int + +// IndexesLayout is used to specify the layout of the indexes. +// It's mapping from index ID to the layout of the index. +type IndexesLayout map[int64]IndexRowLayoutOption + +func (idx IndexesLayout) applyRemoveRecordOpt(opt *RemoveRecordOpt) { + opt.indexesLayoutOffset = idx +} + // CommonMutateOptFunc is a function to provide common options for mutating a table. type CommonMutateOptFunc func(*commonMutateOpt) @@ -367,6 +408,7 @@ type Table interface { // Indices returns the indices of the table. // The caller must be aware of that not all the returned indices are public. Indices() []Index + DeletableIndices() []Index // WritableConstraint returns constraints of the table in writable states. WritableConstraint() []*Constraint @@ -383,7 +425,7 @@ type Table interface { UpdateRecord(ctx MutateContext, txn kv.Transaction, h kv.Handle, currData, newData []types.Datum, touched []bool, opts ...UpdateRecordOption) error // RemoveRecord removes a row in the table. - RemoveRecord(ctx MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum) error + RemoveRecord(ctx MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum, opts ...RemoveRecordOption) error // Allocators returns all allocators. Allocators(ctx AllocatorContext) autoid.Allocators diff --git a/pkg/table/tables/cache.go b/pkg/table/tables/cache.go index 29a1187791f1f..c8fe53a824475 100644 --- a/pkg/table/tables/cache.go +++ b/pkg/table/tables/cache.go @@ -264,9 +264,9 @@ func (c *cachedTable) UpdateRecord(ctx table.MutateContext, txn kv.Transaction, } // RemoveRecord implements table.Table RemoveRecord interface. -func (c *cachedTable) RemoveRecord(sctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum) error { +func (c *cachedTable) RemoveRecord(sctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum, opts ...table.RemoveRecordOption) error { txnCtxAddCachedTable(sctx, c.Meta().ID, c) - return c.TableCommon.RemoveRecord(sctx, txn, h, r) + return c.TableCommon.RemoveRecord(sctx, txn, h, r, opts...) } // TestMockRenewLeaseABA2 is used by test function TestRenewLeaseABAFailPoint. diff --git a/pkg/table/tables/index.go b/pkg/table/tables/index.go index 8be28248092e7..dda35219affc5 100644 --- a/pkg/table/tables/index.go +++ b/pkg/table/tables/index.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" + "github.com/pingcap/tidb/pkg/util/intest" "github.com/pingcap/tidb/pkg/util/rowcodec" "github.com/pingcap/tidb/pkg/util/tracing" ) @@ -633,13 +634,30 @@ func getKeyInTxn(ctx context.Context, txn kv.Transaction, key kv.Key) ([]byte, e return val, nil } +// FetchValues implements table.Index interface. func (c *index) FetchValues(r []types.Datum, vals []types.Datum) ([]types.Datum, error) { - needLength := len(c.idxInfo.Columns) + return fetchIndexRow(c.idxInfo, r, vals, nil) +} + +func fetchIndexRow(idxInfo *model.IndexInfo, r, vals []types.Datum, opt table.IndexRowLayoutOption) ([]types.Datum, error) { + needLength := len(idxInfo.Columns) if vals == nil || cap(vals) < needLength { vals = make([]types.Datum, needLength) } vals = vals[:needLength] - for i, ic := range c.idxInfo.Columns { + // If the context has extra info, use the extra layout info to get index columns. + if len(opt) != 0 { + intest.Assert(len(opt) == len(idxInfo.Columns), "offsets length is not equal to index columns length, offset len: %d, index len: %d", len(opt), len(idxInfo.Columns)) + for i, offset := range opt { + if offset < 0 || offset > len(r) { + return nil, table.ErrIndexOutBound.GenWithStackByArgs(idxInfo.Name, offset, r) + } + vals[i] = r[offset] + } + return vals, nil + } + // Otherwise use the full column layout. + for i, ic := range idxInfo.Columns { if ic.Offset < 0 || ic.Offset >= len(r) { return nil, table.ErrIndexOutBound.GenWithStackByArgs(ic.Name, ic.Offset, r) } diff --git a/pkg/table/tables/partition.go b/pkg/table/tables/partition.go index 8dc157b50afd4..1d3e148ae6625 100644 --- a/pkg/table/tables/partition.go +++ b/pkg/table/tables/partition.go @@ -1685,15 +1685,16 @@ func (t *partitionTableWithGivenSets) GetAllPartitionIDs() []int64 { } // RemoveRecord implements table.Table RemoveRecord interface. -func (t *partitionedTable) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum) error { +func (t *partitionedTable) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum, opts ...table.RemoveRecordOption) error { + opt := table.NewRemoveRecordOpt(opts...) ectx := ctx.GetExprCtx() pid, err := t.locatePartition(ectx.GetEvalCtx(), r) if err != nil { return errors.Trace(err) } - tbl := t.GetPartition(pid) - err = tbl.RemoveRecord(ctx, txn, h, r) + tbl := t.getPartition(pid) + err = tbl.removeRecord(ctx, txn, h, r, opt) if err != nil { return errors.Trace(err) } @@ -1703,8 +1704,8 @@ func (t *partitionedTable) RemoveRecord(ctx table.MutateContext, txn kv.Transact if err != nil { return errors.Trace(err) } - tbl = t.GetPartition(pid) - err = tbl.RemoveRecord(ctx, txn, h, r) + tbl = t.getPartition(pid) + err = tbl.removeRecord(ctx, txn, h, r, opt) if err != nil { return errors.Trace(err) } diff --git a/pkg/table/tables/tables.go b/pkg/table/tables/tables.go index 57f80ffd9cedb..dfcb12dd33b97 100644 --- a/pkg/table/tables/tables.go +++ b/pkg/table/tables/tables.go @@ -315,8 +315,8 @@ func GetWritableIndexByName(idxName string, t table.Table) table.Index { return nil } -// deletableIndices implements table.Table deletableIndices interface. -func (t *TableCommon) deletableIndices() []table.Index { +// DeletableIndices implements table.Table DeletableIndices interface. +func (t *TableCommon) DeletableIndices() []table.Index { // All indices are deletable because we don't need to check StateNone. return t.indices } @@ -608,7 +608,7 @@ func (t *TableCommon) rebuildUpdateRecordIndices( h kv.Handle, touched []bool, oldData []types.Datum, newData []types.Datum, opt *table.UpdateRecordOpt, ) error { - for _, idx := range t.deletableIndices() { + for _, idx := range t.DeletableIndices() { if t.meta.IsCommonHandle && idx.Meta().Primary { continue } @@ -1168,7 +1168,12 @@ func GetChangingColVal(ctx exprctx.BuildContext, cols []*table.Column, col *tabl } // RemoveRecord implements table.Table RemoveRecord interface. -func (t *TableCommon) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum) error { +func (t *TableCommon) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum, opts ...table.RemoveRecordOption) error { + opt := table.NewRemoveRecordOpt(opts...) + return t.removeRecord(ctx, txn, h, r, opt) +} + +func (t *TableCommon) removeRecord(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, r []types.Datum, opt *table.RemoveRecordOpt) error { memBuffer := txn.GetMemBuffer() sh := memBuffer.Staging() defer memBuffer.Cleanup(sh) @@ -1188,7 +1193,11 @@ func (t *TableCommon) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, } // The table has non-public column and this column is doing the operation of "modify/change column". - if len(t.Columns) > len(r) && t.Columns[len(r)].ChangeStateInfo != nil { + // DELETE will use deletable columns, which is the same as the full columns of the table. + // INSERT and UPDATE will only use the writable columns. So they will not see columns under MODIFY/CHANGE state. + // This if block is for the INSERT and UPDATE. + // And, only DELETE will make opt.HasIndexesLayout() to be true currently. + if !opt.HasIndexesLayout() && len(t.Columns) > len(r) && t.Columns[len(r)].ChangeStateInfo != nil { // The changing column datum derived from related column should be casted here. // Otherwise, the existed changing indexes will not be deleted. relatedColDatum := r[t.Columns[len(r)].ChangeStateInfo.DependencyColumnOffset] @@ -1200,7 +1209,7 @@ func (t *TableCommon) RemoveRecord(ctx table.MutateContext, txn kv.Transaction, } r = append(r, value) } - err = t.removeRowIndices(ctx, txn, h, r) + err = t.removeRowIndices(ctx, txn, h, r, opt) if err != nil { return err } @@ -1366,12 +1375,17 @@ func (t *TableCommon) removeRowData(ctx table.MutateContext, txn kv.Transaction, } // removeRowIndices removes all the indices of a row. -func (t *TableCommon) removeRowIndices(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, rec []types.Datum) error { - for _, v := range t.deletableIndices() { +func (t *TableCommon) removeRowIndices(ctx table.MutateContext, txn kv.Transaction, h kv.Handle, rec []types.Datum, opt *table.RemoveRecordOpt) (err error) { + for _, v := range t.DeletableIndices() { if v.Meta().Primary && (t.Meta().IsCommonHandle || t.Meta().PKIsHandle) { continue } - vals, err := v.FetchValues(rec, nil) + var vals []types.Datum + if opt.HasIndexesLayout() { + vals, err = fetchIndexRow(v.Meta(), rec, nil, opt.GetIndexLayout(v.Meta().ID)) + } else { + vals, err = fetchIndexRow(v.Meta(), rec, nil, nil) + } if err != nil { logutil.BgLogger().Info("remove row index failed", zap.Any("index", v.Meta()), zap.Uint64("txnStartTS", txn.StartTS()), zap.String("handle", h.String()), zap.Any("record", rec), zap.Error(err)) return err diff --git a/pkg/tablecodec/tablecodec.go b/pkg/tablecodec/tablecodec.go index ffbfe30dc62f0..2e2fd9b45d1b3 100644 --- a/pkg/tablecodec/tablecodec.go +++ b/pkg/tablecodec/tablecodec.go @@ -739,23 +739,31 @@ func CutIndexPrefix(key kv.Key) []byte { return key[prefixLen+idLen:] } -// CutIndexKeyNew cuts encoded index key into colIDs to bytes slices. -// The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data -// if it is non-unique index. -func CutIndexKeyNew(key kv.Key, length int) (values [][]byte, b []byte, err error) { +// CutIndexKeyTo cuts encoded index key into colIDs to bytes slices. +// The caller should prepare the memory of the result values. +func CutIndexKeyTo(key kv.Key, values [][]byte) (b []byte, err error) { b = key[prefixLen+idLen:] - values = make([][]byte, 0, length) + length := len(values) for i := 0; i < length; i++ { var val []byte val, b, err = codec.CutOne(b) if err != nil { - return nil, nil, errors.Trace(err) + return nil, errors.Trace(err) } - values = append(values, val) + values[i] = val } return } +// CutIndexKeyNew cuts encoded index key into colIDs to bytes slices. +// The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data +// if it is non-unique index. +func CutIndexKeyNew(key kv.Key, length int) (values [][]byte, b []byte, err error) { + values = make([][]byte, length) + b, err = CutIndexKeyTo(key, values) + return +} + // CutCommonHandle cuts encoded common handle key into colIDs to bytes slices. // The returned value b is the remaining bytes of the key which would be empty if it is unique index or handle data // if it is non-unique index. @@ -789,20 +797,29 @@ const ( // If it is common handle, it returns the encoded column values. // If it is int handle, it is encoded as int Datum or uint Datum decided by the unsigned. func reEncodeHandle(handle kv.Handle, unsigned bool) ([][]byte, error) { + handleColLen := 1 + if !handle.IsInt() { + handleColLen = handle.NumCols() + } + result := make([][]byte, 0, handleColLen) + return reEncodeHandleTo(handle, unsigned, nil, result) +} + +func reEncodeHandleTo(handle kv.Handle, unsigned bool, buf []byte, result [][]byte) ([][]byte, error) { if !handle.IsInt() { handleColLen := handle.NumCols() - cHandleBytes := make([][]byte, 0, handleColLen) for i := 0; i < handleColLen; i++ { - cHandleBytes = append(cHandleBytes, handle.EncodedCol(i)) + result = append(result, handle.EncodedCol(i)) } - return cHandleBytes, nil + return result, nil } handleDatum := types.NewIntDatum(handle.IntValue()) if unsigned { handleDatum.SetUint64(handleDatum.GetUint64()) } - intHandleBytes, err := codec.EncodeValue(time.UTC, nil, handleDatum) - return [][]byte{intHandleBytes}, err + intHandleBytes, err := codec.EncodeValue(time.UTC, buf, handleDatum) + result = append(result, intHandleBytes) + return result, err } // reEncodeHandleConsiderNewCollation encodes the handle as a Datum so it can be properly decoded later. @@ -908,8 +925,8 @@ func buildRestoredColumn(allCols []rowcodec.ColInfo) []rowcodec.ColInfo { return restoredColumns } -func decodeIndexKvOldCollation(key, value []byte, colsLen int, hdStatus HandleStatus) ([][]byte, error) { - resultValues, b, err := CutIndexKeyNew(key, colsLen) +func decodeIndexKvOldCollation(key, value []byte, hdStatus HandleStatus, buf []byte, resultValues [][]byte) ([][]byte, error) { + b, err := CutIndexKeyTo(key, resultValues) if err != nil { return nil, errors.Trace(err) } @@ -923,19 +940,17 @@ func decodeIndexKvOldCollation(key, value []byte, colsLen int, hdStatus HandleSt if err != nil { return nil, err } - handleBytes, err := reEncodeHandle(handle, hdStatus == HandleIsUnsigned) + resultValues, err = reEncodeHandleTo(handle, hdStatus == HandleIsUnsigned, buf, resultValues) if err != nil { return nil, errors.Trace(err) } - resultValues = append(resultValues, handleBytes...) } else { // In unique int handle index. handle = decodeIntHandleInIndexValue(value) - handleBytes, err := reEncodeHandle(handle, hdStatus == HandleIsUnsigned) + resultValues, err = reEncodeHandleTo(handle, hdStatus == HandleIsUnsigned, buf, resultValues) if err != nil { return nil, errors.Trace(err) } - resultValues = append(resultValues, handleBytes...) } return resultValues, nil } @@ -951,13 +966,25 @@ func getIndexVersion(value []byte) int { return 0 } +// DecodeIndexKVEx looks like DecodeIndexKV, the difference is that it tries to reduce allocations. +func DecodeIndexKVEx(key, value []byte, colsLen int, hdStatus HandleStatus, columns []rowcodec.ColInfo, buf []byte, preAlloc [][]byte) ([][]byte, error) { + if len(value) <= MaxOldEncodeValueLen { + return decodeIndexKvOldCollation(key, value, hdStatus, buf, preAlloc) + } + if getIndexVersion(value) == 1 { + return decodeIndexKvForClusteredIndexVersion1(key, value, colsLen, hdStatus, columns) + } + return decodeIndexKvGeneral(key, value, colsLen, hdStatus, columns) +} + // DecodeIndexKV uses to decode index key values. // // `colsLen` is expected to be index columns count. // `columns` is expected to be index columns + handle columns(if hdStatus is not HandleNotNeeded). func DecodeIndexKV(key, value []byte, colsLen int, hdStatus HandleStatus, columns []rowcodec.ColInfo) ([][]byte, error) { if len(value) <= MaxOldEncodeValueLen { - return decodeIndexKvOldCollation(key, value, colsLen, hdStatus) + preAlloc := make([][]byte, colsLen, colsLen+len(columns)) + return decodeIndexKvOldCollation(key, value, hdStatus, nil, preAlloc) } if getIndexVersion(value) == 1 { return decodeIndexKvForClusteredIndexVersion1(key, value, colsLen, hdStatus, columns) @@ -967,9 +994,13 @@ func DecodeIndexKV(key, value []byte, colsLen int, hdStatus HandleStatus, column // DecodeIndexHandle uses to decode the handle from index key/value. func DecodeIndexHandle(key, value []byte, colsLen int) (kv.Handle, error) { - _, b, err := CutIndexKeyNew(key, colsLen) - if err != nil { - return nil, errors.Trace(err) + var err error + b := key[prefixLen+idLen:] + for i := 0; i < colsLen; i++ { + _, b, err = codec.CutOne(b) + if err != nil { + return nil, errors.Trace(err) + } } if len(b) > 0 { return decodeHandleInIndexKey(b) diff --git a/pkg/util/dbterror/exeerrors/errors.go b/pkg/util/dbterror/exeerrors/errors.go index a99f25b8d2cf8..86bfe0a8ef83e 100644 --- a/pkg/util/dbterror/exeerrors/errors.go +++ b/pkg/util/dbterror/exeerrors/errors.go @@ -62,6 +62,7 @@ var ( ErrLazyUniquenessCheckFailure = dbterror.ClassExecutor.NewStd(mysql.ErrLazyUniquenessCheckFailure) ErrMemoryExceedForQuery = dbterror.ClassExecutor.NewStd(mysql.ErrMemoryExceedForQuery) ErrMemoryExceedForInstance = dbterror.ClassExecutor.NewStd(mysql.ErrMemoryExceedForInstance) + ErrDeleteNotFoundColumn = dbterror.ClassExecutor.NewStd(mysql.ErrDeleteNotFoundColumn) ErrBRIEBackupFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEBackupFailed) ErrBRIERestoreFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIERestoreFailed) diff --git a/pkg/util/dbterror/plannererrors/planner_terror.go b/pkg/util/dbterror/plannererrors/planner_terror.go index 3c13ac4579c1b..8bfb79be777da 100644 --- a/pkg/util/dbterror/plannererrors/planner_terror.go +++ b/pkg/util/dbterror/plannererrors/planner_terror.go @@ -98,6 +98,7 @@ var ( ErrCTERecursiveForbiddenJoinOrder = dbterror.ClassOptimizer.NewStd(mysql.ErrCTERecursiveForbiddenJoinOrder) ErrInvalidRequiresSingleReference = dbterror.ClassOptimizer.NewStd(mysql.ErrInvalidRequiresSingleReference) ErrSQLInReadOnlyMode = dbterror.ClassOptimizer.NewStd(mysql.ErrReadOnlyMode) + ErrDeleteNotFoundColumn = dbterror.ClassOptimizer.NewStd(mysql.ErrDeleteNotFoundColumn) // Since we cannot know if user logged in with a password, use message of ErrAccessDeniedNoPassword instead ErrAccessDenied = dbterror.ClassOptimizer.NewStdErr(mysql.ErrAccessDenied, mysql.MySQLErrName[mysql.ErrAccessDeniedNoPassword]) ErrBadNull = dbterror.ClassOptimizer.NewStd(mysql.ErrBadNull) diff --git a/pkg/util/processinfo.go b/pkg/util/processinfo.go index 357f469c0e045..92559da329ea9 100644 --- a/pkg/util/processinfo.go +++ b/pkg/util/processinfo.go @@ -155,7 +155,7 @@ func (pi *ProcessInfo) ToRow(tz *time.Location) []any { cpuUsages = pi.SQLCPUUsage.GetCPUUsages() } return append(pi.ToRowForShow(true), pi.Digest, bytesConsumed, diskConsumed, - pi.txnStartTs(tz), pi.ResourceGroupName, pi.SessionAlias, affectedRows, cpuUsages.TidbCPUTime.Seconds(), cpuUsages.TikvCPUTime.Seconds()) + pi.txnStartTs(tz), pi.ResourceGroupName, pi.SessionAlias, affectedRows, cpuUsages.TidbCPUTime.Nanoseconds(), cpuUsages.TikvCPUTime.Nanoseconds()) } // ascServerStatus is a slice of all defined server status in ascending order. diff --git a/tests/integrationtest/r/ddl/db_change.result b/tests/integrationtest/r/ddl/db_change.result index 7c170cb56bfc3..964bf8a9e3c69 100644 --- a/tests/integrationtest/r/ddl/db_change.result +++ b/tests/integrationtest/r/ddl/db_change.result @@ -33,3 +33,21 @@ alter table t rename column b to b2; Error 3837 (HY000): Column 'b' has an expression index dependency and cannot be dropped or renamed alter table t drop column b; Error 3837 (HY000): Column 'b' has an expression index dependency and cannot be dropped or renamed +drop table if exists t; +create table t(a int, b int, c int); +insert into t values(NULL, NULL, NULL); +alter table t modify column a timestamp not null; +select floor((unix_timestamp() - unix_timestamp(a)) / 2) from t; +floor((unix_timestamp() - unix_timestamp(a)) / 2) +0 +set @@time_zone='UTC'; +alter table t modify column b timestamp not null; +select floor((unix_timestamp() - unix_timestamp(b)) / 2) from t; +floor((unix_timestamp() - unix_timestamp(b)) / 2) +0 +set @@time_zone='Asia/Tokyo'; +alter table t modify column c timestamp not null; +select floor((unix_timestamp() - unix_timestamp(c)) / 2) from t; +floor((unix_timestamp() - unix_timestamp(c)) / 2) +0 +set @@time_zone='SYSTEM' diff --git a/tests/integrationtest/r/executor/import_into.result b/tests/integrationtest/r/executor/import_into.result index 5c5250765c6c9..77246505cd4a0 100644 --- a/tests/integrationtest/r/executor/import_into.result +++ b/tests/integrationtest/r/executor/import_into.result @@ -170,3 +170,16 @@ import into t from ''; Error 8156 (HY000): The value of INFILE must not be empty when LOAD DATA from LOCAL import into t from '/a.csv' format 'xx'; Error 8157 (HY000): The FORMAT 'xx' is not supported +drop table if exists temp; +create temporary table temp (id int); +import into temp from '/file.csv'; +Error 1105 (HY000): IMPORT INTO does not support temporary table +drop table if exists gtemp; +create global temporary table gtemp (id int) on commit delete rows; +import into gtemp from '/file.csv'; +Error 1105 (HY000): IMPORT INTO does not support temporary table +drop table if exists cachetbl; +create table cachetbl (id int); +alter table cachetbl cache; +import into cachetbl from '/file.csv'; +Error 1105 (HY000): IMPORT INTO does not support cached table diff --git a/tests/integrationtest/t/ddl/db_change.test b/tests/integrationtest/t/ddl/db_change.test index ef8b94ee60738..2798c0c66d404 100644 --- a/tests/integrationtest/t/ddl/db_change.test +++ b/tests/integrationtest/t/ddl/db_change.test @@ -31,3 +31,18 @@ alter table t rename column b to b2; -- error 3837 alter table t drop column b; +# The generated current time should be correct after DDL reorging in different timezones. +# see issue: https://github.com/pingcap/tidb/issues/56051 +drop table if exists t; +create table t(a int, b int, c int); +insert into t values(NULL, NULL, NULL); +alter table t modify column a timestamp not null; +# the timestamp diff should be less than 2s +select floor((unix_timestamp() - unix_timestamp(a)) / 2) from t; +set @@time_zone='UTC'; +alter table t modify column b timestamp not null; +select floor((unix_timestamp() - unix_timestamp(b)) / 2) from t; +set @@time_zone='Asia/Tokyo'; +alter table t modify column c timestamp not null; +select floor((unix_timestamp() - unix_timestamp(c)) / 2) from t; +set @@time_zone='SYSTEM' diff --git a/tests/integrationtest/t/executor/import_into.test b/tests/integrationtest/t/executor/import_into.test index 68b3a4fa679b2..2d6c803eb1164 100644 --- a/tests/integrationtest/t/executor/import_into.test +++ b/tests/integrationtest/t/executor/import_into.test @@ -174,3 +174,19 @@ import into t from ''; -- error 8157 import into t from '/a.csv' format 'xx'; +# import into temporary or cached table is not supported +drop table if exists temp; +create temporary table temp (id int); +-- error 1105 +import into temp from '/file.csv'; + +drop table if exists gtemp; +create global temporary table gtemp (id int) on commit delete rows; +-- error 1105 +import into gtemp from '/file.csv'; + +drop table if exists cachetbl; +create table cachetbl (id int); +alter table cachetbl cache; +-- error 1105 +import into cachetbl from '/file.csv';