From b5269f7761c05657b58056995e3055ebc4c9d4c6 Mon Sep 17 00:00:00 2001 From: hui lai <1353307710@qq.com> Date: Wed, 28 Aug 2024 20:13:18 +0800 Subject: [PATCH 01/34] [opt](test) remove duplicate case and make concurrent to accelerate regression test (#40021) --- .../test_stream_load_properties.out | 38 ----- .../stream_load/test_stream_load_2pc.groovy | 12 +- .../test_stream_load_properties.groovy | 135 +----------------- 3 files changed, 7 insertions(+), 178 deletions(-) diff --git a/regression-test/data/load_p0/stream_load/test_stream_load_properties.out b/regression-test/data/load_p0/stream_load/test_stream_load_properties.out index bc2a3c31a392666..05f0d92d6276869 100644 --- a/regression-test/data/load_p0/stream_load/test_stream_load_properties.out +++ b/regression-test/data/load_p0/stream_load/test_stream_load_properties.out @@ -1342,44 +1342,6 @@ 90 2023-08-27 true 22 16456 -1476824962 -3279894870153540825 8990195191470116763 26651.906 2.06860148942546E8 \N \N 2022-10-07T03:11:03 2023-03-18 2023-04-15T00:38:33 T L QW0GQ3GoMtHgxPQOWGfVaveynahNpsNs09siMFA1OtO6QEDBQTdivmGyq7bFzejAqwbbVQQpREAmeLjcFSXLnQuou2KbwYD \N \N \N \N true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N \N \N \N 91 2023-08-27 true 90 2465 702240964 6373830997821598984 305860046137409400 15991.356 1.599972327386147E9 \N \N 2023-04-26T19:31:10 2023-07-21 \N 2 B7YKYBYT8w0YC926bZ8Yz1VzyiWw2NWDAiTlEoPVyz9AXGti2Npg1FxWqWk4hEaALw0ZBSuiAIPj41lq36g5QRpPmAjNPK \N \N \N \N true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N \N \N \N --- !sql_json_read_json_by_line -- -2 [0, 0, 0, 0, 0, 0] [117, 117, 117, 117, 117, 117] [-4744, -4744, -4744, -4744, -4744, -4744] [-1593211961, -1593211961, -1593211961, -1593211961, -1593211961, -1593211961] [-3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780] [8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567] [-30948.857, -30948.857, -30948.857, -30948.857, -30948.857] [804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905] [-74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000] [13024168, 13024168, 13024168, 13024168, 13024168, 13024168] ["2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22"] ["2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12"] ["2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21"] ["2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56"] ["g", "g", "g", "g", "g", "g"] ["a", "a", "a", "a", "a", "a"] ["S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -3 [0, 0, 0, 0, 0, 0] [65, 65, 65, 65, 65, 65] [-4963, -4963, -4963, -4963, -4963, -4963] [-1415431954, -1415431954, -1415431954, -1415431954, -1415431954, -1415431954] [-3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000] [8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376] [-5058.13, -5058.13, -5058.13, -5058.13, -5058.13] [1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352] [3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000] [-3634150, -3634150, -3634150, -3634150, -3634150, -3634150] ["2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26"] ["2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44"] ["2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27"] ["2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11"] ["i", "i", "i", "i", "i", "i"] ["G", "G", "G", "G", "G", "G"] ["XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -4 [0, 0, 0, 0, 0, 0] [-67, -67, -67, -67, -67, -67] [-30372, -30372, -30372, -30372, -30372, -30372] [181502941, 181502941, 181502941, 181502941, 181502941, 181502941] [-2062236823576972800, -2062236823576972800, -2062236823576972800, -2062236823576972800, -2062236823576972800, -2062236823576972800] [6357002962400127842, 6357002962400127842, 6357002962400127842, 6357002962400127842, 6357002962400127842, 6357002962400127842] [21235.783, 21235.783, 21235.783, 21235.783, 21235.783] [-1101694755.713891, -1101694755.713891, -1101694755.713891, -1101694755.713891, -1101694755.713891, -1101694755.713891] [58088067.000000000, 58088067.000000000, 58088067.000000000, 58088067.000000000, 58088067.000000000, 58088067.000000000] [-66079723, -66079723, -66079723, -66079723, -66079723, -66079723] ["2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24"] ["2023-06-16 18:07:12", "2023-06-16 18:07:12", "2023-06-16 18:07:12", "2023-06-16 18:07:12", "2023-06-16 18:07:12", "2023-06-16 18:07:12"] ["2022-11-19", "2022-11-19", "2022-11-19", "2022-11-19", "2022-11-19", "2022-11-19"] ["2023-08-23 08:19:12", "2023-08-23 08:19:12", "2023-08-23 08:19:12", "2023-08-23 08:19:12", "2023-08-23 08:19:12", "2023-08-23 08:19:12"] ["G", "G", "G", "G", "G", "G"] ["a", "a", "a", "a", "a", "a"] ["tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -4 [0, 0, 0, 0, 0, 0] [63, 63, 63, 63, 63, 63] [11399, 11399, 11399, 11399, 11399, 11399] [-583523026, -583523026, -583523026, -583523026, -583523026, -583523026] [8801164674137231293, 8801164674137231293, 8801164674137231293, 8801164674137231293, 8801164674137231293, 8801164674137231293] [-8287675635310193906, -8287675635310193906, -8287675635310193906, -8287675635310193906, -8287675635310193906, -8287675635310193906] [23243.16, 23243.16, 23243.16, 23243.16, 23243.16] [716719993.249115, 716719993.249115, 716719993.249115, 716719993.249115, 716719993.249115, 716719993.249115] [-40335903.000000000, -40335903.000000000, -40335903.000000000, -40335903.000000000, -40335903.000000000, -40335903.000000000] [74087997, 74087997, 74087997, 74087997, 74087997, 74087997] ["2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13"] ["2023-08-06 19:03:27", "2023-08-06 19:03:27", "2023-08-06 19:03:27", "2023-08-06 19:03:27", "2023-08-06 19:03:27", "2023-08-06 19:03:27"] ["2023-03-28", "2023-03-28", "2023-03-28", "2023-03-28", "2023-03-28", "2023-03-28"] ["2022-09-27 09:19:47", "2022-09-27 09:19:47", "2022-09-27 09:19:47", "2022-09-27 09:19:47", "2022-09-27 09:19:47", "2022-09-27 09:19:47"] ["z", "z", "z", "z", "z", "z"] ["1", "1", "1", "1", "1", "1"] ["xdc5RcpnOAqeK6Hmz", "xdc5RcpnOAqeK6Hmz", "xdc5RcpnOAqeK6Hmz", "xdc5RcpnOAqeK6Hmz", "xdc5RcpnOAqeK6Hmz", "xdc5RcpnOAqeK6Hmz"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -8 [1, 1, 1, 1, 1, 1] [-11, -11, -11, -11, -11, -11] [-9648, -9648, -9648, -9648, -9648, -9648] [-505356927, -505356927, -505356927, -505356927, -505356927, -505356927] [7604760670442035037, 7604760670442035037, 7604760670442035037, 7604760670442035037, 7604760670442035037, 7604760670442035037] [1634770507625165798, 1634770507625165798, 1634770507625165798, 1634770507625165798, 1634770507625165798, 1634770507625165798] [10822.962, 10822.962, 10822.962, 10822.962, 10822.962] [1987551048.863071, 1987551048.863071, 1987551048.863071, 1987551048.863071, 1987551048.863071, 1987551048.863071] [64879544.000000000, 64879544.000000000, 64879544.000000000, 64879544.000000000, 64879544.000000000, 64879544.000000000] [-55896622, -55896622, -55896622, -55896622, -55896622, -55896622] ["2023-08-08", "2023-08-08", "2023-08-08", "2023-08-08", "2023-08-08", "2023-08-08"] ["2023-01-05 00:55:23", "2023-01-05 00:55:23", "2023-01-05 00:55:23", "2023-01-05 00:55:23", "2023-01-05 00:55:23", "2023-01-05 00:55:23"] ["2023-06-16", "2023-06-16", "2023-06-16", "2023-06-16", "2023-06-16", "2023-06-16"] ["2023-04-06 03:40:26", "2023-04-06 03:40:26", "2023-04-06 03:40:26", "2023-04-06 03:40:26", "2023-04-06 03:40:26", "2023-04-06 03:40:26"] ["U", "U", "U", "U", "U", "U"] ["G", "G", "G", "G", "G", "G"] ["iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -16 [0, 0, 0, 0, 0, 0] [-43, -43, -43, -43, -43, -43] [13560, 13560, 13560, 13560, 13560, 13560] [-1743686513, -1743686513, -1743686513, -1743686513, -1743686513, -1743686513] [7234719406392208769, 7234719406392208769, 7234719406392208769, 7234719406392208769, 7234719406392208769, 7234719406392208769] [-3871745630024229413, -3871745630024229413, -3871745630024229413, -3871745630024229413, -3871745630024229413, -3871745630024229413] [12225.427, 12225.427, 12225.427, 12225.427, 12225.427] [-1352141342.04191, -1352141342.04191, -1352141342.04191, -1352141342.04191, -1352141342.04191, -1352141342.04191] [-35959452.000000000, -35959452.000000000, -35959452.000000000, -35959452.000000000, -35959452.000000000, -35959452.000000000] [-4256846, -4256846, -4256846, -4256846, -4256846, -4256846] ["2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24"] ["2022-12-25 20:40:43", "2022-12-25 20:40:43", "2022-12-25 20:40:43", "2022-12-25 20:40:43", "2022-12-25 20:40:43", "2022-12-25 20:40:43"] ["2023-01-05", "2023-01-05", "2023-01-05", "2023-01-05", "2023-01-05", "2023-01-05"] ["2023-01-27 19:34:04", "2023-01-27 19:34:04", "2023-01-27 19:34:04", "2023-01-27 19:34:04", "2023-01-27 19:34:04", "2023-01-27 19:34:04"] ["V", "V", "V", "V", "V", "V"] ["a", "a", "a", "a", "a", "a"] ["8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -16 [1, 1, 1, 1, 1, 1] [-7, -7, -7, -7, -7, -7] [18655, 18655, 18655, 18655, 18655, 18655] [6240356, 6240356, 6240356, 6240356, 6240356, 6240356] [4552179257266841393, 4552179257266841393, 4552179257266841393, 4552179257266841393, 4552179257266841393, 4552179257266841393] [323868824766329978, 323868824766329978, 323868824766329978, 323868824766329978, 323868824766329978, 323868824766329978] [2972.2478, 2972.2478, 2972.2478, 2972.2478, 2972.2478] [-1177167334.995008, -1177167334.995008, -1177167334.995008, -1177167334.995008, -1177167334.995008, -1177167334.995008] [-84949097.000000000, -84949097.000000000, -84949097.000000000, -84949097.000000000, -84949097.000000000, -84949097.000000000] [29815991, 29815991, 29815991, 29815991, 29815991, 29815991] ["2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24"] ["2022-09-10 01:11:11", "2022-09-10 01:11:11", "2022-09-10 01:11:11", "2022-09-10 01:11:11", "2022-09-10 01:11:11", "2022-09-10 01:11:11"] ["2022-09-10", "2022-09-10", "2022-09-10", "2022-09-10", "2022-09-10", "2022-09-10"] ["2023-06-06 20:38:07", "2023-06-06 20:38:07", "2023-06-06 20:38:07", "2023-06-06 20:38:07", "2023-06-06 20:38:07", "2023-06-06 20:38:07"] ["9", "9", "9", "9", "9", "9"] ["n", "n", "n", "n", "n", "n"] ["s093vhFomjJOSwwn3Dii8iK9wevxUFwRUVJE6GAGOZEErIMLY68bUYV1h5d57QWfp7AxBkTbxPqbEh4ZR2Z3wROrAFQjpUtN4Y9Y8H4nC5xj14n5hXqCdJYpzkX", "s093vhFomjJOSwwn3Dii8iK9wevxUFwRUVJE6GAGOZEErIMLY68bUYV1h5d57QWfp7AxBkTbxPqbEh4ZR2Z3wROrAFQjpUtN4Y9Y8H4nC5xj14n5hXqCdJYpzkX", "s093vhFomjJOSwwn3Dii8iK9wevxUFwRUVJE6GAGOZEErIMLY68bUYV1h5d57QWfp7AxBkTbxPqbEh4ZR2Z3wROrAFQjpUtN4Y9Y8H4nC5xj14n5hXqCdJYpzkX", "s093vhFomjJOSwwn3Dii8iK9wevxUFwRUVJE6GAGOZEErIMLY68bUYV1h5d57QWfp7AxBkTbxPqbEh4ZR2Z3wROrAFQjpUtN4Y9Y8H4nC5xj14n5hXqCdJYpzkX", "s093vhFomjJOSwwn3Dii8iK9wevxUFwRUVJE6GAGOZEErIMLY68bUYV1h5d57QWfp7AxBkTbxPqbEh4ZR2Z3wROrAFQjpUtN4Y9Y8H4nC5xj14n5hXqCdJYpzkX", "s093vhFomjJOSwwn3Dii8iK9wevxUFwRUVJE6GAGOZEErIMLY68bUYV1h5d57QWfp7AxBkTbxPqbEh4ZR2Z3wROrAFQjpUtN4Y9Y8H4nC5xj14n5hXqCdJYpzkX"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -17 [1, 1, 1, 1, 1, 1] [126, 126, 126, 126, 126, 126] [28165, 28165, 28165, 28165, 28165, 28165] [2032059721, 2032059721, 2032059721, 2032059721, 2032059721, 2032059721] [-2686776977990574879, -2686776977990574879, -2686776977990574879, -2686776977990574879, -2686776977990574879, -2686776977990574879] [-8498149444423102876, -8498149444423102876, -8498149444423102876, -8498149444423102876, -8498149444423102876, -8498149444423102876] [-21681.223, -21681.223, -21681.223, -21681.223, -21681.223] [95345732.447678, 95345732.447678, 95345732.447678, 95345732.447678, 95345732.447678, 95345732.447678] [89062729.000000000, 89062729.000000000, 89062729.000000000, 89062729.000000000, 89062729.000000000, 89062729.000000000] [-2464406, -2464406, -2464406, -2464406, -2464406, -2464406] ["2023-08-16", "2023-08-16", "2023-08-16", "2023-08-16", "2023-08-16", "2023-08-16"] ["2023-03-05 10:43:36", "2023-03-05 10:43:36", "2023-03-05 10:43:36", "2023-03-05 10:43:36", "2023-03-05 10:43:36", "2023-03-05 10:43:36"] ["2023-08-20", "2023-08-20", "2023-08-20", "2023-08-20", "2023-08-20", "2023-08-20"] ["2023-04-11 00:33:44", "2023-04-11 00:33:44", "2023-04-11 00:33:44", "2023-04-11 00:33:44", "2023-04-11 00:33:44", "2023-04-11 00:33:44"] ["U", "U", "U", "U", "U", "U"] ["a", "a", "a", "a", "a", "a"] ["XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -27 [0, 0, 0, 0, 0, 0] [-7, -7, -7, -7, -7, -7] [18307, 18307, 18307, 18307, 18307, 18307] [76399879, 76399879, 76399879, 76399879, 76399879, 76399879] [8050764818738996699, 8050764818738996699, 8050764818738996699, 8050764818738996699, 8050764818738996699, 8050764818738996699] [2402219865213589999, 2402219865213589999, 2402219865213589999, 2402219865213589999, 2402219865213589999, 2402219865213589999] [-16678.924, -16678.924, -16678.924, -16678.924, -16678.924] [-2053879544.844726, -2053879544.844726, -2053879544.844726, -2053879544.844726, -2053879544.844726, -2053879544.844726] [12639967.000000000, 12639967.000000000, 12639967.000000000, 12639967.000000000, 12639967.000000000, 12639967.000000000] [24677022, 24677022, 24677022, 24677022, 24677022, 24677022] ["2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01"] ["2023-01-09 07:35:55", "2023-01-09 07:35:55", "2023-01-09 07:35:55", "2023-01-09 07:35:55", "2023-01-09 07:35:55", "2023-01-09 07:35:55"] ["2023-07-31", "2023-07-31", "2023-07-31", "2023-07-31", "2023-07-31", "2023-07-31"] ["2023-03-15 06:40:44", "2023-03-15 06:40:44", "2023-03-15 06:40:44", "2023-03-15 06:40:44", "2023-03-15 06:40:44", "2023-03-15 06:40:44"] ["d", "d", "d", "d", "d", "d"] ["a", "a", "a", "a", "a", "a"] ["dGkS", "dGkS", "dGkS", "dGkS", "dGkS", "dGkS"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -41 [0, 0, 0, 0, 0, 0] [-25, -25, -25, -25, -25, -25] [28704, 28704, 28704, 28704, 28704, 28704] [-437867812, -437867812, -437867812, -437867812, -437867812, -437867812] [5508042206505207079, 5508042206505207079, 5508042206505207079, 5508042206505207079, 5508042206505207079, 5508042206505207079] [462527544684407597, 462527544684407597, 462527544684407597, 462527544684407597, 462527544684407597, 462527544684407597] [13629.614, 13629.614, 13629.614, 13629.614, 13629.614] [213722401.337962, 213722401.337962, 213722401.337962, 213722401.337962, 213722401.337962, 213722401.337962] [86704879.000000000, 86704879.000000000, 86704879.000000000, 86704879.000000000, 86704879.000000000, 86704879.000000000] [-2662959, -2662959, -2662959, -2662959, -2662959, -2662959] ["2023-08-04", "2023-08-04", "2023-08-04", "2023-08-04", "2023-08-04", "2023-08-04"] ["2022-12-28 16:55:08", "2022-12-28 16:55:08", "2022-12-28 16:55:08", "2022-12-28 16:55:08", "2022-12-28 16:55:08", "2022-12-28 16:55:08"] ["2023-05-13", "2023-05-13", "2023-05-13", "2023-05-13", "2023-05-13", "2023-05-13"] ["2023-03-10 18:15:07", "2023-03-10 18:15:07", "2023-03-10 18:15:07", "2023-03-10 18:15:07", "2023-03-10 18:15:07", "2023-03-10 18:15:07"] ["4", "4", "4", "4", "4", "4"] ["a", "a", "a", "a", "a", "a"] ["2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -44 [0, 0, 0, 0, 0, 0] [61, 61, 61, 61, 61, 61] [-23419, -23419, -23419, -23419, -23419, -23419] [378600280, 378600280, 378600280, 378600280, 378600280, 378600280] [6788166268039991679, 6788166268039991679, 6788166268039991679, 6788166268039991679, 6788166268039991679, 6788166268039991679] [-2814786606977504852, -2814786606977504852, -2814786606977504852, -2814786606977504852, -2814786606977504852, -2814786606977504852] [-20151.432, -20151.432, -20151.432, -20151.432, -20151.432] [-1248229001.218229, -1248229001.218229, -1248229001.218229, -1248229001.218229, -1248229001.218229, -1248229001.218229] [-45191385.000000000, -45191385.000000000, -45191385.000000000, -45191385.000000000, -45191385.000000000, -45191385.000000000] [-27910227, -27910227, -27910227, -27910227, -27910227, -27910227] ["2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13"] ["2023-05-18 04:25:32", "2023-05-18 04:25:32", "2023-05-18 04:25:32", "2023-05-18 04:25:32", "2023-05-18 04:25:32", "2023-05-18 04:25:32"] ["2023-03-20", "2023-03-20", "2023-03-20", "2023-03-20", "2023-03-20", "2023-03-20"] ["2022-12-21 11:34:00", "2022-12-21 11:34:00", "2022-12-21 11:34:00", "2022-12-21 11:34:00", "2022-12-21 11:34:00", "2022-12-21 11:34:00"] ["4", "4", "4", "4", "4", "4"] ["I", "I", "I", "I", "I", "I"] ["NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -48 [0, 0, 0, 0, 0, 0] [88, 88, 88, 88, 88, 88] [-18899, -18899, -18899, -18899, -18899, -18899] [1953750640, 1953750640, 1953750640, 1953750640, 1953750640, 1953750640] [-6083034186246180312, -6083034186246180312, -6083034186246180312, -6083034186246180312, -6083034186246180312, -6083034186246180312] [7861718260607212662, 7861718260607212662, 7861718260607212662, 7861718260607212662, 7861718260607212662, 7861718260607212662] [20562.791, 20562.791, 20562.791, 20562.791, 20562.791] [-1597994654.903396, -1597994654.903396, -1597994654.903396, -1597994654.903396, -1597994654.903396, -1597994654.903396] [91765768.000000000, 91765768.000000000, 91765768.000000000, 91765768.000000000, 91765768.000000000, 91765768.000000000] [-47488138, -47488138, -47488138, -47488138, -47488138, -47488138] ["2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12"] ["2022-11-04 07:47:24", "2022-11-04 07:47:24", "2022-11-04 07:47:24", "2022-11-04 07:47:24", "2022-11-04 07:47:24", "2022-11-04 07:47:24"] ["2023-07-16", "2023-07-16", "2023-07-16", "2023-07-16", "2023-07-16", "2023-07-16"] ["2022-08-29 04:51:07", "2022-08-29 04:51:07", "2022-08-29 04:51:07", "2022-08-29 04:51:07", "2022-08-29 04:51:07", "2022-08-29 04:51:07"] ["u", "u", "u", "u", "u", "u"] ["P", "P", "P", "P", "P", "P"] ["wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -49 [0, 0, 0, 0, 0, 0] [126, 126, 126, 126, 126, 126] [31661, 31661, 31661, 31661, 31661, 31661] [359703581, 359703581, 359703581, 359703581, 359703581, 359703581] [-2399575246807057939, -2399575246807057939, -2399575246807057939, -2399575246807057939, -2399575246807057939, -2399575246807057939] [7684667782059034391, 7684667782059034391, 7684667782059034391, 7684667782059034391, 7684667782059034391, 7684667782059034391] [-504.68152, -504.68152, -504.68152, -504.68152, -504.68152] [2121528178.488779, 2121528178.488779, 2121528178.488779, 2121528178.488779, 2121528178.488779, 2121528178.488779] [-99977803.000000000, -99977803.000000000, -99977803.000000000, -99977803.000000000, -99977803.000000000, -99977803.000000000] [25652192, 25652192, 25652192, 25652192, 25652192, 25652192] ["2023-08-25", "2023-08-25", "2023-08-25", "2023-08-25", "2023-08-25", "2023-08-25"] ["2023-07-14 17:16:11", "2023-07-14 17:16:11", "2023-07-14 17:16:11", "2023-07-14 17:16:11", "2023-07-14 17:16:11", "2023-07-14 17:16:11"] ["2022-12-28", "2022-12-28", "2022-12-28", "2022-12-28", "2022-12-28", "2022-12-28"] ["2023-04-10 04:24:11", "2023-04-10 04:24:11", "2023-04-10 04:24:11", "2023-04-10 04:24:11", "2023-04-10 04:24:11", "2023-04-10 04:24:11"] ["i", "i", "i", "i", "i", "i"] ["0", "0", "0", "0", "0", "0"] ["X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -52 [0, 0, 0, 0, 0, 0] [-7, -7, -7, -7, -7, -7] [-6985, -6985, -6985, -6985, -6985, -6985] [826683531, 826683531, 826683531, 826683531, 826683531, 826683531] [-8966681855246736361, -8966681855246736361, -8966681855246736361, -8966681855246736361, -8966681855246736361, -8966681855246736361] [4814686163176635446, 4814686163176635446, 4814686163176635446, 4814686163176635446, 4814686163176635446, 4814686163176635446] [-6490.247, -6490.247, -6490.247, -6490.247, -6490.247] [1076976372.033826, 1076976372.033826, 1076976372.033826, 1076976372.033826, 1076976372.033826, 1076976372.033826] [-25002204.000000000, -25002204.000000000, -25002204.000000000, -25002204.000000000, -25002204.000000000, -25002204.000000000] [-97805693, -97805693, -97805693, -97805693, -97805693, -97805693] ["2023-08-02", "2023-08-02", "2023-08-02", "2023-08-02", "2023-08-02", "2023-08-02"] ["2022-11-21 04:05:22", "2022-11-21 04:05:22", "2022-11-21 04:05:22", "2022-11-21 04:05:22", "2022-11-21 04:05:22", "2022-11-21 04:05:22"] ["2023-05-19", "2023-05-19", "2023-05-19", "2023-05-19", "2023-05-19", "2023-05-19"] ["2023-03-12 05:07:09", "2023-03-12 05:07:09", "2023-03-12 05:07:09", "2023-03-12 05:07:09", "2023-03-12 05:07:09", "2023-03-12 05:07:09"] ["p", "p", "p", "p", "p", "p"] ["a", "a", "a", "a", "a", "a"] ["WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -55 [1, 1, 1, 1, 1, 1] [65, 65, 65, 65, 65, 65] [18805, 18805, 18805, 18805, 18805, 18805] [229725878, 229725878, 229725878, 229725878, 229725878, 229725878] [2742856458318615325, 2742856458318615325, 2742856458318615325, 2742856458318615325, 2742856458318615325, 2742856458318615325] [5907702768956232371, 5907702768956232371, 5907702768956232371, 5907702768956232371, 5907702768956232371, 5907702768956232371] [12354.624, 12354.624, 12354.624, 12354.624, 12354.624] [1697579881.947477, 1697579881.947477, 1697579881.947477, 1697579881.947477, 1697579881.947477, 1697579881.947477] [20409908.000000000, 20409908.000000000, 20409908.000000000, 20409908.000000000, 20409908.000000000, 20409908.000000000] [-69625379, -69625379, -69625379, -69625379, -69625379, -69625379] ["2023-08-09", "2023-08-09", "2023-08-09", "2023-08-09", "2023-08-09", "2023-08-09"] ["2022-12-17 11:47:54", "2022-12-17 11:47:54", "2022-12-17 11:47:54", "2022-12-17 11:47:54", "2022-12-17 11:47:54", "2022-12-17 11:47:54"] ["2023-03-19", "2023-03-19", "2023-03-19", "2023-03-19", "2023-03-19", "2023-03-19"] ["2023-03-13 10:31:52", "2023-03-13 10:31:52", "2023-03-13 10:31:52", "2023-03-13 10:31:52", "2023-03-13 10:31:52", "2023-03-13 10:31:52"] ["G", "G", "G", "G", "G", "G"] ["a", "a", "a", "a", "a", "a"] ["BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -61 [1, 1, 1, 1, 1, 1] [121, 121, 121, 121, 121, 121] [31806, 31806, 31806, 31806, 31806, 31806] [-1410915562, -1410915562, -1410915562, -1410915562, -1410915562, -1410915562] [-250403393155768717, -250403393155768717, -250403393155768717, -250403393155768717, -250403393155768717, -250403393155768717] [4301573778529723431, 4301573778529723431, 4301573778529723431, 4301573778529723431, 4301573778529723431, 4301573778529723431] [10719.892, 10719.892, 10719.892, 10719.892, 10719.892] [1073780599.127242, 1073780599.127242, 1073780599.127242, 1073780599.127242, 1073780599.127242, 1073780599.127242] [78876206.000000000, 78876206.000000000, 78876206.000000000, 78876206.000000000, 78876206.000000000, 78876206.000000000] [-77837482, -77837482, -77837482, -77837482, -77837482, -77837482] ["2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01"] ["2023-01-31 15:44:07", "2023-01-31 15:44:07", "2023-01-31 15:44:07", "2023-01-31 15:44:07", "2023-01-31 15:44:07", "2023-01-31 15:44:07"] ["2023-03-15", "2023-03-15", "2023-03-15", "2023-03-15", "2023-03-15", "2023-03-15"] ["2023-02-06 12:53:16", "2023-02-06 12:53:16", "2023-02-06 12:53:16", "2023-02-06 12:53:16", "2023-02-06 12:53:16", "2023-02-06 12:53:16"] ["U", "U", "U", "U", "U", "U"] ["a", "a", "a", "a", "a", "a"] ["y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -76 [1, 1, 1, 1, 1, 1] [-63, -63, -63, -63, -63, -63] [25799, 25799, 25799, 25799, 25799, 25799] [-1387912656, -1387912656, -1387912656, -1387912656, -1387912656, -1387912656] [8967926767558546181, 8967926767558546181, 8967926767558546181, 8967926767558546181, 8967926767558546181, 8967926767558546181] [-3537865898119184476, -3537865898119184476, -3537865898119184476, -3537865898119184476, -3537865898119184476, -3537865898119184476] [5311.188, 5311.188, 5311.188, 5311.188, 5311.188] [173628749.847049, 173628749.847049, 173628749.847049, 173628749.847049, 173628749.847049, 173628749.847049] [-28850258.000000000, -28850258.000000000, -28850258.000000000, -28850258.000000000, -28850258.000000000, -28850258.000000000] [-99202733, -99202733, -99202733, -99202733, -99202733, -99202733] ["2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12"] ["2023-05-28 15:56:22", "2023-05-28 15:56:22", "2023-05-28 15:56:22", "2023-05-28 15:56:22", "2023-05-28 15:56:22", "2023-05-28 15:56:22"] ["2023-06-11", "2023-06-11", "2023-06-11", "2023-06-11", "2023-06-11", "2023-06-11"] ["2022-11-29 16:23:10", "2022-11-29 16:23:10", "2022-11-29 16:23:10", "2022-11-29 16:23:10", "2022-11-29 16:23:10", "2022-11-29 16:23:10"] ["7", "7", "7", "7", "7", "7"] ["j", "j", "j", "j", "j", "j"] ["fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -84 [0, 0, 0, 0, 0, 0] [-10, -10, -10, -10, -10, -10] [9493, 9493, 9493, 9493, 9493, 9493] [-547874696, -547874696, -547874696, -547874696, -547874696, -547874696] [-115057683458952756, -115057683458952756, -115057683458952756, -115057683458952756, -115057683458952756, -115057683458952756] [4473017779279230085, 4473017779279230085, 4473017779279230085, 4473017779279230085, 4473017779279230085, 4473017779279230085] [13718.372, 13718.372, 13718.372, 13718.372, 13718.372] [-978213266.02697, -978213266.02697, -978213266.02697, -978213266.02697, -978213266.02697, -978213266.02697] [-88516589.000000000, -88516589.000000000, -88516589.000000000, -88516589.000000000, -88516589.000000000, -88516589.000000000] [-62683124, -62683124, -62683124, -62683124, -62683124, -62683124] ["2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26"] ["2022-12-13 00:33:43", "2022-12-13 00:33:43", "2022-12-13 00:33:43", "2022-12-13 00:33:43", "2022-12-13 00:33:43", "2022-12-13 00:33:43"] ["2023-03-25", "2023-03-25", "2023-03-25", "2023-03-25", "2023-03-25", "2023-03-25"] ["2022-09-15 10:53:14", "2022-09-15 10:53:14", "2022-09-15 10:53:14", "2022-09-15 10:53:14", "2022-09-15 10:53:14", "2022-09-15 10:53:14"] ["J", "J", "J", "J", "J", "J"] ["a", "a", "a", "a", "a", "a"] ["yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] - --- !sql_json_read_json_by_line -- -2 [0, 0, 0, 0, 0, 0] [117, 117, 117, 117, 117, 117] [-4744, -4744, -4744, -4744, -4744, -4744] [-1593211961, -1593211961, -1593211961, -1593211961, -1593211961, -1593211961] [-3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780] [8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567] [-30948.857, -30948.857, -30948.857, -30948.857, -30948.857] [804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905] [-74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000] [13024168, 13024168, 13024168, 13024168, 13024168, 13024168] ["2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22"] ["2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12"] ["2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21"] ["2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56"] ["g", "g", "g", "g", "g", "g"] ["a", "a", "a", "a", "a", "a"] ["S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -3 [0, 0, 0, 0, 0, 0] [65, 65, 65, 65, 65, 65] [-4963, -4963, -4963, -4963, -4963, -4963] [-1415431954, -1415431954, -1415431954, -1415431954, -1415431954, -1415431954] [-3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000] [8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376] [-5058.13, -5058.13, -5058.13, -5058.13, -5058.13] [1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352] [3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000] [-3634150, -3634150, -3634150, -3634150, -3634150, -3634150] ["2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26"] ["2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44"] ["2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27"] ["2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11"] ["i", "i", "i", "i", "i", "i"] ["G", "G", "G", "G", "G", "G"] ["XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -4 [0, 0, 0, 0, 0, 0] [-67, -67, -67, -67, -67, -67] [-30372, -30372, -30372, -30372, -30372, -30372] [181502941, 181502941, 181502941, 181502941, 181502941, 181502941] [-2062236823576972800, -2062236823576972800, -2062236823576972800, -2062236823576972800, -2062236823576972800, -2062236823576972800] [6357002962400127842, 6357002962400127842, 6357002962400127842, 6357002962400127842, 6357002962400127842, 6357002962400127842] [21235.783, 21235.783, 21235.783, 21235.783, 21235.783] [-1101694755.713891, -1101694755.713891, -1101694755.713891, -1101694755.713891, -1101694755.713891, -1101694755.713891] [58088067.000000000, 58088067.000000000, 58088067.000000000, 58088067.000000000, 58088067.000000000, 58088067.000000000] [-66079723, -66079723, -66079723, -66079723, -66079723, -66079723] ["2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24"] ["2023-06-16 18:07:12", "2023-06-16 18:07:12", "2023-06-16 18:07:12", "2023-06-16 18:07:12", "2023-06-16 18:07:12", "2023-06-16 18:07:12"] ["2022-11-19", "2022-11-19", "2022-11-19", "2022-11-19", "2022-11-19", "2022-11-19"] ["2023-08-23 08:19:12", "2023-08-23 08:19:12", "2023-08-23 08:19:12", "2023-08-23 08:19:12", "2023-08-23 08:19:12", "2023-08-23 08:19:12"] ["G", "G", "G", "G", "G", "G"] ["a", "a", "a", "a", "a", "a"] ["tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W", "tX4eqSuxvREnF4UIk8OnDyDZM1yT2G2IVzcNB4Lejgxr1W"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -8 [1, 1, 1, 1, 1, 1] [-11, -11, -11, -11, -11, -11] [-9648, -9648, -9648, -9648, -9648, -9648] [-505356927, -505356927, -505356927, -505356927, -505356927, -505356927] [7604760670442035037, 7604760670442035037, 7604760670442035037, 7604760670442035037, 7604760670442035037, 7604760670442035037] [1634770507625165798, 1634770507625165798, 1634770507625165798, 1634770507625165798, 1634770507625165798, 1634770507625165798] [10822.962, 10822.962, 10822.962, 10822.962, 10822.962] [1987551048.863071, 1987551048.863071, 1987551048.863071, 1987551048.863071, 1987551048.863071, 1987551048.863071] [64879544.000000000, 64879544.000000000, 64879544.000000000, 64879544.000000000, 64879544.000000000, 64879544.000000000] [-55896622, -55896622, -55896622, -55896622, -55896622, -55896622] ["2023-08-08", "2023-08-08", "2023-08-08", "2023-08-08", "2023-08-08", "2023-08-08"] ["2023-01-05 00:55:23", "2023-01-05 00:55:23", "2023-01-05 00:55:23", "2023-01-05 00:55:23", "2023-01-05 00:55:23", "2023-01-05 00:55:23"] ["2023-06-16", "2023-06-16", "2023-06-16", "2023-06-16", "2023-06-16", "2023-06-16"] ["2023-04-06 03:40:26", "2023-04-06 03:40:26", "2023-04-06 03:40:26", "2023-04-06 03:40:26", "2023-04-06 03:40:26", "2023-04-06 03:40:26"] ["U", "U", "U", "U", "U", "U"] ["G", "G", "G", "G", "G", "G"] ["iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ", "iyAI9214vOaKrPo1EmVesccN0PdeCC0rKXzRJv33KpnnJbG0o0FXubzuBfrYTQZ"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -16 [0, 0, 0, 0, 0, 0] [-43, -43, -43, -43, -43, -43] [13560, 13560, 13560, 13560, 13560, 13560] [-1743686513, -1743686513, -1743686513, -1743686513, -1743686513, -1743686513] [7234719406392208769, 7234719406392208769, 7234719406392208769, 7234719406392208769, 7234719406392208769, 7234719406392208769] [-3871745630024229413, -3871745630024229413, -3871745630024229413, -3871745630024229413, -3871745630024229413, -3871745630024229413] [12225.427, 12225.427, 12225.427, 12225.427, 12225.427] [-1352141342.04191, -1352141342.04191, -1352141342.04191, -1352141342.04191, -1352141342.04191, -1352141342.04191] [-35959452.000000000, -35959452.000000000, -35959452.000000000, -35959452.000000000, -35959452.000000000, -35959452.000000000] [-4256846, -4256846, -4256846, -4256846, -4256846, -4256846] ["2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24", "2023-08-24"] ["2022-12-25 20:40:43", "2022-12-25 20:40:43", "2022-12-25 20:40:43", "2022-12-25 20:40:43", "2022-12-25 20:40:43", "2022-12-25 20:40:43"] ["2023-01-05", "2023-01-05", "2023-01-05", "2023-01-05", "2023-01-05", "2023-01-05"] ["2023-01-27 19:34:04", "2023-01-27 19:34:04", "2023-01-27 19:34:04", "2023-01-27 19:34:04", "2023-01-27 19:34:04", "2023-01-27 19:34:04"] ["V", "V", "V", "V", "V", "V"] ["a", "a", "a", "a", "a", "a"] ["8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj", "8PRRNG2OCMIFR5k9nuC0O4TDT6hxwj"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -17 [1, 1, 1, 1, 1, 1] [126, 126, 126, 126, 126, 126] [28165, 28165, 28165, 28165, 28165, 28165] [2032059721, 2032059721, 2032059721, 2032059721, 2032059721, 2032059721] [-2686776977990574879, -2686776977990574879, -2686776977990574879, -2686776977990574879, -2686776977990574879, -2686776977990574879] [-8498149444423102876, -8498149444423102876, -8498149444423102876, -8498149444423102876, -8498149444423102876, -8498149444423102876] [-21681.223, -21681.223, -21681.223, -21681.223, -21681.223] [95345732.447678, 95345732.447678, 95345732.447678, 95345732.447678, 95345732.447678, 95345732.447678] [89062729.000000000, 89062729.000000000, 89062729.000000000, 89062729.000000000, 89062729.000000000, 89062729.000000000] [-2464406, -2464406, -2464406, -2464406, -2464406, -2464406] ["2023-08-16", "2023-08-16", "2023-08-16", "2023-08-16", "2023-08-16", "2023-08-16"] ["2023-03-05 10:43:36", "2023-03-05 10:43:36", "2023-03-05 10:43:36", "2023-03-05 10:43:36", "2023-03-05 10:43:36", "2023-03-05 10:43:36"] ["2023-08-20", "2023-08-20", "2023-08-20", "2023-08-20", "2023-08-20", "2023-08-20"] ["2023-04-11 00:33:44", "2023-04-11 00:33:44", "2023-04-11 00:33:44", "2023-04-11 00:33:44", "2023-04-11 00:33:44", "2023-04-11 00:33:44"] ["U", "U", "U", "U", "U", "U"] ["a", "a", "a", "a", "a", "a"] ["XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb", "XJ3RV27MQpsPD30k1xV0RGSuomCT1z5oEfOiF2gNisoMyFhoClXJJ8eIZSeasLKeJwflG4In7xn54n9oI16rpRQJkeb"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -27 [0, 0, 0, 0, 0, 0] [-7, -7, -7, -7, -7, -7] [18307, 18307, 18307, 18307, 18307, 18307] [76399879, 76399879, 76399879, 76399879, 76399879, 76399879] [8050764818738996699, 8050764818738996699, 8050764818738996699, 8050764818738996699, 8050764818738996699, 8050764818738996699] [2402219865213589999, 2402219865213589999, 2402219865213589999, 2402219865213589999, 2402219865213589999, 2402219865213589999] [-16678.924, -16678.924, -16678.924, -16678.924, -16678.924] [-2053879544.844726, -2053879544.844726, -2053879544.844726, -2053879544.844726, -2053879544.844726, -2053879544.844726] [12639967.000000000, 12639967.000000000, 12639967.000000000, 12639967.000000000, 12639967.000000000, 12639967.000000000] [24677022, 24677022, 24677022, 24677022, 24677022, 24677022] ["2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01"] ["2023-01-09 07:35:55", "2023-01-09 07:35:55", "2023-01-09 07:35:55", "2023-01-09 07:35:55", "2023-01-09 07:35:55", "2023-01-09 07:35:55"] ["2023-07-31", "2023-07-31", "2023-07-31", "2023-07-31", "2023-07-31", "2023-07-31"] ["2023-03-15 06:40:44", "2023-03-15 06:40:44", "2023-03-15 06:40:44", "2023-03-15 06:40:44", "2023-03-15 06:40:44", "2023-03-15 06:40:44"] ["d", "d", "d", "d", "d", "d"] ["a", "a", "a", "a", "a", "a"] ["dGkS", "dGkS", "dGkS", "dGkS", "dGkS", "dGkS"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -41 [0, 0, 0, 0, 0, 0] [-25, -25, -25, -25, -25, -25] [28704, 28704, 28704, 28704, 28704, 28704] [-437867812, -437867812, -437867812, -437867812, -437867812, -437867812] [5508042206505207079, 5508042206505207079, 5508042206505207079, 5508042206505207079, 5508042206505207079, 5508042206505207079] [462527544684407597, 462527544684407597, 462527544684407597, 462527544684407597, 462527544684407597, 462527544684407597] [13629.614, 13629.614, 13629.614, 13629.614, 13629.614] [213722401.337962, 213722401.337962, 213722401.337962, 213722401.337962, 213722401.337962, 213722401.337962] [86704879.000000000, 86704879.000000000, 86704879.000000000, 86704879.000000000, 86704879.000000000, 86704879.000000000] [-2662959, -2662959, -2662959, -2662959, -2662959, -2662959] ["2023-08-04", "2023-08-04", "2023-08-04", "2023-08-04", "2023-08-04", "2023-08-04"] ["2022-12-28 16:55:08", "2022-12-28 16:55:08", "2022-12-28 16:55:08", "2022-12-28 16:55:08", "2022-12-28 16:55:08", "2022-12-28 16:55:08"] ["2023-05-13", "2023-05-13", "2023-05-13", "2023-05-13", "2023-05-13", "2023-05-13"] ["2023-03-10 18:15:07", "2023-03-10 18:15:07", "2023-03-10 18:15:07", "2023-03-10 18:15:07", "2023-03-10 18:15:07", "2023-03-10 18:15:07"] ["4", "4", "4", "4", "4", "4"] ["a", "a", "a", "a", "a", "a"] ["2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI", "2dxEKrIfvZTmYHO6jXR7HMuJAJrj1dJD2WRroeHL20dGolyHdcI"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -44 [0, 0, 0, 0, 0, 0] [61, 61, 61, 61, 61, 61] [-23419, -23419, -23419, -23419, -23419, -23419] [378600280, 378600280, 378600280, 378600280, 378600280, 378600280] [6788166268039991679, 6788166268039991679, 6788166268039991679, 6788166268039991679, 6788166268039991679, 6788166268039991679] [-2814786606977504852, -2814786606977504852, -2814786606977504852, -2814786606977504852, -2814786606977504852, -2814786606977504852] [-20151.432, -20151.432, -20151.432, -20151.432, -20151.432] [-1248229001.218229, -1248229001.218229, -1248229001.218229, -1248229001.218229, -1248229001.218229, -1248229001.218229] [-45191385.000000000, -45191385.000000000, -45191385.000000000, -45191385.000000000, -45191385.000000000, -45191385.000000000] [-27910227, -27910227, -27910227, -27910227, -27910227, -27910227] ["2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13", "2023-08-13"] ["2023-05-18 04:25:32", "2023-05-18 04:25:32", "2023-05-18 04:25:32", "2023-05-18 04:25:32", "2023-05-18 04:25:32", "2023-05-18 04:25:32"] ["2023-03-20", "2023-03-20", "2023-03-20", "2023-03-20", "2023-03-20", "2023-03-20"] ["2022-12-21 11:34:00", "2022-12-21 11:34:00", "2022-12-21 11:34:00", "2022-12-21 11:34:00", "2022-12-21 11:34:00", "2022-12-21 11:34:00"] ["4", "4", "4", "4", "4", "4"] ["I", "I", "I", "I", "I", "I"] ["NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1", "NZXyjFFg0Yunu6fDXpyr9OlUtMNbaJbSENshFUaeCdMR64vgELMWSxvUxwLfmhzoDY1z6bLCyjuGmMUk9hhnF9hKsFkgpbcPo2nz1"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -48 [0, 0, 0, 0, 0, 0] [88, 88, 88, 88, 88, 88] [-18899, -18899, -18899, -18899, -18899, -18899] [1953750640, 1953750640, 1953750640, 1953750640, 1953750640, 1953750640] [-6083034186246180312, -6083034186246180312, -6083034186246180312, -6083034186246180312, -6083034186246180312, -6083034186246180312] [7861718260607212662, 7861718260607212662, 7861718260607212662, 7861718260607212662, 7861718260607212662, 7861718260607212662] [20562.791, 20562.791, 20562.791, 20562.791, 20562.791] [-1597994654.903396, -1597994654.903396, -1597994654.903396, -1597994654.903396, -1597994654.903396, -1597994654.903396] [91765768.000000000, 91765768.000000000, 91765768.000000000, 91765768.000000000, 91765768.000000000, 91765768.000000000] [-47488138, -47488138, -47488138, -47488138, -47488138, -47488138] ["2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12"] ["2022-11-04 07:47:24", "2022-11-04 07:47:24", "2022-11-04 07:47:24", "2022-11-04 07:47:24", "2022-11-04 07:47:24", "2022-11-04 07:47:24"] ["2023-07-16", "2023-07-16", "2023-07-16", "2023-07-16", "2023-07-16", "2023-07-16"] ["2022-08-29 04:51:07", "2022-08-29 04:51:07", "2022-08-29 04:51:07", "2022-08-29 04:51:07", "2022-08-29 04:51:07", "2022-08-29 04:51:07"] ["u", "u", "u", "u", "u", "u"] ["P", "P", "P", "P", "P", "P"] ["wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv", "wodMoILg2POwMnYHhY33utnoZ325ocWKXPLvo1Cxx8C2Wj8maoUhfwozaHgjzoOqeW9lMj2nNinNDJV2dnXsNfa0hVeNzonA7s84gYHSyHXDzvjv"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -49 [0, 0, 0, 0, 0, 0] [126, 126, 126, 126, 126, 126] [31661, 31661, 31661, 31661, 31661, 31661] [359703581, 359703581, 359703581, 359703581, 359703581, 359703581] [-2399575246807057939, -2399575246807057939, -2399575246807057939, -2399575246807057939, -2399575246807057939, -2399575246807057939] [7684667782059034391, 7684667782059034391, 7684667782059034391, 7684667782059034391, 7684667782059034391, 7684667782059034391] [-504.68152, -504.68152, -504.68152, -504.68152, -504.68152] [2121528178.488779, 2121528178.488779, 2121528178.488779, 2121528178.488779, 2121528178.488779, 2121528178.488779] [-99977803.000000000, -99977803.000000000, -99977803.000000000, -99977803.000000000, -99977803.000000000, -99977803.000000000] [25652192, 25652192, 25652192, 25652192, 25652192, 25652192] ["2023-08-25", "2023-08-25", "2023-08-25", "2023-08-25", "2023-08-25", "2023-08-25"] ["2023-07-14 17:16:11", "2023-07-14 17:16:11", "2023-07-14 17:16:11", "2023-07-14 17:16:11", "2023-07-14 17:16:11", "2023-07-14 17:16:11"] ["2022-12-28", "2022-12-28", "2022-12-28", "2022-12-28", "2022-12-28", "2022-12-28"] ["2023-04-10 04:24:11", "2023-04-10 04:24:11", "2023-04-10 04:24:11", "2023-04-10 04:24:11", "2023-04-10 04:24:11", "2023-04-10 04:24:11"] ["i", "i", "i", "i", "i", "i"] ["0", "0", "0", "0", "0", "0"] ["X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7", "X53UUdHkzSVyWMmGyI5i6M4ohehRhx1NR02IjJsuKy64Tp0KJvsHgjJ64F2qHOpNQ17EUnIPJlmpCV32vDZkJwXjhhY1eObFH2Ru7gHqsmFCQ2zy7"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -52 [0, 0, 0, 0, 0, 0] [-7, -7, -7, -7, -7, -7] [-6985, -6985, -6985, -6985, -6985, -6985] [826683531, 826683531, 826683531, 826683531, 826683531, 826683531] [-8966681855246736361, -8966681855246736361, -8966681855246736361, -8966681855246736361, -8966681855246736361, -8966681855246736361] [4814686163176635446, 4814686163176635446, 4814686163176635446, 4814686163176635446, 4814686163176635446, 4814686163176635446] [-6490.247, -6490.247, -6490.247, -6490.247, -6490.247] [1076976372.033826, 1076976372.033826, 1076976372.033826, 1076976372.033826, 1076976372.033826, 1076976372.033826] [-25002204.000000000, -25002204.000000000, -25002204.000000000, -25002204.000000000, -25002204.000000000, -25002204.000000000] [-97805693, -97805693, -97805693, -97805693, -97805693, -97805693] ["2023-08-02", "2023-08-02", "2023-08-02", "2023-08-02", "2023-08-02", "2023-08-02"] ["2022-11-21 04:05:22", "2022-11-21 04:05:22", "2022-11-21 04:05:22", "2022-11-21 04:05:22", "2022-11-21 04:05:22", "2022-11-21 04:05:22"] ["2023-05-19", "2023-05-19", "2023-05-19", "2023-05-19", "2023-05-19", "2023-05-19"] ["2023-03-12 05:07:09", "2023-03-12 05:07:09", "2023-03-12 05:07:09", "2023-03-12 05:07:09", "2023-03-12 05:07:09", "2023-03-12 05:07:09"] ["p", "p", "p", "p", "p", "p"] ["a", "a", "a", "a", "a", "a"] ["WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p", "WG0vFztUWdoC7f8D14NbhHAK302bHf6s1JKNU2hiIjZ5ABhHwikfSzCAKdr04s6bhGkkssdVzRuSSheQ0rFUGkueuLch5p"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -55 [1, 1, 1, 1, 1, 1] [65, 65, 65, 65, 65, 65] [18805, 18805, 18805, 18805, 18805, 18805] [229725878, 229725878, 229725878, 229725878, 229725878, 229725878] [2742856458318615325, 2742856458318615325, 2742856458318615325, 2742856458318615325, 2742856458318615325, 2742856458318615325] [5907702768956232371, 5907702768956232371, 5907702768956232371, 5907702768956232371, 5907702768956232371, 5907702768956232371] [12354.624, 12354.624, 12354.624, 12354.624, 12354.624] [1697579881.947477, 1697579881.947477, 1697579881.947477, 1697579881.947477, 1697579881.947477, 1697579881.947477] [20409908.000000000, 20409908.000000000, 20409908.000000000, 20409908.000000000, 20409908.000000000, 20409908.000000000] [-69625379, -69625379, -69625379, -69625379, -69625379, -69625379] ["2023-08-09", "2023-08-09", "2023-08-09", "2023-08-09", "2023-08-09", "2023-08-09"] ["2022-12-17 11:47:54", "2022-12-17 11:47:54", "2022-12-17 11:47:54", "2022-12-17 11:47:54", "2022-12-17 11:47:54", "2022-12-17 11:47:54"] ["2023-03-19", "2023-03-19", "2023-03-19", "2023-03-19", "2023-03-19", "2023-03-19"] ["2023-03-13 10:31:52", "2023-03-13 10:31:52", "2023-03-13 10:31:52", "2023-03-13 10:31:52", "2023-03-13 10:31:52", "2023-03-13 10:31:52"] ["G", "G", "G", "G", "G", "G"] ["a", "a", "a", "a", "a", "a"] ["BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7", "BU2JJTmOKfXr9q8SjWLRUhxn2Me7HFpkMAJxCJCRvnnUJg2l3zXXOnLavxUNt7"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -61 [1, 1, 1, 1, 1, 1] [121, 121, 121, 121, 121, 121] [31806, 31806, 31806, 31806, 31806, 31806] [-1410915562, -1410915562, -1410915562, -1410915562, -1410915562, -1410915562] [-250403393155768717, -250403393155768717, -250403393155768717, -250403393155768717, -250403393155768717, -250403393155768717] [4301573778529723431, 4301573778529723431, 4301573778529723431, 4301573778529723431, 4301573778529723431, 4301573778529723431] [10719.892, 10719.892, 10719.892, 10719.892, 10719.892] [1073780599.127242, 1073780599.127242, 1073780599.127242, 1073780599.127242, 1073780599.127242, 1073780599.127242] [78876206.000000000, 78876206.000000000, 78876206.000000000, 78876206.000000000, 78876206.000000000, 78876206.000000000] [-77837482, -77837482, -77837482, -77837482, -77837482, -77837482] ["2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01", "2023-08-01"] ["2023-01-31 15:44:07", "2023-01-31 15:44:07", "2023-01-31 15:44:07", "2023-01-31 15:44:07", "2023-01-31 15:44:07", "2023-01-31 15:44:07"] ["2023-03-15", "2023-03-15", "2023-03-15", "2023-03-15", "2023-03-15", "2023-03-15"] ["2023-02-06 12:53:16", "2023-02-06 12:53:16", "2023-02-06 12:53:16", "2023-02-06 12:53:16", "2023-02-06 12:53:16", "2023-02-06 12:53:16"] ["U", "U", "U", "U", "U", "U"] ["a", "a", "a", "a", "a", "a"] ["y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb", "y25ujOZPm64KwGfKXGPMgqaUAdIhi8GtBb"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -76 [1, 1, 1, 1, 1, 1] [-63, -63, -63, -63, -63, -63] [25799, 25799, 25799, 25799, 25799, 25799] [-1387912656, -1387912656, -1387912656, -1387912656, -1387912656, -1387912656] [8967926767558546181, 8967926767558546181, 8967926767558546181, 8967926767558546181, 8967926767558546181, 8967926767558546181] [-3537865898119184476, -3537865898119184476, -3537865898119184476, -3537865898119184476, -3537865898119184476, -3537865898119184476] [5311.188, 5311.188, 5311.188, 5311.188, 5311.188] [173628749.847049, 173628749.847049, 173628749.847049, 173628749.847049, 173628749.847049, 173628749.847049] [-28850258.000000000, -28850258.000000000, -28850258.000000000, -28850258.000000000, -28850258.000000000, -28850258.000000000] [-99202733, -99202733, -99202733, -99202733, -99202733, -99202733] ["2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12", "2023-08-12"] ["2023-05-28 15:56:22", "2023-05-28 15:56:22", "2023-05-28 15:56:22", "2023-05-28 15:56:22", "2023-05-28 15:56:22", "2023-05-28 15:56:22"] ["2023-06-11", "2023-06-11", "2023-06-11", "2023-06-11", "2023-06-11", "2023-06-11"] ["2022-11-29 16:23:10", "2022-11-29 16:23:10", "2022-11-29 16:23:10", "2022-11-29 16:23:10", "2022-11-29 16:23:10", "2022-11-29 16:23:10"] ["7", "7", "7", "7", "7", "7"] ["j", "j", "j", "j", "j", "j"] ["fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0", "fjbvP5vNgAwYBIAEnONGcVbrBpGM3mqVeBjDQs4KQlLEgNbnHVIIscgaRuPdDjU0"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -84 [0, 0, 0, 0, 0, 0] [-10, -10, -10, -10, -10, -10] [9493, 9493, 9493, 9493, 9493, 9493] [-547874696, -547874696, -547874696, -547874696, -547874696, -547874696] [-115057683458952756, -115057683458952756, -115057683458952756, -115057683458952756, -115057683458952756, -115057683458952756] [4473017779279230085, 4473017779279230085, 4473017779279230085, 4473017779279230085, 4473017779279230085, 4473017779279230085] [13718.372, 13718.372, 13718.372, 13718.372, 13718.372] [-978213266.02697, -978213266.02697, -978213266.02697, -978213266.02697, -978213266.02697, -978213266.02697] [-88516589.000000000, -88516589.000000000, -88516589.000000000, -88516589.000000000, -88516589.000000000, -88516589.000000000] [-62683124, -62683124, -62683124, -62683124, -62683124, -62683124] ["2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26"] ["2022-12-13 00:33:43", "2022-12-13 00:33:43", "2022-12-13 00:33:43", "2022-12-13 00:33:43", "2022-12-13 00:33:43", "2022-12-13 00:33:43"] ["2023-03-25", "2023-03-25", "2023-03-25", "2023-03-25", "2023-03-25", "2023-03-25"] ["2022-09-15 10:53:14", "2022-09-15 10:53:14", "2022-09-15 10:53:14", "2022-09-15 10:53:14", "2022-09-15 10:53:14", "2022-09-15 10:53:14"] ["J", "J", "J", "J", "J", "J"] ["a", "a", "a", "a", "a", "a"] ["yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq", "yQtIngWQxQTtvo1z2xxWMOT6JdaZT599ZTsOKOxwERkicW5YhScbCNrQAz8BHaarqK8AHPwvT2uXRlFKu6uZLIONVqdMR0Irap9OzNVIJCJmkCq"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] - -- !sql_json_read_json_by_line -- 2 [0, 0, 0, 0, 0, 0] [117, 117, 117, 117, 117, 117] [-4744, -4744, -4744, -4744, -4744, -4744] [-1593211961, -1593211961, -1593211961, -1593211961, -1593211961, -1593211961] [-3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780, -3869640069299678780] [8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567, 8491817458398170567] [-30948.857, -30948.857, -30948.857, -30948.857, -30948.857] [804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905, 804341131.229905] [-74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000, -74019648.000000000] [13024168, 13024168, 13024168, 13024168, 13024168, 13024168] ["2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22", "2023-08-22"] ["2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12", "2022-09-30 07:47:12"] ["2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21", "2023-04-21"] ["2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56", "2022-11-24 15:07:56"] ["g", "g", "g", "g", "g", "g"] ["a", "a", "a", "a", "a", "a"] ["S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl", "S9JEYFrLN4zr1vX1yPUE6ovSX431nJdCuttpBUOVMrp844vBfHStO7laHNc5sI9MehAi8GbGDGV3t322DPMy7SBlquU5D7jsGISMNpX4IWbn3Yrsl"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] 3 [0, 0, 0, 0, 0, 0] [65, 65, 65, 65, 65, 65] [-4963, -4963, -4963, -4963, -4963, -4963] [-1415431954, -1415431954, -1415431954, -1415431954, -1415431954, -1415431954] [-3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000, -3804309860450207000] [8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376, 8209240008557215376] [-5058.13, -5058.13, -5058.13, -5058.13, -5058.13] [1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352, 1034763010.616352] [3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000, 3858273.000000000] [-3634150, -3634150, -3634150, -3634150, -3634150, -3634150] ["2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26", "2023-08-26"] ["2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44", "2023-03-06 07:47:44"] ["2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27", "2023-04-27"] ["2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11", "2022-12-20 15:40:11"] ["i", "i", "i", "i", "i", "i"] ["G", "G", "G", "G", "G", "G"] ["XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ", "XuyX5eljhfMNqMmuOGkFNvyoKFyFMzGWPuGQPxAi6NYV6JA2aooYGJ0CgQ"] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy index 263281929d9db51..a193a3f503db6f0 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy @@ -27,7 +27,7 @@ suite("test_stream_load_2pc", "p0") { // for test dup, mow, uniq, agg tables, we use sql statement concat these parameters // due to dynamic partition is different from others, it's the reason why we concat create sql statement - def tables = ["stream_load_dup_tbl_basic", "stream_load_mow_tbl_basic", "stream_load_uniq_tbl_basic", "stream_load_agg_tbl_basic"] + def tables = ["stream_load_dup_tbl_basic_2pc", "stream_load_mow_tbl_basic_2pc", "stream_load_uniq_tbl_basic_2pc", "stream_load_agg_tbl_basic_2pc"] def columns_stream_load = [ """k00, k01, k02, k03, k04, k05, k06, k07, k08, k09, k10, k11, k12, k13, k14, k15, k16, k17, k18""", @@ -39,7 +39,7 @@ suite("test_stream_load_2pc", "p0") { def create_table_sql = [ """ - CREATE TABLE stream_load_dup_tbl_basic + CREATE TABLE ${tables[0]} ( k00 INT NOT NULL, k01 DATE NOT NULL, @@ -95,7 +95,7 @@ suite("test_stream_load_2pc", "p0") { DUPLICATE KEY(k00)""", """ - CREATE TABLE stream_load_mow_tbl_basic + CREATE TABLE ${tables[1]} ( k00 INT NOT NULL, k01 DATE NULL, @@ -148,7 +148,7 @@ suite("test_stream_load_2pc", "p0") { """, """ - CREATE TABLE stream_load_uniq_tbl_basic + CREATE TABLE ${tables[2]} ( k00 INT NOT NULL, k01 DATE NOT NULL, @@ -198,7 +198,7 @@ suite("test_stream_load_2pc", "p0") { """, """ - CREATE TABLE stream_load_agg_tbl_basic + CREATE TABLE ${tables[3]} ( k00 INT NOT NULL, k01 DATE NOT NULL, @@ -566,7 +566,7 @@ suite("test_stream_load_2pc", "p0") { def expected = [1, 3, 3, 5, 21] // we recreate table for each partition, then load data with stream load and check the result for (i = 0; i < tables.size(); ++i) { - if (isCloudMode() && tables[i].equals("stream_load_mow_tbl_basic")) { + if (isCloudMode() && tables[i].equals("stream_load_mow_tbl_basic_2pc")) { log.info("Skip stream load mow table in cloud mode") continue; } diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy index 1e4e207821b5462..4f52caa10c55177 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy @@ -31,7 +31,7 @@ import org.apache.http.client.RedirectStrategy; import org.apache.http.impl.client.LaxRedirectStrategy; import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_stream_load_properties", "p0,nonConcurrent") { +suite("test_stream_load_properties", "p0") { def tables = [ "dup_tbl_basic", @@ -1100,139 +1100,6 @@ suite("test_stream_load_properties", "p0,nonConcurrent") { } } - // test read_json_by_line with enable_simdjson_reader=false/true - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - def get_be_param = { paramName -> - // assuming paramName on all BEs have save value - String backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - for (Object ele in (List) configList) { - assert ele instanceof List - if (((List) ele)[0] == paramName) { - return ((List) ele)[2] - } - } - } - - def set_be_param = { paramName, paramValue -> - // for eache BE node, set paramName=paramValue - for (String id in backendId_to_backendIP.keySet()) { - def beIp = backendId_to_backendIP.get(id) - def bePort = backendId_to_backendHttpPort.get(id) - def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, paramValue)) - assertTrue(out.contains("OK")) - } - } - - // read and save original value of enable_simdjson_reader - boolean enable_simdjson_reader = Boolean.parseBoolean(get_be_param.call("enable_simdjson_reader")) - - i = 0 - try { - // set enable_simdjson_reader=false and test - set_be_param.call("enable_simdjson_reader", "false") - - for (String tableName in tables) { - sql new File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text - sql new File("""${context.file.parent}/ddl/${tableName}_create.sql""").text - - streamLoad { - table "stream_load_" + tableName - set 'format', 'json' - set 'columns', columns[i] - set 'read_json_by_line', 'true' - if (i <= 3) { - file json_by_line_files[0] - } else { - file json_by_line_files[1] - } - time 10000 // limit inflight 10s - - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception - } - log.info("Stream load result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - assertEquals(jsonLoadedRows[i], json.NumberTotalRows) - assertEquals(jsonLoadedRows[i], json.NumberLoadedRows) - assertEquals(0, json.NumberFilteredRows) - assertEquals(0, json.NumberUnselectedRows) - } - } - sql "sync" - def tableName1 = "stream_load_" + tableName - if (i <= 3) { - qt_sql_json_read_by_line "select * from ${tableName1} order by k00,k01" - } else { - qt_sql_json_read_json_by_line "select * from ${tableName1} order by k00" - } - i++ - } - } finally { - for (String tableName in tables) { - sql new File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text - } - } - - i = 0 - try { - // set enable_simdjson_reader=true and test - set_be_param.call("enable_simdjson_reader", "true") - - for (String tableName in tables) { - sql new File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text - sql new File("""${context.file.parent}/ddl/${tableName}_create.sql""").text - - streamLoad { - table "stream_load_" + tableName - set 'format', 'json' - set 'columns', columns[i] - set 'read_json_by_line', 'true' - if (i <= 3) { - file json_by_line_files[0] - } else { - file json_by_line_files[1] - } - time 10000 // limit inflight 10s - - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception - } - log.info("Stream load result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - assertEquals(jsonLoadedRows[i], json.NumberTotalRows) - assertEquals(jsonLoadedRows[i], json.NumberLoadedRows) - assertEquals(0, json.NumberFilteredRows) - assertEquals(0, json.NumberUnselectedRows) - } - } - sql "sync" - def tableName1 = "stream_load_" + tableName - if (i <= 3) { - qt_sql_json_read_by_line "select * from ${tableName1} order by k00,k01" - } else { - qt_sql_json_read_json_by_line "select * from ${tableName1} order by k00" - } - i++ - } - } finally { - for (String tableName in tables) { - sql new File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text - } - } - // restore BEs to original value - set_be_param.call("enable_simdjson_reader", enable_simdjson_reader ? "true" : "false") - // test invalid jsonpaths i = 0 try { From f174a304f2723b77bcf3de2614af1ba54613604f Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Wed, 28 Aug 2024 21:05:11 +0800 Subject: [PATCH 02/34] [enhancement](exception) pageio method should return error when allocate memory failed (#40020) ## Proposed changes 1. pageio method should return error when allocate memory failed just for point query. 2. variant parsing logic should return error when allocate memory failed. --------- Co-authored-by: yiguolei --- be/src/olap/rowset/segment_creator.cpp | 2 +- be/src/olap/rowset/segment_creator.h | 6 +++++- be/src/olap/rowset/segment_v2/page_io.cpp | 4 ++-- be/src/olap/rowset/segment_v2/page_io.h | 11 ++++++++++- be/src/vec/common/schema_util.cpp | 14 ++++---------- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/be/src/olap/rowset/segment_creator.cpp b/be/src/olap/rowset/segment_creator.cpp index c657238c0208866..1afd3215db42f62 100644 --- a/be/src/olap/rowset/segment_creator.cpp +++ b/be/src/olap/rowset/segment_creator.cpp @@ -83,7 +83,7 @@ Status SegmentFlusher::flush_single_block(const vectorized::Block* block, int32_ return Status::OK(); } -Status SegmentFlusher::_parse_variant_columns(vectorized::Block& block) { +Status SegmentFlusher::_internal_parse_variant_columns(vectorized::Block& block) { size_t num_rows = block.rows(); if (num_rows == 0) { return Status::OK(); diff --git a/be/src/olap/rowset/segment_creator.h b/be/src/olap/rowset/segment_creator.h index 961e161853c1b72..c862fce87a43bdc 100644 --- a/be/src/olap/rowset/segment_creator.h +++ b/be/src/olap/rowset/segment_creator.h @@ -141,7 +141,11 @@ class SegmentFlusher { bool need_buffering(); private: - Status _parse_variant_columns(vectorized::Block& block); + // This method will catch exception when allocate memory failed + Status _parse_variant_columns(vectorized::Block& block) { + RETURN_IF_CATCH_EXCEPTION({ return _internal_parse_variant_columns(block); }); + } + Status _internal_parse_variant_columns(vectorized::Block& block); Status _add_rows(std::unique_ptr& segment_writer, const vectorized::Block* block, size_t row_offset, size_t row_num); Status _add_rows(std::unique_ptr& segment_writer, diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp index cea4a23f7421783..07d5656ee8a44b1 100644 --- a/be/src/olap/rowset/segment_v2/page_io.cpp +++ b/be/src/olap/rowset/segment_v2/page_io.cpp @@ -111,8 +111,8 @@ Status PageIO::write_page(io::FileWriter* writer, const std::vector& body return Status::OK(); } -Status PageIO::read_and_decompress_page(const PageReadOptions& opts, PageHandle* handle, - Slice* body, PageFooterPB* footer) { +Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle* handle, + Slice* body, PageFooterPB* footer) { opts.sanity_check(); opts.stats->total_pages_num++; diff --git a/be/src/olap/rowset/segment_v2/page_io.h b/be/src/olap/rowset/segment_v2/page_io.h index 736b3e521f6800d..b23af4b0b350e5c 100644 --- a/be/src/olap/rowset/segment_v2/page_io.h +++ b/be/src/olap/rowset/segment_v2/page_io.h @@ -123,8 +123,17 @@ class PageIO { // `handle' holds the memory of page data, // `body' points to page body, // `footer' stores the page footer. + // This method is exception safe, it will failed when allocate memory failed. static Status read_and_decompress_page(const PageReadOptions& opts, PageHandle* handle, - Slice* body, PageFooterPB* footer); + Slice* body, PageFooterPB* footer) { + RETURN_IF_CATCH_EXCEPTION( + { return read_and_decompress_page_(opts, handle, body, footer); }); + } + +private: + // An internal method that not deal with exception. + static Status read_and_decompress_page_(const PageReadOptions& opts, PageHandle* handle, + Slice* body, PageFooterPB* footer); }; } // namespace segment_v2 diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index 4d470ccc3b73b63..c0b48e01307012d 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -530,16 +530,10 @@ Status _parse_variant_columns(Block& block, const std::vector& variant_pos, Status parse_variant_columns(Block& block, const std::vector& variant_pos, const ParseConfig& config) { - try { - // Parse each variant column from raw string column - RETURN_IF_ERROR( - vectorized::schema_util::_parse_variant_columns(block, variant_pos, config)); - } catch (const doris::Exception& e) { - // TODO more graceful, max_filter_ratio - LOG(WARNING) << "encounter execption " << e.to_string(); - return Status::InternalError(e.to_string()); - } - return Status::OK(); + // Parse each variant column from raw string column + RETURN_IF_CATCH_EXCEPTION({ + return vectorized::schema_util::_parse_variant_columns(block, variant_pos, config); + }); } Status encode_variant_sparse_subcolumns(ColumnObject& column) { From 2d17e6ed52277fdc6ad2701a13ac017b0434dd34 Mon Sep 17 00:00:00 2001 From: zhengyu Date: Wed, 28 Aug 2024 21:21:41 +0800 Subject: [PATCH 03/34] [ut](filecache) fix try_any_cast error in UTs (#40039) ## Proposed changes Issue Number: close #xxx --------- Signed-off-by: freemandealer --- be/test/io/fs/s3_file_writer_test.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index 782ef80121a6451..ab76fb54347d275 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -944,10 +944,9 @@ TEST_F(S3FileWriterTest, multi_part_complete_error_2) { sp->set_call_back("S3FileWriter::_complete:2", [](auto&& outcome) { // Deliberately make one upload one part task fail to test if s3 file writer could // handle io error - auto* parts = try_any_cast>*>( - outcome.back()); + auto* parts = try_any_cast*>(outcome.back()); size_t size = parts->size(); - parts->back()->SetPartNumber(size + 2); + parts->back().part_num = (size + 2); }); Defer defer {[&]() { sp->clear_call_back("S3FileWriter::_complete:2"); }}; auto client = s3_fs->client_holder(); @@ -992,8 +991,8 @@ TEST_F(S3FileWriterTest, multi_part_complete_error_1) { sp->set_call_back("S3FileWriter::_complete:1", [](auto&& outcome) { // Deliberately make one upload one part task fail to test if s3 file writer could // handle io error - const auto& points = try_any_cast>*>&>( + const auto& points = try_any_cast< + const std::pair*>&>( outcome.back()); (*points.first) = false; points.second->pop_back(); From 643eb973d06811ded99ac88e4c6899edc786940f Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 28 Aug 2024 21:34:20 +0800 Subject: [PATCH 04/34] [fix](nereids) Fix add column syntax (#39997) The original parser allows the column_definition_list only to have one column_definition, this PR makes the nereids parser compatible with the legacy one. --------- Co-authored-by: morrySnow <101034200+morrySnow@users.noreply.github.com> --- .../org/apache/doris/nereids/DorisParser.g4 | 2 +- .../nereids_syntax_p0/ddl/add_column.groovy | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/nereids_syntax_p0/ddl/add_column.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 2b0e5f1a561227f..1c695f29c329d23 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -237,7 +237,7 @@ addRollupClause alterTableClause : ADD COLUMN columnDef columnPosition? toRollup? properties=propertyClause? #addColumnClause - | ADD COLUMN LEFT_PAREN columnDef (COMMA columnDef) RIGHT_PAREN + | ADD COLUMN LEFT_PAREN columnDef (COMMA columnDef)* RIGHT_PAREN toRollup? properties=propertyClause? #addColumnsClause | DROP COLUMN name=identifier fromRollup? properties=propertyClause? #dropColumnClause | MODIFY COLUMN columnDef columnPosition? fromRollup? diff --git a/regression-test/suites/nereids_syntax_p0/ddl/add_column.groovy b/regression-test/suites/nereids_syntax_p0/ddl/add_column.groovy new file mode 100644 index 000000000000000..f38b4883fc6a9ea --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/ddl/add_column.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("add_column") { + def tableName ="test" + + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + + sql """ + drop table if exists test + """ + + sql """ + CREATE TABLE IF NOT EXISTS `test` ( + `id` bigint(20) NOT NULL, + `count` bigint(20) NOT NULL, + ) ENGINE=OLAP + UNIQUE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 5 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + """ + + sql """ + ALTER TABLE `test` + ADD COLUMN (`cost` VARCHAR(256) DEFAULT "add"); + """ +} + From 781a9942205271f51775334595a99d894c3416f6 Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Wed, 28 Aug 2024 21:40:02 +0800 Subject: [PATCH 05/34] [chore](build) Skip cloud module for arm64 (aarch64) in build-for-release.sh (#39804) --- build-for-release.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build-for-release.sh b/build-for-release.sh index 4d9a257bd2eb67e..2347f9ebaab5e11 100755 --- a/build-for-release.sh +++ b/build-for-release.sh @@ -164,7 +164,11 @@ cp -R "${ORI_OUTPUT}"/apache_hdfs_broker "${OUTPUT_EXT}"/apache_hdfs_broker cp -R "${ORI_OUTPUT}"/be/* "${OUTPUT_BE}"/ # CLOUD -cp -R "${ORI_OUTPUT}"/ms/* "${OUTPUT_CLOUD}"/ +if [[ "${ARCH}" == "arm64" ]]; then + echo "WARNING: Cloud module is not supported on ARM platform, will skip building it." +else + cp -R "${ORI_OUTPUT}"/ms/* "${OUTPUT_CLOUD}"/ +fi if [[ "${TAR}" -eq 1 ]]; then echo "Begin to compress" From ddc6b81c840ef94f6123cce18b81e901354a661e Mon Sep 17 00:00:00 2001 From: qiye Date: Wed, 28 Aug 2024 21:57:30 +0800 Subject: [PATCH 06/34] [improve](ES Catalog)Only push down literal expr in binary predicate (#39845) ## Proposed changes 1. Add support for reverse binary predicate, such as ` 3 >= k3`. 2. Only push down literal expr in binary predicate This only affects old planner, new planner dose not have any bug. We add check here to make the push down to ES more robust. --- .../doris/datasource/es/QueryBuilders.java | 77 ++++++++++++++----- .../elasticsearch/QueryBuildersTest.java | 27 +++++++ 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java index 3a54e012a32733d..19930bb2b14a880 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java @@ -31,6 +31,7 @@ import org.apache.doris.analysis.LargeIntLiteral; import org.apache.doris.analysis.LikePredicate; import org.apache.doris.analysis.LikePredicate.Operator; +import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.SlotRef; import org.apache.doris.catalog.EsResource; import org.apache.doris.thrift.TExprOpcode; @@ -127,9 +128,24 @@ public static QueryBuilder toEsDsl(Expr expr) { .build()); } - private static QueryBuilder parseBinaryPredicate(Expr expr, TExprOpcode opCode, String column, + private static TExprOpcode flipOpCode(TExprOpcode opCode) { + switch (opCode) { + case GE: + return TExprOpcode.LE; + case GT: + return TExprOpcode.LT; + case LE: + return TExprOpcode.GE; + case LT: + return TExprOpcode.GT; + default: + return opCode; + } + } + + private static QueryBuilder parseBinaryPredicate(LiteralExpr expr, TExprOpcode opCode, String column, boolean needDateCompat) { - Object value = toDorisLiteral(expr.getChild(1)); + Object value = toDorisLiteral(expr); if (needDateCompat) { value = compatDefaultDate(value); } @@ -223,6 +239,20 @@ private static QueryBuilder parseFunctionCallExpr(Expr expr) { return new QueryBuilders.EsQueryBuilder(stringValue); } + private static String getColumnFromExpr(Expr expr) { + // Type transformed cast can not pushdown + if (expr instanceof CastExpr) { + Expr withoutCastExpr = exprWithoutCast(expr); + if (withoutCastExpr.getType().equals(expr.getType()) + || (withoutCastExpr.getType().isFloatingPointType() && expr.getType().isFloatingPointType())) { + return ((SlotRef) withoutCastExpr).getColumnName(); + } + } else if (expr instanceof SlotRef) { + return ((SlotRef) expr).getColumnName(); + } + return null; + } + /** * Doris expr to es dsl. **/ @@ -241,32 +271,43 @@ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map= 3 - if (withoutCastExpr.getType().equals(leftExpr.getType()) || (withoutCastExpr.getType().isFloatingPointType() - && leftExpr.getType().isFloatingPointType())) { - column = ((SlotRef) withoutCastExpr).getColumnName(); - } else { - notPushDownList.add(expr); - return null; - } - } else if (leftExpr instanceof SlotRef) { - column = ((SlotRef) leftExpr).getColumnName(); - } else { + String column = getColumnFromExpr(leftExpr); + + if (StringUtils.isEmpty(column)) { + Expr rightExpr = expr.getChild(1); + column = getColumnFromExpr(rightExpr); + opCode = flipOpCode(opCode); + isFlip = true; + } + + if (StringUtils.isEmpty(column)) { notPushDownList.add(expr); return null; } + // Check whether the date type need compat, it must before keyword replace. List needCompatDateFields = builderOptions.getNeedCompatDateFields(); boolean needDateCompat = needCompatDateFields != null && needCompatDateFields.contains(column); // Replace col with col.keyword if mapping exist. column = fieldsContext.getOrDefault(column, column); if (expr instanceof BinaryPredicate) { - return parseBinaryPredicate(expr, opCode, column, needDateCompat); + BinaryPredicate binaryPredicate = (BinaryPredicate) expr; + Expr value; + if (isFlip) { + value = binaryPredicate.getChild(0); + } else { + value = binaryPredicate.getChild(1); + } + // only push down literal expr to ES + if (value instanceof LiteralExpr) { + LiteralExpr literalExpr = (LiteralExpr) value; + return parseBinaryPredicate(literalExpr, opCode, column, needDateCompat); + } else { + notPushDownList.add(expr); + return null; + } } if (expr instanceof IsNullPredicate) { return parseIsNullPredicate(expr, column); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java index 3cf9261b9325add..ca5344990e7ec75 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java @@ -209,6 +209,33 @@ public void testCastConvertEsDsl() { new FloatLiteral(3.0, Type.DOUBLE)); QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions); Assertions.assertEquals(3, notPushDownList.size()); + + SlotRef k4 = new SlotRef(null, "k4"); + k4.setType(Type.FLOAT); + CastExpr castFloatExpr = new CastExpr(Type.FLOAT, k4); + BinaryPredicate castFloatPredicate = new BinaryPredicate(Operator.GE, new FloatLiteral(3.0, Type.FLOAT), + castFloatExpr); + QueryBuilders.QueryBuilder queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + Assertions.assertEquals("{\"range\":{\"k4\":{\"lte\":3.0}}}", queryBuilder.toJson()); + Assertions.assertEquals(3, notPushDownList.size()); + + castFloatPredicate = new BinaryPredicate(Operator.LE, new FloatLiteral(3.0, Type.FLOAT), + castFloatExpr); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + Assertions.assertEquals("{\"range\":{\"k4\":{\"gte\":3.0}}}", queryBuilder.toJson()); + Assertions.assertEquals(3, notPushDownList.size()); + + castFloatPredicate = new BinaryPredicate(Operator.LT, new FloatLiteral(3.0, Type.FLOAT), + castFloatExpr); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + Assertions.assertEquals("{\"range\":{\"k4\":{\"gt\":3.0}}}", queryBuilder.toJson()); + Assertions.assertEquals(3, notPushDownList.size()); + + castFloatPredicate = new BinaryPredicate(Operator.GT, new FloatLiteral(3.0, Type.FLOAT), + castFloatExpr); + queryBuilder = QueryBuilders.toEsDsl(castFloatPredicate, notPushDownList, fieldsContext, builderOptions); + Assertions.assertEquals("{\"range\":{\"k4\":{\"lt\":3.0}}}", queryBuilder.toJson()); + Assertions.assertEquals(3, notPushDownList.size()); } From d09a39b0530855624f6f59ae116b57b3009094e9 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 28 Aug 2024 22:05:11 +0800 Subject: [PATCH 07/34] [fix](memory) Fix OwnedSlice free memory (#40043) ``` I20240828 11:41:58.529152 20190 mem_tracker_limiter.cpp:193] [Address Sanitizer] memory buf not exist, mem tracker label: Load#Id=ac42f13dd1430d2c-bd60f5a4829d0792, consumption: 13054587, peak consumption: 13054587, buf: 0, size: 0, stack_trace: 0# doris::OwnedSlice::~OwnedSlice() 1# doris::segment_v2::ScalarColumnWriter::finish_current_page() 2# doris::segment_v2::ScalarColumnWriter::finish() 3# doris::segment_v2::VerticalSegmentWriter::write_batch() 4# doris::SegmentFlusher::_add_rows(std::unique_ptr >&, doris::vectorized::Block const*, unsigned long, unsigned long) 5# doris::SegmentFlusher::flush_single_block(doris::vectorized::Block const*, int, long*) 6# doris::SegmentCreator::flush_single_block(doris::vectorized::Block const*, int, long*) 7# doris::BaseBetaRowsetWriter::flush_memtable(doris::vectorized::Block*, int, long*) 8# doris::FlushToken::_do_flush_memtable(doris::MemTable*, int, long*) 9# doris::FlushToken::_flush_memtable(std::unique_ptr >, int, long) 10# doris::MemtableFlushTask::run() 11# doris::ThreadPool::dispatch_thread() 12# doris::Thread::supervise_thread(void*) 13# ? 14# clone ``` ``` I20240828 11:41:58.613629 20183 mem_tracker_limiter.cpp:182] [Address Sanitizer] free memory buf size inaccurate, mem tracker label: Load#Id=433657e8b3834e94-ac178bb7ab8ff661, consumption: 3239536, peak consumption: 6385184, buf: 0x6030015390a0, size: 32, old buf: 0x6030015390a0, old size: 20, new stack_trace: 0# doris::OwnedSlice::~OwnedSlice() 1# doris::segment_v2::IndexedColumnWriter::_finish_current_data_page(unsigned long&) 2# doris::segment_v2::IndexedColumnWriter::finish(doris::segment_v2::IndexedColumnMetaPB*) 3# doris::PrimaryKeyIndexBuilder::finalize(doris::segment_v2::PrimaryKeyIndexMetaPB*) 4# doris::segment_v2::VerticalSegmentWriter::_write_primary_key_index() 5# doris::segment_v2::VerticalSegmentWriter::finalize_columns_index(unsigned long*) 6# doris::segment_v2::VerticalSegmentWriter::finalize(unsigned long*, unsigned long*) 7# doris::SegmentFlusher::_flush_segment_writer(std::unique_ptr >&, std::shared_ptr, long*) 8# doris::SegmentFlusher::flush_single_block(doris::vectorized::Block const*, int, long*) 9# doris::SegmentCreator::flush_single_block(doris::vectorized::Block const*, int, long*) 10# doris::BetaRowsetWriterV2::flush_memtable(doris::vectorized::Block*, int, long*) 11# doris::FlushToken::_do_flush_memtable(doris::MemTable*, int, long*) 12# doris::FlushToken::_flush_memtable(std::unique_ptr >, int, long) 13# doris::MemtableFlushTask::run() 14# doris::ThreadPool::dispatch_thread() 15# doris::Thread::supervise_thread(void*) 16# ? 17# clone , old stack_trace: 0# Allocator::alloc_impl(unsigned long, unsigned long) 1# doris::faststring::build() 2# doris::segment_v2::BinaryPrefixPageBuilder::finish(doris::OwnedSlice*) 3# doris::segment_v2::IndexedColumnWriter::_finish_current_data_page(unsigned long&) 4# doris::segment_v2::IndexedColumnWriter::finish(doris::segment_v2::IndexedColumnMetaPB*) 5# doris::PrimaryKeyIndexBuilder::finalize(doris::segment_v2::PrimaryKeyIndexMetaPB*) 6# doris::segment_v2::VerticalSegmentWriter::_write_primary_key_index() 7# doris::segment_v2::VerticalSegmentWriter::finalize_columns_index(unsigned long*) 8# doris::segment_v2::VerticalSegmentWriter::finalize(unsigned long*, unsigned long*) 9# doris::SegmentFlusher::_flush_segment_writer(std::unique_ptr >&, std::shared_ptr, long*) 10# doris::SegmentFlusher::flush_single_block(doris::vectorized::Block const*, int, long*) 11# doris::SegmentCreator::flush_single_block(doris::vectorized::Block const*, int, long*) 12# doris::BetaRowsetWriterV2::flush_memtable(doris::vectorized::Block*, int, long*) 13# doris::FlushToken::_do_flush_memtable(doris::MemTable*, int, long*) 14# doris::FlushToken::_flush_memtable(std::unique_ptr >, int, long) 15# doris::MemtableFlushTask::run() 16# doris::ThreadPool::dispatch_thread() 17# doris::Thread::supervise_thread(void*) 18# ? 19# clon ``` --- be/src/util/faststring.h | 3 ++- be/src/util/slice.h | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/be/src/util/faststring.h b/be/src/util/faststring.h index 3ec0acbda01d79e..f82bcfdb0d9585e 100644 --- a/be/src/util/faststring.h +++ b/be/src/util/faststring.h @@ -85,7 +85,8 @@ class faststring : private Allocator(Allocator::alloc(len_)); + ret = reinterpret_cast(Allocator::alloc(capacity_)); + DCHECK(len_ <= capacity_); memcpy(ret, data_, len_); } OwnedSlice result(ret, len_, capacity_); diff --git a/be/src/util/slice.h b/be/src/util/slice.h index 1c8579ffed45d59..b38b1147894f9e5 100644 --- a/be/src/util/slice.h +++ b/be/src/util/slice.h @@ -362,7 +362,12 @@ class OwnedSlice : private Allocator Date: Wed, 28 Aug 2024 22:11:13 +0800 Subject: [PATCH 08/34] [improve](statistics)Clean expired TableStatsMeta. (#39779) Drop catalog and older version of drop database (before https://github.com/apache/doris/pull/39685), will not remove table stats object in memory. Need a background thread to clean the garbage. --- fe/fe-core/src/main/cup/sql_parser.cup | 11 ++- .../doris/analysis/ShowTableStatsStmt.java | 40 +++++++-- .../org/apache/doris/qe/ShowExecutor.java | 11 +++ .../doris/statistics/AnalysisManager.java | 4 + .../doris/statistics/StatisticsCleaner.java | 56 ++++++++++++ .../doris/statistics/TableStatsMeta.java | 25 ++++++ .../hive/test_drop_expired_table_stats.groovy | 90 +++++++++++++++++++ 7 files changed, 227 insertions(+), 10 deletions(-) create mode 100644 regression-test/suites/external_table_p0/hive/test_drop_expired_table_stats.groovy diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 37907e44915b485..921fdd590ed6070 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -4604,14 +4604,19 @@ show_param ::= RESULT = new ShowSyncJobStmt(dbName); :} /* show table stats */ - | KW_TABLE opt_cached:cached KW_STATS table_name:tbl opt_partition_names:partitionNames opt_col_list:cols + | KW_TABLE KW_STATS table_name:tbl opt_partition_names:partitionNames opt_col_list:cols {: - RESULT = new ShowTableStatsStmt(tbl, cols, partitionNames, cached, null); + RESULT = new ShowTableStatsStmt(tbl, cols, partitionNames, null); + :} + /* show table id stats */ + | KW_TABLE KW_STATS INTEGER_LITERAL:tableId + {: + RESULT = new ShowTableStatsStmt(tableId); :} /* show index stats */ | KW_INDEX KW_STATS table_name:tbl ident:id {: - RESULT = new ShowTableStatsStmt(tbl, null, null, false, id); + RESULT = new ShowTableStatsStmt(tbl, null, null, id); :} /* show column stats */ | KW_COLUMN opt_cached:cached KW_STATS table_name:tbl opt_col_list:cols opt_partition_names:partitionNames diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java index 968574b750c5e11..6070d76103b7400 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java @@ -89,18 +89,29 @@ public class ShowTableStatsStmt extends ShowStmt { private final TableName tableName; private final List columnNames; private final PartitionNames partitionNames; - private final boolean cached; private final String indexName; + private final long tableId; + private final boolean useTableId; private TableIf table; + public ShowTableStatsStmt(long tableId) { + this.tableName = null; + this.columnNames = null; + this.partitionNames = null; + this.indexName = null; + this.tableId = tableId; + this.useTableId = true; + } + public ShowTableStatsStmt(TableName tableName, List columnNames, - PartitionNames partitionNames, boolean cached, String indexName) { + PartitionNames partitionNames, String indexName) { this.tableName = tableName; this.columnNames = columnNames; this.partitionNames = partitionNames; - this.cached = cached; this.indexName = indexName; + this.tableId = -1; + this.useTableId = false; } public TableName getTableName() { @@ -110,6 +121,13 @@ public TableName getTableName() { @Override public void analyze(Analyzer analyzer) throws UserException { super.analyze(analyzer); + if (useTableId) { + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "Permission denied", + ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP()); + } + return; + } tableName.analyze(analyzer); if (partitionNames != null) { partitionNames.analyze(analyzer); @@ -171,6 +189,14 @@ public TableIf getTable() { return table; } + public boolean isUseTableId() { + return useTableId; + } + + public long getTableId() { + return tableId; + } + public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) { if (indexName != null) { return constructIndexResultSet(tableStatistic); @@ -185,6 +211,10 @@ public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) { } } + public ShowResultSet constructEmptyResultSet() { + return new ShowResultSet(getMetaData(), new ArrayList<>()); + } + public ShowResultSet constructResultSet(long rowCount) { List> result = Lists.newArrayList(); if (partitionNames != null) { @@ -313,8 +343,4 @@ public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta tableStati } return new ShowResultSet(getMetaData(), result); } - - public boolean isCached() { - return cached; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index f5dd6e441a950ae..9c807a4b1bf9fed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2712,6 +2712,17 @@ private void handleShowDataSkew() throws AnalysisException { private void handleShowTableStats() { ShowTableStatsStmt showTableStatsStmt = (ShowTableStatsStmt) stmt; TableIf tableIf = showTableStatsStmt.getTable(); + // Handle use table id to show table stats. Mainly for online debug. + if (showTableStatsStmt.isUseTableId()) { + long tableId = showTableStatsStmt.getTableId(); + TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableId); + if (tableStats == null) { + resultSet = showTableStatsStmt.constructEmptyResultSet(); + } else { + resultSet = showTableStatsStmt.constructResultSet(tableStats); + } + return; + } TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId()); /* tableStats == null means it's not analyzed, in this case show the estimated row count. diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index a9fcd7b6f67ad95..55b0bdc4efd3c7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -1381,6 +1381,10 @@ public void removeTableStats(long tableId) { } } + public Set getIdToTblStatsKeys() { + return new HashSet<>(idToTblStats.keySet()); + } + public ColStatsMeta findColStatsMeta(long tblId, String indexName, String colName) { TableStatsMeta tableStats = findTableStatsStatus(tblId); if (tableStats == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java index 50c5accb0ba2af3..9775b6ecb73f790 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java @@ -17,6 +17,7 @@ package org.apache.doris.statistics; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MaterializedIndexMeta; @@ -27,6 +28,7 @@ import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.persist.TableStatsDeletionLog; import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.collect.Maps; @@ -74,6 +76,7 @@ protected void runAfterCatalogReady() { } public synchronized void clear() { + clearTableStats(); try { if (!init()) { return; @@ -99,6 +102,59 @@ private void clearStats(OlapTable statsTbl, boolean isTableColumnStats) { } while (!expiredStats.isEmpty()); } + private void clearTableStats() { + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + Set tableIds = analysisManager.getIdToTblStatsKeys(); + InternalCatalog internalCatalog = Env.getCurrentInternalCatalog(); + for (long id : tableIds) { + try { + TableStatsMeta stats = analysisManager.findTableStatsStatus(id); + if (stats == null) { + continue; + } + // If ctlName, dbName and tblName exist, it means the table stats is created under new version. + // First try to find the table by the given names. If table exists, means the tableMeta is valid, + // it should be kept in memory. + try { + StatisticsUtil.findTable(stats.ctlName, stats.dbName, stats.tblName); + continue; + } catch (Exception e) { + LOG.debug("Table {}.{}.{} not found.", stats.ctlName, stats.dbName, stats.tblName); + } + // If we couldn't find table by names, try to find it in internal catalog. This is to support older + // version which the tableStats object doesn't store the names but only table id. + // We may remove external table's tableStats here, but it's not a big problem. + // Because the stats in column_statistics table is still available, + // the only disadvantage is auto analyze may be triggered for this table. + // But it only happens once, the new table stats object will have all the catalog, db and table names. + if (tableExistInInternalCatalog(internalCatalog, id)) { + continue; + } + LOG.info("Table {}.{}.{} with id {} not exist, remove its table stats record.", + stats.ctlName, stats.dbName, stats.tblName, id); + analysisManager.removeTableStats(id); + Env.getCurrentEnv().getEditLog().logDeleteTableStats(new TableStatsDeletionLog(id)); + } catch (Exception e) { + LOG.info(e); + } + } + } + + private boolean tableExistInInternalCatalog(InternalCatalog internalCatalog, long tableId) { + List dbIds = internalCatalog.getDbIds(); + for (long dbId : dbIds) { + Database database = internalCatalog.getDbNullable(dbId); + if (database == null) { + continue; + } + TableIf table = database.getTableNullable(tableId); + if (table != null) { + return true; + } + } + return false; + } + private boolean init() { try { String dbName = FeConstants.INTERNAL_DB_NAME; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java index 97833041922a66b..2d566d71cfcbc1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java @@ -47,9 +47,24 @@ public class TableStatsMeta implements Writable, GsonPostProcessable { + @SerializedName("ctlId") + public final long ctlId; + + @SerializedName("ctln") + public final String ctlName; + + @SerializedName("dbId") + public final long dbId; + + @SerializedName("dbn") + public final String dbName; + @SerializedName("tblId") public final long tblId; + @SerializedName("tbln") + public final String tblName; + @SerializedName("idxId") public final long idxId; @SerializedName("updatedRows") @@ -93,14 +108,24 @@ public class TableStatsMeta implements Writable, GsonPostProcessable { @VisibleForTesting public TableStatsMeta() { + ctlId = 0; + ctlName = null; + dbId = 0; + dbName = null; tblId = 0; + tblName = null; idxId = 0; } // It's necessary to store these fields separately from AnalysisInfo, since the lifecycle between AnalysisInfo // and TableStats is quite different. public TableStatsMeta(long rowCount, AnalysisInfo analyzedJob, TableIf table) { + this.ctlId = table.getDatabase().getCatalog().getId(); + this.ctlName = table.getDatabase().getCatalog().getName(); + this.dbId = table.getDatabase().getId(); + this.dbName = table.getDatabase().getFullName(); this.tblId = table.getId(); + this.tblName = table.getName(); this.idxId = -1; this.rowCount = rowCount; update(analyzedJob, table); diff --git a/regression-test/suites/external_table_p0/hive/test_drop_expired_table_stats.groovy b/regression-test/suites/external_table_p0/hive/test_drop_expired_table_stats.groovy new file mode 100644 index 000000000000000..dbbe014e28bb6b1 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_drop_expired_table_stats.groovy @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_drop_expired_table_stats", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return + } + + for (String hivePrefix : ["hive2", "hive3"]) { + String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp") + String extHiveHmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = hivePrefix + "_test_drop_expired_table_stats" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + + + sql """use stats_test""" + sql """analyze table employee_gz with sync""" + def result = sql """show table stats employee_gz""" + assertEquals(1, result.size()) + + def ctlId + def dbId + def tblId + result = sql """show catalogs""" + + for (int i = 0; i < result.size(); i++) { + if (result[i][1] == catalog_name) { + ctlId = result[i][0] + } + } + logger.info("catalog id is " + ctlId) + result = sql """show proc '/catalogs/$ctlId'""" + for (int i = 0; i < result.size(); i++) { + if (result[i][1] == 'stats_test') { + dbId = result[i][0] + } + } + logger.info("db id is " + dbId) + result = sql """show proc '/catalogs/$ctlId/$dbId'""" + for (int i = 0; i < result.size(); i++) { + if (result[i][1] == 'employee_gz') { + tblId = result[i][0] + } + } + logger.info("table id is " + tblId) + result = sql """show table stats $tblId""" + logger.info("Table stats " + result) + assertEquals(1, result.size()) + + sql """drop catalog ${catalog_name}""" + result = sql """show table stats $tblId""" + logger.info("Table stats " + result) + assertEquals(1, result.size()) + + try { + sql """drop expired stats""" + } catch (Exception e) { + logger.info("Drop expired stats exception. " + e.getMessage()) + } + result = sql """show table stats $tblId""" + logger.info("Table stats " + result) + assertEquals(0, result.size()) + } +} + From 9327f2ca8195dd475d8b8a976c9666ef2e9cc474 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 28 Aug 2024 22:59:43 +0800 Subject: [PATCH 09/34] [chore](backup) limit the involved tablets in a backup job (#39987) and to avoid FE OOM caused by saving too much metadata. Assuming the average tablet size is 50MB, the default value of 300000 can support 14TB of data per backup job. The docs PR: https://github.com/apache/doris-website/pull/1056 --- .../src/main/java/org/apache/doris/common/Config.java | 9 +++++++++ .../main/java/org/apache/doris/backup/BackupJob.java | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 69d382c7a5c2e74..355683230d51edb 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1532,6 +1532,15 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int max_backup_restore_job_num_per_db = 10; + /** + * Control the max num of tablets per backup job involved. + */ + @ConfField(mutable = true, masterOnly = true, description = { + "用于控制每次 backup job 允许备份的 tablet 上限，以避免 OOM", + "Control the max num of tablets per backup job involved, to avoid OOM" + }) + public static int max_backup_tablets_per_job = 300000; + /** * whether to ignore table that not support type when backup, and not report exception. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index d7d550e5208f2c7..353bab7429f97b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -516,6 +516,17 @@ private void prepareAndSendSnapshotTask() { } } + // Limit the max num of tablets involved in a backup job, to avoid OOM. + if (unfinishedTaskIds.size() > Config.max_backup_tablets_per_job) { + String msg = String.format("the num involved tablets %d exceeds the limit %d, " + + "which might cause the FE OOM, change config `max_backup_tablets_per_job` " + + "to change this limitation", + unfinishedTaskIds.size(), Config.max_backup_tablets_per_job); + LOG.warn(msg); + status = new Status(ErrCode.COMMON_ERROR, msg); + return; + } + backupMeta = new BackupMeta(copiedTables, copiedResources); // send tasks From d5c7b76e84c68407d4e78f5e6ecbd4ad86771130 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Wed, 28 Aug 2024 23:01:12 +0800 Subject: [PATCH 10/34] [chore](ci) rm disable_auto_compaction=true (#40068) Co-authored-by: stephen --- regression-test/pipeline/external/conf/be.conf | 2 +- regression-test/pipeline/p0/conf/be.conf | 2 +- regression-test/pipeline/p1/conf/be.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/regression-test/pipeline/external/conf/be.conf b/regression-test/pipeline/external/conf/be.conf index 1e3cf1b6b862e56..a7edbd7b55a59e6 100644 --- a/regression-test/pipeline/external/conf/be.conf +++ b/regression-test/pipeline/external/conf/be.conf @@ -48,7 +48,7 @@ max_garbage_sweep_interval=180 log_buffer_level = -1 enable_stream_load_record = true storage_root_path=/mnt/ssd01/cluster_storage/doris.SSD/P0/cluster1;/mnt/ssd01/cluster_storage/doris.SSD -disable_auto_compaction=true + priority_networks=172.19.0.0/24 enable_fuzzy_mode=true max_depth_of_expr_tree=200 diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf index 9eae04c663ada86..e4745ccb5a332fc 100644 --- a/regression-test/pipeline/p0/conf/be.conf +++ b/regression-test/pipeline/p0/conf/be.conf @@ -50,7 +50,7 @@ log_buffer_level = -1 enable_stream_load_record = true stream_load_record_batch_size = 500 storage_root_path=/mnt/ssd01/cluster_storage/doris.SSD/P0/cluster1;/mnt/ssd01/cluster_storage/doris.SSD -disable_auto_compaction=true + priority_networks=172.19.0.0/24 enable_fuzzy_mode=true max_depth_of_expr_tree=200 diff --git a/regression-test/pipeline/p1/conf/be.conf b/regression-test/pipeline/p1/conf/be.conf index ce0a61bef73e89c..1c0fd53d4958ca6 100644 --- a/regression-test/pipeline/p1/conf/be.conf +++ b/regression-test/pipeline/p1/conf/be.conf @@ -48,7 +48,7 @@ max_garbage_sweep_interval=180 log_buffer_level = -1 enable_stream_load_record = true storage_root_path=/mnt/ssd01/cluster_storage/doris.SSD/P0/cluster1 -disable_auto_compaction=true + priority_networks=172.19.0.0/24 enable_fuzzy_mode=true max_depth_of_expr_tree=200 From 8f533e9ee4329d26f00bf04c66cd6e5e2d5e209a Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 28 Aug 2024 23:04:17 +0800 Subject: [PATCH 11/34] [fix](doris compose) fix docker py exception (#39976) ``` Exception: java.lang.Exception: Exit value: 1 != 0, stdout: { "code": 1, "err": "Traceback (most recent call last):\n File \"/home/ubuntu/regression-test/doris_master/NEREIDS_ASAN/p0/docker/runtime/doris-compose/command.py\", line 336, in run\n cluster = CLUSTER.Cluster.load(args.NAME)\n File \"/home/ubuntu/regression-test/doris_master/NEREIDS_ASAN/p0/docker/runtime/doris-compose/cluster.py\", line 656, in load\n raise Exception(\nException: Failed to load cluster, its directory /tmp/doris/test_coordidator_be_restart not exists.\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.10/dist-packages/requests/adapters.py\", line 633, in send\n conn = self.get_connection_with_tls_context(\n File \"/usr/local/lib/python3.10/dist-packages/requests/adapters.py\", line 489, in get_connection_with_tls_context\n conn = self.poolmanager.connection_from_host(\n File \"/usr/lib/python3/dist-packages/urllib3/poolmanager.py\", line 245, in connection_from_host\n return self.connection_from_context(request_context)\n File \"/usr/lib/python3/dist-packages/urllib3/poolmanager.py\", line 257, in connection_from_context\n raise URLSchemeUnknown(scheme)\nurllib3.exceptions.URLSchemeUnknown: Not supported URL scheme http+docker\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/usr/lib/python3/dist-packages/docker/api/client.py\", line 214, in _retrieve_server_version\n return self.version(api_version=False)[\"ApiVersion\"]\n File \"/usr/lib/python3/dist-packages/docker/api/daemon.py\", line 181, in version\n return self._result(self._get(url), json=True)\n File \"/usr/lib/python3/dist-packages/docker/utils/decorators.py\", line 46, in inner\n return f(self, *args, **kwargs)\n File \"/usr/lib/python3/dist-packages/docker/api/client.py\", line 237, in _get\n return self.get(url, **self._set_request_timeout(kwargs))\n File \"/usr/local/lib/python3.10/dist-packages/requests/sessions.py\", line 602, in get\n return self.request(\"GET\", url, **kwargs)\n File \"/usr/local/lib/python3.10/dist-packages/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n File \"/usr/local/lib/python3.10/dist-packages/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n File \"/usr/local/lib/python3.10/dist-packages/requests/adapters.py\", line 637, in send\n raise InvalidURL(e, request=request)\nrequests.exceptions.InvalidURL: Not supported URL scheme http+docker\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/home/ubuntu/regression-test/doris_master/NEREIDS_ASAN/p0/docker/runtime/doris-compose/doris-compose.py\", line 54, in \n data = run(args, disable_log, help)\n File \"/home/ubuntu/regression-test/doris_master/NEREIDS_ASAN/p0/docker/runtime/doris-compose/doris-compose.py\", line 38, in run\n result = cmd.run(args)\n File \"/home/ubuntu/regression-test/doris_master/NEREIDS_ASAN/p0/docker/runtime/doris-compose/command.py\", line 386, in run\n cluster = CLUSTER.Cluster.new(args.NAME, args.IMAGE, args.cloud,\n File \"/home/ubuntu/regression-test/doris_master/NEREIDS_ASAN/p0/docker/runtime/doris-compose/cluster.py\", line 640, in new\n subnet = gen_subnet_prefix16()\n File \"/home/ubuntu/regression-test/doris_master/NEREIDS_ASAN/p0/docker/runtime/doris-compose/cluster.py\", line 87, in gen_subnet_prefix16\n used_subnet = utils.get_docker_subnets_prefix16()\n File \"/home/ubuntu/regression-test/doris_master/NEREIDS_ASAN/p0/docker/runtime/doris-compose/utils.py\", line 226, in get_docker_subnets_prefix16\n client = docker.from_env()\n File \"/usr/lib/python3/dist-packages/docker/client.py\", line 96, in from_env\n return cls(\n File \"/usr/lib/python3/dist-packages/docker/client.py\", line 45, in __init__\n self.api = APIClient(*args, **kwargs)\n File \"/usr/lib/python3/dist-packages/docker/api/client.py\", line 197, in __init__\n self._version = self._retrieve_server_version()\n File \"/usr/lib/python3/dist-packages/docker/api/client.py\", line 221, in _retrieve_server_version\n raise DockerException(\ndocker.errors.DockerException: Error while fetching server API version: Not supported URL scheme http+docker\n" } , stderr: /usr/local/lib/python3.10/dist-packages/paramiko/pkey.py:100: CryptographyDeprecationWarning: TripleDES has been moved to cryptography.hazmat.decrepit.ciphers.algorithms.TripleDES and will be removed from this module in 48.0.0. "cipher": algorithms.TripleDES, /usr/local/lib/python3.10/dist-packages/paramiko/transport.py:258: CryptographyDeprecationWarning: TripleDES has been moved to cryptography.hazmat.decrepit.ciphers.algorithms.TripleDES and will be removed from this module in 48.0.0. "class": algorithms.TripleDES, at org.apache.doris.regression.suite.SuiteCluster.runCmd(SuiteCluster.groovy:603) ``` --- docker/runtime/doris-compose/Dockerfile | 17 +++-------------- docker/runtime/doris-compose/cluster.py | 16 +++++++++++++++- docker/runtime/doris-compose/requirements.txt | 1 + .../doris/regression/RegressionTest.groovy | 3 +++ .../doris/regression/suite/SuiteCluster.groovy | 2 +- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/docker/runtime/doris-compose/Dockerfile b/docker/runtime/doris-compose/Dockerfile index 3934ee5cb756d33..2aabe196205c238 100644 --- a/docker/runtime/doris-compose/Dockerfile +++ b/docker/runtime/doris-compose/Dockerfile @@ -29,16 +29,7 @@ ARG JDK_IMAGE=openjdk:17-jdk-slim FROM ${JDK_IMAGE} -RUN </echo/g' /opt/apache-doris/cloud/bin/start.sh +RUN mkdir /opt/apache-doris/fdb +RUN if [ -d /opt/apache-doris/cloud/bin ]; then \ + sed -i 's/\/echo/g' /opt/apache-doris/cloud/bin/start.sh ; \ fi -EOF # fe and be COPY output /opt/apache-doris/ diff --git a/docker/runtime/doris-compose/cluster.py b/docker/runtime/doris-compose/cluster.py index 9194716409650dd..f2e46b798cb4840 100644 --- a/docker/runtime/doris-compose/cluster.py +++ b/docker/runtime/doris-compose/cluster.py @@ -342,10 +342,16 @@ def compose(self): for path in ("/etc/localtime", "/etc/timezone", "/usr/share/zoneinfo") if os.path.exists(path) ] + if self.cluster.coverage_dir: volumes.append("{}:{}/coverage".format(self.cluster.coverage_dir, DOCKER_DORIS_PATH)) + extra_hosts = [ + "{}:{}".format(node.get_name(), node.get_ip()) + for node in self.cluster.get_all_nodes() + ] + content = { "cap_add": ["SYS_PTRACE"], "hostname": self.get_name(), @@ -357,6 +363,7 @@ def compose(self): "ipv4_address": self.get_ip(), } }, + "extra_hosts": extra_hosts, "ports": self.docker_ports(), "ulimits": { "core": -1 @@ -699,7 +706,14 @@ def get_node(self, node_type, id): raise Exception("No found {} with id {}".format(node_type, id)) return Node.new(self, node_type, id, meta) - def get_all_nodes(self, node_type): + def get_all_nodes(self, node_type=None): + if node_type is None: + nodes = [] + for nt, group in self.groups.items(): + for id, meta in group.get_all_nodes().items(): + nodes.append(Node.new(self, nt, id, meta)) + return nodes + group = self.groups.get(node_type, None) if not group: raise Exception("Unknown node_type: {}".format(node_type)) diff --git a/docker/runtime/doris-compose/requirements.txt b/docker/runtime/doris-compose/requirements.txt index ac177eddf82ad02..05258de2df607cc 100644 --- a/docker/runtime/doris-compose/requirements.txt +++ b/docker/runtime/doris-compose/requirements.txt @@ -22,3 +22,4 @@ jsonpickle prettytable pymysql python-dateutil +requests<=2.31.0 diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy index 332772271c67f94..92e92a9b736c32f 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy @@ -453,6 +453,9 @@ class RegressionTest { log.warn("install doris compose requirements failed: code=${proc.exitValue()}, " + "output: ${sout.toString()}, error: ${serr.toString()}") } + + def pipList = 'pip list'.execute().text + log.info("python library: ${pipList}") } static void printPassed() { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy index 9789b751fed51b0..6a322039985cb61 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy @@ -587,7 +587,7 @@ class SuiteCluster { } private Object runCmd(String cmd, int timeoutSecond = 60) throws Exception { - def fullCmd = String.format('python %s %s --output-json', config.dorisComposePath, cmd) + def fullCmd = String.format('python -W ignore %s %s --output-json', config.dorisComposePath, cmd) logger.info('Run doris compose cmd: {}', fullCmd) def proc = fullCmd.execute() def outBuf = new StringBuilder() From 3770e421a833795de7b8eff1c87d4fd338f0f785 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Wed, 28 Aug 2024 23:10:44 +0800 Subject: [PATCH 12/34] [Fix](group commit) Fix cloud group commit be select strategy (#39986) ## Proposed changes In #35558, we optimized be select for group commit. However, we forgot to apply this strategy to cloud. This PR applys it. --- .../apache/doris/httpv2/rest/LoadAction.java | 140 +++++++++++------- .../apache/doris/load/StreamLoadHandler.java | 8 +- .../doris/load/loadv2/MysqlLoadManager.java | 2 +- 3 files changed, 92 insertions(+), 58 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java index c2d50460ea49548..b0a714da149564f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -85,7 +85,7 @@ public class LoadAction extends RestBaseController { @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", method = RequestMethod.PUT) public Object load(HttpServletRequest request, HttpServletResponse response, - @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) { + @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) { if (needRedirect(request.getScheme())) { return redirectToHttps(request); } @@ -102,21 +102,29 @@ public Object load(HttpServletRequest request, HttpServletResponse response, @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_stream_load", method = RequestMethod.PUT) public Object streamLoad(HttpServletRequest request, - HttpServletResponse response, - @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) { - LOG.info("streamload action, db: {}, tbl: {}, headers: {}", db, table, getAllHeaders(request)); + HttpServletResponse response, + @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) { + LOG.info("streamload action, db: {}, tbl: {}, headers: {}", db, table, getAllHeaders(request)); boolean groupCommit = false; String groupCommitStr = request.getHeader("group_commit"); - if (groupCommitStr != null && groupCommitStr.equalsIgnoreCase("async_mode")) { - groupCommit = true; - try { - if (isGroupCommitBlock(db, table)) { - String msg = "insert table " + table + GroupCommitPlanner.SCHEMA_CHANGE; - return new RestBaseResult(msg); + if (groupCommitStr != null) { + if (!groupCommitStr.equalsIgnoreCase("async_mode") && !groupCommitStr.equalsIgnoreCase("sync_mode") + && !groupCommitStr.equalsIgnoreCase("off_mode")) { + return new RestBaseResult("Header `group_commit` can only be `sync_mode`, `async_mode` or `off_mode`."); + } + if (!groupCommitStr.equalsIgnoreCase("off_mode")) { + groupCommit = true; + if (groupCommitStr.equalsIgnoreCase("async_mode")) { + try { + if (isGroupCommitBlock(db, table)) { + String msg = "insert table " + table + GroupCommitPlanner.SCHEMA_CHANGE; + return new RestBaseResult(msg); + } + } catch (Exception e) { + LOG.info("exception:" + e); + return new RestBaseResult(e.getMessage()); + } } - } catch (Exception e) { - LOG.info("exception:" + e); - return new RestBaseResult(e.getMessage()); } } if (needRedirect(request.getScheme())) { @@ -147,21 +155,32 @@ public Object streamLoadWithSql(HttpServletRequest request, HttpServletResponse boolean groupCommit = false; long tableId = -1; String groupCommitStr = request.getHeader("group_commit"); - if (groupCommitStr != null && groupCommitStr.equalsIgnoreCase("async_mode")) { - groupCommit = true; - try { - String[] pair = parseDbAndTb(sql); - Database db = Env.getCurrentInternalCatalog() - .getDbOrException(pair[0], s -> new TException("database is invalid for dbName: " + s)); - Table tbl = db.getTableOrException(pair[1], s -> new TException("table is invalid: " + s)); - tableId = tbl.getId(); - if (isGroupCommitBlock(pair[0], pair[1])) { - String msg = "insert table " + pair[1] + GroupCommitPlanner.SCHEMA_CHANGE; - return new RestBaseResult(msg); + if (groupCommitStr != null) { + if (!groupCommitStr.equalsIgnoreCase("async_mode") && !groupCommitStr.equalsIgnoreCase("sync_mode") + && !groupCommitStr.equalsIgnoreCase("off_mode")) { + return new RestBaseResult("Header `group_commit` can only be `sync_mode`, `async_mode` or `off_mode`."); + } + if (!groupCommitStr.equalsIgnoreCase("off_mode")) { + try { + groupCommit = true; + String[] pair = parseDbAndTb(sql); + Database db = Env.getCurrentInternalCatalog() + .getDbOrException(pair[0], s -> new TException("database is invalid for dbName: " + s)); + Table tbl = db.getTableOrException(pair[1], s -> new TException("table is invalid: " + s)); + tableId = tbl.getId(); + + // async mode needs to write WAL, we need to block load during waiting WAL. + if (groupCommitStr.equalsIgnoreCase("async_mode")) { + if (isGroupCommitBlock(pair[0], pair[1])) { + String msg = "insert table " + pair[1] + GroupCommitPlanner.SCHEMA_CHANGE; + return new RestBaseResult(msg); + } + + } + } catch (Exception e) { + LOG.info("exception:" + e); + return new RestBaseResult(e.getMessage()); } - } catch (Exception e) { - LOG.info("exception:" + e); - return new RestBaseResult(e.getMessage()); } } executeCheckPassword(request, response); @@ -223,8 +242,8 @@ private String[] parseDbAndTb(String sql) throws Exception { @RequestMapping(path = "/api/{" + DB_KEY + "}/_stream_load_2pc", method = RequestMethod.PUT) public Object streamLoad2PC(HttpServletRequest request, - HttpServletResponse response, - @PathVariable(value = DB_KEY) String db) { + HttpServletResponse response, + @PathVariable(value = DB_KEY) String db) { LOG.info("streamload action 2PC, db: {}, headers: {}", db, getAllHeaders(request)); if (needRedirect(request.getScheme())) { return redirectToHttps(request); @@ -236,9 +255,9 @@ public Object streamLoad2PC(HttpServletRequest request, @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_stream_load_2pc", method = RequestMethod.PUT) public Object streamLoad2PC_table(HttpServletRequest request, - HttpServletResponse response, - @PathVariable(value = DB_KEY) String db, - @PathVariable(value = TABLE_KEY) String table) { + HttpServletResponse response, + @PathVariable(value = DB_KEY) String db, + @PathVariable(value = TABLE_KEY) String table) { LOG.info("streamload action 2PC, db: {}, tbl: {}, headers: {}", db, table, getAllHeaders(request)); if (needRedirect(request.getScheme())) { return redirectToHttps(request); @@ -382,7 +401,7 @@ private TNetworkAddress selectRedirectBackend(HttpServletRequest request, boolea if (Strings.isNullOrEmpty(cloudClusterName)) { throw new LoadException("No cloud cluster name selected."); } - return selectCloudRedirectBackend(cloudClusterName, request, groupCommit); + return selectCloudRedirectBackend(cloudClusterName, request, groupCommit, tableId); } else { return selectLocalRedirectBackend(groupCommit, request, tableId); } @@ -409,21 +428,7 @@ private TNetworkAddress selectLocalRedirectBackend(boolean groupCommit, HttpServ throw new LoadException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG + ", policy: " + policy); } if (groupCommit) { - ConnectContext ctx = new ConnectContext(); - ctx.setEnv(Env.getCurrentEnv()); - ctx.setThreadLocalInfo(); - ctx.setRemoteIP(request.getRemoteAddr()); - // We set this variable to fulfill required field 'user' in - // TMasterOpRequest(FrontendService.thrift) - ctx.setQualifiedUser(Auth.ADMIN_USER); - ctx.setThreadLocalInfo(); - - try { - backend = Env.getCurrentEnv().getGroupCommitManager() - .selectBackendForGroupCommit(tableId, ctx, false); - } catch (DdlException e) { - throw new RuntimeException(e); - } + backend = selectBackendForGroupCommit("", request, tableId, false); } else { backend = Env.getCurrentSystemInfo().getBackend(backendIds.get(0)); } @@ -433,9 +438,15 @@ private TNetworkAddress selectLocalRedirectBackend(boolean groupCommit, HttpServ return new TNetworkAddress(backend.getHost(), backend.getHttpPort()); } - private TNetworkAddress selectCloudRedirectBackend(String clusterName, HttpServletRequest req, boolean groupCommit) + private TNetworkAddress selectCloudRedirectBackend(String clusterName, HttpServletRequest req, boolean groupCommit, + long tableId) throws LoadException { - Backend backend = StreamLoadHandler.selectBackend(clusterName, groupCommit); + Backend backend = null; + if (groupCommit) { + backend = selectBackendForGroupCommit(clusterName, req, tableId, true); + } else { + backend = StreamLoadHandler.selectBackend(clusterName); + } String redirectPolicy = req.getHeader(LoadAction.HEADER_REDIRECT_POLICY); // User specified redirect policy @@ -443,7 +454,7 @@ private TNetworkAddress selectCloudRedirectBackend(String clusterName, HttpServl return new TNetworkAddress(backend.getHost(), backend.getHttpPort()); } redirectPolicy = redirectPolicy == null || redirectPolicy.isEmpty() - ? Config.streamload_redirect_policy : redirectPolicy; + ? Config.streamload_redirect_policy : redirectPolicy; Pair publicHostPort = null; Pair privateHostPort = null; @@ -563,7 +574,7 @@ private boolean checkClusterToken(String token) { // temporarily addressing the users' needs for audit logs. // So this function is not widely tested under general scenario private Object executeWithClusterToken(HttpServletRequest request, String db, - String table, boolean isStreamLoad) { + String table, boolean isStreamLoad) { try { ConnectContext ctx = new ConnectContext(); ctx.setEnv(Env.getCurrentEnv()); @@ -647,4 +658,29 @@ private String getAllHeaders(HttpServletRequest request) { } return headers.toString(); } + + private Backend selectBackendForGroupCommit(String clusterName, HttpServletRequest req, long tableId, + boolean isCloud) + throws LoadException { + ConnectContext ctx = new ConnectContext(); + ctx.setEnv(Env.getCurrentEnv()); + ctx.setThreadLocalInfo(); + ctx.setRemoteIP(req.getRemoteAddr()); + // We set this variable to fulfill required field 'user' in + // TMasterOpRequest(FrontendService.thrift) + ctx.setQualifiedUser(Auth.ADMIN_USER); + ctx.setThreadLocalInfo(); + if (isCloud) { + ctx.setCloudCluster(clusterName); + } + + Backend backend = null; + try { + backend = Env.getCurrentEnv().getGroupCommitManager() + .selectBackendForGroupCommit(tableId, ctx, isCloud); + } catch (DdlException e) { + throw new LoadException(e.getMessage(), e); + } + return backend; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java index 0f44ec3f785b346..9e51e2cffd0fe57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java @@ -84,13 +84,12 @@ public StreamLoadHandler(TStreamLoadPutRequest request, AtomicInteger indexId, * Select a random backend in the given cloud cluster. * * @param clusterName cloud cluster name - * @param groupCommit if this selection is for group commit * @throws LoadException if there is no available backend */ - public static Backend selectBackend(String clusterName, boolean groupCommit) throws LoadException { + public static Backend selectBackend(String clusterName) throws LoadException { List backends = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) .getBackendsByClusterName(clusterName) - .stream().filter(be -> be.isAlive() && (!groupCommit || groupCommit && !be.isDecommissioned())) + .stream().filter(Backend::isAlive) .collect(Collectors.toList()); if (backends.isEmpty()) { @@ -101,8 +100,7 @@ public static Backend selectBackend(String clusterName, boolean groupCommit) thr // TODO: add a more sophisticated algorithm to select backend SecureRandom rand = new SecureRandom(); int randomIndex = rand.nextInt(backends.size()); - Backend backend = backends.get(randomIndex); - return backend; + return backends.get(randomIndex); } public void setCloudCluster() throws UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java index 7b156aeff8fe932..975b1ca11750eea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java @@ -447,7 +447,7 @@ public HttpPut generateRequestForMySqlLoad( private String selectBackendForMySqlLoad(String database, String table) throws LoadException { Backend backend = null; if (Config.isCloudMode()) { - backend = StreamLoadHandler.selectBackend(ConnectContext.get().getCloudCluster(), false); + backend = StreamLoadHandler.selectBackend(ConnectContext.get().getCloudCluster()); } else { BeSelectionPolicy policy = new BeSelectionPolicy.Builder().needLoadAvailable().build(); List backendIds = Env.getCurrentSystemInfo().selectBackendIdsByPolicy(policy, 1); From d91a6c98685884571fcec64737e33efce5043041 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Wed, 28 Aug 2024 23:11:51 +0800 Subject: [PATCH 13/34] [Fix](group commit) Fix table not found fault when disable group commit (#39731) ## Proposed changes selectRedirectBackend param table id is only needed by group commit. In non group commit mode, if table id is null, we need to ignore it. --- .../apache/doris/httpv2/rest/LoadAction.java | 24 +++++++---- ...eam_load_with_nonexist_db_and_table.groovy | 41 +++++++++++++++++++ 2 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java index b0a714da149564f..c9681eb784981f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -313,17 +313,20 @@ private Object executeWithoutPassword(HttpServletRequest request, return new RestBaseResult(e.getMessage()); } } else { - Optional database = Env.getCurrentEnv().getCurrentCatalog().getDb(dbName); - if (!database.isPresent()) { - return new RestBaseResult("Database not founded."); - } + long tableId = -1; + if (groupCommit) { + Optional database = Env.getCurrentEnv().getCurrentCatalog().getDb(dbName); + if (!database.isPresent()) { + return new RestBaseResult("Database not found."); + } - Optional olapTable = ((Database) database.get()).getTable(tableName); - if (!olapTable.isPresent()) { - return new RestBaseResult("OlapTable not founded."); - } + Optional olapTable = ((Database) database.get()).getTable(tableName); + if (!olapTable.isPresent()) { + return new RestBaseResult("OlapTable not found."); + } - long tableId = ((OlapTable) olapTable.get()).getId(); + tableId = ((OlapTable) olapTable.get()).getId(); + } redirectAddr = selectRedirectBackend(request, groupCommit, tableId); } @@ -403,6 +406,9 @@ private TNetworkAddress selectRedirectBackend(HttpServletRequest request, boolea } return selectCloudRedirectBackend(cloudClusterName, request, groupCommit, tableId); } else { + if (groupCommit && tableId == -1) { + throw new LoadException("Group commit table id wrong."); + } return selectLocalRedirectBackend(groupCommit, request, tableId); } } diff --git a/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy b/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy new file mode 100644 index 000000000000000..ba806967bf79b1e --- /dev/null +++ b/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load_with_nonexist_db_and_table.groovy @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_group_commit_stream_load_with_nonexist_db_and_table") { + def tableName = "test_group_commit_stream_load_with_nonexist_db_and_table" + sql "create database if not exists ${tableName}" + + try { + def command = "curl --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}" + + " -H group_commit:sync_mode" + + " -H column_separator:," + + " -T ${context.config.dataPath}/load_p0/stream_load/test_stream_load1.csv" + + " http://${context.config.feHttpAddress}/api/${tableName}/${tableName}/_stream_load" + log.info("stream load command: ${command}") + + def process = command.execute() + code = process.waitFor() + out = process.text + log.info("stream lad result: ${out}".toString()) + assertTrue(out.toString().contains("table not found")) + } catch (Exception e) { + logger.info("failed: " + e.getMessage()) + assertTrue(false) + } finally { + + } +} From f6d3280eedeef08f0eaba023364420a9819962c5 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Wed, 28 Aug 2024 23:14:26 +0800 Subject: [PATCH 14/34] [Enhancement](txn) Block new insert into if schema change happens during transaction (#39483) --- be/src/olap/schema_change.cpp | 1 + .../insert/BatchInsertIntoTableCommand.java | 11 ++ .../plans/physical/PhysicalResultSink.java | 4 - .../org/apache/doris/qe/StmtExecutor.java | 1 + .../doris/transaction/TransactionEntry.java | 18 +++ .../insert_p0/{ => transaction}/test_txn.out | 0 .../{ => transaction}/txn_insert.out | 0 .../txn_insert_inject_case.out | 0 .../txn_insert_restart_fe.out | 0 ...n_insert_restart_fe_with_schema_change.out | 0 .../txn_insert_values_with_schema_change.out | 0 .../txn_insert_with_drop.out | 0 .../txn_insert_with_schema_change.out | 0 .../txn_insert_with_specify_columns.out | 0 ...y_columns_schema_change_add_key_column.out | 15 ++ ...columns_schema_change_add_value_column.out | 15 ++ ...y_columns_schema_change_reorder_column.out | 19 +++ .../{ => transaction}/test_txn.groovy | 0 .../{ => transaction}/txn_insert.groovy | 6 +- .../txn_insert_concurrent_insert.groovy | 4 +- .../txn_insert_inject_case.groovy | 2 +- .../txn_insert_restart_fe.groovy | 0 ...nsert_restart_fe_with_schema_change.groovy | 0 ...xn_insert_values_with_schema_change.groovy | 2 +- .../txn_insert_with_drop.groovy | 0 .../txn_insert_with_schema_change.groovy | 2 +- .../txn_insert_with_specify_columns.groovy | 0 ...olumns_schema_change_add_key_column.groovy | 127 +++++++++++++++++ ...umns_schema_change_add_value_column.groovy | 129 ++++++++++++++++++ ...olumns_schema_change_reorder_column.groovy | 129 ++++++++++++++++++ 30 files changed, 473 insertions(+), 12 deletions(-) rename regression-test/data/insert_p0/{ => transaction}/test_txn.out (100%) rename regression-test/data/insert_p0/{ => transaction}/txn_insert.out (100%) rename regression-test/data/insert_p0/{ => transaction}/txn_insert_inject_case.out (100%) rename regression-test/data/insert_p0/{ => transaction}/txn_insert_restart_fe.out (100%) rename regression-test/data/insert_p0/{ => transaction}/txn_insert_restart_fe_with_schema_change.out (100%) rename regression-test/data/insert_p0/{ => transaction}/txn_insert_values_with_schema_change.out (100%) rename regression-test/data/insert_p0/{ => transaction}/txn_insert_with_drop.out (100%) rename regression-test/data/insert_p0/{ => transaction}/txn_insert_with_schema_change.out (100%) rename regression-test/data/insert_p0/{ => transaction}/txn_insert_with_specify_columns.out (100%) create mode 100644 regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_key_column.out create mode 100644 regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_value_column.out create mode 100644 regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_reorder_column.out rename regression-test/suites/insert_p0/{ => transaction}/test_txn.groovy (100%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert.groovy (99%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert_concurrent_insert.groovy (97%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert_inject_case.groovy (99%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert_restart_fe.groovy (100%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert_restart_fe_with_schema_change.groovy (100%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert_values_with_schema_change.groovy (98%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert_with_drop.groovy (100%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert_with_schema_change.groovy (99%) rename regression-test/suites/insert_p0/{ => transaction}/txn_insert_with_specify_columns.groovy (100%) create mode 100644 regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_key_column.groovy create mode 100644 regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_value_column.groovy create mode 100644 regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_reorder_column.groovy diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 6905ea89be014fb..3278b8888239e41 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -777,6 +777,7 @@ SchemaChangeJob::SchemaChangeJob(StorageEngine& local_storage_engine, // The admin should upgrade all BE and then upgrade FE. // Should delete the old code after upgrade finished. Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& request) { + DBUG_EXECUTE_IF("SchemaChangeJob._do_process_alter_tablet.sleep", { sleep(10); }) Status res; signal::tablet_id = _base_tablet->get_table_id(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java index e9cade9775568c9..414f6ed07f7ea1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.StmtType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.ErrorCode; @@ -111,6 +112,16 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { Preconditions.checkArgument(plan.isPresent(), "insert into command must contain OlapTableSinkNode"); sink = ((PhysicalOlapTableSink) plan.get()); Table targetTable = sink.getTargetTable(); + if (ctx.getTxnEntry().isFirstTxnInsert()) { + ctx.getTxnEntry().setTxnSchemaVersion(((OlapTable) targetTable).getBaseSchemaVersion()); + ctx.getTxnEntry().setFirstTxnInsert(false); + } else { + if (((OlapTable) targetTable).getBaseSchemaVersion() != ctx.getTxnEntry().getTxnSchemaVersion()) { + throw new AnalysisException("There are schema changes in one transaction, " + + "you can commit this transaction with formal data or rollback " + + "this whole transaction."); + } + } // should set columns of sink since we maybe generate some invisible columns List fullSchema = sink.getTargetTable().getFullSchema(); List targetSchema = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalResultSink.java index aceb1f137749120..8fb6dfb286e2dec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalResultSink.java @@ -58,10 +58,6 @@ public PhysicalResultSink(List outputExprs, Optional getOutputExprs() { - return outputExprs; - } - @Override public PhysicalResultSink withChildren(List children) { Preconditions.checkArgument(children.size() == 1, diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 90e7404a0387cdc..812d9d004173755 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -2066,6 +2066,7 @@ private void handleTransactionStmt() throws Exception { .setTxnConf(new TTxnParams().setNeedTxn(true).setThriftRpcTimeoutMs(5000).setTxnId(-1).setDb("") .setTbl("").setMaxFilterRatio(context.getSessionVariable().getEnableInsertStrict() ? 0 : context.getSessionVariable().getInsertMaxFilterRatio())); + context.getTxnEntry().setFirstTxnInsert(true); StringBuilder sb = new StringBuilder(); sb.append("{'label':'").append(context.getTxnEntry().getLabel()).append("', 'status':'") .append(TransactionStatus.PREPARE.name()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java index 6771d9c3156b599..4a2beea2d0ba892 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java @@ -80,6 +80,8 @@ public class TransactionEntry { private List dataToSend = new ArrayList<>(); private long rowsInTransaction = 0; private Types.PUniqueId pLoadId; + private boolean isFirstTxnInsert = false; + private volatile int txnSchemaVersion = -1; // for insert into select for multi tables private boolean isTransactionBegan = false; @@ -181,6 +183,22 @@ public void setpLoadId(Types.PUniqueId pLoadId) { this.pLoadId = pLoadId; } + public boolean isFirstTxnInsert() { + return isFirstTxnInsert; + } + + public void setFirstTxnInsert(boolean firstTxnInsert) { + isFirstTxnInsert = firstTxnInsert; + } + + public int getTxnSchemaVersion() { + return txnSchemaVersion; + } + + public void setTxnSchemaVersion(int txnSchemaVersion) { + this.txnSchemaVersion = txnSchemaVersion; + } + // Used for insert into select, return the sub_txn_id for this insert public long beginTransaction(TableIf table, SubTransactionType subTransactionType) throws Exception { if (isInsertValuesTxnBegan()) { diff --git a/regression-test/data/insert_p0/test_txn.out b/regression-test/data/insert_p0/transaction/test_txn.out similarity index 100% rename from regression-test/data/insert_p0/test_txn.out rename to regression-test/data/insert_p0/transaction/test_txn.out diff --git a/regression-test/data/insert_p0/txn_insert.out b/regression-test/data/insert_p0/transaction/txn_insert.out similarity index 100% rename from regression-test/data/insert_p0/txn_insert.out rename to regression-test/data/insert_p0/transaction/txn_insert.out diff --git a/regression-test/data/insert_p0/txn_insert_inject_case.out b/regression-test/data/insert_p0/transaction/txn_insert_inject_case.out similarity index 100% rename from regression-test/data/insert_p0/txn_insert_inject_case.out rename to regression-test/data/insert_p0/transaction/txn_insert_inject_case.out diff --git a/regression-test/data/insert_p0/txn_insert_restart_fe.out b/regression-test/data/insert_p0/transaction/txn_insert_restart_fe.out similarity index 100% rename from regression-test/data/insert_p0/txn_insert_restart_fe.out rename to regression-test/data/insert_p0/transaction/txn_insert_restart_fe.out diff --git a/regression-test/data/insert_p0/txn_insert_restart_fe_with_schema_change.out b/regression-test/data/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.out similarity index 100% rename from regression-test/data/insert_p0/txn_insert_restart_fe_with_schema_change.out rename to regression-test/data/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.out diff --git a/regression-test/data/insert_p0/txn_insert_values_with_schema_change.out b/regression-test/data/insert_p0/transaction/txn_insert_values_with_schema_change.out similarity index 100% rename from regression-test/data/insert_p0/txn_insert_values_with_schema_change.out rename to regression-test/data/insert_p0/transaction/txn_insert_values_with_schema_change.out diff --git a/regression-test/data/insert_p0/txn_insert_with_drop.out b/regression-test/data/insert_p0/transaction/txn_insert_with_drop.out similarity index 100% rename from regression-test/data/insert_p0/txn_insert_with_drop.out rename to regression-test/data/insert_p0/transaction/txn_insert_with_drop.out diff --git a/regression-test/data/insert_p0/txn_insert_with_schema_change.out b/regression-test/data/insert_p0/transaction/txn_insert_with_schema_change.out similarity index 100% rename from regression-test/data/insert_p0/txn_insert_with_schema_change.out rename to regression-test/data/insert_p0/transaction/txn_insert_with_schema_change.out diff --git a/regression-test/data/insert_p0/txn_insert_with_specify_columns.out b/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns.out similarity index 100% rename from regression-test/data/insert_p0/txn_insert_with_specify_columns.out rename to regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns.out diff --git a/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_key_column.out b/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_key_column.out new file mode 100644 index 000000000000000..b06ce07b4a89dc4 --- /dev/null +++ b/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_key_column.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_desc1 -- +c1 int Yes true \N +c2 int Yes false \N NONE +c3 int Yes false \N NONE + +-- !select_desc2 -- +c1 int Yes true \N +new_col int Yes true \N +c2 int Yes false \N NONE +c3 int Yes false \N NONE + +-- !select1 -- +1 \N 2 3 + diff --git a/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_value_column.out b/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_value_column.out new file mode 100644 index 000000000000000..560051c88002279 --- /dev/null +++ b/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_value_column.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_desc1 -- +c1 int Yes true \N +c2 int Yes false \N NONE +c3 int Yes false \N NONE + +-- !select_desc2 -- +c1 int Yes true \N +c2 int Yes false \N NONE +c3 int Yes false \N NONE +new_col int Yes false \N NONE + +-- !select1 -- +1 2 3 \N + diff --git a/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_reorder_column.out b/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_reorder_column.out new file mode 100644 index 000000000000000..2cac2aa11bc811f --- /dev/null +++ b/regression-test/data/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_reorder_column.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_desc1 -- +c1 int Yes true \N +c2 bigint Yes false \N NONE +c3 int Yes false \N NONE + +-- !select_desc2 -- +c1 int Yes true \N +c3 int Yes false \N NONE +c2 bigint Yes false \N NONE + +-- !select_desc3 -- +c1 int Yes true \N +c3 int Yes false \N NONE +c2 bigint Yes false \N NONE + +-- !select1 -- +1 3 2 + diff --git a/regression-test/suites/insert_p0/test_txn.groovy b/regression-test/suites/insert_p0/transaction/test_txn.groovy similarity index 100% rename from regression-test/suites/insert_p0/test_txn.groovy rename to regression-test/suites/insert_p0/transaction/test_txn.groovy diff --git a/regression-test/suites/insert_p0/txn_insert.groovy b/regression-test/suites/insert_p0/transaction/txn_insert.groovy similarity index 99% rename from regression-test/suites/insert_p0/txn_insert.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert.groovy index 14b056540b6cfd2..1f595d891730681 100644 --- a/regression-test/suites/insert_p0/txn_insert.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert.groovy @@ -287,7 +287,7 @@ suite("txn_insert") { if (observer_fe_url != null) { logger.info("observer url: $observer_fe_url") connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = observer_fe_url) { - result = sql """ select count() from regression_test_insert_p0.${table}_0 """ + result = sql """ select count() from regression_test_insert_p0_transaction.${table}_0 """ logger.info("select from observer result: $result") assertEquals(79, result[0][0]) } @@ -403,7 +403,7 @@ suite("txn_insert") { } // 13. txn insert does not commit or rollback by user, and txn is aborted because connection is closed - def dbName = "regression_test_insert_p0" + def dbName = "regression_test_insert_p0_transaction" def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true" logger.info("url: ${url}") def get_txn_id_from_server_info = { serverInfo -> @@ -776,7 +776,7 @@ suite("txn_insert") { } } - def db_name = "regression_test_insert_p0" + def db_name = "regression_test_insert_p0_transaction" def tables = sql """ show tables from $db_name """ logger.info("tables: $tables") for (def table_info : tables) { diff --git a/regression-test/suites/insert_p0/txn_insert_concurrent_insert.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_concurrent_insert.groovy similarity index 97% rename from regression-test/suites/insert_p0/txn_insert_concurrent_insert.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert_concurrent_insert.groovy index e520d0b9d953d12..6c19fc4ee3410c2 100644 --- a/regression-test/suites/insert_p0/txn_insert_concurrent_insert.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert_concurrent_insert.groovy @@ -79,7 +79,7 @@ suite("txn_insert_concurrent_insert") { } sql """ sync """ - def dbName = "regression_test_insert_p0" + def dbName = "regression_test_insert_p0_transaction" def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true" logger.info("url: ${url}") @@ -119,7 +119,7 @@ suite("txn_insert_concurrent_insert") { logger.info("result: ${result}") assertEquals(2606192, result[0][0]) - def db_name = "regression_test_insert_p0" + def db_name = "regression_test_insert_p0_transaction" def tables = sql """ show tables from $db_name """ logger.info("tables: $tables") for (def table_info : tables) { diff --git a/regression-test/suites/insert_p0/txn_insert_inject_case.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy similarity index 99% rename from regression-test/suites/insert_p0/txn_insert_inject_case.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy index 50a7729656a500b..5478aeef69c70de 100644 --- a/regression-test/suites/insert_p0/txn_insert_inject_case.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy @@ -152,7 +152,7 @@ suite("txn_insert_inject_case", "nonConcurrent") { // 2. commit failed sql """ truncate table ${table}_0 """ - def dbName = "regression_test_insert_p0" + def dbName = "regression_test_insert_p0_transaction" def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true" logger.info("url: ${url}") def get_txn_id_from_server_info = { serverInfo -> diff --git a/regression-test/suites/insert_p0/txn_insert_restart_fe.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe.groovy similarity index 100% rename from regression-test/suites/insert_p0/txn_insert_restart_fe.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert_restart_fe.groovy diff --git a/regression-test/suites/insert_p0/txn_insert_restart_fe_with_schema_change.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.groovy similarity index 100% rename from regression-test/suites/insert_p0/txn_insert_restart_fe_with_schema_change.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.groovy diff --git a/regression-test/suites/insert_p0/txn_insert_values_with_schema_change.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_values_with_schema_change.groovy similarity index 98% rename from regression-test/suites/insert_p0/txn_insert_values_with_schema_change.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert_values_with_schema_change.groovy index e408d418289ef34..245f2d07e5332b2 100644 --- a/regression-test/suites/insert_p0/txn_insert_values_with_schema_change.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert_values_with_schema_change.groovy @@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit suite("txn_insert_values_with_schema_change") { def table = "txn_insert_values_with_schema_change" - def dbName = "regression_test_insert_p0" + def dbName = "regression_test_insert_p0_transaction" def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true" logger.info("url: ${url}") List errors = new ArrayList<>() diff --git a/regression-test/suites/insert_p0/txn_insert_with_drop.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_with_drop.groovy similarity index 100% rename from regression-test/suites/insert_p0/txn_insert_with_drop.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert_with_drop.groovy diff --git a/regression-test/suites/insert_p0/txn_insert_with_schema_change.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_with_schema_change.groovy similarity index 99% rename from regression-test/suites/insert_p0/txn_insert_with_schema_change.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert_with_schema_change.groovy index 7f4eef8a92a7c88..f2fc1a009f9a78f 100644 --- a/regression-test/suites/insert_p0/txn_insert_with_schema_change.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert_with_schema_change.groovy @@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit suite("txn_insert_with_schema_change") { def table = "txn_insert_with_schema_change" - def dbName = "regression_test_insert_p0" + def dbName = "regression_test_insert_p0_transaction" def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true" logger.info("url: ${url}") List errors = new ArrayList<>() diff --git a/regression-test/suites/insert_p0/txn_insert_with_specify_columns.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns.groovy similarity index 100% rename from regression-test/suites/insert_p0/txn_insert_with_specify_columns.groovy rename to regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns.groovy diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_key_column.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_key_column.groovy new file mode 100644 index 000000000000000..de9162cabc73ed9 --- /dev/null +++ b/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_key_column.groovy @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.sql.Connection +import java.sql.DriverManager +import java.sql.Statement +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +suite("txn_insert_with_specify_columns_schema_change_add_key_column", "nonConcurrent") { + if(!isCloudMode()) { + def table = "txn_insert_with_specify_columns_schema_change_add_key_column" + + def dbName = "regression_test_insert_p0_transaction" + def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true" + logger.info("url: ${url}") + List errors = new ArrayList<>() + CountDownLatch insertLatch = new CountDownLatch(1) + CountDownLatch insertLatch2 = new CountDownLatch(1) + + sql """ DROP TABLE IF EXISTS $table force """ + sql """ + create table $table ( + c1 INT NULL, + c2 INT NULL, + c3 INT NULL + ) ENGINE=OLAP + UNIQUE KEY(c1) + DISTRIBUTED BY HASH(c1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1"); + """ + sql """ insert into ${table} (c3, c2, c1) values (3, 2, 1) """ + + def getAlterTableState = { job_state -> + def retry = 0 + sql "use ${dbName};" + while (true) { + sleep(2000) + def state = sql " show alter table column where tablename = '${table}' order by CreateTime desc limit 1" + logger.info("alter table state: ${state}") + if (state.size() > 0 && state[0][9] == job_state) { + return + } + retry++ + if (retry >= 10) { + break + } + } + assertTrue(false, "alter table job state is ${last_state}, not ${job_state} after retry ${retry} times") + } + + def txnInsert = { + try (Connection conn = DriverManager.getConnection(url, context.config.jdbcUser, context.config.jdbcPassword); + Statement statement = conn.createStatement()) { + try { + qt_select_desc1 """desc $table""" + + insertLatch.await(2, TimeUnit.MINUTES) + + statement.execute("begin") + statement.execute("insert into ${table} (c3, c2, c1) values (33, 22, 11),(333, 222, 111);") + + insertLatch2.await(2, TimeUnit.MINUTES) + qt_select_desc2 """desc $table""" + statement.execute("insert into ${table} (c3, c2, c1) values(3333, 2222, 1111);") + statement.execute("insert into ${table} (c3, c2, c1) values(33333, 22222, 11111),(333333, 222222, 111111);") + statement.execute("commit") + } catch (Exception e) { + logger.info("txn insert failed", e) + assertTrue(e.getMessage().contains("There are schema changes in one transaction, you can commit this transaction with formal data or rollback this whole transaction.")) + statement.execute("rollback") + } + } + } + + def schemaChange = { sql -> + try (Connection conn = DriverManager.getConnection(url, context.config.jdbcUser, context.config.jdbcPassword); + Statement statement = conn.createStatement()) { + statement.execute(sql) + getAlterTableState("RUNNING") + insertLatch.countDown() + getAlterTableState("FINISHED") + insertLatch2.countDown() + } catch (Throwable e) { + logger.error("schema change failed", e) + errors.add("schema change failed " + e.getMessage()) + } + } + + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + try { + GetDebugPoint().enableDebugPointForAllBEs("SchemaChangeJob._do_process_alter_tablet.sleep") + Thread schema_change_thread = new Thread(() -> schemaChange("alter table ${table} add column new_col int key after c1;")) + Thread insert_thread = new Thread(() -> txnInsert()) + schema_change_thread.start() + insert_thread.start() + schema_change_thread.join() + insert_thread.join() + + logger.info("errors: " + errors) + assertEquals(0, errors.size()) + getAlterTableState("FINISHED") + order_qt_select1 """select * from ${table} order by c1, c2, c3""" + } catch (Exception e) { + logger.info("failed: " + e.getMessage()) + assertTrue(false) + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + } + } +} diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_value_column.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_value_column.groovy new file mode 100644 index 000000000000000..1d072f572c6a22a --- /dev/null +++ b/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_add_value_column.groovy @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.sql.Connection +import java.sql.DriverManager +import java.sql.Statement +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +suite("txn_insert_with_specify_columns_schema_change_add_value_column", "nonConcurrent") { + if(!isCloudMode()) { + def table = "txn_insert_with_specify_columns_schema_change_add_value_column" + + def dbName = "regression_test_insert_p0_transaction" + def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true" + logger.info("url: ${url}") + List errors = new ArrayList<>() + CountDownLatch insertLatch = new CountDownLatch(1) + CountDownLatch schemaChangeLatch = new CountDownLatch(1) + + sql """ DROP TABLE IF EXISTS $table force """ + sql """ + create table $table ( + c1 INT NULL, + c2 INT NULL, + c3 INT NULL + ) ENGINE=OLAP + UNIQUE KEY(c1) + DISTRIBUTED BY HASH(c1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1"); + """ + sql """ insert into ${table} (c3, c2, c1) values (3, 2, 1) """ + + def getAlterTableState = { job_state -> + def retry = 0 + sql "use ${dbName};" + while (true) { + sleep(2000) + def state = sql " show alter table column where tablename = '${table}' order by CreateTime desc limit 1" + logger.info("alter table state: ${state}") + if (state.size() > 0 && state[0][9] == job_state) { + return + } + retry++ + if (retry >= 10) { + break + } + } + assertTrue(false, "alter table job state is ${last_state}, not ${job_state} after retry ${retry} times") + } + + def txnInsert = { + try (Connection conn = DriverManager.getConnection(url, context.config.jdbcUser, context.config.jdbcPassword); + Statement statement = conn.createStatement()) { + try { + qt_select_desc1 """desc $table""" + + statement.execute("begin") + statement.execute("insert into ${table} (c3, c2, c1) values (33, 22, 11),(333, 222, 111);") + + schemaChangeLatch.countDown() + + insertLatch.await(2, TimeUnit.MINUTES) + + qt_select_desc2 """desc $table""" + statement.execute("insert into ${table} (c3, c2, c1) values(3333, 2222, 1111);") + statement.execute("insert into ${table} (c3, c2, c1) values(33333, 22222, 11111),(333333, 222222, 111111);") + statement.execute("commit") + } catch (Exception e) { + logger.info("txn insert failed", e) + assertTrue(e.getMessage().contains("There are schema changes in one transaction, you can commit this transaction with formal data or rollback this whole transaction.")) + statement.execute("rollback") + } + } + } + + def schemaChange = { sql -> + try (Connection conn = DriverManager.getConnection(url, context.config.jdbcUser, context.config.jdbcPassword); + Statement statement = conn.createStatement()) { + schemaChangeLatch.await(2, TimeUnit.MINUTES) + statement.execute(sql) + getAlterTableState("FINISHED") + insertLatch.countDown() + } catch (Throwable e) { + logger.error("schema change failed", e) + errors.add("schema change failed " + e.getMessage()) + } + } + + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + try { + GetDebugPoint().enableDebugPointForAllBEs("SchemaChangeJob._do_process_alter_tablet.sleep") + Thread schema_change_thread = new Thread(() -> schemaChange("alter table ${table} add column new_col int after c3;")) + Thread insert_thread = new Thread(() -> txnInsert()) + schema_change_thread.start() + insert_thread.start() + + schema_change_thread.join() + insert_thread.join() + + logger.info("errors: " + errors) + assertEquals(0, errors.size()) + getAlterTableState("FINISHED") + order_qt_select1 """select * from ${table} order by c1, c2, c3""" + } catch (Exception e) { + logger.info("failed: " + e.getMessage()) + assertTrue(false) + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + } + } + +} diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_reorder_column.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_reorder_column.groovy new file mode 100644 index 000000000000000..e4c16fdf8c3033f --- /dev/null +++ b/regression-test/suites/insert_p0/transaction/txn_insert_with_specify_columns_schema_change_reorder_column.groovy @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.sql.Connection +import java.sql.DriverManager +import java.sql.Statement +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +suite("txn_insert_with_specify_columns_schema_change_reorder_column", "nonConcurrent") { + if(!isCloudMode()){ + def table = "txn_insert_with_specify_columns_schema_change_reorder_column" + + def dbName = "regression_test_insert_p0_transaction" + def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, dbName).replace("&useServerPrepStmts=true", "") + "&useLocalSessionState=true" + logger.info("url: ${url}") + List errors = new ArrayList<>() + CountDownLatch insertLatch = new CountDownLatch(1) + CountDownLatch insertLatch2 = new CountDownLatch(1) + + sql """ DROP TABLE IF EXISTS $table force """ + sql """ + create table $table ( + c1 INT NULL, + c2 BIGINT NULL, + c3 INT NULL + ) ENGINE=OLAP + UNIQUE KEY(c1) + DISTRIBUTED BY HASH(c1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1"); + """ + sql """ insert into ${table} (c3, c2, c1) values (3, 2, 1) """ + + def getAlterTableState = { job_state -> + def retry = 0 + sql "use ${dbName};" + while (true) { + sleep(2000) + def state = sql " show alter table column where tablename = '${table}' order by CreateTime desc limit 1" + logger.info("alter table state: ${state}") + if (state.size() > 0 && state[0][9] == job_state) { + return + } + retry++ + if (retry >= 10) { + break + } + } + assertTrue(false, "alter table job state is ${last_state}, not ${job_state} after retry ${retry} times") + } + + def txnInsert = { + try (Connection conn = DriverManager.getConnection(url, context.config.jdbcUser, context.config.jdbcPassword); + Statement statement = conn.createStatement()) { + try { + qt_select_desc1 """desc $table""" + + insertLatch.await(2, TimeUnit.MINUTES) + + statement.execute("begin") + statement.execute("insert into ${table} (c3, c2, c1) values (33, 22, 11),(333, 222, 111);") + + insertLatch2.await(2, TimeUnit.MINUTES) + + qt_select_desc2 """desc $table""" + statement.execute("insert into ${table} (c3, c2, c1) values(3333, 2222, 1111);") + statement.execute("insert into ${table} (c3, c2, c1) values(33333, 22222, 11111),(333333, 222222, 111111);") + statement.execute("commit") + } catch (Throwable e) { + logger.error("txn insert failed", e) + assertTrue(e.getMessage().contains("There are schema changes in one transaction, you can commit this transaction with formal data or rollback this whole transaction.")) + statement.execute("rollback") + } + } + } + + def schemaChange = { sql -> + try (Connection conn = DriverManager.getConnection(url, context.config.jdbcUser, context.config.jdbcPassword); + Statement statement = conn.createStatement()) { + statement.execute(sql) + getAlterTableState("RUNNING") + insertLatch.countDown() + getAlterTableState("FINISHED") + insertLatch2.countDown() + } catch (Throwable e) { + logger.error("schema change failed", e) + errors.add("schema change failed " + e.getMessage()) + } + } + + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + try { + GetDebugPoint().enableDebugPointForAllBEs("SchemaChangeJob._do_process_alter_tablet.sleep") + Thread schema_change_thread = new Thread(() -> schemaChange("alter table ${table} order by (c1,c3,c2);")) + Thread insert_thread = new Thread(() -> txnInsert()) + schema_change_thread.start() + insert_thread.start() + schema_change_thread.join() + insert_thread.join() + + logger.info("errors: " + errors) + assertEquals(0, errors.size()) + getAlterTableState("FINISHED") + qt_select_desc3 """desc $table""" + order_qt_select1 """select * from ${table} order by c1, c2, c3""" + } catch (Exception e) { + logger.info("failed: " + e.getMessage()) + assertTrue(false) + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + } + } +} From bddfd5a48d3b6292e2d948e0c068028ec4f1baa0 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Wed, 28 Aug 2024 23:16:37 +0800 Subject: [PATCH 15/34] [fix](compaction) fix heap-use-after-free caused by compaction sample info resize (#40058) --- be/src/olap/merger.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index 3d8121ef0e47051..02d77372b4b3c7c 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -460,7 +460,10 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr tablet, ReaderType reader_t vectorized::RowSourcesBuffer row_sources_buf( tablet->tablet_id(), dst_rowset_writer->context().tablet_path, reader_type); - tablet->sample_infos.resize(column_groups.size(), {0, 0, 0}); + { + std::unique_lock lock(tablet->sample_info_lock); + tablet->sample_infos.resize(column_groups.size(), {0, 0, 0}); + } // compact group one by one for (auto i = 0; i < column_groups.size(); ++i) { VLOG_NOTICE << "row source size: " << row_sources_buf.total_size(); @@ -468,10 +471,16 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr tablet, ReaderType reader_t int64_t batch_size = config::compaction_batch_size != -1 ? config::compaction_batch_size : estimate_batch_size(i, tablet, merge_way_num); - RETURN_IF_ERROR(vertical_compact_one_group( + CompactionSampleInfo sample_info; + Status st = vertical_compact_one_group( tablet, reader_type, tablet_schema, is_key, column_groups[i], &row_sources_buf, src_rowset_readers, dst_rowset_writer, max_rows_per_segment, stats_output, - key_group_cluster_key_idxes, batch_size, &(tablet->sample_infos[i]))); + key_group_cluster_key_idxes, batch_size, &sample_info); + { + std::unique_lock lock(tablet->sample_info_lock); + tablet->sample_infos[i] = sample_info; + } + RETURN_IF_ERROR(st); if (is_key) { RETURN_IF_ERROR(row_sources_buf.flush()); } From f652783832f8a876365d571da2118724abdb0a66 Mon Sep 17 00:00:00 2001 From: zzzxl <33418555+zzzxl1993@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:38:08 +0800 Subject: [PATCH 16/34] [fix](inverted index) fix error handling in fast_execute (#40024) 1. fix error handling in fast_execute 2. Fixed the issue where fast_execute was still being executed when set enable_inverted_index_query = false. --- be/src/vec/exprs/vexpr.cpp | 13 +++++++++++++ be/src/vec/exprs/vexpr.h | 1 + 2 files changed, 14 insertions(+) diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp index c615c91dc75ef17..b66c8aa80a7e6fe 100644 --- a/be/src/vec/exprs/vexpr.cpp +++ b/be/src/vec/exprs/vexpr.cpp @@ -210,6 +210,7 @@ Status VExpr::prepare(RuntimeState* state, const RowDescriptor& row_desc, VExprC RETURN_IF_ERROR(i->prepare(state, row_desc, context)); } --context->_depth_num; + _enable_inverted_index_query = state->query_options().enable_inverted_index_query; return Status::OK(); } @@ -603,6 +604,10 @@ Status VExpr::get_result_from_const(vectorized::Block* block, const std::string& bool VExpr::fast_execute(Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count, const std::string& function_name) { + if (!_enable_inverted_index_query) { + return false; + } + std::string result_column_name = gen_predicate_result_sign(block, arguments, function_name); if (!block.has(result_column_name)) { DBUG_EXECUTE_IF("segment_iterator.fast_execute", { @@ -651,11 +656,19 @@ std::string VExpr::gen_predicate_result_sign(Block& block, const ColumnNumbers& std::set values; for (size_t i = 1; i < arguments.size(); i++) { const auto& entry = block.get_by_position(arguments[i]); + if (!is_column_const(*entry.column)) { + return pred_result_sign; + } values.insert(entry.type->to_string(*entry.column, 0)); } pred_result_sign += boost::join(values, ","); + } else if (function_name == "collection_in" || function_name == "collection_not_in") { + return pred_result_sign; } else { const auto& entry = block.get_by_position(arguments[1]); + if (!is_column_const(*entry.column)) { + return pred_result_sign; + } pred_result_sign += entry.type->to_string(*entry.column, 0); } } diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h index 7ee4e0a5dd4a6ad..0409327aef1dc0a 100644 --- a/be/src/vec/exprs/vexpr.h +++ b/be/src/vec/exprs/vexpr.h @@ -328,6 +328,7 @@ class VExpr { // ensuring uniqueness during index traversal uint32_t _index_unique_id = 0; bool _can_fast_execute = false; + bool _enable_inverted_index_query = true; }; } // namespace vectorized From 55ed12774fd9ac65257672e83e8fc66f8b9ea3d9 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:44:15 +0800 Subject: [PATCH 17/34] [improvement](mtmv) Optimize get available mv logic to avoid unnecessary rewrite (#39734) Optimize get available mv logic to avoid unnecessary rewrite such as mv def sql is as following CREATE MATERIALIZED VIEW mv1_0 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS select lineitem.L_LINENUMBER, orders.O_CUSTKEY from lineitem inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY if we run query as following, mv1_0 should not participate in query rewrite by materialized view to avoid unnecessary rewrite select L_LINENUMBER from mv1_0; --- .../mv/InitMaterializationContextHook.java | 3 +- .../nereids/trees/plans/PlanVisitorTest.java | 16 ++ .../doris/regression/suite/Suite.groovy | 24 +++ .../mv/direct_query/direct_query.groovy | 162 ++++++++++++++++++ 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index c48303dbf0b9772..e1f98afae3f7797 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -78,7 +78,8 @@ public void initMaterializationContext(CascadesContext cascadesContext) { * @param cascadesContext current cascadesContext in the planner */ protected void doInitMaterializationContext(CascadesContext cascadesContext) { - TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), true); + // Only collect the table or mv which query use directly, to avoid useless mv partition in rewrite + TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false); try { Plan rewritePlan = cascadesContext.getRewritePlan(); // Keep use one connection context when in query, if new connect context, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java index ec184080bdf4017..60f6e19faab6350 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java @@ -243,6 +243,22 @@ public BitSet getDisableNereidsRules() { .map(TableIf::getName) .collect(Collectors.toSet()), expectedMvsWithNoExpand); + + TableCollectorContext allTableTypeWithExpand = + new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.values()), true); + physicalPlan.accept(TableCollector.INSTANCE, allTableTypeWithExpand); + // when collect in plan with expand, should collect table which is expended + Set expectedTablesWithExpand = new HashSet<>(); + expectedTablesWithExpand.add("mv1"); + expectedTablesWithExpand.add("table1"); + expectedTablesWithExpand.add("table2"); + expectedTablesWithExpand.add("table3"); + Assertions.assertEquals( + allTableTypeWithExpand.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTablesWithExpand); }); dropMvByNereids("drop materialized view mv1"); } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index ac8795729e1e94b..f5e6e5ab1bab710 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1502,6 +1502,30 @@ class Suite implements GroovyInterceptable { return result.values().toList() } + def create_async_mv = { db, mv_name, mv_sql -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + sql "analyze table ${mv_name} with sync;" + } + + def mv_not_part_in = { query_sql, mv_name -> + explain { + sql(" memo plan ${query_sql}") + notContains("${mv_name} chose") + notContains("${mv_name} not chose") + notContains("${mv_name} fail") + } + } + def mv_rewrite_success = { query_sql, mv_name -> explain { sql(" memo plan ${query_sql}") diff --git a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy new file mode 100644 index 000000000000000..404154300d2724c --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy @@ -0,0 +1,162 @@ +package mv.direct_query +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("direct_query_mv") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) (PARTITION `day_2` VALUES LESS THAN ('2023-12-30')) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) (PARTITION `day_1` VALUES LESS THAN ('2023-12-30')) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table lineitem with sync;""" + sql """analyze table orders with sync;""" + sql """analyze table partsupp with sync;""" + + + create_async_mv(db, "mv1_0", + """ + select lineitem.L_LINENUMBER, orders.O_CUSTKEY + from lineitem + inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY + """) + // select mv directly, mv should not part in rewrite + mv_not_part_in("""select L_LINENUMBER from mv1_0;""", "mv1_0") + + + create_async_mv(db, "mv2_0", + """ + select L_LINENUMBER, count(O_CUSTKEY) + from mv1_0 + group by L_LINENUMBER; + """) + // mv2 use mv1, though query not use mv1 directly, mv2 should part in rewrite and shoule be chosen + mv_rewrite_success(""" + select L_LINENUMBER, count(O_CUSTKEY) + from lineitem + inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY + group by L_LINENUMBER; + """, + "mv2_0"); + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0""" +} From 5f08e5d5c028d4369144b0f58a37fc40ff4b2dbc Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 29 Aug 2024 10:46:57 +0800 Subject: [PATCH 18/34] [Opt](parquet/orc-reader) Opt get dict ids in _rewrite_dict_predicates(). (#39893) ## Proposed changes The original implementation needed to obtain the dict code through dict value when generating dictionary filter conditions. In order to obtain the dict code, a `dict_value_to_code` hash map needs to be constructed. We found that when the amount of data is large, it is time-consuming to construct the `dict_value_to_code` hash map, and the current implementation will also construct it when there is no filter condition. Because the dict code is the position of the dictionary element, this PR records the position when filtering the dict value, so as to directly obtain the dict code. --- be/src/vec/exec/format/orc/vorc_reader.cpp | 49 +++++++------------ .../parquet/byte_array_dict_decoder.cpp | 11 ----- .../format/parquet/byte_array_dict_decoder.h | 4 -- be/src/vec/exec/format/parquet/decoder.h | 5 -- .../parquet/fix_length_dict_decoder.hpp | 14 ------ .../parquet/vparquet_column_chunk_reader.h | 5 -- .../format/parquet/vparquet_column_reader.cpp | 5 -- .../format/parquet/vparquet_column_reader.h | 7 --- .../format/parquet/vparquet_group_reader.cpp | 43 +++++++--------- 9 files changed, 36 insertions(+), 107 deletions(-) diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index e21bc6d67811f60..cffa934cc2c740a 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -2054,7 +2054,6 @@ Status OrcReader::on_string_dicts_loaded( orc::StringDictionary* dict = file_column_name_to_dict_map_iter->second; std::vector dict_values; - std::unordered_map dict_value_to_code; size_t max_value_length = 0; uint64_t dictionaryCount = dict->dictionaryOffset.size() - 1; if (dictionaryCount == 0) { @@ -2074,7 +2073,6 @@ Status OrcReader::on_string_dicts_loaded( max_value_length = length; } dict_values.emplace_back(dict_value); - dict_value_to_code[dict_value] = i; } dict_value_column->insert_many_strings_overflow(&dict_values[0], dict_values.size(), max_value_length); @@ -2113,31 +2111,37 @@ Status OrcReader::on_string_dicts_loaded( ++index; } - // 2.2 Execute conjuncts and filter block. - std::vector columns_to_filter(1, dict_pos); - int column_to_keep = temp_block.columns(); + // 2.2 Execute conjuncts. if (dict_pos != 0) { // VExprContext.execute has an optimization, the filtering is executed when block->rows() > 0 // The following process may be tricky and time-consuming, but we have no other way. temp_block.get_by_position(0).column->assume_mutable()->resize(dict_value_column_size); } - RETURN_IF_CATCH_EXCEPTION(RETURN_IF_ERROR(VExprContext::execute_conjuncts_and_filter_block( - ctxs, &temp_block, columns_to_filter, column_to_keep))); + IColumn::Filter result_filter(temp_block.rows(), 1); + bool can_filter_all; + RETURN_IF_ERROR(VExprContext::execute_conjuncts(ctxs, nullptr, &temp_block, &result_filter, + &can_filter_all)); if (dict_pos != 0) { // We have to clean the first column to insert right data. temp_block.get_by_position(0).column->assume_mutable()->clear(); } - // Check some conditions. - ColumnPtr& dict_column = temp_block.get_by_position(dict_pos).column; - // If dict_column->size() == 0, can filter this stripe. - if (dict_column->size() == 0) { + // If can_filter_all = true, can filter this stripe. + if (can_filter_all) { *is_stripe_filtered = true; return Status::OK(); } + // 3. Get dict codes. + std::vector dict_codes; + for (size_t i = 0; i < result_filter.size(); ++i) { + if (result_filter[i]) { + dict_codes.emplace_back(i); + } + } + // About Performance: if dict_column size is too large, it will generate a large IN filter. - if (dict_column->size() > MAX_DICT_CODE_PREDICATE_TO_REWRITE) { + if (dict_codes.size() > MAX_DICT_CODE_PREDICATE_TO_REWRITE) { it = _dict_filter_cols.erase(it); for (auto& ctx : ctxs) { _non_dict_filter_conjuncts.emplace_back(ctx); @@ -2145,26 +2149,9 @@ Status OrcReader::on_string_dicts_loaded( continue; } - // 3. Get dict codes. - std::vector dict_codes; - if (dict_column->is_nullable()) { - const ColumnNullable* nullable_column = - static_cast(dict_column.get()); - const ColumnString* nested_column = static_cast( - nullable_column->get_nested_column_ptr().get()); - for (int i = 0; i < nested_column->size(); ++i) { - StringRef dict_value = nested_column->get_data_at(i); - dict_codes.emplace_back(dict_value_to_code[dict_value]); - } - } else { - for (int i = 0; i < dict_column->size(); ++i) { - StringRef dict_value = dict_column->get_data_at(i); - dict_codes.emplace_back(dict_value_to_code[dict_value]); - } - } - // 4. Rewrite conjuncts. - RETURN_IF_ERROR(_rewrite_dict_conjuncts(dict_codes, slot_id, dict_column->is_nullable())); + RETURN_IF_ERROR(_rewrite_dict_conjuncts( + dict_codes, slot_id, temp_block.get_by_position(dict_pos).column->is_nullable())); ++it; } return Status::OK(); diff --git a/be/src/vec/exec/format/parquet/byte_array_dict_decoder.cpp b/be/src/vec/exec/format/parquet/byte_array_dict_decoder.cpp index 8b9532e68d03912..7d9f708011c4e51 100644 --- a/be/src/vec/exec/format/parquet/byte_array_dict_decoder.cpp +++ b/be/src/vec/exec/format/parquet/byte_array_dict_decoder.cpp @@ -44,7 +44,6 @@ Status ByteArrayDictDecoder::set_dict(std::unique_ptr& dict, int32_t total_length += l; } - _dict_value_to_code.reserve(num_values); // For insert_many_strings_overflow _dict_data.resize(total_length + ColumnString::MAX_STRINGS_OVERFLOW_SIZE); _max_value_length = 0; @@ -55,7 +54,6 @@ Status ByteArrayDictDecoder::set_dict(std::unique_ptr& dict, int32_t offset_cursor += 4; memcpy(&_dict_data[offset], dict_item_address + offset_cursor, l); _dict_items.emplace_back(&_dict_data[offset], l); - _dict_value_to_code[StringRef(&_dict_data[offset], l)] = i; offset_cursor += l; offset += l; if (offset_cursor > length) { @@ -77,15 +75,6 @@ Status ByteArrayDictDecoder::read_dict_values_to_column(MutableColumnPtr& doris_ return Status::OK(); } -Status ByteArrayDictDecoder::get_dict_codes(const ColumnString* string_column, - std::vector* dict_codes) { - for (int i = 0; i < string_column->size(); ++i) { - StringRef dict_value = string_column->get_data_at(i); - dict_codes->emplace_back(_dict_value_to_code[dict_value]); - } - return Status::OK(); -} - MutableColumnPtr ByteArrayDictDecoder::convert_dict_column_to_string_column( const ColumnInt32* dict_column) { auto res = ColumnString::create(); diff --git a/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h b/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h index 744a62165fb7fe3..bb83d41813bb785 100644 --- a/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h +++ b/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h @@ -54,9 +54,6 @@ class ByteArrayDictDecoder final : public BaseDictDecoder { Status read_dict_values_to_column(MutableColumnPtr& doris_column) override; - Status get_dict_codes(const ColumnString* column_string, - std::vector* dict_codes) override; - MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) override; protected: @@ -64,6 +61,5 @@ class ByteArrayDictDecoder final : public BaseDictDecoder { std::vector _dict_items; std::vector _dict_data; size_t _max_value_length; - std::unordered_map _dict_value_to_code; }; } // namespace doris::vectorized diff --git a/be/src/vec/exec/format/parquet/decoder.h b/be/src/vec/exec/format/parquet/decoder.h index 57fecf4abfb7a26..1654878af80a29f 100644 --- a/be/src/vec/exec/format/parquet/decoder.h +++ b/be/src/vec/exec/format/parquet/decoder.h @@ -78,11 +78,6 @@ class Decoder { return Status::NotSupported("read_dict_values_to_column is not supported"); } - virtual Status get_dict_codes(const ColumnString* column_string, - std::vector* dict_codes) { - return Status::NotSupported("get_dict_codes is not supported"); - } - virtual MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) { LOG(FATAL) << "Method convert_dict_column_to_string_column is not supported"; __builtin_unreachable(); diff --git a/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp b/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp index 65e329ae89b5a46..0bcc0bd5e73a402 100644 --- a/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp +++ b/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp @@ -109,10 +109,8 @@ class FixLengthDictDecoder final : public BaseDictDecoder { _dict = std::move(dict); char* dict_item_address = reinterpret_cast(_dict.get()); _dict_items.resize(num_values); - _dict_value_to_code.reserve(num_values); for (size_t i = 0; i < num_values; ++i) { _dict_items[i] = dict_item_address; - _dict_value_to_code[StringRef(_dict_items[i], _type_length)] = i; dict_item_address += _type_length; } return Status::OK(); @@ -128,17 +126,6 @@ class FixLengthDictDecoder final : public BaseDictDecoder { return Status::OK(); } - Status get_dict_codes(const ColumnString* string_column, - std::vector* dict_codes) override { - size_t size = string_column->size(); - dict_codes->reserve(size); - for (int i = 0; i < size; ++i) { - StringRef dict_value = string_column->get_data_at(i); - dict_codes->emplace_back(_dict_value_to_code[dict_value]); - } - return Status::OK(); - } - MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) override { auto res = ColumnString::create(); std::vector dict_values(dict_column->size()); @@ -149,7 +136,6 @@ class FixLengthDictDecoder final : public BaseDictDecoder { res->insert_many_strings(&dict_values[0], dict_values.size()); return res; } - std::unordered_map _dict_value_to_code; // For dictionary encoding std::vector _dict_items; }; diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h index 79ee3cd646306c3..a00a46837257621 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h @@ -183,11 +183,6 @@ class ColumnChunkReader { ->read_dict_values_to_column(doris_column); } - Status get_dict_codes(const ColumnString* column_string, std::vector* dict_codes) { - return _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)]->get_dict_codes( - column_string, dict_codes); - } - MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) { return _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)] ->convert_dict_column_to_string_column(dict_column); diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp index c31c63ee87c36c0..9c368b6a7a6fa5f 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp @@ -454,11 +454,6 @@ Status ScalarColumnReader::read_dict_values_to_column(MutableColumnPtr& doris_co return Status::OK(); } -Status ScalarColumnReader::get_dict_codes(const ColumnString* column_string, - std::vector* dict_codes) { - return _chunk_reader->get_dict_codes(column_string, dict_codes); -} - MutableColumnPtr ScalarColumnReader::convert_dict_column_to_string_column( const ColumnInt32* dict_column) { return _chunk_reader->convert_dict_column_to_string_column(dict_column); diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h index f0eadb8bcd61c59..4c6e5b1eac9f60a 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h @@ -128,11 +128,6 @@ class ParquetColumnReader { return Status::NotSupported("read_dict_values_to_column is not supported"); } - virtual Status get_dict_codes(const ColumnString* column_string, - std::vector* dict_codes) { - return Status::NotSupported("get_dict_codes is not supported"); - } - virtual MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) { LOG(FATAL) << "Method convert_dict_column_to_string_column is not supported"; __builtin_unreachable(); @@ -180,8 +175,6 @@ class ScalarColumnReader : public ParquetColumnReader { ColumnSelectVector& select_vector, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) override; Status read_dict_values_to_column(MutableColumnPtr& doris_column, bool* has_dict) override; - Status get_dict_codes(const ColumnString* column_string, - std::vector* dict_codes) override; MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) override; const std::vector& get_rep_level() const override { return _rep_levels; } const std::vector& get_def_level() const override { return _def_levels; } diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp index 576ed58a284f8fa..b993f4cd31e3133 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp @@ -841,7 +841,7 @@ Status RowGroupReader::_rewrite_dict_predicates() { ++index; } - // 2.2 Execute conjuncts and filter block. + // 2.2 Execute conjuncts. VExprContextSPtrs ctxs; auto iter = _slot_id_to_filter_conjuncts->find(slot_id); if (iter != _slot_id_to_filter_conjuncts->end()) { @@ -854,33 +854,39 @@ Status RowGroupReader::_rewrite_dict_predicates() { return Status::NotFound(msg.str()); } - std::vector columns_to_filter(1, dict_pos); - int column_to_keep = temp_block.columns(); if (dict_pos != 0) { // VExprContext.execute has an optimization, the filtering is executed when block->rows() > 0 // The following process may be tricky and time-consuming, but we have no other way. temp_block.get_by_position(0).column->assume_mutable()->resize(dict_value_column_size); } + IColumn::Filter result_filter(temp_block.rows(), 1); + bool can_filter_all; { SCOPED_RAW_TIMER(&_predicate_filter_time); - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts_and_filter_block( - ctxs, &temp_block, columns_to_filter, column_to_keep)); + RETURN_IF_ERROR(VExprContext::execute_conjuncts(ctxs, nullptr, &temp_block, + &result_filter, &can_filter_all)); } if (dict_pos != 0) { // We have to clean the first column to insert right data. temp_block.get_by_position(0).column->assume_mutable()->clear(); } - // Check some conditions. - ColumnPtr& dict_column = temp_block.get_by_position(dict_pos).column; - // If dict_column->size() == 0, can filter this row group. - if (dict_column->size() == 0) { + // If can_filter_all = true, can filter this row group. + if (can_filter_all) { _is_row_group_filtered = true; return Status::OK(); } + // 3. Get dict codes. + std::vector dict_codes; + for (size_t i = 0; i < result_filter.size(); ++i) { + if (result_filter[i]) { + dict_codes.emplace_back(i); + } + } + // About Performance: if dict_column size is too large, it will generate a large IN filter. - if (dict_column->size() > MAX_DICT_CODE_PREDICATE_TO_REWRITE) { + if (dict_codes.size() > MAX_DICT_CODE_PREDICATE_TO_REWRITE) { it = _dict_filter_cols.erase(it); for (auto& ctx : ctxs) { _filter_conjuncts.push_back(ctx); @@ -888,22 +894,9 @@ Status RowGroupReader::_rewrite_dict_predicates() { continue; } - // 3. Get dict codes. - std::vector dict_codes; - if (dict_column->is_nullable()) { - const ColumnNullable* nullable_column = - static_cast(dict_column.get()); - const ColumnString* nested_column = static_cast( - nullable_column->get_nested_column_ptr().get()); - RETURN_IF_ERROR(_column_readers[dict_filter_col_name]->get_dict_codes( - assert_cast(nested_column), &dict_codes)); - } else { - RETURN_IF_ERROR(_column_readers[dict_filter_col_name]->get_dict_codes( - assert_cast(dict_column.get()), &dict_codes)); - } - // 4. Rewrite conjuncts. - RETURN_IF_ERROR(_rewrite_dict_conjuncts(dict_codes, slot_id, dict_column->is_nullable())); + RETURN_IF_ERROR(_rewrite_dict_conjuncts( + dict_codes, slot_id, temp_block.get_by_position(dict_pos).column->is_nullable())); ++it; } return Status::OK(); From 52536421cc7830ef8a26f6ae10e494aaed77a3b7 Mon Sep 17 00:00:00 2001 From: daidai <2017501503@qq.com> Date: Thu, 29 Aug 2024 10:52:12 +0800 Subject: [PATCH 19/34] [fix](ut)fix be enable_http_auth ut (#40071) ## Proposed changes before pr #39577 --- be/test/http/http_auth_test.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/be/test/http/http_auth_test.cpp b/be/test/http/http_auth_test.cpp index 6077f871b7076ed..f39e61aa6e2cf7c 100644 --- a/be/test/http/http_auth_test.cpp +++ b/be/test/http/http_auth_test.cpp @@ -73,8 +73,7 @@ TEST_F(HttpAuthTest, enable_all_http_auth) { { auto evhttp_req = evhttp_request_new(nullptr, nullptr); HttpRequest req2(evhttp_req); - auto auth = encode_basic_auth("doris", "passwd"); - req2._headers.emplace(HttpHeaders::AUTHORIZATION, auth); + req2._headers.emplace(HttpHeaders::AUTHORIZATION, "Basic cm9vdDo="); EXPECT_EQ(s_auth_handler.on_header(&req2), -1); } @@ -82,8 +81,7 @@ TEST_F(HttpAuthTest, enable_all_http_auth) { { auto evhttp_req = evhttp_request_new(nullptr, nullptr); HttpRequest req3(evhttp_req); - auto auth = encode_basic_auth("doris", "passwd"); - req3._headers.emplace(HttpHeaders::AUTHORIZATION, auth); + req3._headers.emplace(HttpHeaders::AUTHORIZATION, "Basic cm9vdDo="); req3._params.emplace("table", "T"); EXPECT_EQ(s_auth_handler.on_header(&req3), 0); evhttp_request_free(evhttp_req); From 83e6eacdef2c9ccd51acd824956a44c0e6a750eb Mon Sep 17 00:00:00 2001 From: zclllhhjj Date: Thu, 29 Aug 2024 10:55:18 +0800 Subject: [PATCH 20/34] [Fix](load) fix commit txn timeout when loading to table with many tablet (#40031) fix commit txn timeout when loading to table with many tablet --- .../apache/doris/load/BrokerFileGroup.java | 16 +++++--- .../transaction/DatabaseTransactionMgr.java | 40 +++++++++++++------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java index d182e1d08d4d216..366983d4139cdec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java @@ -152,12 +152,16 @@ public void parse(Database db, DataDescription dataDescription) throws DdlExcept } } - boolean isPartitionRestoring = olapTable.getPartitions().stream().anyMatch( - partition -> partition.getState() == PartitionState.RESTORE - ); - // restore table - if (!isPartitionRestoring && olapTable.getState() == OlapTableState.RESTORE) { - throw new DdlException("Table [" + olapTable.getName() + "] is under restore"); + // only do check when here's restore on this table now + if (olapTable.getState() == OlapTableState.RESTORE) { + boolean hasPartitionRestoring = olapTable.getPartitions().stream() + .anyMatch(partition -> partition.getState() == PartitionState.RESTORE); + // tbl RESTORE && all partition NOT RESTORE -> whole table restore + // tbl RESTORE && some partition RESTORE -> just partitions restore, NOT WHOLE TABLE + // so check wether the whole table restore here + if (!hasPartitionRestoring) { + throw new DdlException("Table [" + olapTable.getName() + "] is under restore"); + } } if (olapTable.getKeysType() != KeysType.AGG_KEYS && dataDescription.isNegative()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 3680f081d038dd0..5e9c22bede7e82c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -509,7 +509,9 @@ private void checkCommitStatus(List tableList, TransactionState transacti List tabletIds = tabletCommitInfos.stream() .map(TabletCommitInfo::getTabletId).collect(Collectors.toList()); List tabletMetaList = tabletInvertedIndex.getTabletMetaList(tabletIds); + HashMap tableIdtoRestoring = new HashMap<>(); for (int i = 0; i < tabletMetaList.size(); i++) { + // get partition and table of this tablet TabletMeta tabletMeta = tabletMetaList.get(i); if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) { continue; @@ -518,29 +520,43 @@ private void checkCommitStatus(List
tableList, TransactionState transacti long tableId = tabletMeta.getTableId(); OlapTable tbl = (OlapTable) idToTable.get(tableId); if (tbl == null) { - // this can happen when tableId == -1 (tablet being dropping) - // or table really not exist. + // this can happen when tableId == -1 (tablet being dropping) or table really not exist. continue; } + // check relative partition restore here long partitionId = tabletMeta.getPartitionId(); if (tbl.getPartition(partitionId) == null) { - // this can happen when partitionId == -1 (tablet being dropping) - // or partition really not exist. + // this can happen when partitionId == -1 (tablet being dropping) or partition really not exist. continue; - } else if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) { + } + if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) { // partition in restore process which can not load data throw new LoadException("Table [" + tbl.getName() + "], Partition [" + tbl.getPartition(partitionId).getName() + "] is in restore process. Can not load into it"); } - boolean isPartitionRestoring = tbl.getPartitions().stream().anyMatch( - partition -> partition.getState() == PartitionState.RESTORE - ); - // restore table - if (!isPartitionRestoring && tbl.getState() == OlapTableState.RESTORE) { - throw new LoadException("Table " + tbl.getName() + " is in restore process. " - + "Can not load into it"); + // only do check when here's restore on this table now + if (tbl.getState() == OlapTableState.RESTORE) { + boolean hasPartitionRestoring = false; + if (tableIdtoRestoring.containsKey(tableId)) { + hasPartitionRestoring = tableIdtoRestoring.get(tableId); + } else { + for (Partition partition : tbl.getPartitions()) { + if (partition.getState() == PartitionState.RESTORE) { + hasPartitionRestoring = true; + break; + } + } + tableIdtoRestoring.put(tableId, hasPartitionRestoring); + } + // tbl RESTORE && all partition NOT RESTORE -> whole table restore + // tbl RESTORE && some partition RESTORE -> just partitions restore, NOT WHOLE TABLE + // so check wether the whole table restore here + if (!hasPartitionRestoring) { + throw new LoadException( + "Table " + tbl.getName() + " is in restore process. " + "Can not load into it"); + } } if (!tableToPartition.containsKey(tableId)) { From 9ce66718f075fb4b3b79fce36f0b7f01d580919d Mon Sep 17 00:00:00 2001 From: Lijia Liu Date: Thu, 29 Aug 2024 10:59:06 +0800 Subject: [PATCH 21/34] [improvement](nereids) Support * EXCEPT, * REPLACE and other expressions to appear in select statement at the same time (#40014) Follow #36777 Sometimes, users want to use * EXCEPT and other expressions in select clause. For example: SELECT * EXCEPT (value), sum(value) FROM tbl GROUP BY 1 Co-authored-by: liutang123 --- .../org/apache/doris/nereids/DorisParser.g4 | 14 +- .../doris/nereids/analyzer/UnboundStar.java | 91 ++++++++++-- .../nereids/parser/LogicalPlanBuilder.java | 134 +++++++----------- .../LogicalPlanBuilderForCreateView.java | 49 ------- .../rules/analysis/BindExpression.java | 57 +++++--- .../trees/plans/logical/LogicalAggregate.java | 4 + .../trees/plans/logical/LogicalProject.java | 52 +++---- .../trees/expressions/SelectExceptTest.java | 98 +++++++------ .../trees/expressions/SelectReplaceTest.java | 18 ++- .../nereids/trees/plans/PlanToStringTest.java | 2 +- .../data/correctness/test_select_except.out | 9 ++ .../correctness/test_select_except.groovy | 13 ++ 12 files changed, 288 insertions(+), 253 deletions(-) create mode 100644 regression-test/data/correctness/test_select_except.out diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 1c695f29c329d23..5480c2a3215b5db 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -739,11 +739,6 @@ selectClause selectColumnClause : namedExpressionSeq - | ASTERISK exceptOrReplace+ - ; - -exceptOrReplace - : (EXCEPT | REPLACE) LEFT_PAREN namedExpressionSeq RIGHT_PAREN ; whereClause @@ -1134,8 +1129,8 @@ primaryExpression | name=CAST LEFT_PAREN expression AS castDataType RIGHT_PAREN #cast | constant #constantDefault | interval #intervalLiteral - | ASTERISK #star - | qualifiedName DOT ASTERISK #star + | ASTERISK (exceptOrReplace)* #star + | qualifiedName DOT ASTERISK (exceptOrReplace)* #star | CHAR LEFT_PAREN arguments+=expression (COMMA arguments+=expression)* (USING charSet=identifierOrText)? @@ -1158,6 +1153,11 @@ primaryExpression | primaryExpression COLLATE (identifier | STRING_LITERAL | DEFAULT) #collate ; +exceptOrReplace + : EXCEPT LEFT_PAREN namedExpressionSeq RIGHT_PAREN #except + | REPLACE LEFT_PAREN namedExpressionSeq RIGHT_PAREN #replace + ; + castDataType : dataType |(SIGNED|UNSIGNED) (INT|INTEGER)? diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index 2875036eb074931..6d8ed904ec109d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * Star expression. @@ -39,21 +40,78 @@ public class UnboundStar extends NamedExpression implements LeafExpression, Unbo // the start and end position of the sql substring(e.g. "*", "table.*") private final Optional> indexInSqlString; + private final List exceptedSlots; + + private final List replacedAlias; + public UnboundStar(List qualifier) { super(ImmutableList.of()); this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); this.indexInSqlString = Optional.empty(); + this.exceptedSlots = ImmutableList.of(); + this.replacedAlias = ImmutableList.of(); } public UnboundStar(List qualifier, Optional> indexInSqlString) { super(ImmutableList.of()); this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); this.indexInSqlString = indexInSqlString; + this.exceptedSlots = ImmutableList.of(); + this.replacedAlias = ImmutableList.of(); + } + + /** + * The star expression is used in the select list, and the exceptedSlots is the column that is not selected. + * + * @param qualifier for example, "table1.*" + * @param exceptedSlots for example, * EXCEPT(a, b) + * @param replacedAlias for example, * REPLACE(a + 1 AS a) + */ + public UnboundStar(List qualifier, List exceptedSlots, + List replacedAlias) { + super(ImmutableList.of()); + this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); + this.indexInSqlString = Optional.empty(); + this.exceptedSlots = Objects.requireNonNull(ImmutableList.copyOf(exceptedSlots), + "except columns can not be null"); + this.replacedAlias = Objects.requireNonNull(ImmutableList.copyOf(replacedAlias), + "replace columns can not be null"); + } + + /** + * The star expression is used in the select list, and the exceptedSlots is the column that is not selected. + * + * @param qualifier for example, "table1.*" + * @param exceptedSlots for example, * EXCEPT(a, b) + * @param replacedAlias for example, * REPLACE(a + 1 AS a) + * @param indexInSqlString see {@link UnboundStar#indexInSqlString} + */ + public UnboundStar(List qualifier, List exceptedSlots, List replacedAlias, + Optional> indexInSqlString) { + super(ImmutableList.of()); + this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); + this.indexInSqlString = indexInSqlString; + this.exceptedSlots = Objects.requireNonNull(ImmutableList.copyOf(exceptedSlots), + "except columns can not be null"); + this.replacedAlias = Objects.requireNonNull(ImmutableList.copyOf(replacedAlias), + "replace columns can not be null"); } @Override public String toSql() { - return Utils.qualifiedName(qualifier, "*"); + StringBuilder builder = new StringBuilder(); + builder.append(Utils.qualifiedName(qualifier, "*")); + if (!exceptedSlots.isEmpty()) { + String exceptStr = exceptedSlots.stream().map(NamedExpression::toSql) + .collect(Collectors.joining(", ", " EXCEPT(", ")")); + builder.append(exceptStr); + } + if (!replacedAlias.isEmpty()) { + String replaceStr = replacedAlias.stream().map(NamedExpression::toSql) + .collect(Collectors.joining(", ", " REPLACE(", ")")); + builder.append(replaceStr); + } + return builder.toString(); } @Override @@ -66,6 +124,15 @@ public String toString() { return toSql(); } + public Optional> getIndexInSqlString() { + return indexInSqlString; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitUnboundStar(this, context); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -78,24 +145,24 @@ public boolean equals(Object o) { return false; } UnboundStar that = (UnboundStar) o; - return qualifier.equals(that.qualifier); - } - - public Optional> getIndexInSqlString() { - return indexInSqlString; + return qualifier.equals(that.qualifier) && exceptedSlots.equals(that.exceptedSlots) && replacedAlias.equals( + that.replacedAlias); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), qualifier); + return Objects.hash(super.hashCode(), qualifier, exceptedSlots, replacedAlias); } - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitUnboundStar(this, context); + public UnboundStar withIndexInSql(Pair index) { + return new UnboundStar(qualifier, exceptedSlots, replacedAlias, Optional.ofNullable(index)); } - public UnboundStar withIndexInSql(Pair index) { - return new UnboundStar(qualifier, Optional.ofNullable(index)); + public List getReplacedAlias() { + return replacedAlias; + } + + public List getExceptedSlots() { + return exceptedSlots; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index d82fd817370c08e..af7d6cd63affd27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -96,6 +96,8 @@ import org.apache.doris.nereids.DorisParser.DropMTMVContext; import org.apache.doris.nereids.DorisParser.DropProcedureContext; import org.apache.doris.nereids.DorisParser.ElementAtContext; +import org.apache.doris.nereids.DorisParser.ExceptContext; +import org.apache.doris.nereids.DorisParser.ExceptOrReplaceContext; import org.apache.doris.nereids.DorisParser.ExistContext; import org.apache.doris.nereids.DorisParser.ExplainContext; import org.apache.doris.nereids.DorisParser.ExportContext; @@ -160,6 +162,7 @@ import org.apache.doris.nereids.DorisParser.RefreshTriggerContext; import org.apache.doris.nereids.DorisParser.RegularQuerySpecificationContext; import org.apache.doris.nereids.DorisParser.RelationContext; +import org.apache.doris.nereids.DorisParser.ReplaceContext; import org.apache.doris.nereids.DorisParser.ResumeMTMVContext; import org.apache.doris.nereids.DorisParser.RollupDefContext; import org.apache.doris.nereids.DorisParser.RollupDefsContext; @@ -488,7 +491,6 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.RuleNode; import org.antlr.v4.runtime.tree.TerminalNode; -import org.springframework.util.CollectionUtils; import java.math.BigDecimal; import java.math.BigInteger; @@ -1355,10 +1357,6 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte LogicalPlan selectPlan; LogicalPlan relation; if (ctx.fromClause() == null) { - SelectColumnClauseContext columnCtx = selectCtx.selectColumnClause(); - if (!CollectionUtils.isEmpty(columnCtx.exceptOrReplace())) { - throw new ParseException("select with modifiers cannot be used in one row relation", selectCtx); - } relation = new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of(new UnboundAlias(Literal.of(0)))); } else { @@ -1521,7 +1519,46 @@ public Expression visitStar(StarContext ctx) { } else { target = ImmutableList.of(); } - return new UnboundStar(target); + List exceptOrReplaceList = ctx.exceptOrReplace(); + if (exceptOrReplaceList != null && !exceptOrReplaceList.isEmpty()) { + List finalExpectSlots = ImmutableList.of(); + List finalReplacedAlias = ImmutableList.of(); + for (ExceptOrReplaceContext exceptOrReplace : exceptOrReplaceList) { + if (exceptOrReplace instanceof ExceptContext) { + if (!finalExpectSlots.isEmpty()) { + throw new ParseException("only one except clause is supported", ctx); + } + ExceptContext exceptContext = (ExceptContext) exceptOrReplace; + List expectSlots = getNamedExpressions(exceptContext.namedExpressionSeq()); + boolean allSlots = expectSlots.stream().allMatch(UnboundSlot.class::isInstance); + if (expectSlots.isEmpty() || !allSlots) { + throw new ParseException( + "only column name is supported in except clause", ctx); + } + finalExpectSlots = expectSlots; + } else if (exceptOrReplace instanceof ReplaceContext) { + if (!finalReplacedAlias.isEmpty()) { + throw new ParseException("only one replace clause is supported", ctx); + } + ReplaceContext replaceContext = (ReplaceContext) exceptOrReplace; + List expectAlias = getNamedExpressions(replaceContext.namedExpressionSeq()); + boolean allAlias = expectAlias.stream() + .allMatch(e -> e instanceof UnboundAlias + && ((UnboundAlias) e).getAlias().isPresent()); + if (expectAlias.isEmpty() || !allAlias) { + throw new ParseException( + "only alias is supported in select-replace clause", ctx); + } + finalReplacedAlias = expectAlias; + } else { + throw new ParseException("Unsupported except or replace clause: " + exceptOrReplace.getText(), + ctx); + } + } + return new UnboundStar(target, finalExpectSlots, finalReplacedAlias); + } else { + return new UnboundStar(target); + } }); } @@ -3032,46 +3069,8 @@ protected LogicalPlan withSelectQuerySpecification( if (!(aggregate instanceof Aggregate) && havingClause.isPresent()) { // create a project node for pattern match of ProjectToGlobalAggregate rule // then ProjectToGlobalAggregate rule can insert agg node as LogicalHaving node's child - LogicalPlan project; - if (!CollectionUtils.isEmpty(selectColumnCtx.exceptOrReplace())) { - List except = ImmutableList.of(); - List replace = ImmutableList.of(); - boolean hasExcept = false; - boolean hasReplace = false; - for (DorisParser.ExceptOrReplaceContext er : selectColumnCtx.exceptOrReplace()) { - if ((hasReplace && er.REPLACE() != null) || (hasExcept && er.EXCEPT() != null)) { - throw new ParseException("only one except or replace is supported", selectColumnCtx); - } - if (er.EXCEPT() != null) { - hasExcept = true; - except = getNamedExpressions(er.namedExpressionSeq()); - if (!except.stream().allMatch(UnboundSlot.class::isInstance)) { - throw new ParseException( - "only column name is supported in except clause", selectColumnCtx); - } - } else if (er.REPLACE() != null) { - hasReplace = true; - replace = getNamedExpressions(er.namedExpressionSeq()); - boolean isEmpty = replace.isEmpty(); - boolean allAlias = replace.stream() - .allMatch(e -> e instanceof UnboundAlias - && ((UnboundAlias) e).getAlias().isPresent()); - if (isEmpty || !allAlias) { - throw new ParseException( - "only alias is supported in select-replace clause", selectColumnCtx); - } - } else { - throw new ParseException("only except or replace is supported", selectColumnCtx); - } - } - UnboundStar star = new UnboundStar(ImmutableList.of()); - project = new LogicalProject<>( - ImmutableList.of(star), except, replace, isDistinct, aggregate); - } else { - List projects = getNamedExpressions(selectColumnCtx.namedExpressionSeq()); - project = new LogicalProject<>( - projects, ImmutableList.of(), ImmutableList.of(), isDistinct, aggregate); - } + List projects = getNamedExpressions(selectColumnCtx.namedExpressionSeq()); + LogicalPlan project = new LogicalProject<>(projects, isDistinct, aggregate); return new LogicalHaving<>(ExpressionUtils.extractConjunctionToSet( getExpression((havingClause.get().booleanExpression()))), project); } else { @@ -3268,49 +3267,18 @@ protected LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContex if (aggCtx.isPresent()) { if (isDistinct) { return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())), - Collections.emptyList(), Collections.emptyList(), isDistinct, input); + isDistinct, input); } else { return input; } } else { - if (!CollectionUtils.isEmpty(selectCtx.exceptOrReplace())) { - List except = ImmutableList.of(); - List replace = ImmutableList.of(); - boolean hasExcept = false; - boolean hasReplace = false; - for (DorisParser.ExceptOrReplaceContext er : selectCtx.exceptOrReplace()) { - if ((hasReplace && er.REPLACE() != null) || (hasExcept && er.EXCEPT() != null)) { - throw new ParseException("only one except or replace is supported", selectCtx); - } - if (er.EXCEPT() != null) { - hasExcept = true; - except = getNamedExpressions(er.namedExpressionSeq()); - if (!except.stream().allMatch(UnboundSlot.class::isInstance)) { - throw new ParseException( - "only column name is supported in except clause", selectCtx); - } - } else if (er.REPLACE() != null) { - hasReplace = true; - replace = getNamedExpressions(er.namedExpressionSeq()); - boolean isEmpty = replace.isEmpty(); - boolean allAlias = replace.stream() - .allMatch(e -> e instanceof UnboundAlias - && ((UnboundAlias) e).getAlias().isPresent()); - if (isEmpty || !allAlias) { - throw new ParseException( - "only alias is supported in select-replace clause", selectCtx); - } - } else { - throw new ParseException("only except or replace is supported", selectCtx); - } + List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); + if (input instanceof UnboundOneRowRelation) { + if (projects.stream().anyMatch(project -> project instanceof UnboundStar)) { + throw new ParseException("SELECT * must have a FROM clause"); } - UnboundStar star = new UnboundStar(ImmutableList.of()); - return new LogicalProject<>(ImmutableList.of(star), except, replace, isDistinct, input); - } else { - List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); - return new LogicalProject<>( - projects, Collections.emptyList(), Collections.emptyList(), isDistinct, input); } + return new LogicalProject<>(projects, isDistinct, input); } }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForCreateView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForCreateView.java index 2109ea5969d0bd9..cb5005d48835f00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForCreateView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForCreateView.java @@ -24,17 +24,13 @@ import org.apache.doris.nereids.DorisParser.ColumnReferenceContext; import org.apache.doris.nereids.DorisParser.DereferenceContext; import org.apache.doris.nereids.DorisParser.GroupingElementContext; -import org.apache.doris.nereids.DorisParser.HavingClauseContext; import org.apache.doris.nereids.DorisParser.IdentifierContext; import org.apache.doris.nereids.DorisParser.LateralViewContext; import org.apache.doris.nereids.DorisParser.MultipartIdentifierContext; import org.apache.doris.nereids.DorisParser.NamedExpressionContext; -import org.apache.doris.nereids.DorisParser.SelectClauseContext; -import org.apache.doris.nereids.DorisParser.SelectColumnClauseContext; import org.apache.doris.nereids.DorisParser.StarContext; import org.apache.doris.nereids.DorisParser.TableAliasContext; import org.apache.doris.nereids.DorisParser.TableNameContext; -import org.apache.doris.nereids.DorisParser.WhereClauseContext; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -42,19 +38,15 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import org.antlr.v4.runtime.ParserRuleContext; -import org.springframework.util.CollectionUtils; import java.util.Map; -import java.util.Optional; /**LogicalPlanBuilderForCreateView*/ public class LogicalPlanBuilderForCreateView extends LogicalPlanBuilder { @@ -94,47 +86,6 @@ public LogicalSubQueryAlias visitAliasQuery(AliasQueryContext ctx) { return super.visitAliasQuery(ctx); } - @Override - protected LogicalPlan withSelectQuerySpecification( - ParserRuleContext ctx, - LogicalPlan inputRelation, - SelectClauseContext selectClause, - Optional whereClause, - Optional aggClause, - Optional havingClause) { - LogicalPlan plan = super.withSelectQuerySpecification(ctx, inputRelation, selectClause, whereClause, - aggClause, havingClause); - SelectColumnClauseContext selectColumnCtx = selectClause.selectColumnClause(); - if ((!aggClause.isPresent() || isRepeat(aggClause.get())) && havingClause.isPresent() - && !CollectionUtils.isEmpty(selectColumnCtx.exceptOrReplace()) - && plan instanceof LogicalHaving && plan.child(0) instanceof LogicalProject) { - LogicalHaving> having = (LogicalHaving) plan; - LogicalProject project = having.child(); - UnboundStar star = (UnboundStar) project.getProjects().get(0); - star = star.withIndexInSql(Pair.of(selectColumnCtx.start.getStartIndex(), - selectColumnCtx.stop.getStopIndex())); - project = project.withProjects(ImmutableList.of(star)); - return (LogicalPlan) plan.withChildren(project); - } else { - return plan; - } - } - - @Override - protected LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext selectCtx, - Optional aggCtx, boolean isDistinct) { - LogicalPlan plan = super.withProjection(input, selectCtx, aggCtx, isDistinct); - if (!aggCtx.isPresent() && !CollectionUtils.isEmpty(selectCtx.exceptOrReplace()) - && plan instanceof LogicalProject) { - LogicalProject project = (LogicalProject) plan; - UnboundStar star = (UnboundStar) project.getProjects().get(0); - star = star.withIndexInSql(Pair.of(selectCtx.start.getStartIndex(), selectCtx.stop.getStopIndex())); - return project.withProjects(ImmutableList.of(star)); - } else { - return plan; - } - } - @Override protected LogicalPlan withTableAlias(LogicalPlan plan, TableAliasContext ctx) { if (ctx.strictIdentifier() == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index cb36128d6c3338f..ea1d923f83a3f97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -620,29 +620,26 @@ private Plan bindProject(MatchingContext> ctx) { SimpleExprAnalyzer analyzer = buildSimpleExprAnalyzer( project, cascadesContext, project.children(), true, true); - List excepts = project.getExcepts(); - Supplier> boundExcepts = Suppliers.memoize( - () -> analyzer.analyzeToSet(project.getExcepts())); - - List replaces = project.getReplaces(); - Supplier> boundReplaces = Suppliers.memoize( - () -> analyzer.analyzeToList(project.getReplaces())); - - Builder boundProjections = ImmutableList.builderWithExpectedSize(project.arity()); + Builder boundProjections = ImmutableList.builderWithExpectedSize(project.getProjects().size()); StatementContext statementContext = ctx.statementContext; for (Expression expression : project.getProjects()) { Expression expr = analyzer.analyze(expression); if (!(expr instanceof BoundStar)) { boundProjections.add((NamedExpression) expr); } else { + UnboundStar unboundStar = (UnboundStar) expression; + List excepts = unboundStar.getExceptedSlots(); + Set boundExcepts = Suppliers.memoize(() -> analyzer.analyzeToSet(excepts)).get(); BoundStar boundStar = (BoundStar) expr; - List slots = boundStar.getSlots(); - if (!excepts.isEmpty()) { - slots = Utils.filterImmutableList(slots, slot -> !boundExcepts.get().contains((Slot) slot)); - } + + List slots = exceptStarSlots(boundExcepts, boundStar); + + List replaces = unboundStar.getReplacedAlias(); if (!replaces.isEmpty()) { final Map replaceMap = new HashMap<>(); final Set replaced = new HashSet<>(); + Supplier> boundReplaces = Suppliers.memoize( + () -> analyzer.analyzeToList(replaces)); for (NamedExpression replace : boundReplaces.get()) { Preconditions.checkArgument(replace instanceof Alias); Alias alias = (Alias) replace; @@ -654,7 +651,7 @@ private Plan bindProject(MatchingContext> ctx) { replaceMap.put(slot, alias); } - Collection c = CollectionUtils.intersection(boundExcepts.get(), replaceMap.keySet()); + Collection c = CollectionUtils.intersection(boundExcepts, replaceMap.keySet()); if (!c.isEmpty()) { throw new AnalysisException("Replace column name: " + c + " is in excepts"); } @@ -676,7 +673,6 @@ private Plan bindProject(MatchingContext> ctx) { // for create view stmt expand star List slotsForLambda = slots; - UnboundStar unboundStar = (UnboundStar) expression; unboundStar.getIndexInSqlString().ifPresent(pair -> { statementContext.addIndexInSqlToString(pair, toSqlWithBackquote(slotsForLambda)); }); @@ -701,6 +697,17 @@ private Plan bindFilter(MatchingContext> ctx) { return new LogicalFilter<>(boundConjuncts.build(), filter.child()); } + private List exceptStarSlots(Set boundExcepts, BoundStar boundStar) { + List slots = boundStar.getSlots(); + if (!boundExcepts.isEmpty()) { + slots = Utils.filterImmutableList(slots, slot -> !boundExcepts.contains(slot)); + if (slots.isEmpty()) { + throw new AnalysisException("All slots in * EXCEPT clause are excepted"); + } + } + return slots; + } + private Plan bindAggregate(MatchingContext> ctx) { LogicalAggregate agg = ctx.root; CascadesContext cascadesContext = ctx.cascadesContext; @@ -709,13 +716,29 @@ private Plan bindAggregate(MatchingContext> ctx) { agg, cascadesContext, agg.children(), true, true); List boundAggOutput = aggOutputAnalyzer.analyzeToList(agg.getOutputExpressions()); List boundProjections = new ArrayList<>(boundAggOutput.size()); - for (NamedExpression output : boundAggOutput) { + for (int i = 0; i < boundAggOutput.size(); i++) { + NamedExpression output = boundAggOutput.get(i); if (output instanceof BoundStar) { - boundProjections.addAll(((BoundStar) output).getSlots()); + UnboundStar unboundStar = (UnboundStar) agg.getOutputExpression(i); + if (!unboundStar.getReplacedAlias().isEmpty()) { + // If user use * replace in agg, witch slot the group by clause should reference is ambiguous. + // For example, select * replace (col % 2 as col) from t group by col + // The sql should be rewritten like follows: + // select * except (col), col % 2 as col1 from t group by ..., col1 + // or + // select * except (col), col % 2 as col1 from t group by ..., col -- group by origin col in t + throw new AnalysisException("* REPLACE in agg clause is not supported, use * EXCEPT instead"); + } + List excepts = unboundStar.getExceptedSlots(); + Set boundExcepts = Suppliers.memoize( + () -> aggOutputAnalyzer.analyzeToSet(excepts)).get(); + List slots = exceptStarSlots(boundExcepts, (BoundStar) output); + boundProjections.addAll(slots); } else { boundProjections.add(output); } } + Supplier aggOutputScopeWithoutAggFun = buildAggOutputScopeWithoutAggFun(boundProjections, cascadesContext); List boundGroupBy = bindGroupBy( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 9f65988da95358b..2798b1aef0102b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -157,6 +157,10 @@ public List getOutputExpressions() { return outputExpressions; } + public NamedExpression getOutputExpression(int index) { + return outputExpressions.get(index); + } + public String getOutputExprsSql() { return outputExpressions.stream().map(ExpressionTrait::toSql).collect(Collectors.joining(", ")); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index f6e076d87b89623..1484030e9c25a9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -58,28 +58,24 @@ public class LogicalProject extends LogicalUnary projects; private final Supplier> projectsSet; - private final List excepts; - private final List replaces; private final boolean isDistinct; public LogicalProject(List projects, CHILD_TYPE child) { - this(projects, ImmutableList.of(), ImmutableList.of(), false, ImmutableList.of(child)); + this(projects, false, ImmutableList.of(child)); } - public LogicalProject(List projects, List excepts, List replaces, - boolean isDistinct, List child) { - this(projects, excepts, replaces, isDistinct, Optional.empty(), Optional.empty(), child); + public LogicalProject(List projects, boolean isDistinct, List child) { + this(projects, isDistinct, Optional.empty(), Optional.empty(), child); } - public LogicalProject(List projects, List excepts, List replaces, - boolean isDistinct, Plan child) { - this(projects, excepts, replaces, isDistinct, + public LogicalProject(List projects, boolean isDistinct, Plan child) { + this(projects, isDistinct, Optional.empty(), Optional.empty(), ImmutableList.of(child)); } - private LogicalProject(List projects, List excepts, - List replaces, boolean isDistinct, Optional groupExpression, - Optional logicalProperties, List child) { + private LogicalProject(List projects, boolean isDistinct, + Optional groupExpression, Optional logicalProperties, + List child) { super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties, child); Preconditions.checkArgument(projects != null, "projects can not be null"); // only ColumnPrune rule may produce empty projects, this happens in rewrite phase @@ -90,8 +86,6 @@ private LogicalProject(List projects, List exc ? ImmutableList.of(ExpressionUtils.selectMinimumColumn(child.get(0).getOutput())) : projects; this.projectsSet = Suppliers.memoize(() -> ImmutableSet.copyOf(this.projects)); - this.excepts = Utils.fastToImmutableList(excepts); - this.replaces = Utils.fastToImmutableList(replaces); this.isDistinct = isDistinct; } @@ -105,14 +99,6 @@ public List getProjects() { return projects; } - public List getExcepts() { - return excepts; - } - - public List getReplaces() { - return replaces; - } - @Override public List computeOutput() { Builder slots = ImmutableList.builderWithExpectedSize(projects.size()); @@ -126,9 +112,7 @@ public List computeOutput() { public String toString() { return Utils.toSqlString("LogicalProject[" + id.asInt() + "]", "distinct", isDistinct, - "projects", projects, - "excepts", excepts, - "replaces", replaces + "projects", projects ); } @@ -152,8 +136,6 @@ public boolean equals(Object o) { } LogicalProject that = (LogicalProject) o; boolean equal = projectsSet.get().equals(that.projectsSet.get()) - && excepts.equals(that.excepts) - && replaces.equals(that.replaces) && isDistinct == that.isDistinct; // TODO: should add exprId for UnBoundStar and BoundStar for equality comparison if (!projects.isEmpty() && (projects.get(0) instanceof UnboundStar || projects.get(0) instanceof BoundStar)) { @@ -164,18 +146,18 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(projectsSet.get(), excepts, replaces, isDistinct); + return Objects.hash(projectsSet.get(), isDistinct); } @Override public LogicalProject withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalProject<>(projects, excepts, replaces, isDistinct, Utils.fastToImmutableList(children)); + return new LogicalProject<>(projects, isDistinct, Utils.fastToImmutableList(children)); } @Override public LogicalProject withGroupExpression(Optional groupExpression) { - return new LogicalProject<>(projects, excepts, replaces, isDistinct, + return new LogicalProject<>(projects, isDistinct, groupExpression, Optional.of(getLogicalProperties()), children); } @@ -183,20 +165,20 @@ public LogicalProject withGroupExpression(Optional groupE public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalProject<>(projects, excepts, replaces, isDistinct, + return new LogicalProject<>(projects, isDistinct, groupExpression, logicalProperties, children); } public LogicalProject withProjects(List projects) { - return new LogicalProject<>(projects, excepts, replaces, isDistinct, children); + return new LogicalProject<>(projects, isDistinct, children); } public LogicalProject withProjectsAndChild(List projects, Plan child) { - return new LogicalProject<>(projects, excepts, replaces, isDistinct, ImmutableList.of(child)); + return new LogicalProject<>(projects, isDistinct, ImmutableList.of(child)); } public LogicalProject withDistinct(boolean isDistinct) { - return new LogicalProject<>(projects, excepts, replaces, isDistinct, children); + return new LogicalProject<>(projects, isDistinct, children); } public boolean isDistinct() { @@ -218,8 +200,6 @@ public JSONObject toJson() { JSONObject logicalProject = super.toJson(); JSONObject properties = new JSONObject(); properties.put("Projects", projects.toString()); - properties.put("Excepts", excepts.toString()); - properties.put("Replaces", replaces.toString()); properties.put("IsDistinct", isDistinct); logicalProject.put("Properties", properties); return logicalProject; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java index b0f297144781154..d4dce7e80648a13 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java @@ -19,7 +19,10 @@ import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundStar; +import org.apache.doris.nereids.datasets.tpch.AnalyzeCheckTestBase; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.ParseException; +import org.apache.doris.nereids.pattern.PatternDescriptor; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.MemoPatternMatchSupported; @@ -31,22 +34,35 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -class SelectExceptTest implements MemoPatternMatchSupported { +class SelectExceptTest extends AnalyzeCheckTestBase implements MemoPatternMatchSupported { + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + connectContext.setDatabase("test"); + String t = "create table t1(" + + "id int, \n" + + "value int)\n" + + "distributed by hash(id) buckets 1\n" + + "properties('replication_num' = '1');"; + createTables(t); + } + @Test void testExcept() { LogicalOlapScan olapScan = PlanConstructor.newLogicalOlapScan(0, "t1", 1); LogicalProject project = new LogicalProject<>( - ImmutableList.of(new UnboundStar(ImmutableList.of("db", "t1"))), - ImmutableList.of(new UnboundSlot("db", "t1", "id")), - ImmutableList.of(), - false, - olapScan); + ImmutableList.of( + new UnboundStar(ImmutableList.of("db", "t1"), + ImmutableList.of(new UnboundSlot("db", "t1", "id")), + ImmutableList.of() + )), false, olapScan); PlanChecker.from(MemoTestUtils.createConnectContext()) .analyze(project) .matches( logicalProject( logicalOlapScan() - ).when(proj -> proj.getExcepts().size() == 1 && proj.getProjects().size() == 1) + ).when(proj -> proj.getProjects().size() == 1 && proj.getProjects().get(0).getName().equals("name")) ); } @@ -59,52 +75,52 @@ void testParse() { logicalCheckPolicy( unboundRelation() ) - ).when(project -> project.getExcepts().size() == 2 - && project.getProjects().get(0) instanceof UnboundStar) + ).when(project -> project.getProjects().size() == 1 + && project.getProjects().get(0) instanceof UnboundStar + && ((UnboundStar) project.getProjects().get(0)).getExceptedSlots().size() == 2) )); String sql2 = "select k1, k2, v1, v2 except(v1, v2) from t1"; Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext()) - .checkParse(sql2, (checker) -> checker.matches( - logicalProject( - logicalCheckPolicy( - unboundRelation() - ) - ).when(project -> project.getExcepts().size() == 2 - && project.getProjects().get(0) instanceof UnboundStar) - ))); + .parse(sql2)); String sql3 = "select * except(v1, v2)"; Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext()) - .checkParse(sql3, (checker) -> checker.matches( - logicalProject( - logicalCheckPolicy( - unboundRelation() - ) - ).when(project -> project.getExcepts().size() == 2 - && project.getProjects().get(0) instanceof UnboundStar) - ))); + .parse(sql3)); String sql4 = "select * except() from t1"; Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext()) - .checkParse(sql4, (checker) -> checker.matches( - logicalProject( - logicalCheckPolicy( - unboundRelation() - ) - ).when(project -> project.getExcepts().size() == 2 - && project.getProjects().get(0) instanceof UnboundStar) - ))); + .parse(sql4)); String sql5 = "select * except(v1 + v2, v3 as k3) from t1"; Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext()) - .checkParse(sql5, (checker) -> checker.matches( - logicalProject( - logicalCheckPolicy( - unboundRelation() - ) - ).when(project -> project.getExcepts().size() == 2 - && project.getProjects().get(0) instanceof UnboundStar) - ))); + .parse(sql5)); + + String sql6 = "select * except(id name) from t1"; + Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .parse(sql6)); + } + + @Test + public void testExceptAnalyze() { + PatternDescriptor expected = logicalAggregate( + logicalProject( + logicalOlapScan() + ).when(proj -> proj.getProjects().size() == 2) + ).when(agg -> agg.getOutputExpressions().size() == 2 + && agg.getOutputExpression(0).getName().equals("id") + && agg.getOutputExpression(1).getName().equals("value") + && agg.getGroupByExpressions().size() == 1 + ); + String sql1 = "select * except(value), sum(value) as value from t1 group by id"; + PlanChecker.from(connectContext) + .parse(sql1).analyze().matches(expected); + + String sql2 = "select * except(value), sum(value) as value from t1 group by 1"; + PlanChecker.from(connectContext) + .parse(sql2).analyze().matches(expected); + + String sql3 = "select * except(id, value) from t1"; // All slots in * EXCEPT clause are excepted + Assertions.assertThrows(AnalysisException.class, () -> PlanChecker.from(connectContext).parse(sql3).analyze()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectReplaceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectReplaceTest.java index 47c079d45548c0b..a002fe1a9c04a97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectReplaceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectReplaceTest.java @@ -88,24 +88,28 @@ void testReplace() { String sql6 = "select * replace (1 as fake) from t1"; Assertions.assertThrows(NereidsException.class, () -> PlanChecker.from(connectContext).checkPlannerResult(sql6)); + + // agg not support replace + String sql7 = "select * replace (v2 + 1 as v2) from t2 group by id, k1, v2"; + Assertions.assertThrows(NereidsException.class, + () -> PlanChecker.from(connectContext).checkPlannerResult(sql7)); } @Test public void testReplace2() { LogicalOlapScan olapScan = PlanConstructor.newLogicalOlapScan(0, "t1", 1); LogicalProject project = new LogicalProject<>( - ImmutableList.of(new UnboundStar(ImmutableList.of("db", "t1"))), - ImmutableList.of(), - ImmutableList.of(new UnboundAlias(new UnboundSlot("name"), "id")), - false, - olapScan); + ImmutableList.of( + new UnboundStar(ImmutableList.of("db", "t1"), + ImmutableList.of(), + ImmutableList.of(new UnboundAlias(new UnboundSlot("name"), "id")) + )), false, olapScan); PlanChecker.from(MemoTestUtils.createConnectContext()) .analyze(project) .matches( logicalProject( logicalOlapScan() - ).when(proj -> proj.getReplaces().size() == 1 - && proj.getProjects().get(0).getName().equals("id")) + ).when(proj -> proj.getProjects().get(0).getName().equals("id")) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java index 38525eaf0a3f6f3..312afd22b9269d1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java @@ -91,7 +91,7 @@ public void testLogicalOlapScan() { public void testLogicalProject(@Mocked Plan child) { LogicalProject plan = new LogicalProject<>(ImmutableList.of( new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList())), child); - Assertions.assertTrue(plan.toString().matches("LogicalProject\\[\\d+\\] \\( distinct=false, projects=\\[a#\\d+], excepts=\\[], replaces=\\[] \\)")); + Assertions.assertTrue(plan.toString().matches("LogicalProject\\[\\d+\\] \\( distinct=false, projects=\\[a#\\d+] \\)")); } @Test diff --git a/regression-test/data/correctness/test_select_except.out b/regression-test/data/correctness/test_select_except.out new file mode 100644 index 000000000000000..cdec4aa6410d419 --- /dev/null +++ b/regression-test/data/correctness/test_select_except.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !except_agg -- +1 5 +2 4 + +-- !except_agg_ordinal -- +1 5 +2 4 + diff --git a/regression-test/suites/correctness/test_select_except.groovy b/regression-test/suites/correctness/test_select_except.groovy index 68eb3066312f062..de36dd1d048f074 100644 --- a/regression-test/suites/correctness/test_select_except.groovy +++ b/regression-test/suites/correctness/test_select_except.groovy @@ -63,6 +63,19 @@ suite("test_select_except") { """ exception "errCode" } + test { + sql """ + select * except (siteid, citycode, username, pv) from tbl_select_except""" + exception "errCode" + } + qt_except_agg """ + select * except (siteid, username, pv), sum(pv) + from tbl_select_except + group by citycode order by citycode""" + qt_except_agg_ordinal """ + select * except (siteid, username, pv), sum(pv) + from tbl_select_except + group by 1 order by citycode""" } finally { sql "drop table if exists tbl_select_except" } From 0a638e472ca59efc836a81cc8f6e7d691a3c93a2 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 29 Aug 2024 11:06:05 +0800 Subject: [PATCH 22/34] [case](mtmv) fix test_base_mtmv failed (#39794) errCode = 2, detailMessage = Table[t_test_base_mtmv_user]'s state(SCHEMA_CHANGE) is not NORMAL. Do not allow doing ALTER ops --- regression-test/suites/mtmv_p0/test_base_mtmv.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/regression-test/suites/mtmv_p0/test_base_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_mtmv.groovy index 0015a0fa13061de..fce6ae523b57c9f 100644 --- a/regression-test/suites/mtmv_p0/test_base_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_base_mtmv.groovy @@ -71,12 +71,14 @@ suite("test_base_mtmv","mtmv") { sql """ alter table ${tableName} add COLUMN new_col INT AFTER username; """ + assertEquals("FINISHED", getAlterColumnFinalState("${tableName}")) order_qt_add_column "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" // rename column sql """ alter table ${tableName} rename COLUMN new_col new_col_1; """ + assertEquals("FINISHED", getAlterColumnFinalState("${tableName}")) order_qt_rename_column "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" sql """ REFRESH MATERIALIZED VIEW ${mvName} AUTO @@ -88,6 +90,7 @@ suite("test_base_mtmv","mtmv") { sql """ alter table ${tableName} modify COLUMN new_col_1 BIGINT; """ + assertEquals("FINISHED", getAlterColumnFinalState("${tableName}")) order_qt_modify_column "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" sql """ REFRESH MATERIALIZED VIEW ${mvName} AUTO @@ -99,6 +102,7 @@ suite("test_base_mtmv","mtmv") { sql """ alter table ${tableName} drop COLUMN new_col_1; """ + assertEquals("FINISHED", getAlterColumnFinalState("${tableName}")) order_qt_drop_column "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" sql """ REFRESH MATERIALIZED VIEW ${mvName} AUTO From 4e6f5b0e33c994d5e80e988720989c4f02d77eb6 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 29 Aug 2024 11:06:50 +0800 Subject: [PATCH 23/34] [case](mtmv) MTMV upgrade downgrade case (#39789) --- .../test_upgrade_downgrade_mtmv.out | 7 +++ .../suites/mtmv_up_down_p0/load.groovy | 47 +++++++++++++++++++ .../test_upgrade_downgrade_mtmv.groovy | 38 +++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 regression-test/data/mtmv_up_down_p0/test_upgrade_downgrade_mtmv.out create mode 100644 regression-test/suites/mtmv_up_down_p0/load.groovy create mode 100644 regression-test/suites/mtmv_up_down_p0/test_upgrade_downgrade_mtmv.groovy diff --git a/regression-test/data/mtmv_up_down_p0/test_upgrade_downgrade_mtmv.out b/regression-test/data/mtmv_up_down_p0/test_upgrade_downgrade_mtmv.out new file mode 100644 index 000000000000000..3c49cb862ed6702 --- /dev/null +++ b/regression-test/data/mtmv_up_down_p0/test_upgrade_downgrade_mtmv.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !refresh_init -- +1 1 1 name1 + +-- !mtmv_sync -- +true + diff --git a/regression-test/suites/mtmv_up_down_p0/load.groovy b/regression-test/suites/mtmv_up_down_p0/load.groovy new file mode 100644 index 000000000000000..45a309ae0a772d8 --- /dev/null +++ b/regression-test/suites/mtmv_up_down_p0/load.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_downgrade_prepare_mtmv","p0,mtmv,restart_fe,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + String suiteName = "mtmv_up_down" + String hivePrefix = "hive2"; + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalog_name = "${hivePrefix}_${suiteName}_catalog" + String mvName = "${hivePrefix}_${suiteName}_mtmv" + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + create MATERIALIZED VIEW ${mvName} + REFRESH auto ON MANUAL + partition by(part1) + DISTRIBUTED BY RANDOM BUCKETS AUTO + PROPERTIES ('replication_num' = '1') + as select * from ${catalog_name}.multi_catalog.one_partition a inner join ${catalog_name}.multi_catalog.region b on a.id=b.r_regionkey; + """ + waitingMTMVTaskFinishedByMvName(mvName) +} diff --git a/regression-test/suites/mtmv_up_down_p0/test_upgrade_downgrade_mtmv.groovy b/regression-test/suites/mtmv_up_down_p0/test_upgrade_downgrade_mtmv.groovy new file mode 100644 index 000000000000000..a49bda7120217ae --- /dev/null +++ b/regression-test/suites/mtmv_up_down_p0/test_upgrade_downgrade_mtmv.groovy @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_downgrade_mtmv","p0,mtmv,restart_fe,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + String suiteName = "mtmv_up_down" + String dbName = context.config.getDbNameByFile(context.file) + String hivePrefix = "hive2"; + String catalog_name = "${hivePrefix}_${suiteName}_catalog" + String mvName = "${hivePrefix}_${suiteName}_mtmv" + // test data is normal + order_qt_refresh_init "SELECT * FROM ${mvName}" + // test is sync + order_qt_mtmv_sync "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + sql """ + REFRESH MATERIALIZED VIEW ${mvName} complete + """ + // test can refresh success + waitingMTMVTaskFinishedByMvName(mvName) +} From 964c33d30eb52f738d8629011f90e750feddf820 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 29 Aug 2024 11:10:48 +0800 Subject: [PATCH 24/34] [Fix](Variant) variant fallthrough with inverted index (#40069) When reading from segment, the schema type is variant, if we check type valid in `get_inverted_index`, the result should always return nullptr(since variant type it self does not support inverted index), but the actual storage could be `string` or etc.So we should ignore the type check and return the correct inverted index iterators introduced by #36163 --- be/src/olap/rowset/segment_v2/segment.cpp | 6 ++++-- be/src/olap/rowset/segment_v2/segment_iterator.cpp | 7 ++++++- regression-test/data/variant_github_events_p2/load.out | 10 ++++++++++ .../suites/variant_github_events_p2/load.groovy | 9 +++++++-- .../suites/variant_p0/with_index/load.groovy | 1 + .../suites/variant_p0/with_index/var_index.groovy | 2 ++ 6 files changed, 30 insertions(+), 5 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index ac54c9252c95cb4..64f58e546c22bac 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -203,7 +203,8 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o ColumnReader* reader = nullptr; if (col.is_extracted_column()) { auto relative_path = col.path_info_ptr()->copy_pop_front(); - const auto* node = _sub_column_tree[col.unique_id()].find_exact(relative_path); + int32_t unique_id = col.unique_id() > 0 ? col.unique_id() : col.parent_unique_id(); + const auto* node = _sub_column_tree[unique_id].find_exact(relative_path); reader = node != nullptr ? node->data.reader.get() : nullptr; } else { reader = _column_readers.contains(col.unique_id()) @@ -775,8 +776,9 @@ ColumnReader* Segment::_get_column_reader(const TabletColumn& col) { // init column iterator by path info if (col.has_path_info() || col.is_variant_type()) { auto relative_path = col.path_info_ptr()->copy_pop_front(); + int32_t unique_id = col.unique_id() > 0 ? col.unique_id() : col.parent_unique_id(); const auto* node = col.has_path_info() - ? _sub_column_tree[col.unique_id()].find_exact(relative_path) + ? _sub_column_tree[unique_id].find_exact(relative_path) : nullptr; if (node != nullptr) { return node->data.reader.get(); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index d4ae09b1bb739a5..c79621647414c7d 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1488,10 +1488,15 @@ Status SegmentIterator::_init_inverted_index_iterators() { } for (auto cid : _schema->column_ids()) { if (_inverted_index_iterators[cid] == nullptr) { + // Not check type valid, since we need to get inverted index for related variant type when reading the segment. + // If check type valid, we can not get inverted index for variant type, and result nullptr.The result for calling + // get_inverted_index with variant suffix should return corresponding inverted index meta. + bool check_inverted_index_by_type = false; // Use segment’s own index_meta, for compatibility with future indexing needs to default to lowercase. RETURN_IF_ERROR(_segment->new_inverted_index_iterator( _opts.tablet_schema->column(cid), - _segment->_tablet_schema->get_inverted_index(_opts.tablet_schema->column(cid)), + _segment->_tablet_schema->get_inverted_index(_opts.tablet_schema->column(cid), + check_inverted_index_by_type), _opts, &_inverted_index_iterators[cid])); } } diff --git a/regression-test/data/variant_github_events_p2/load.out b/regression-test/data/variant_github_events_p2/load.out index 8d5e3327e3c12e9..4bee99a71a99ea9 100644 --- a/regression-test/data/variant_github_events_p2/load.out +++ b/regression-test/data/variant_github_events_p2/load.out @@ -11,3 +11,13 @@ 5451 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?","gravatar_id":"","id":3437916,"login":"misol","url":"https://api.github.com/users/misol"},"created_at":"2015-01-01T02:48:28Z","id":"2489433218","org":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?","gravatar_id":"","id":1429259,"login":"xpressengine","url":"https://api.github.com/orgs/xpressengine"},"payload":{"action":"created","comment":{"body":"Html5 도 같이 지원하는 업로더였으면 좋겠어요! 구글링 해보면 꽤 나와요 :)","created_at":"2015-01-01T02:48:27Z","html_url":"https://github.com/xpressengine/xe-core/issues/1086#issuecomment-68479093","id":68479093,"issue_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086","updated_at":"2015-01-01T02:48:27Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/comments/68479093","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}},"issue":{"comments":4,"comments_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/comments","created_at":"2014-12-12T11:48:03Z","events_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/events","html_url":"https://github.com/xpressengine/xe-core/issues/1086","id":51797879,"labels_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/labels{/name}","locked":0,"number":1086,"state":"open","title":"파일 업로더 교체","updated_at":"2015-01-01T02:48:27Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086","user":{"avatar_url":"https://avatars.githubusercontent.com/u/53764?v=3","events_url":"https://api.github.com/users/bnu/events{/privacy}","followers_url":"https://api.github.com/users/bnu/followers","following_url":"https://api.github.com/users/bnu/following{/other_user}","gists_url":"https://api.github.com/users/bnu/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/bnu","id":53764,"login":"bnu","organizations_url":"https://api.github.com/users/bnu/orgs","received_events_url":"https://api.github.com/users/bnu/received_events","repos_url":"https://api.github.com/users/bnu/repos","site_admin":0,"starred_url":"https://api.github.com/users/bnu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bnu/subscriptions","type":"User","url":"https://api.github.com/users/bnu"}}},"public":1,"repo":{"id":7953576,"name":"xpressengine/xe-core","url":"https://api.github.com/repos/xpressengine/xe-core"},"type":"IssueCommentEvent"} 5995 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?","gravatar_id":"","id":3437916,"login":"misol","url":"https://api.github.com/users/misol"},"created_at":"2015-01-01T01:47:44Z","id":"2489414108","org":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?","gravatar_id":"","id":1429259,"login":"xpressengine","url":"https://api.github.com/orgs/xpressengine"},"payload":{"action":"opened","number":1120,"pull_request":{"_links":{"comments":{"href":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/comments"},"commits":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/commits"},"html":{"href":"https://github.com/xpressengine/xe-core/pull/1120"},"issue":{"href":"https://api.github.com/repos/xpressengine/xe-core/issues/1120"},"review_comment":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/comments/{number}"},"review_comments":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/comments"},"self":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120"},"statuses":{"href":"https://api.github.com/repos/xpressengine/xe-core/statuses/d2b05732abfd85020335ce272abd37c0ad1c6654"}},"additions":4748,"base":{"label":"xpressengine:develop","ref":"develop","repo":{"archive_url":"https://api.github.com/repos/xpressengine/xe-core/{archive_format}{/ref}","assignees_url":"https://api.github.com/repos/xpressengine/xe-core/assignees{/user}","blobs_url":"https://api.github.com/repos/xpressengine/xe-core/git/blobs{/sha}","branches_url":"https://api.github.com/repos/xpressengine/xe-core/branches{/branch}","clone_url":"https://github.com/xpressengine/xe-core.git","collaborators_url":"https://api.github.com/repos/xpressengine/xe-core/collaborators{/collaborator}","comments_url":"https://api.github.com/repos/xpressengine/xe-core/comments{/number}","commits_url":"https://api.github.com/repos/xpressengine/xe-core/commits{/sha}","compare_url":"https://api.github.com/repos/xpressengine/xe-core/compare/{base}...{head}","contents_url":"https://api.github.com/repos/xpressengine/xe-core/contents/{+path}","contributors_url":"https://api.github.com/repos/xpressengine/xe-core/contributors","created_at":"2013-02-01T07:16:05Z","default_branch":"master","description":"PHP Open Source CMS","downloads_url":"https://api.github.com/repos/xpressengine/xe-core/downloads","events_url":"https://api.github.com/repos/xpressengine/xe-core/events","fork":0,"forks":143,"forks_count":143,"forks_url":"https://api.github.com/repos/xpressengine/xe-core/forks","full_name":"xpressengine/xe-core","git_commits_url":"https://api.github.com/repos/xpressengine/xe-core/git/commits{/sha}","git_refs_url":"https://api.github.com/repos/xpressengine/xe-core/git/refs{/sha}","git_tags_url":"https://api.github.com/repos/xpressengine/xe-core/git/tags{/sha}","git_url":"git://github.com/xpressengine/xe-core.git","has_downloads":1,"has_issues":1,"has_pages":0,"has_wiki":1,"homepage":"http://www.xpressengine.com","hooks_url":"https://api.github.com/repos/xpressengine/xe-core/hooks","html_url":"https://github.com/xpressengine/xe-core","id":7953576,"issue_comment_url":"https://api.github.com/repos/xpressengine/xe-core/issues/comments/{number}","issue_events_url":"https://api.github.com/repos/xpressengine/xe-core/issues/events{/number}","issues_url":"https://api.github.com/repos/xpressengine/xe-core/issues{/number}","keys_url":"https://api.github.com/repos/xpressengine/xe-core/keys{/key_id}","labels_url":"https://api.github.com/repos/xpressengine/xe-core/labels{/name}","language":"PHP","languages_url":"https://api.github.com/repos/xpressengine/xe-core/languages","merges_url":"https://api.github.com/repos/xpressengine/xe-core/merges","milestones_url":"https://api.github.com/repos/xpressengine/xe-core/milestones{/number}","name":"xe-core","notifications_url":"https://api.github.com/repos/xpressengine/xe-core/notifications{?since,all,participating}","open_issues":156,"open_issues_count":156,"owner":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?v=3","events_url":"https://api.github.com/users/xpressengine/events{/privacy}","followers_url":"https://api.github.com/users/xpressengine/followers","following_url":"https://api.github.com/users/xpressengine/following{/other_user}","gists_url":"https://api.github.com/users/xpressengine/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/xpressengine","id":1429259,"login":"xpressengine","organizations_url":"https://api.github.com/users/xpressengine/orgs","received_events_url":"https://api.github.com/users/xpressengine/received_events","repos_url":"https://api.github.com/users/xpressengine/repos","site_admin":0,"starred_url":"https://api.github.com/users/xpressengine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xpressengine/subscriptions","type":"Organization","url":"https://api.github.com/users/xpressengine"},"private":0,"pulls_url":"https://api.github.com/repos/xpressengine/xe-core/pulls{/number}","pushed_at":"2014-12-31T08:52:46Z","releases_url":"https://api.github.com/repos/xpressengine/xe-core/releases{/id}","size":90250,"ssh_url":"git@github.com:xpressengine/xe-core.git","stargazers_count":149,"stargazers_url":"https://api.github.com/repos/xpressengine/xe-core/stargazers","statuses_url":"https://api.github.com/repos/xpressengine/xe-core/statuses/{sha}","subscribers_url":"https://api.github.com/repos/xpressengine/xe-core/subscribers","subscription_url":"https://api.github.com/repos/xpressengine/xe-core/subscription","svn_url":"https://github.com/xpressengine/xe-core","tags_url":"https://api.github.com/repos/xpressengine/xe-core/tags","teams_url":"https://api.github.com/repos/xpressengine/xe-core/teams","trees_url":"https://api.github.com/repos/xpressengine/xe-core/git/trees{/sha}","updated_at":"2014-12-30T00:05:52Z","url":"https://api.github.com/repos/xpressengine/xe-core","watchers":149,"watchers_count":149},"sha":"c3430d1c724f42154ca5dd648637c4df796d1708","user":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?v=3","events_url":"https://api.github.com/users/xpressengine/events{/privacy}","followers_url":"https://api.github.com/users/xpressengine/followers","following_url":"https://api.github.com/users/xpressengine/following{/other_user}","gists_url":"https://api.github.com/users/xpressengine/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/xpressengine","id":1429259,"login":"xpressengine","organizations_url":"https://api.github.com/users/xpressengine/orgs","received_events_url":"https://api.github.com/users/xpressengine/received_events","repos_url":"https://api.github.com/users/xpressengine/repos","site_admin":0,"starred_url":"https://api.github.com/users/xpressengine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xpressengine/subscriptions","type":"Organization","url":"https://api.github.com/users/xpressengine"}},"body":"https://github.com/xpressengine/xe-core/issues/634\\r\\n\\r\\n# Internet Explorer send punycode URL(ASCII) URL and non-alphabet\\r\\nUnicode URL URL as a referer. 인터넷 익스플로러는 리퍼러 주소로 퓨니코드 주소와 유니코드 URL을 섞어\\r\\n쓰고 있습니다. AJAX 통신에는 리퍼러로 Unicode를 사용하고 요청 호스트로는 퓨니코드 URL을 사용(이건 다국어 주소\\r\\n형식으로 접속하려면 이렇게 했어야 할 것)합니다.\\r\\n- XE strictly compare referer and server host for denying CSRF, but\\r\\npunycode URL and Unicode URL should be dealt as a same one. 그런데 XE는 리퍼러의\\r\\n호스트와 서버 호스트를 비교합니다. punycode로 쓰인 주소와 Unicode로 쓰인 주소는 같은 주소를 지시하더라도 문자열이\\r\\n다릅니다. 같은 주소를 지칭하는 다른 문자열을 punycode로 변환해서 같은 주소라고 인식할 수 있게 수정했습니다.\\r\\n- Fix checkCSRF function to deal both form as a same one.\\r\\n- Convert Unicode URL input to punycode URL on the Admin Default URL\\r\\nSettings. 관리자가 유니코드 형식으로 기본 주소를 입력하더라도, 퓨니코드로 변환해 저장하도록 했습니다. 퓨니코드로 저장하는\\r\\n것이 여러모로 유용하기 때문입니다.\\r\\n- For converting punycode URL, include IDNA coverting class. 퓨니코드와 유니코드\\r\\n간 변환을 위해서 IDNA 변환 클래스(LGPL사용권)를 포함시켰습니다.\\r\\n\\r\\n**이 수정을 하면 *한글 도메인에서 글 작성이 가능*합니다. 하지만, *파일 업로드의 경우는 SWF Uploader 이슈로 파일 업로드가 불가능*합니다. 이 문제는, HTML5를 지원하는 파일 업로더를 이용하면 해결됩니다. (워드 프레스 등의 해결법) HTML5를 지원하는 파일 업로더는 AXISJ 업로더 등을 포함해서 XE 공식 홈페이지 자료실에서 다운받아 사용할 수 있습니다.(에디터 변경)**","changed_files":8,"comments":0,"comments_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/comments","commits":1,"commits_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/commits","created_at":"2015-01-01T01:47:43Z","deletions":1,"diff_url":"https://github.com/xpressengine/xe-core/pull/1120.diff","head":{"label":"misol:support_punycode_domain","ref":"support_punycode_domain","repo":{"archive_url":"https://api.github.com/repos/misol/xe-core/{archive_format}{/ref}","assignees_url":"https://api.github.com/repos/misol/xe-core/assignees{/user}","blobs_url":"https://api.github.com/repos/misol/xe-core/git/blobs{/sha}","branches_url":"https://api.github.com/repos/misol/xe-core/branches{/branch}","clone_url":"https://github.com/misol/xe-core.git","collaborators_url":"https://api.github.com/repos/misol/xe-core/collaborators{/collaborator}","comments_url":"https://api.github.com/repos/misol/xe-core/comments{/number}","commits_url":"https://api.github.com/repos/misol/xe-core/commits{/sha}","compare_url":"https://api.github.com/repos/misol/xe-core/compare/{base}...{head}","contents_url":"https://api.github.com/repos/misol/xe-core/contents/{+path}","contributors_url":"https://api.github.com/repos/misol/xe-core/contributors","created_at":"2014-12-31T14:41:05Z","default_branch":"master","description":"PHP Open Source CMS","downloads_url":"https://api.github.com/repos/misol/xe-core/downloads","events_url":"https://api.github.com/repos/misol/xe-core/events","fork":1,"forks":0,"forks_count":0,"forks_url":"https://api.github.com/repos/misol/xe-core/forks","full_name":"misol/xe-core","git_commits_url":"https://api.github.com/repos/misol/xe-core/git/commits{/sha}","git_refs_url":"https://api.github.com/repos/misol/xe-core/git/refs{/sha}","git_tags_url":"https://api.github.com/repos/misol/xe-core/git/tags{/sha}","git_url":"git://github.com/misol/xe-core.git","has_downloads":1,"has_issues":0,"has_pages":0,"has_wiki":1,"homepage":"http://www.xpressengine.com","hooks_url":"https://api.github.com/repos/misol/xe-core/hooks","html_url":"https://github.com/misol/xe-core","id":28667946,"issue_comment_url":"https://api.github.com/repos/misol/xe-core/issues/comments/{number}","issue_events_url":"https://api.github.com/repos/misol/xe-core/issues/events{/number}","issues_url":"https://api.github.com/repos/misol/xe-core/issues{/number}","keys_url":"https://api.github.com/repos/misol/xe-core/keys{/key_id}","labels_url":"https://api.github.com/repos/misol/xe-core/labels{/name}","language":"PHP","languages_url":"https://api.github.com/repos/misol/xe-core/languages","merges_url":"https://api.github.com/repos/misol/xe-core/merges","milestones_url":"https://api.github.com/repos/misol/xe-core/milestones{/number}","name":"xe-core","notifications_url":"https://api.github.com/repos/misol/xe-core/notifications{?since,all,participating}","open_issues":0,"open_issues_count":0,"owner":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"},"private":0,"pulls_url":"https://api.github.com/repos/misol/xe-core/pulls{/number}","pushed_at":"2015-01-01T01:36:28Z","releases_url":"https://api.github.com/repos/misol/xe-core/releases{/id}","size":90250,"ssh_url":"git@github.com:misol/xe-core.git","stargazers_count":0,"stargazers_url":"https://api.github.com/repos/misol/xe-core/stargazers","statuses_url":"https://api.github.com/repos/misol/xe-core/statuses/{sha}","subscribers_url":"https://api.github.com/repos/misol/xe-core/subscribers","subscription_url":"https://api.github.com/repos/misol/xe-core/subscription","svn_url":"https://github.com/misol/xe-core","tags_url":"https://api.github.com/repos/misol/xe-core/tags","teams_url":"https://api.github.com/repos/misol/xe-core/teams","trees_url":"https://api.github.com/repos/misol/xe-core/git/trees{/sha}","updated_at":"2014-12-31T14:41:10Z","url":"https://api.github.com/repos/misol/xe-core","watchers":0,"watchers_count":0},"sha":"d2b05732abfd85020335ce272abd37c0ad1c6654","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}},"html_url":"https://github.com/xpressengine/xe-core/pull/1120","id":26739793,"issue_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120","locked":0,"mergeable_state":"unknown","merged":0,"number":1120,"patch_url":"https://github.com/xpressengine/xe-core/pull/1120.patch","review_comment_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/comments/{number}","review_comments":0,"review_comments_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/comments","state":"open","statuses_url":"https://api.github.com/repos/xpressengine/xe-core/statuses/d2b05732abfd85020335ce272abd37c0ad1c6654","title":"fix for Not-Alphabet URL document writing (#634)","updated_at":"2015-01-01T01:47:43Z","url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}}},"public":1,"repo":{"id":7953576,"name":"xpressengine/xe-core","url":"https://api.github.com/repos/xpressengine/xe-core"},"type":"PullRequestEvent"} +-- !sql -- +\N +\N +\N +\N +4748 + +-- !sql -- +135 + diff --git a/regression-test/suites/variant_github_events_p2/load.groovy b/regression-test/suites/variant_github_events_p2/load.groovy index 4e81dc2237c3804..e1742231afc9ab9 100644 --- a/regression-test/suites/variant_github_events_p2/load.groovy +++ b/regression-test/suites/variant_github_events_p2/load.groovy @@ -169,7 +169,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ // // build inverted index at middle of loading the data // ADD INDEX - sql """ ALTER TABLE github_events ADD INDEX idx_var (`v`) USING INVERTED PROPERTIES("parser" = "chinese", "parser_mode" = "fine_grained", "support_phrase" = "true") """ + sql """ ALTER TABLE github_events ADD INDEX idx_var (`v`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") """ wait_for_latest_op_on_table_finish("github_events", timeout) // 2022 @@ -214,7 +214,8 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ } while (running) } - + sql """set enable_match_without_inverted_index = false""" + // filter by bloom filter qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1;""" qt_sql """select * from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1 limit 10""" sql """select * from github_events order by k limit 10""" @@ -230,4 +231,8 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ sql """insert into github_events2 select * from github_events order by k""" sql """select v['payload']['commits'] from github_events order by k ;""" sql """select v['payload']['commits'] from github_events2 order by k ;""" + + // query with inverted index + qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where v["repo"]["name"] match 'xpressengine' order by 1;""" + qt_sql """select count() from github_events where v["repo"]["name"] match 'apache' order by 1;""" } \ No newline at end of file diff --git a/regression-test/suites/variant_p0/with_index/load.groovy b/regression-test/suites/variant_p0/with_index/load.groovy index 2ff781a2008ab01..93737e8a5b9dc8f 100644 --- a/regression-test/suites/variant_p0/with_index/load.groovy +++ b/regression-test/suites/variant_p0/with_index/load.groovy @@ -61,6 +61,7 @@ suite("regression_test_variant_with_index", "nonConcurrent"){ properties("replication_num" = "1", "disable_auto_compaction" = "true"); """ sql """insert into var_with_index values(1, '{"a" : 0, "b": 3}', 'hello world'), (2, '{"a" : 123}', 'world'),(3, '{"a" : 123}', 'hello world')""" + sql """set enable_match_without_inverted_index = false""" qt_sql_inv_1 """select v["a"] from var_with_index where inv match 'hello' order by k""" qt_sql_inv_2 """select v["a"] from var_with_index where inv match 'hello' and cast(v['a'] as int) > 0 order by k""" qt_sql_inv_3 """select * from var_with_index where inv match 'hello' and cast(v["a"] as int) > 0 order by k""" diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index 5f061f3208bab8e..a21d242954504a9 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -33,7 +33,9 @@ suite("regression_test_variant_var_index", "p0"){ sql """insert into var_index values(2, '{"a" : 18811, "b" : "hello world", "c" : 1181111}')""" sql """insert into var_index values(3, '{"a" : 18811, "b" : "hello wworld", "c" : 11111}')""" sql """insert into var_index values(4, '{"a" : 1234, "b" : "hello xxx world", "c" : 8181111}')""" + sql """set enable_match_without_inverted_index = false""" qt_sql """select * from var_index where cast(v["a"] as smallint) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 1024 order by k""" + sql """set enable_match_without_inverted_index = true""" sql """insert into var_index values(5, '{"a" : 123456789, "b" : 123456, "c" : 8181111}')""" qt_sql """select * from var_index where cast(v["a"] as int) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 11111 order by k""" // insert double/float/array/json From df079db57e495dc5f9b0c1ecd8f21818579d8358 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Thu, 29 Aug 2024 11:15:02 +0800 Subject: [PATCH 25/34] [chore](ci) remove ci pipeline cloud_p1 (#40087) It is enough to run p1 daily. --- .asf.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 821947fa1a0451b..e71e55de23fc83c 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -52,9 +52,7 @@ github: - Clang Formatter - CheckStyle - P0 Regression (Doris Regression) - - P1 Regression (Doris Regression) - External Regression (Doris External Regression) - - cloud_p1 (Doris Cloud Regression) - cloud_p0 (Doris Cloud Regression) - FE UT (Doris FE UT) - BE UT (Doris BE UT) @@ -114,7 +112,6 @@ github: - Clang Formatter - CheckStyle - P0 Regression (Doris Regression) - - P1 Regression (Doris Regression) - External Regression (Doris External Regression) - FE UT (Doris FE UT) - BE UT (Doris BE UT) From 9dbd4d951839d53c0663a1028d776246c2f7ab2a Mon Sep 17 00:00:00 2001 From: xzj7019 <131111794+xzj7019@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:26:51 +0800 Subject: [PATCH 26/34] [fix](coordinator) fix union all instance number (#39999) In following plan pattern, fragment 11 has 3 child fragments aligning with 3 exchange nodes and the union plan node only have 2 plan children, i.e, the aggr node and the exchange node(4). In current union node's parallelism computing, it used the plan children's size to do iteration to find the max parallel but ignore the last fragment's. Actually, it should use fragment's children(exclude unpartitioned ones) instead of plan node's children to do the iteration. 11:VUNION(703) | |----10:VAGGREGATE (update finalize)(697) | | group by: brand_name[#23], skc[#24] | | sortByGroupKey:false | | cardinality=1 | | final projections: '1', brand_name[#25], skc[#26] | | final project output tuple id: 11 | | distribute expr lists: skc[#24] | | | 9:VHASH JOIN(691) | | join op: INNER JOIN(PARTITIONED)[] | | equal join conjunct: (skc[#19] = skc_code[#12]) | | cardinality=1 | | vec output tuple id: 9 | | output tuple id: 9 | | vIntermediate tuple ids: 8 | | hash output slot ids: 18 19 | | final projections: brand_name[#20], skc[#21] | | final project output tuple id: 9 | | distribute expr lists: skc[#19] | | distribute expr lists: skc_code[#12] | | | |----6:VEXCHANGE | | offset: 0 | | distribute expr lists: skc_code[#12] | | | 8:VEXCHANGE | offset: 0 | distribute expr lists: | 4:VEXCHANGE offset: 0 distribute expr lists: --- .../java/org/apache/doris/qe/Coordinator.java | 43 ++++----- .../query_p0/union/test_union_instance.groovy | 88 +++++++++++++++++++ 2 files changed, 111 insertions(+), 20 deletions(-) create mode 100644 regression-test/suites/query_p0/union/test_union_instance.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 018602fafd39481..0cbe3eebbab9f1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -1788,7 +1788,6 @@ protected void computeFragmentHosts() throws Exception { } Pair pairNodes = findLeftmostNode(fragment.getPlanRoot()); - PlanNode fatherNode = pairNodes.first; PlanNode leftMostNode = pairNodes.second; /* @@ -1803,25 +1802,8 @@ protected void computeFragmentHosts() throws Exception { // (Case B) // there is no leftmost scan; we assign the same hosts as those of our // input fragment which has a higher instance_number - - int inputFragmentIndex = 0; - int maxParallelism = 0; - // If the fragment has three children, then the first child and the second child are - // the children(both exchange node) of shuffle HashJoinNode, - // and the third child is the right child(ExchangeNode) of broadcast HashJoinNode. - // We only need to pay attention to the maximum parallelism among - // the two ExchangeNodes of shuffle HashJoinNode. - int childrenCount = (fatherNode != null) ? fatherNode.getChildren().size() : 1; - for (int j = 0; j < childrenCount; j++) { - int currentChildFragmentParallelism - = fragmentExecParamsMap.get(fragment.getChild(j).getFragmentId()).instanceExecParams.size(); - if (currentChildFragmentParallelism > maxParallelism) { - maxParallelism = currentChildFragmentParallelism; - inputFragmentIndex = j; - } - } - - PlanFragmentId inputFragmentId = fragment.getChild(inputFragmentIndex).getFragmentId(); + int maxParallelFragmentIndex = findMaxParallelFragmentIndex(fragment); + PlanFragmentId inputFragmentId = fragment.getChild(maxParallelFragmentIndex).getFragmentId(); // AddAll() soft copy() int exchangeInstances = -1; if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) { @@ -1978,6 +1960,27 @@ protected void computeFragmentHosts() throws Exception { } } + private int findMaxParallelFragmentIndex(PlanFragment fragment) { + Preconditions.checkState(!fragment.getChildren().isEmpty(), "fragment has no children"); + + // exclude broadcast join right side's child fragments + List childFragmentCandidates = fragment.getChildren().stream() + .filter(e -> e.getOutputPartition() != DataPartition.UNPARTITIONED) + .collect(Collectors.toList()); + + int maxParallelism = 0; + int maxParaIndex = 0; + for (int i = 0; i < childFragmentCandidates.size(); i++) { + PlanFragmentId childFragmentId = childFragmentCandidates.get(i).getFragmentId(); + int currentChildFragmentParallelism = fragmentExecParamsMap.get(childFragmentId).instanceExecParams.size(); + if (currentChildFragmentParallelism > maxParallelism) { + maxParallelism = currentChildFragmentParallelism; + maxParaIndex = i; + } + } + return maxParaIndex; + } + private TNetworkAddress getGroupCommitBackend(Map addressToBackendID) { // Used for Nereids planner Group commit insert BE select. TNetworkAddress execHostport = new TNetworkAddress(groupCommitBackend.getHost(), diff --git a/regression-test/suites/query_p0/union/test_union_instance.groovy b/regression-test/suites/query_p0/union/test_union_instance.groovy new file mode 100644 index 000000000000000..17a0d3ef1dd6146 --- /dev/null +++ b/regression-test/suites/query_p0/union/test_union_instance.groovy @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_union_instance") { + multi_sql """ + drop table if exists t1; + drop table if exists t2; + drop table if exists t3; + drop table if exists t4; + CREATE TABLE `t1` ( + `c1` int NULL, + `c2` int NULL, + `c3` int NULL + ) ENGINE=OLAP + DUPLICATE KEY(`c1`, `c2`, `c3`) + DISTRIBUTED BY HASH(`c1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1"); + + insert into t1 values (1,1,1); + insert into t1 values (2,2,2); + insert into t1 values (3,3,3); + + CREATE TABLE `t2` ( + `year_week` varchar(45) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`year_week`) + DISTRIBUTED BY HASH(`year_week`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1"); + + CREATE TABLE `t3` ( + `unique_key` varchar(2999) NULL, + `brand_name` varchar(96) NULL, + `skc` varchar(150) NULL + ) ENGINE=OLAP + UNIQUE KEY(`unique_key`) + DISTRIBUTED BY HASH(`unique_key`) BUCKETS 20 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1"); + + CREATE TABLE `t4` ( + `skc_code` varchar(150) NULL + ) ENGINE=OLAP + UNIQUE KEY(`skc_code`) + DISTRIBUTED BY HASH(`skc_code`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1"); + + set parallel_pipeline_task_num=1; + set disable_nereids_rules='PRUNE_EMPTY_PARTITION'; + """ + explain { + sql """ + SELECT `t`.`year_week` AS `year_week` + ,'' AS `brand_name` + , '' AS `skc_code` + FROM `t1` a + CROSS JOIN `t2` t + union all + SELECT '1' AS `year_week` + ,`a`.`brand_name` AS `brand_name` + ,`a`.`skc` AS `skc_code` + FROM `t3` a + INNER JOIN[shuffle] `t4` b ON `a`.`skc` = `b`.`skc_code` + GROUP BY 1, 2, 3; + """ + contains "3:VNESTED LOOP JOIN" + contains "4:VEXCHANGE" + contains "8:VEXCHANGE" + contains "6:VEXCHANGE" + contains "11:VUNION" + } +} From 664ca8ad780164963ea870e160dda6396c63464a Mon Sep 17 00:00:00 2001 From: Sun Chenyang Date: Thu, 29 Aug 2024 11:38:54 +0800 Subject: [PATCH 27/34] [fix](Nereids) retrieve the table using SlotReference in the Match (#39652) ## Proposed changes Sometimes, there is no table in SlotRef, so we need to use SlotReference to get the table. --- .../glue/translator/ExpressionTranslator.java | 41 +++-- .../inverted_index_p0/test_match_and_join.out | 4 + .../test_match_and_join.groovy | 146 ++++++++++++++++++ 3 files changed, 170 insertions(+), 21 deletions(-) create mode 100644 regression-test/data/inverted_index_p0/test_match_and_join.out create mode 100644 regression-test/suites/inverted_index_p0/test_match_and_join.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 47d11d56c7c9ec0..e876b30280b4a25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -37,11 +37,10 @@ import org.apache.doris.analysis.LambdaFunctionExpr; import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.analysis.OrderByElement; -import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TimestampArithmeticExpr; -import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Function; import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.Index; @@ -189,19 +188,11 @@ public Expr visitLessThanEqual(LessThanEqual lessThanEqual, PlanTranslatorContex return le; } - private OlapTable getOlapTableFromSlotDesc(SlotDescriptor slotDesc) { - if (slotDesc != null && slotDesc.isScanSlot()) { - TupleDescriptor slotParent = slotDesc.getParent(); - return (OlapTable) slotParent.getTable(); - } - return null; - } - - private OlapTable getOlapTableDirectly(SlotRef left) { - if (left.getTableDirect() instanceof OlapTable) { - return (OlapTable) left.getTableDirect(); - } - return null; + private OlapTable getOlapTableDirectly(SlotReference slot) { + return slot.getTable() + .filter(OlapTable.class::isInstance) + .map(OlapTable.class::cast) + .orElse(null); } @Override @@ -213,12 +204,20 @@ public Expr visitElementAt(ElementAt elementAt, PlanTranslatorContext context) { public Expr visitMatch(Match match, PlanTranslatorContext context) { Index invertedIndex = null; // Get the first slot from match's left expr - SlotRef left = (SlotRef) match.left().getInputSlots().stream().findFirst().get().accept(this, context); - OlapTable olapTbl = Optional.ofNullable(getOlapTableFromSlotDesc(left.getDesc())) - .orElse(getOlapTableDirectly(left)); - + SlotReference slot = match.getInputSlots().stream() + .findFirst() + .filter(SlotReference.class::isInstance) + .map(SlotReference.class::cast) + .orElseThrow(() -> new AnalysisException( + "No SlotReference found in Match, SQL is " + match.toSql())); + + Column column = slot.getColumn() + .orElseThrow(() -> new AnalysisException( + "SlotReference in Match failed to get Column, SQL is " + match.toSql())); + + OlapTable olapTbl = getOlapTableDirectly(slot); if (olapTbl == null) { - throw new AnalysisException("slotRef in matchExpression failed to get OlapTable"); + throw new AnalysisException("SlotReference in Match failed to get OlapTable, SQL is " + match.toSql()); } List indexes = olapTbl.getIndexes(); @@ -226,7 +225,7 @@ public Expr visitMatch(Match match, PlanTranslatorContext context) { for (Index index : indexes) { if (index.getIndexType() == IndexDef.IndexType.INVERTED) { List columns = index.getColumns(); - if (columns != null && !columns.isEmpty() && left.getColumnName().equals(columns.get(0))) { + if (columns != null && !columns.isEmpty() && column.getName().equals(columns.get(0))) { invertedIndex = index; break; } diff --git a/regression-test/data/inverted_index_p0/test_match_and_join.out b/regression-test/data/inverted_index_p0/test_match_and_join.out new file mode 100644 index 000000000000000..d05128e455a86c7 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_match_and_join.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +810 + diff --git a/regression-test/suites/inverted_index_p0/test_match_and_join.groovy b/regression-test/suites/inverted_index_p0/test_match_and_join.groovy new file mode 100644 index 000000000000000..f4d0bcf38877cf6 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_match_and_join.groovy @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_match_and_join") { + + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + + sql """ drop table if exists table_30_undef_partitions2_keys3_properties4_distributed_by55 """ + sql """ + create table table_30_undef_partitions2_keys3_properties4_distributed_by55 ( + pk int, + col_int_undef_signed_index_inverted int null , + col_int_undef_signed int null , + col_int_undef_signed_not_null int not null , + col_int_undef_signed_not_null_index_inverted int not null , + col_date_undef_signed date null , + col_date_undef_signed_index_inverted date null , + col_date_undef_signed_not_null date not null , + col_date_undef_signed_not_null_index_inverted date not null , + col_varchar_1024__undef_signed varchar(1024) null , + col_varchar_1024__undef_signed_index_inverted varchar(1024) null , + col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_true varchar(1024) null , + col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_false varchar(1024) null , + col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_true varchar(1024) null , + col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_false varchar(1024) null , + col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained varchar(1024) null , + col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained varchar(1024) null , + col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_false varchar(1024) null , + col_varchar_1024__undef_signed_not_null varchar(1024) not null , + col_varchar_1024__undef_signed_not_null_index_inverted varchar(1024) not null , + col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_true_parser_english varchar(1024) not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_english_support_phrase_false varchar(1024) not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_true varchar(1024) not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_false varchar(1024) not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained varchar(1024) not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained varchar(1024) not null , + col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_false_parser_chinese varchar(1024) not null , + INDEX col_int_undef_signed_index_inverted_idx (`col_int_undef_signed_index_inverted`) USING INVERTED, + INDEX col_int_undef_signed_not_null_index_inverted_idx (`col_int_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_date_undef_signed_index_inverted_idx (`col_date_undef_signed_index_inverted`) USING INVERTED, + INDEX col_date_undef_signed_not_null_index_inverted_idx (`col_date_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_index_inverted_idx (`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_index_inv_0_idx (`col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_true`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_index_inv_1_idx (`col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_index_inv_2_idx (`col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_true`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_index_inv_3_idx (`col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_index_inv_4_idx (`col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "true", "parser_mode" = "coarse_grained"), + INDEX col_varchar_1024__undef_signed_index_inv_5_idx (`col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "true", "parser_mode" = "fine_grained"), + INDEX col_varchar_1024__undef_signed_index_inv_6_idx (`col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx (`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_not_null__7_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_true_parser_english`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_not_null__8_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_english_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_not_null__9_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_true`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_not_null__10_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_not_null__11_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "true", "parser_mode" = "coarse_grained"), + INDEX col_varchar_1024__undef_signed_not_null__12_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "true", "parser_mode" = "fine_grained"), + INDEX col_varchar_1024__undef_signed_not_null__13_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_false_parser_chinese`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "false") + ) engine=olap + DUPLICATE KEY(pk, col_int_undef_signed_index_inverted) + distributed by hash(pk) buckets 10 + properties("replication_num" = "1"); + """ + + sql """ + insert into table_30_undef_partitions2_keys3_properties4_distributed_by55(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_index_inverted,col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_true,col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_false,col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_true,col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_false,col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained,col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained,col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_false,col_varchar_1024__undef_signed_not_null,col_varchar_1024__undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_true_parser_english,col_varchar_1024__undef_signed_not_null_index_inverted_parser_english_support_phrase_false,col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_true,col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_false,col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained,col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained,col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_false_parser_chinese) values (0,0,-1603201726,2,1,'2023-12-14','2024-02-18','2023-12-17','2011-04-03','and I''ll didn''t would her him back see','放心','','hey','see','--','注册','?','谈好','they','型号呢里交流你是谁柏霖焦作市中旬郑娜出团','?','g','肥姐也冲击随其自然每次鹏飞叫做亲戚华阳','we because who look about you''re as','h','金海','your so up at right got because who'),(1,2,null,1073463547,-1,'2023-12-11',null,'2024-02-18','2023-12-19','未税',null,'?','have don''t get now but','核对过','will','as','could','say I''m','?','?','签订','节点快乐今天儿童经销总代理出面','v','how','get he''s can yeah no her yes oh he''s would','青云不知道阿奎那干嘛维泽','r'),(2,1,1777219286,-2,2011685283,null,'2026-01-18','2025-06-18','2023-12-18','t','','使人','been',null,'?','','分公司平志伟长光','all oh are they his','调价东风路','c','先锋','宝龙','','彭巧语稳定','that''s out it about','good','want okay now he go do you''re something'),(3,1,6,7,0,'2025-06-18','2023-12-13','2024-02-18','2025-02-18','','?','分享合创人参金鳞','直管问起','','','走在网络失误','ok',null,'yeah back how yeah come he i hey were do','序列号','不懂','?','go','丰富','are don''t','then','泥石谁'),(4,-2,3,-1564396755,1197391189,'2026-02-18','2024-02-18','2023-12-16','2023-12-09','now','k','come you''re had I''ll in are look he''s on to','go',null,'ok you was','雅虎相关天天自私休息许昌有种效果','南天',null,'有些时候','?','could','his','--','of','we he can on and all','--','had'),(5,null,8,-2,0,null,'2023-12-14','2023-12-13','2024-02-18','','say','胡华威','','g','out yeah her didn''t for what like can''t not','about','r','--','?','something','?','--','l','再来全程若虚','一颗','','oh know was how when just not'),(6,-1,null,-2,-2,'2025-06-18','2023-12-15','2024-02-18','2024-01-17','x','','have as are know of a','b','time','at','her here know you your were as mean','a','--','know','in','家里应该山东能不鞥','我刚回来呀','--','--','?','原装双千兆莱克特上面妹妹飞机多久笔记本治疗那捕鲸船','?'),(7,null,null,-1723016302,-2,'2025-02-18','2024-01-19','2023-12-16','2025-02-17','--','晓得','on he''s with','--','电厂','小浪底接触','look','有点','','?','f','怎么样份款','want there did yes','老公你这个满意老刘勇气四天离谱有数广大不敢','back been at like','say not really','of all are could the know would','are'),(8,38379557,null,-1,-1,'2024-01-31','2027-01-09','2025-02-18','2024-01-09','--','分钟','will','惭愧李静加入我刚来呀颜色','look i got we really ok','谈谈','who','something',null,'','didn''t if','','j','证明金信会刊单挑刘金军港','--','--','going it',''),(9,968361924,0,-2,-2,null,'2023-12-16','2023-12-09','2023-12-19','got who tell in didn''t','been I''ll i if don''t it don''t then had','--','?','听着逍遥预计和兴涨价暴风影音就在埋在拼搏','原厂海信大品牌腾龙瑞丽盛源几天就走许愿树详谈','didn''t','don''t look me I''ll','','优惠最起码热卖不已本公司嘉通机会底下','mean','t','not not of come don''t go i it','up what come','going can say the time look of when','so','为什么',''),(10,-1,-1,752321536,-819779964,'2025-06-18','2024-01-31','2023-12-20','2023-12-12','--','?','挑战散落的反应都是准备都还多久大公司','','金蝶','',null,'you''re','热线在那普惠质保金烟火城市','?','one','','get it','b','负责','with','he i have about hey or you','中科部分一鸣国标'),(11,-1,3,-2,-1,'2023-12-13',null,'2025-02-17','2023-12-14','讨论组','?','they','鼠标','think he your want know','I''ll','it see in','--','see','图美创世纪','还跟他快运小曹别人','样机','有机会算错','go have had as all do I''ll','f','got all can didn''t something back had no','so','time'),(12,null,-1,5,0,'2023-12-15','2023-12-13','2023-12-20','2023-12-19',null,'?','询问无论汉威叁万领导地方亮度','注明','at just look from been at time','been','know your you''re','with now when in know been as I''ll','my he time with come','着呢很多','--','大学老师照片快运','?','could it if at tell was oh and yeah','--','all','不会被班车学校交叉照片治疗严格分数加载浪漫','?'),(13,-1,null,210811703,9,'2026-02-18','2002-12-15','2025-06-18','2026-02-18','不愿意','','know what','甘心是不是办事处数字见你难说发吧发吧晚上给把太原','but','贸易通','it really mean all how could when with all',null,'know something','now were all can''t can my tell oh they','didn''t up','真理','that''s know yeah i','d','as he they would been something his I''ll','张总曙光主机油漆正常','','was'),(14,-1,1291248968,1,-1,'2024-01-31','2024-01-19','2023-12-17','2024-02-18',null,'','when not not because back there so',null,'as ok but about up at see could come',null,'美女及其胡提高性能女孩健民没沈晓海不含','?','can','--','what say not she about is','齐村据对中病毒你问火球绿城只能唯一的','','便宜','--','','--','--'),(15,-1,-1,-1,-186244435,'2024-02-18','2024-02-18','2023-12-11','2027-01-09','so i come','right','because','做好','who','调价付费生活炒货谁家有货看着要不然心事报价格大话',null,'市场','','about do go know all','弄个','汇宝单单自己国栋底下刻录新款一周心情','停留批单','重新发一天第三方管理一千多绝对','me this she you''re were on','--','I''m','we to is he was not she'),(16,6,-2,-623692175,1438177278,'2025-06-18','2025-02-17','2024-01-09','2024-01-09','?','x','周星驰四川力天','','','老婆明白泰杰斯操作合计难说地市做方案','o','mean','--','外面不完利万','?','?','x','k','吃过饭仪器','as is like','--','--'),(17,1,-2,-2,-2,null,'2023-12-10','2023-12-16','2024-01-17',null,'g','did I''m her would could','回忆订票周星驰礼拜天代表王佩丽进一步房地产','think','can','look or want and no yeah','','','长信请你','组哟','同方','--','g','?','加密','外运','提示到来派人郑州还有要不然高亮屏你呢岂能被人'),(18,6,6,236306652,5,'2024-01-17','2023-12-15','2026-01-18','2023-12-11','安全正好标底圆圆跨区分钟不多','你用手机吧','it''s him back or it''s just','?','只有南阳路回家诊断他说你还没有回来啊北京报价格','了吗舍得良子停机都想共同三润','a','about','hey','he''s out to because ok mean','say','','his','命名层次还不错附上作风','光电清理着那个家不止它是热销中','think all and','核对轩明令你','about'),(19,-1,null,-1,-1,null,'2023-12-11','2023-12-11','2023-12-17','?',null,'what going from','莱克特','--','惊喜礼物刘海开头所谓开封吃肉我刚回来呀','子卡','y','who I''m now all to like got why','春江寸草心合作过半天班安小瑞达证明一次','--','out','有人你好呀重要机架短暂天地','yeah but on be did or there when','she','get here it''s see could like what would','','yeah her your'),(20,1256894923,-1,-1772431512,-1,'2023-12-12',null,'2027-01-09','2023-12-13','还得东风系列则会有谈好主演少秋经营第二','一夜开除做完疲劳单挑','or they','?','','no',null,'导入液晶移动关键','?','主办','心情常用生意说说提示','go one been yeah as','','会员卡','not get','伤心双飞燕安装','it''s','售后'),(21,-2,2,-1,-1,'2023-12-14','2023-12-11','2012-01-08','2025-02-17',null,'but','--','','would','教练史丽威达机柜唯一对了五万条快乐在乎志彬','a','电子卡将','a','','going','','拟在建爱找到严重商人大量晓燕写出来重要性营销聪明','独立','背景','l','d','back that were because will some going it did on'),(22,132295595,null,-1,-1,'2023-12-12','2025-06-18','2026-01-18','2026-02-18','相约总代也要想起来接收','?','?','?',null,'方式','?','认可','工业大学','?','one but','?','have','什么弯弯的月亮一套节省动物上有盘点始终','盛源分辨率另外图站','我给你打电话中病毒珍贵苦楚','小曹文宁正式版刀片柯艳琦','--'),(23,null,-1,1444541305,2,'2023-12-14','2023-12-14','2024-01-17','2023-12-17','would here from just going','out','about now back know it''s','丁丁圣荣松伟我还港湾金海','well it well of here is really','--','已经创世纪','?','','your','拿住网页委托收款海星报名说不出电厂通过短句','弟子快点','ok on this the see about at you you','no something don''t because see will why at she yeah','兴致订货治好出团预算赵总冲动','out why he''s her is that''s well','mean','?'),(24,-1482687319,5,-1,-1,null,'2023-12-10','2023-12-16','2023-12-20','','多少钱一群立刻你问若虚顺风晚上',null,'业茂','--','?','','使用','d','will in well','拿住','are','做好','say not it will for come no but so','--','位置综合学员能不鞥很好用文宁','no','not'),(25,null,4,8,5,'2023-12-17','2024-02-18','2024-01-31','2023-12-19','?','one they okay have when you''re','共同应该股份有限公司处理一下','--','离开','动作','would','客运量特此证明地区真理初次肆万小楼听雨北京','成交这块工商局','--','but','think','地区','because','--','?','网易有空吗晚上给把一颗真正以以上的授狗那里指点','图像'),(26,3,-792244912,-1,-2,'2023-12-20','2027-01-16','2027-01-09','2023-12-12','惊喜总代理商部队款且','?','x','王总签不签都浪潮年前分享应用独立雪松路黄委会冲动','yeah','简单嘉运达那么饲料核对过蠢材最新版本处理一下百川售后','at','','科贸被授予谈下来冀海潮交流群','here been yes because','先打款不忙','','then right it all not','?','--','why come going on out','k','资格'),(27,-1,-1,-2,1216258389,'2023-12-09','2024-01-19','2024-01-09','2023-12-20','--','?','喝酒','几下','?','know','?','洗手间','?','小键盘','will go did','身份证造成怎么着你吃法三业','o','--','k','--','号码学员星星珊瑚版','传真'),(28,7,-1844462362,6,-2,null,'2024-01-09','2023-12-14','2023-12-20','不然佩利飞机','对吧','come get why how to is what','he','do','发票浪漫没人','你问','--','?','说句','?','u','are','y','I''ll','oh in he''s i yeah no','what it''s','大学路商联'),(29,845502225,1862705239,-2,-1,'2023-12-10','2024-01-31','2024-01-17','2023-12-18','',null,null,'about ok about come for here','?','','--','口语铭岳刚出门热备不够','I''ll','没有直销免费想要周鹏留个鑫海源签字高亮屏漫长','办好未税图站','--','联恒你们已经上班了柏霖方向楼下群管理员保证良子能用吗神龙','just','','--','?','some'); + """ + + sql """ drop table if exists table_300_undef_partitions2_keys3_properties4_distributed_by5 """ + sql """ + create table table_300_undef_partitions2_keys3_properties4_distributed_by5 ( + col_int_undef_signed int/*agg_type_placeholder*/ null , + col_int_undef_signed_index_inverted int/*agg_type_placeholder*/ null , + col_int_undef_signed_not_null int/*agg_type_placeholder*/ not null , + col_int_undef_signed_not_null_index_inverted int/*agg_type_placeholder*/ not null , + col_date_undef_signed date/*agg_type_placeholder*/ null , + col_date_undef_signed_index_inverted date/*agg_type_placeholder*/ null , + col_date_undef_signed_not_null date/*agg_type_placeholder*/ not null , + col_date_undef_signed_not_null_index_inverted date/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_index_inverted varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_true varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_false varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_true varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_false varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_false varchar(1024)/*agg_type_placeholder*/ null , + col_varchar_1024__undef_signed_not_null varchar(1024)/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed_not_null_index_inverted varchar(1024)/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_true_parser_english varchar(1024)/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_english_support_phrase_false varchar(1024)/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_true varchar(1024)/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_false varchar(1024)/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained varchar(1024)/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained varchar(1024)/*agg_type_placeholder*/ not null , + col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_false_parser_chinese varchar(1024)/*agg_type_placeholder*/ not null , + pk int/*agg_type_placeholder*/, + INDEX col_int_undef_signed_index_inverted_idx (`col_int_undef_signed_index_inverted`) USING INVERTED, + INDEX col_int_undef_signed_not_null_index_inverted_idx (`col_int_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_date_undef_signed_index_inverted_idx (`col_date_undef_signed_index_inverted`) USING INVERTED, + INDEX col_date_undef_signed_not_null_index_inverted_idx (`col_date_undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_index_inverted_idx (`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_index_inv_0_idx (`col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_true`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_index_inv_1_idx (`col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_index_inv_2_idx (`col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_true`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_index_inv_3_idx (`col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_index_inv_4_idx (`col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "true", "parser_mode" = "coarse_grained"), + INDEX col_varchar_1024__undef_signed_index_inv_5_idx (`col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "true", "parser_mode" = "fine_grained"), + INDEX col_varchar_1024__undef_signed_index_inv_6_idx (`col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx (`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED, + INDEX col_varchar_1024__undef_signed_not_null__7_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_true_parser_english`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_not_null__8_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_english_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_not_null__9_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_true`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true"), + INDEX col_varchar_1024__undef_signed_not_null__10_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_false`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "false"), + INDEX col_varchar_1024__undef_signed_not_null__11_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "true", "parser_mode" = "coarse_grained"), + INDEX col_varchar_1024__undef_signed_not_null__12_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "true", "parser_mode" = "fine_grained"), + INDEX col_varchar_1024__undef_signed_not_null__13_idx (`col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_false_parser_chinese`) USING INVERTED PROPERTIES("parser" = "chinese", "support_phrase" = "false") + ) engine=olap + distributed by hash(pk) buckets 10 + properties("replication_num" = "1"); + """ + + sql """ + insert into table_300_undef_partitions2_keys3_properties4_distributed_by5(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_index_inverted,col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_true,col_varchar_1024__undef_signed_index_inverted_parser_english_support_phrase_false,col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_true,col_varchar_1024__undef_signed_index_inverted_parser_unicode_support_phrase_false,col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained,col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained,col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_false,col_varchar_1024__undef_signed_not_null,col_varchar_1024__undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_true_parser_english,col_varchar_1024__undef_signed_not_null_index_inverted_parser_english_support_phrase_false,col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_true,col_varchar_1024__undef_signed_not_null_index_inverted_parser_unicode_support_phrase_false,col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained,col_varchar_1024__undef_signed_not_null_index_inverted_parser_chinese_support_phrase_true_parser_mode_fine_grained,col_varchar_1024__undef_signed_not_null_index_inverted_support_phrase_false_parser_chinese) values (0,-2,-1901730183,5,-1,'2023-12-14',null,'2023-12-16','2023-12-20','肯定','华康','--','from but when okay would when','','弹出','from my','下月很难说奇偶上班不止单位接受','加密狗铁路董事长重命名策略一点河南压力为你','提示硕博伟泽聊聊冰河心理学对吧','中小金汇通','then can','','will there don''t had did want been some','f','过年','her','--'),(1,1382192944,-1,-234802337,-1,'2025-02-18','2025-02-17','2023-12-12','2027-01-16','国务卿','','look back i in you me say look go','孙瑞霞很想成本还是记录那位','well do he say can it''s oh don''t','--','--','got','think','there','合计回公司遮盖很大置业','','成绩','from','what','原因的','','是的交单记得拆机有点不好漫长司军几关系'),(2,-1,-2,2,-2,'2004-04-02','2023-12-19','2023-12-15','2023-12-12','up','--','汇众孤独办款名家生后打算武汉瀚海答案','--','f','could',null,'差价图形工作站中卢有伴随熟悉大量中创热备软件产生休息','','是你呀有车吗','不过图腾水哦这话大家冀海潮','good say like','my he some yes come so','--','or','?','a come a','see'),(3,-2,-2,-2017523107,-2,'2023-12-17','2027-01-16','2024-01-31','2023-12-17','你家报表表情','不走除非飞信许昌后勤教练新买这一生不下令你','你杀时间回来',null,'time there he well it with me','排列','表情','招工','right','请他','I''m mean be from','','?','完美','here my','b','关于','c'),(4,1065841778,-2,-1,-1,'2024-02-18',null,'2025-06-18','2023-12-20','左右','','?','面子故事离开邮箱合适','ok why here time in that','等级','i','--',null,'be look out of had we to what be don''t','no can hey to get','or as you''re about','go all her some well','只有塔式海花见你名称也冲击幸福跟进中核对过只有','see could i','?','?','海星动态刻意海星刘畅上不了学习'),(5,1458245006,null,-1,-2,'2026-01-18','2023-12-13','2024-01-31','2024-01-08','?','v','when to could','--','发布神秘一小时试过备用金','g','用管','v','这是','is','g','','then','--','','','王哥','p'),(6,3,-1,-239832171,-1,'2024-01-08','2024-01-17','2023-12-12','2026-02-18','is well if all time have you''re','n','?','have',null,'键盘鼠标',null,'第二上门十分新远方表达商丘','the going were at mean','there like been on and yes what or','--','--','go','it''s','花月夜','什么时间','行业屏保长得','我在政治哦同志们参加'),(7,-2,-2,-1,1754315155,'2024-02-18','2023-12-19','2025-02-18','2023-12-14','','--','','显卡宇瑞鑫海源关系','or don''t no ok is',null,'科技大厦经常实现航务新增而是阿奎那纸业自动','x','寝室','then','用户','i how how','一两智博在不在数据公寓电粉','礼拜天','well you here so been what okay see something','--','河南总经销','?'),(8,112676751,-127780972,-1937697475,5,'2024-01-17','2023-12-11','2023-12-16','2024-01-09','三阳','you''re','who','残品作风折旧到底怎么办正规承认又给','','have','r','r','j','if is in','空间冗余','his','','浇水过节费先付新增综合杨学说不','你不认识华阳跟着预计长光理光许昌想象','e','','千金屏保九九马马虎虎怡海'),(9,3,-957042490,3,-1,'2023-12-09','2023-12-20','2023-12-16','2023-12-09',null,'每一小时',null,'was',null,'','广州市','me or you''re look right got when you''re about','it''s okay this up how are for','t','was she don''t at i know can when because','go','how here ok okay','got her he it''s could can''t back had about have','she','?','he''s him they well I''m yeah',''),(10,-200104483,-1,4,-2,'2023-12-16','2025-06-18','2024-02-18','2024-02-18','see','j','今晚','how','态度','he','will','him','不多','had about did','眼睁睁','--','?','打电话','到来','利盟','w','歘俩'),(11,-1,null,-2,-2,null,'2023-12-10','2025-02-18','2024-01-08',null,'生物','你吃法','my','这几太难','状态第二次郑娜备份文件三个业务下班','零捌','time I''ll about on they because don''t good why will','就算格飞五洲一块小妹特性吃过饭叁仟肯定写上去','--','大雪服务站光纤跃民独显从你那里石龙汇宝真是','','on get you''re will','him','why','分开信阳','填写','him of yeah my get they can'),(12,7,8,1567572056,-2,'2023-12-13','2025-02-18','2025-06-18','2023-12-11','命名操心你怎么去的保密中档说吧淀粉英联生命中这几天','some','则会有','充满',null,'领出','i','would oh me are','was','字库文件','?','老公好早一定','见面蓝牙','?','不如孙海洋一辈子蓝牙晓燕','--','it didn''t or hey some think do like here','自私空间'),(13,null,-2079628735,-2,1,'2023-12-11','2023-12-15','2023-12-18','2024-01-09','','--','?','停产','?','农村一起',null,'','一种下了期待刘总圣诞节金汇通千万','见到李东贸易通','?','?','have get did the i to then','for','--','?','线上输入不再梦想朱晓明','who with don''t for that ok him can one'),(14,7,-1104140591,-2,-1,'2023-12-16','2024-01-08','2025-02-17','2023-12-10',null,null,'oh ok','a','?','也许','?','could you','','对你以上有人多家','请他信海维修总之张海燕孤独过去列表奖金','say','or','it''s','o','so that''s she she yeah the right if','?','?'),(15,-2,-1,-2,-1,'2023-12-09','2025-02-17','2024-01-17','2023-12-13','今晚','up ok can but i do one','','will','s',null,'v','b','王枫','come','广发','for I''m now can hey when because time come','?','were','go some ok something of','富通你好删过刚刚','when yeah think how I''ll now','--'),(16,1,null,1025070939,-1065499727,'2024-02-18','2025-02-17','2023-12-19','2024-01-31','','富通怡海赶快瑞星左右河南经销','not','姓江','was','何时','v','me can''t now he ok is been from hey we','o','--','--','we','s','b','that for here at you''re can''t right hey yes','come hey on here yes mean going it''s about or','j','中铁'),(17,1406750472,null,-2,5,'2023-12-16','2024-01-09','2024-01-08','2024-02-18','?',null,'the been is but then can''t yes him this really','really okay say tell how','但愿',null,'--','we right that was I''m','?','','','something','五万多条','--','','','在家里主页绝对北环备份文件','几个工业大学带你正道直接费用科峰不需要鑫海源接口'),(18,6,-1,1073335855,6,'2023-12-18','2023-12-19','2025-02-17','2023-12-10','hey','ok','have was my like','万一证明雅虎','his at didn''t it','分销商中层经理网通等于','维泽特配机器备份文件招工河南经销新普怡海','不可能奖励空间冗余加密狗分钟玉清','?','can''t not','c','空间冗余','--','u','--','have at now was yes have out','would then now is','y'),(19,null,-2,-2,-1,'2024-02-18','2025-06-18','2027-01-16','2024-01-31','could good','up i I''ll','mean','--','--','记得一岁无聊也得空间冗余打印机','英特','客户',null,'','got','明月这个扩展名城市科贸','世安误区独显安装愿意联盟比爱白贰台','?','简介','come a that''s or','?','go'),(20,-2,null,-2,634169930,'2023-12-16','2024-01-17','2024-01-17','2023-12-16','be','?','','--','had','天孝','','then ok to that think this all you like will','细节新闻有钱老公张小莉这点','河南总经销个人自己','有些时候在呀自己的刷卡','j','','I''ll all or','that no why your then okay some her I''ll i','','have','帐户宠物关机也是'),(21,-1,null,-1,0,'2025-02-17','2024-01-31','2024-01-08','2024-01-31','等于合作单位看待解难答案三种','','折扣不行立博我那个叫东西上半年沃尔普会员卡','那款武警总队今晚华北区输得起不曾很想熊猫','?',null,'but out ok how','out',null,'it''s','','三阳开泰','on and he''s and one know know yes good or','帮帮','伟泽这么双子安阳咨询服务时间磁盘柜高密度','速度钢琴曲金牌总代理顺河路彩霞金立下一部分广告下了黄河科学','p','why'),(22,-1,1,-2,-2,'2027-01-16','2024-01-09','2023-12-15','2023-12-15','工资','are really I''m','when of was in well a get think well','are','--','know','?','well say about been going why would yeah out go','?','r','did he''s be something out can''t some him be','','痛苦私人水货绿城打击不起网站关键','不说需要帮忙做到经销商历来三润十来年记得','your in no have got','or look','磁盘你们已经上班了丹尼斯李金才泡茶等等赵洪应一共','进入了'),(23,-1,-2,-2,-2,'2023-12-09','2024-02-18','2023-12-16','2023-12-16','本人','屠龙记','at','瑞昌','?','r','he had are going he''s because from would',null,'we','p','你不中小','all he good go would go my up a what','中天','didn''t','why','w','联盟新闻重新化验好处职务其中欧颜色',''),(24,-2,null,-2117558545,720042588,'2027-01-09','2014-06-24','2023-12-19','2025-02-18','?','l','been you i','广大两千量大优惠索尼','c','come go know when','新买','忘不了大部分有误国有企业','this','加为好友','I''m','your were just as of look good a your or','侠诺灵生不敢电视海川回你在家里海花本科市场上','for didn''t','最重要哪呢个五万多条难过姑娘','for about the I''m yeah go hey','我刚回来呀','--'),(25,9,-2,-1,-1562783499,'2024-01-09','2025-06-18','2025-02-17','2024-01-08','珍惜不再','--','下个','?','电话','some see go from don''t that''s be really','杀毒你来找我吧日常也有过政府','鼠标','you''re here back about she we some don''t they','--','中的','--','on say is on what','卖友公路说吧答案','这款','结果','基地连续公司近台南不愿意','q'),(26,null,null,566690289,-1,'2024-01-17','2023-12-19','2023-12-13','2024-02-18','about','隆康王子','就有','n','中层经理','购买','虚妄主奴婢最近忙什么呢找你见到新区外面','be is','','北环私人郑大沧田第一次请他肛门锐成文件夹浪潮','下来','?','z','生活屏保派送','?','--','万方雷林授权发生过焕然去不了生意还以为直供','he one go'),(27,8,-1329519821,5,-1,'2023-12-19','2023-12-14','2023-12-20','2025-06-18','some they want I''ll they yeah going','花园','原因','?','中午随风很好用珊瑚版比人很多输得起售后王燕纯度',null,'i','because i look what','--','his','why','--','that''s','the it for some are don''t here','oh back say they are from','新密韩鸽飞空分电脑一部一万有机会开发强暴找你','你休息吸纳中脱机正式版视讯重要性客户群以上','it''s'),(28,4,7,738661869,1,'2024-02-18','2024-01-19','2025-02-18','2023-12-15','?',null,'都想','王星银河外运保留令你年后使人瑞星','?','hey up','we say','科技市场','刘雪','s','--','something','tell','办事处进去祝福','get with hey that''s back','玉冰','老刘批发价忙碌小型机','三石'),(29,0,143593368,-2,335460636,'2026-02-18','2025-02-18','2025-06-18','2026-01-18',null,'暂时控制卡','?','知道','b','占到热线我给你打吧企业方法首创那你只能员工','','安信情歌内存王晶威达他娘的语文说好','离开逝去自己小苏网卡新一代武汉方法色戒','找不到','time','小楼听雨宝贵不但是忙碌飞扬农科阿奎那一套虚妄','废话王总签不签都各位收集','既然','宫颈糜烂','惊喜金牌','this',''); + """ + + qt_sql """ + select count() from table_30_undef_partitions2_keys3_properties4_distributed_by55 t1, table_300_undef_partitions2_keys3_properties4_distributed_by5 t2 where t1. col_date_undef_signed_index_inverted is not null or (t1. col_varchar_1024__undef_signed_index_inverted_parser_chinese_support_phrase_true_parser_mode_coarse_grained MATCH "儿童") and not (t2. col_int_undef_signed is null) and not (t1. col_int_undef_signed_not_null_index_inverted is null or t1. col_int_undef_signed_not_null_index_inverted is not null); + """ + +} From d7eb3431095d529a42fa12429651ebb1608a360b Mon Sep 17 00:00:00 2001 From: Sun Chenyang Date: Thu, 29 Aug 2024 12:01:38 +0800 Subject: [PATCH 28/34] [fix] (UT) fix be ut InvertedIndexArrayTest (#40090) ## Proposed changes fix be ut InvertedIndexArrayTest --- .../segment_v2/inverted_index_array_test.cpp | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp index 469f5243c79f5ee..8621b81518af80b 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp @@ -58,13 +58,21 @@ class InvertedIndexArrayTest : public testing::Test { public: const std::string kTestDir = "./ut_dir/inverted_index_array_test"; - void check_terms_stats(string dir_str, string file_str) { - CLuceneError err; - CL_NS(store)::IndexInput* index_input = nullptr; - DorisFSDirectory::FSIndexInput::open(io::global_local_filesystem(), file_str.c_str(), - index_input, err, 4096); - std::unique_ptr reader = - std::make_unique(index_input, 4096); + void check_terms_stats(string file_str) { + std::unique_ptr reader; + try { + CLuceneError err; + CL_NS(store)::IndexInput* index_input = nullptr; + auto ok = DorisFSDirectory::FSIndexInput::open( + io::global_local_filesystem(), file_str.c_str(), index_input, err, 4096); + if (!ok) { + throw err; + } + reader = std::make_unique(index_input, 4096); + } catch (...) { + EXPECT_TRUE(false); + } + std::cout << "Term statistics for " << file_str << std::endl; std::cout << "==================================" << std::endl; lucene::store::Directory* dir = reader.get(); @@ -121,8 +129,6 @@ class InvertedIndexArrayTest : public testing::Test { InvertedIndexDescriptor::get_index_file_path_v1(index_path_prefix, index_id, ""); auto fs = io::global_local_filesystem(); - io::FileWriterPtr file_writer; - EXPECT_TRUE(fs->create_file(index_path, &file_writer).ok()); auto index_meta_pb = std::make_unique(); index_meta_pb->set_index_type(IndexType::INVERTED); index_meta_pb->set_index_id(index_id); @@ -204,7 +210,7 @@ class InvertedIndexArrayTest : public testing::Test { EXPECT_EQ(_inverted_index_builder->finish(), Status::OK()); EXPECT_EQ(index_file_writer->close(), Status::OK()); - check_terms_stats(file_writer->path().parent_path(), file_writer->path().filename()); + check_terms_stats(index_path); } }; From e656cb7980ddb440ff614966c0f80fb809dd449b Mon Sep 17 00:00:00 2001 From: qiye Date: Thu, 29 Aug 2024 12:05:52 +0800 Subject: [PATCH 29/34] [fix](build index)Only get file size for inverted index (#39965) close #39963 Master will not have the warning logs due to `file_system.h` removed the log line. But the bug still exists. --- be/src/olap/rowset/beta_rowset.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index b269051e43f4558..209aca7fb03b4c7 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -83,6 +83,10 @@ Status BetaRowset::get_inverted_index_size(size_t* index_size) { if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { auto indices = _schema->indexes(); for (auto& index : indices) { + // only get file_size for inverted index + if (index.index_type() != IndexType::INVERTED) { + continue; + } for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { auto seg_path = DORIS_TRY(segment_path(seg_id)); int64_t file_size = 0; From ea4c77f9622f218ff76d4743e9a9d82cfc3452c4 Mon Sep 17 00:00:00 2001 From: hui lai <1353307710@qq.com> Date: Thu, 29 Aug 2024 14:12:02 +0800 Subject: [PATCH 30/34] [chore](routine load) make error msg clear if routine load name illegal (#40037) Make error msg clear if routine load name illegal(such as job name is too long). --- .../doris/analysis/CreateRoutineLoadStmt.java | 9 +++- .../org/apache/doris/common/FeNameFormat.java | 2 + .../test_routine_load_name.groovy | 44 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_name.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java index 13654509821d12d..1648d943ae4b899 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java @@ -347,7 +347,14 @@ public void analyze(Analyzer analyzer) throws UserException { // check dbName and tableName checkDBTable(analyzer); // check name - FeNameFormat.checkCommonName(NAME_TYPE, name); + try { + FeNameFormat.checkCommonName(NAME_TYPE, name); + } catch (AnalysisException e) { + // 64 is the length of regular expression matching + // (FeNameFormat.COMMON_NAME_REGEX/UNDERSCORE_COMMON_NAME_REGEX) + throw new AnalysisException(e.getMessage() + + " Maybe routine load job name is longer than 64 or contains illegal characters"); + } // check load properties include column separator etc. checkLoadProperties(); // check routine load job properties include desired concurrent number etc. diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index 287d23686607e7f..18dfce2fc081a62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -30,6 +30,8 @@ public class FeNameFormat { private static final String LABEL_REGEX = "^[-_A-Za-z0-9:]{1," + Config.label_regex_length + "}$"; + // if modify the matching length of a regular expression, + // please modify error msg when FeNameFormat.checkCommonName throw exception in CreateRoutineLoadStmt private static final String COMMON_NAME_REGEX = "^[a-zA-Z][a-zA-Z0-9-_]{0,63}$"; private static final String UNDERSCORE_COMMON_NAME_REGEX = "^[_a-zA-Z][a-zA-Z0-9-_]{0,63}$"; private static final String TABLE_NAME_REGEX = "^[a-zA-Z][a-zA-Z0-9-_]*$"; diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_name.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_name.groovy new file mode 100644 index 000000000000000..53606fbdf6ef334 --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_name.groovy @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_routine_load_name","p0") { + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + try { + sql """ + CREATE ROUTINE LOAD test_routine_load_name_too_much_filler_filler_filler_filler_filler_filler_filler + COLUMNS TERMINATED BY "|" + PROPERTIES + ( + "max_batch_interval" = "5", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "multi_table_load_invalid_table", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + } catch (Exception e) { + log.info("exception: ${e.toString()}".toString()) + assertEquals(e.toString().contains("Incorrect ROUTINE LOAD NAME name"), true) + assertEquals(e.toString().contains("required format is"), true) + assertEquals(e.toString().contains("Maybe routine load job name is longer than 64 or contains illegal characters"), true) + } +} \ No newline at end of file From e24bc7f650ab7dbc2e974e68e42cdd5be38ac776 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 29 Aug 2024 14:47:30 +0800 Subject: [PATCH 31/34] [chore](mtmv)add plan log for UpdateMvByPartitionCommand (#40072) fe.conf set `sys_log_verbose_modules = org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand` --- .../trees/plans/commands/UpdateMvByPartitionCommand.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index ba1a054752b55ad..de284bd837748f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -65,6 +65,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Range; import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.HashSet; @@ -79,6 +81,8 @@ * Update mv by partition */ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand { + private static final Logger LOG = LogManager.getLogger(UpdateMvByPartitionCommand.class); + private UpdateMvByPartitionCommand(LogicalPlan logicalQuery) { super(logicalQuery, Optional.empty(), Optional.empty()); } @@ -104,6 +108,10 @@ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionName } LogicalSink sink = UnboundTableSinkCreator.createUnboundTableSink(mv.getFullQualifiers(), ImmutableList.of(), ImmutableList.of(), parts, plan); + if (LOG.isDebugEnabled()) { + LOG.debug("MTMVTask plan for mvName: {}, partitionNames: {}, plan: {}", mv.getName(), partitionNames, + sink.treeString()); + } return new UpdateMvByPartitionCommand(sink); } From 9cb7d83ed858c44dbcc1d845437c818caca78578 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 29 Aug 2024 15:49:50 +0800 Subject: [PATCH 32/34] [fix](auth) ordinary users can see the processes of other users (#39747) Fix when set show_all_fe_connection="true"; At this time, ordinary users can see the processes of other users --- .../main/java/org/apache/doris/qe/ConnectScheduler.java | 9 ++++++++- .../src/main/java/org/apache/doris/qe/ShowExecutor.java | 1 + .../org/apache/doris/service/FrontendServiceImpl.java | 6 +++++- gensrc/thrift/FrontendService.thrift | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectScheduler.java index dfd6f0ebdae4fb6..cafe9edd3a18f05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectScheduler.java @@ -17,6 +17,7 @@ package org.apache.doris.qe; +import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.util.DebugUtil; @@ -173,10 +174,16 @@ public List listConnection(String user, boolean isFul } // used for thrift - public List> listConnectionWithoutAuth(boolean isShowFullSql) { + public List> listConnectionForRpc(UserIdentity userIdentity, boolean isShowFullSql) { List> list = new ArrayList<>(); long nowMs = System.currentTimeMillis(); for (ConnectContext ctx : connectionMap.values()) { + // Check auth + if (!ctx.getCurrentUserIdentity().equals(userIdentity) && !Env.getCurrentEnv() + .getAccessManager() + .checkGlobalPriv(userIdentity, PrivPredicate.GRANT)) { + continue; + } list.add(ctx.toThreadInfo(isShowFullSql).toRow(-1, nowMs)); } return list; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 9c807a4b1bf9fed..1cbfcf8ba37fa94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -533,6 +533,7 @@ private void handleShowProcesslist() { try { TShowProcessListRequest request = new TShowProcessListRequest(); request.setShowFullSql(isShowFullSql); + request.setCurrentUserIdent(ConnectContext.get().getCurrentUserIdentity().toThrift()); List> frontends = FrontendsProcNode.getFrontendWithRpcPort(Env.getCurrentEnv(), false); FrontendService.Client client = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index f64f55a47fe62cd..157ad2807084859 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3963,8 +3963,12 @@ public TShowProcessListResult showProcessList(TShowProcessListRequest request) { if (request.isSetShowFullSql()) { isShowFullSql = request.isShowFullSql(); } + UserIdentity userIdentity = UserIdentity.ROOT; + if (request.isSetCurrentUserIdent()) { + userIdentity = UserIdentity.fromThrift(request.getCurrentUserIdent()); + } List> processList = ExecuteEnv.getInstance().getScheduler() - .listConnectionWithoutAuth(isShowFullSql); + .listConnectionForRpc(userIdentity, isShowFullSql); TShowProcessListResult result = new TShowProcessListResult(); result.setProcessList(processList); return result; diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index b6e4aacf65614ae..2dcdf9b718036f1 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1520,6 +1520,7 @@ struct TGetColumnInfoResult { struct TShowProcessListRequest { 1: optional bool show_full_sql + 2: optional Types.TUserIdentity current_user_ident } struct TShowProcessListResult { From fb509eedd233065166b8fbe62dc152f5ed9d1b41 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 29 Aug 2024 15:50:36 +0800 Subject: [PATCH 33/34] [fix](mtmv) Mtmv support set both immediate and starttime (#39573) extends: #36805 Previously, if the user set immediate execution, the current time would replace the user's set start time --- .../org/apache/doris/mtmv/MTMVJobManager.java | 8 +- .../data/mtmv_p0/test_start_time_mtmv.out | 17 +++++ .../mtmv_p0/test_start_time_mtmv.groovy | 76 +++++++++++++++++++ 3 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/mtmv_p0/test_start_time_mtmv.out create mode 100644 regression-test/suites/mtmv_p0/test_start_time_mtmv.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java index 11089899b309a85..1ace738f1d0d9f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java @@ -105,14 +105,14 @@ private void setScheduleJobConfig(JobExecutionConfiguration jobExecutionConfigur .setInterval(mtmv.getRefreshInfo().getRefreshTriggerInfo().getIntervalTrigger().getInterval()); timerDefinition .setIntervalUnit(mtmv.getRefreshInfo().getRefreshTriggerInfo().getIntervalTrigger().getTimeUnit()); - if (mtmv.getRefreshInfo().getBuildMode().equals(BuildMode.IMMEDIATE)) { - jobExecutionConfiguration.setImmediate(true); - } else if (mtmv.getRefreshInfo().getBuildMode().equals(BuildMode.DEFERRED) && !StringUtils + if (!StringUtils .isEmpty(mtmv.getRefreshInfo().getRefreshTriggerInfo().getIntervalTrigger().getStartTime())) { timerDefinition.setStartTimeMs(TimeUtils.timeStringToLong( mtmv.getRefreshInfo().getRefreshTriggerInfo().getIntervalTrigger().getStartTime())); } - + if (mtmv.getRefreshInfo().getBuildMode().equals(BuildMode.IMMEDIATE)) { + jobExecutionConfiguration.setImmediate(true); + } jobExecutionConfiguration.setTimerDefinition(timerDefinition); } diff --git a/regression-test/data/mtmv_p0/test_start_time_mtmv.out b/regression-test/data/mtmv_p0/test_start_time_mtmv.out new file mode 100644 index 000000000000000..8e17dcacec9139d --- /dev/null +++ b/regression-test/data/mtmv_p0/test_start_time_mtmv.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !init -- +1 1 +2 2 +3 3 + +-- !create -- +EVERY 2 HOUR STARTS 9999-12-13 21:07:09 + +-- !alter -- +EVERY 2 HOUR STARTS 9998-12-13 21:07:09 + +-- !refresh -- +1 1 +2 2 +3 3 + diff --git a/regression-test/suites/mtmv_p0/test_start_time_mtmv.groovy b/regression-test/suites/mtmv_p0/test_start_time_mtmv.groovy new file mode 100644 index 000000000000000..89872adc253b7b4 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_start_time_mtmv.groovy @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_start_time_mtmv","mtmv") { + String suiteName = "test_start_time_mtmv" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 INT, + k3 varchar(32) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into ${tableName} values(1,1),(2,2),(3,3); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD immediate REFRESH AUTO ON SCHEDULE EVERY 2 HOUR STARTS "9999-12-13 21:07:09" + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName}; + """ + + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_init "SELECT * FROM ${mvName}" + + order_qt_create "select RecurringStrategy from jobs('type'='mv') where MvName='${mvName}'" + + sql """ + alter MATERIALIZED VIEW ${mvName} REFRESH auto ON SCHEDULE EVERY 2 HOUR STARTS "9998-12-13 21:07:09"; + """ + + order_qt_alter "select RecurringStrategy from jobs('type'='mv') where MvName='${mvName}'" + + // refresh mv + sql """ + REFRESH MATERIALIZED VIEW ${mvName} complete + """ + + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_refresh "SELECT * FROM ${mvName}" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} From 63b8949d5c5452fcce147295d920a98f6c07d103 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 29 Aug 2024 15:55:06 +0800 Subject: [PATCH 34/34] [enhance](mtmv) Prohibit creating materialized views in external catalogs (#39557) --- .../trees/plans/commands/info/CreateMTMVInfo.java | 4 ++++ regression-test/suites/mtmv_p0/test_hive_mtmv.groovy | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 67aaae7fb870575..6e1a4b8d6ec8f08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -38,6 +38,7 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVPartitionUtil; @@ -151,6 +152,9 @@ public CreateMTMVInfo(boolean ifNotExists, TableNameInfo mvName, public void analyze(ConnectContext ctx) throws Exception { // analyze table name mvName.analyze(ctx); + if (!InternalCatalog.INTERNAL_CATALOG_NAME.equals(mvName.getCtl())) { + throw new AnalysisException("Only support creating asynchronous materialized views in internal catalog"); + } try { FeNameFormat.checkTableName(mvName.getTbl()); } catch (org.apache.doris.common.AnalysisException e) { diff --git a/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy index 4ac5ad9e8904635..7b3abc2300f383d 100644 --- a/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy @@ -38,6 +38,16 @@ suite("test_hive_mtmv", "p0,external,hive,external_docker,external_docker_hive") def dbName = "regression_test_mtmv_p0" sql """drop materialized view if exists ${mvName};""" + test { + sql """CREATE MATERIALIZED VIEW ${catalog_name}.`default`.${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${catalog_name}.`default`.mtmv_base1;""" + exception "internal" + } + sql """ CREATE MATERIALIZED VIEW ${mvName} BUILD DEFERRED REFRESH AUTO ON MANUAL