From c8b0275fe72369c119c226554519fe8b03c018ad Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Tue, 30 Jan 2024 10:36:12 +0800 Subject: [PATCH] [fix](join) incorrect result of mark join --- be/src/pipeline/exec/hashjoin_build_sink.cpp | 8 -- be/src/pipeline/exec/hashjoin_build_sink.h | 7 - .../pipeline/exec/hashjoin_probe_operator.cpp | 1 - .../pipeline/exec/hashjoin_probe_operator.h | 7 - .../vec/common/hash_table/join_hash_table.h | 85 +++++++----- .../vec/exec/join/process_hash_table_probe.h | 2 + .../exec/join/process_hash_table_probe_impl.h | 51 +++++-- be/src/vec/exec/join/vhash_join_node.cpp | 8 -- be/src/vec/exec/join/vhash_join_node.h | 16 +-- .../data/nereids_p0/join/test_mark_join.out | 43 ++++++ .../nereids_p0/join/test_mark_join.groovy | 127 ++++++++++++++++++ 11 files changed, 265 insertions(+), 90 deletions(-) create mode 100644 regression-test/data/nereids_p0/join/test_mark_join.out create mode 100644 regression-test/suites/nereids_p0/join/test_mark_join.groovy diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp index ccf2f2a58942bb9..1f0146d74d3e792 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.cpp +++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp @@ -76,12 +76,6 @@ Status HashJoinBuildSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo _shared_hash_table_dependency->block(); p._shared_hashtable_controller->append_dependency(p.node_id(), _shared_hash_table_dependency); - } else { - if ((p._join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN || - p._join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) && - p._have_other_join_conjunct) { - _build_indexes_null = std::make_shared>(); - } } _memory_usage_counter = ADD_LABEL_COUNTER(profile(), "MemoryUsage"); @@ -492,7 +486,6 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block* state, local_state._shared_state->build_block.get(), &local_state, use_global_rf)); RETURN_IF_ERROR( local_state.process_build_block(state, (*local_state._shared_state->build_block))); - local_state._shared_state->build_indexes_null = local_state._build_indexes_null; if (_shared_hashtable_controller) { _shared_hash_table_context->status = Status::OK(); // arena will be shared with other instances. @@ -538,7 +531,6 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block* _shared_hash_table_context->hash_table_variants)); local_state._shared_state->build_block = _shared_hash_table_context->block; - local_state._build_indexes_null = _shared_hash_table_context->build_indexes_null; local_state._shared_state->build_indexes_null = _shared_hash_table_context->build_indexes_null; const bool use_global_rf = diff --git a/be/src/pipeline/exec/hashjoin_build_sink.h b/be/src/pipeline/exec/hashjoin_build_sink.h index 7521563ecb8b078..3d0bd747f5a342d 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.h +++ b/be/src/pipeline/exec/hashjoin_build_sink.h @@ -120,13 +120,6 @@ class HashJoinBuildSinkLocalState final std::shared_ptr _shared_hash_table_dependency; std::vector _build_col_ids; - /* - * For null aware anti/semi join with other join conjuncts, we do need to care about the rows in - * build side with null keys, - * because the other join conjuncts' result may be changed from null to false(null & false == false). - */ - std::shared_ptr> _build_indexes_null; - RuntimeProfile::Counter* _build_table_timer = nullptr; RuntimeProfile::Counter* _build_expr_call_timer = nullptr; RuntimeProfile::Counter* _build_table_insert_timer = nullptr; diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp index aef2e011fa04833..846c015a531467b 100644 --- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp +++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp @@ -301,7 +301,6 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc Status st; if (local_state._probe_index < local_state._probe_block.rows()) { - local_state._build_indexes_null = local_state._shared_state->build_indexes_null; DCHECK(local_state._has_set_need_null_map_for_probe); RETURN_IF_CATCH_EXCEPTION({ std::visit( diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h index 1bdb9864c406aaa..4b7f9271920c118 100644 --- a/be/src/pipeline/exec/hashjoin_probe_operator.h +++ b/be/src/pipeline/exec/hashjoin_probe_operator.h @@ -125,13 +125,6 @@ class HashJoinProbeLocalState final // For mark join, last probe index of null mark int _last_probe_null_mark; - /* - * For null aware anti/semi join with other join conjuncts, we do need to care about the rows in - * build side with null keys, - * because the other join conjuncts' result may be changed from null to false(null & false == false). - */ - std::shared_ptr> _build_indexes_null; - vectorized::Block _probe_block; vectorized::ColumnRawPtrs _probe_columns; // other expr diff --git a/be/src/vec/common/hash_table/join_hash_table.h b/be/src/vec/common/hash_table/join_hash_table.h index 08311989b5d6459..baf5989d053007b 100644 --- a/be/src/vec/common/hash_table/join_hash_table.h +++ b/be/src/vec/common/hash_table/join_hash_table.h @@ -68,6 +68,7 @@ class JoinHashTable { std::vector& get_visited() { return visited; } + template void build(const Key* __restrict keys, const uint32_t* __restrict bucket_nums, size_t num_elem) { build_keys = keys; @@ -76,7 +77,12 @@ class JoinHashTable { next[i] = first[bucket_num]; first[bucket_num] = i; } - first[bucket_size] = 0; // index = bucket_num means null + if constexpr ((JoinOpType != TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN && + JoinOpType != TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) || + !with_other_conjuncts) { + /// Only null aware join with other conjuncts need to access the null value in hash table + first[bucket_size] = 0; // index = bucket_num means null + } } template @@ -128,51 +134,48 @@ class JoinHashTable { * select 'a' not in ('b', null) => null => 'a' != 'b' and 'a' != null => true and null => null * select 'a' not in ('a', 'b', null) => false */ - auto find_null_aware_with_other_conjuncts( - const Key* __restrict keys, const uint32_t* __restrict build_idx_map, int probe_idx, - uint32_t build_idx, int probe_rows, uint32_t* __restrict probe_idxs, - uint32_t* __restrict build_idxs, std::set& null_result, - const std::vector& build_indexes_null, const size_t build_block_count) { + auto find_null_aware_with_other_conjuncts(const Key* __restrict keys, + const uint32_t* __restrict build_idx_map, + int probe_idx, uint32_t build_idx, int probe_rows, + uint32_t* __restrict probe_idxs, + uint32_t* __restrict build_idxs, + uint8_t* __restrict null_flags, + bool picking_null_keys) { auto matched_cnt = 0; const auto batch_size = max_batch_size; - bool has_matched = false; auto do_the_probe = [&]() { + /// If no any rows match the probe key, here start to handle null keys in build side. + /// The result of "Any = null" is null. + if (build_idx == 0 && !picking_null_keys) { + build_idx = first[bucket_size]; + picking_null_keys = true; // now pick null from build side + } + while (build_idx && matched_cnt < batch_size) { - if (build_idx == bucket_size) { - /// All rows in build side should be executed with other join conjuncts. - for (size_t i = 1; i != build_block_count; ++i) { - build_idxs[matched_cnt] = i; - probe_idxs[matched_cnt] = probe_idx; - matched_cnt++; - } - null_result.emplace(probe_idx); - build_idx = 0; - has_matched = true; - break; - } else if (keys[probe_idx] == build_keys[build_idx]) { + if (picking_null_keys || keys[probe_idx] == build_keys[build_idx]) { build_idxs[matched_cnt] = build_idx; probe_idxs[matched_cnt] = probe_idx; + null_flags[matched_cnt] = picking_null_keys; matched_cnt++; - has_matched = true; } build_idx = next[build_idx]; + + // If `build_idx` is 0, all matched keys are handled, + // now need to handle null keys in build side. + if (!build_idx && !picking_null_keys) { + build_idx = first[bucket_size]; + picking_null_keys = true; // now pick null keys from build side + } } // may over batch_size when emplace 0 into build_idxs if (!build_idx) { - if (!has_matched) { // has no any row matched - for (auto index : build_indexes_null) { - build_idxs[matched_cnt] = index; - probe_idxs[matched_cnt] = probe_idx; - matched_cnt++; - } - } probe_idxs[matched_cnt] = probe_idx; build_idxs[matched_cnt] = 0; + picking_null_keys = false; matched_cnt++; - has_matched = false; } probe_idx++; @@ -184,11 +187,21 @@ class JoinHashTable { while (probe_idx < probe_rows && matched_cnt < batch_size) { build_idx = build_idx_map[probe_idx]; + if (build_idx == bucket_size) { + build_idxs[matched_cnt] = build_idx; + probe_idxs[matched_cnt] = probe_idx; + matched_cnt++; + probe_idx++; + break; + } do_the_probe(); + if (picking_null_keys) { + break; + } } probe_idx -= (build_idx != 0); - return std::tuple {probe_idx, build_idx, matched_cnt}; + return std::tuple {probe_idx, build_idx, matched_cnt, picking_null_keys}; } template @@ -215,13 +228,15 @@ class JoinHashTable { bool has_null_key() { return _has_null_key; } - void pre_build_idxs(std::vector& bucksets, const uint8_t* null_map) { + void pre_build_idxs(std::vector& buckets, const uint8_t* null_map) { if (null_map) { - first[bucket_size] = bucket_size; // distinguish between not matched and null - } - - for (uint32_t i = 0; i < bucksets.size(); i++) { - bucksets[i] = first[bucksets[i]]; + for (unsigned int& bucket : buckets) { + bucket = bucket == bucket_size ? bucket_size : first[bucket]; + } + } else { + for (unsigned int& bucket : buckets) { + bucket = first[bucket]; + } } } diff --git a/be/src/vec/exec/join/process_hash_table_probe.h b/be/src/vec/exec/join/process_hash_table_probe.h index 02bf242e55a1624..0ed743a7d18cce8 100644 --- a/be/src/vec/exec/join/process_hash_table_probe.h +++ b/be/src/vec/exec/join/process_hash_table_probe.h @@ -93,7 +93,9 @@ struct ProcessHashTableProbe { std::vector _probe_indexs; bool _probe_visited = false; + bool _picking_null_keys = false; std::vector _build_indexs; + std::vector _null_flags; std::vector _build_blocks_locs; // only need set the tuple is null in RIGHT_OUTER_JOIN and FULL_OUTER_JOIN ColumnUInt8::Container* _tuple_is_null_left_flags = nullptr; diff --git a/be/src/vec/exec/join/process_hash_table_probe_impl.h b/be/src/vec/exec/join/process_hash_table_probe_impl.h index 9f5167bb555bf29..b72bc2d62c27906 100644 --- a/be/src/vec/exec/join/process_hash_table_probe_impl.h +++ b/be/src/vec/exec/join/process_hash_table_probe_impl.h @@ -131,6 +131,11 @@ typename HashTableType::State ProcessHashTableProbe::_init_p // may over batch size 1 for some outer join case _probe_indexs.resize(_batch_size + 1); _build_indexs.resize(_batch_size + 1); + if constexpr (JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN || + JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) { + _null_flags.resize(_batch_size + 1); + memset(_null_flags.data(), 0, _batch_size + 1); + } if (!_parent->_ready_probe) { _parent->_ready_probe = true; @@ -185,14 +190,41 @@ Status ProcessHashTableProbe::do_process(HashTableType& hash JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) && with_other_conjuncts) { SCOPED_TIMER(_search_hashtable_timer); - auto [new_probe_idx, new_build_idx, new_current_offset] = - hash_table_ctx.hash_table->find_null_aware_with_other_conjuncts( - hash_table_ctx.keys, hash_table_ctx.bucket_nums.data(), probe_index, - build_index, probe_rows, _probe_indexs.data(), _build_indexs.data(), - null_result, *(_parent->_build_indexes_null), _build_block->rows()); - probe_index = new_probe_idx; - build_index = new_build_idx; - current_offset = new_current_offset; + + // If the key of one probe row is null, + // this probe row should match with all rows in build side(match result: null). + if (build_index == hash_table_ctx.hash_table->get_bucket_size() && !_picking_null_keys) { + const auto rows = _build_block->rows(); + /// FIXME: Memory allocation issue due to the possibility of rows being a huge value. + if (rows > _batch_size + 1) { + _build_indexs.resize(rows); + _probe_indexs.resize(rows); + _null_flags.resize(rows); + } + + for (size_t i = 0; i != rows - 1; ++i) { + _probe_indexs[i] = probe_index; + _build_indexs[i] = i + 1; + _null_flags[i] = 1; + } + + _probe_indexs[rows - 1] = probe_index; + _build_indexs[rows - 1] = 0; + _null_flags[rows - 1] = 0; + current_offset = rows; + build_index = 0; + probe_index++; + } else { + auto [new_probe_idx, new_build_idx, new_current_offset, picking_null_keys] = + hash_table_ctx.hash_table->find_null_aware_with_other_conjuncts( + hash_table_ctx.keys, hash_table_ctx.bucket_nums.data(), probe_index, + build_index, probe_rows, _probe_indexs.data(), _build_indexs.data(), + _null_flags.data(), _picking_null_keys); + probe_index = new_probe_idx; + build_index = new_build_idx; + current_offset = new_current_offset; + _picking_null_keys = picking_null_keys; + } } else { SCOPED_TIMER(_search_hashtable_timer); auto [new_probe_idx, new_build_idx, @@ -279,8 +311,7 @@ Status ProcessHashTableProbe::do_mark_join_conjuncts( filter_data[i] = _build_indexs[i] != 0 && _build_indexs[i] != hash_table_bucket_size; if constexpr (is_null_aware_join) { if constexpr (with_other_conjuncts) { - mark_null_map[i] = - null_result.contains(_probe_indexs[i]) && _build_indexs[i] != 0; + mark_null_map[i] = _null_flags[i]; } else { if (filter_data[i]) { last_probe_matched = _probe_indexs[i]; diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp index a813ec565a4b04a..ec630f3fe32edd8 100644 --- a/be/src/vec/exec/join/vhash_join_node.cpp +++ b/be/src/vec/exec/join/vhash_join_node.cpp @@ -180,12 +180,6 @@ Status HashJoinNode::init(const TPlanNode& tnode, RuntimeState* state) { } #endif - if ((_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN || - _join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) && - _have_other_join_conjunct) { - _build_indexes_null = std::make_shared>(); - } - _runtime_filters.resize(_runtime_filter_descs.size()); for (size_t i = 0; i < _runtime_filter_descs.size(); i++) { RETURN_IF_ERROR(state->runtime_filter_mgr()->register_producer_filter( @@ -761,7 +755,6 @@ Status HashJoinNode::sink(doris::RuntimeState* state, vectorized::Block* in_bloc // arena will be shared with other instances. _shared_hash_table_context->arena = _arena; _shared_hash_table_context->block = _build_block; - _shared_hash_table_context->build_indexes_null = _build_indexes_null; _shared_hash_table_context->hash_table_variants = _hash_table_variants; _shared_hash_table_context->short_circuit_for_null_in_probe_side = _has_null_in_build_side; @@ -794,7 +787,6 @@ Status HashJoinNode::sink(doris::RuntimeState* state, vectorized::Block* in_bloc *std::static_pointer_cast( _shared_hash_table_context->hash_table_variants)); _build_block = _shared_hash_table_context->block; - _build_indexes_null = _shared_hash_table_context->build_indexes_null; if (!_shared_hash_table_context->runtime_filters.empty()) { auto ret = std::visit( diff --git a/be/src/vec/exec/join/vhash_join_node.h b/be/src/vec/exec/join/vhash_join_node.h index fb5bf19015d99c0..5984f0469bfe3bb 100644 --- a/be/src/vec/exec/join/vhash_join_node.h +++ b/be/src/vec/exec/join/vhash_join_node.h @@ -117,11 +117,6 @@ struct ProcessHashTableBuild { for (uint32_t i = 1; i < _rows; i++) { if ((*null_map)[i]) { *has_null_key = true; - if constexpr (with_other_conjuncts && - (JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN || - JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN)) { - _parent->_build_indexes_null->emplace_back(i); - } } } if (short_circuit_for_null && *has_null_key) { @@ -136,8 +131,8 @@ struct ProcessHashTableBuild { hash_table_ctx.init_serialized_keys(_build_raw_ptrs, _rows, null_map ? null_map->data() : nullptr, true, true, hash_table_ctx.hash_table->get_bucket_size()); - hash_table_ctx.hash_table->build(hash_table_ctx.keys, hash_table_ctx.bucket_nums.data(), - _rows); + hash_table_ctx.hash_table->template build( + hash_table_ctx.keys, hash_table_ctx.bucket_nums.data(), _rows); hash_table_ctx.bucket_nums.resize(_batch_size); hash_table_ctx.bucket_nums.shrink_to_fit(); @@ -301,13 +296,6 @@ class HashJoinNode final : public VJoinNodeBase { std::vector _probe_column_disguise_null; std::vector _probe_column_convert_to_null; - /* - * For null aware anti/semi join with other join conjuncts, we do need to care about the rows in - * build side with null keys, - * because the other join conjuncts' result maybe change null to false(null & false == false). - */ - std::shared_ptr> _build_indexes_null; - DataTypes _right_table_data_types; DataTypes _left_table_data_types; std::vector _right_table_column_names; diff --git a/regression-test/data/nereids_p0/join/test_mark_join.out b/regression-test/data/nereids_p0/join/test_mark_join.out new file mode 100644 index 000000000000000..4098502b75df661 --- /dev/null +++ b/regression-test/data/nereids_p0/join/test_mark_join.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mark_join1 -- +1 1 true +2 2 true +3 \N true +3 \N true +4 \N \N + +-- !mark_join2 -- +1 1 \N +2 2 \N +3 \N \N +3 \N true +4 \N true + +-- !mark_join3 -- +1 1 false +2 2 false +3 \N false +3 \N false +4 \N false + +-- !mark_join4 -- +1 1 false +2 2 false +3 \N \N +3 \N true +4 \N true + +-- !mark_join5 -- +1 1 false +2 2 false +3 \N true +3 \N true +4 \N \N + +-- !mark_join6 -- +1 1 true +2 2 true +3 \N false +3 \N true +4 \N false + diff --git a/regression-test/suites/nereids_p0/join/test_mark_join.groovy b/regression-test/suites/nereids_p0/join/test_mark_join.groovy new file mode 100644 index 000000000000000..bb1b9c5e87105d4 --- /dev/null +++ b/regression-test/suites/nereids_p0/join/test_mark_join.groovy @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mark_join", "nereids_p0") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql"use nereids_test_query_db" + + sql "drop table if exists `test_mark_join_t1`;" + sql "drop table if exists `test_mark_join_t2`;" + + sql """ + CREATE TABLE IF NOT EXISTS `test_mark_join_t1` ( + k1 int not null, + k2 int, + k3 bigint, + v1 varchar(255) not null, + v2 varchar(255), + v3 varchar(255) + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + CREATE TABLE IF NOT EXISTS `test_mark_join_t2` ( + k1 int not null, + k2 int, + k3 bigint, + v1 varchar(255) not null, + v2 varchar(255), + v3 varchar(255) + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + insert into `test_mark_join_t1` values + (1, 1, 1, 'abc', 'efg', 'hjk'), + (2, 2, 2, 'aabb', 'eeff', 'ccdd'), + (3, null, 3, 'iii', null, null), + (3, null, null, 'hhhh', null, null), + (4, null, 4, 'dddd', 'ooooo', 'kkkkk' + ); + """ + + sql """ + insert into `test_mark_join_t2` values + (1, 1, 1, 'abc', 'efg', 'hjk'), + (2, 2, 2, 'aabb', 'eeff', 'ccdd'), + (3, null, null, 'diid', null, null), + (3, null, 3, 'ooekd', null, null), + (4, 4, null, 'oepeld', null, 'kkkkk' + ); + """ + + qt_mark_join1 """ + select + k1, k2 + , k1 not in (select test_mark_join_t2.k2 from test_mark_join_t2 where test_mark_join_t2.k3 < test_mark_join_t1.k3) vv + from test_mark_join_t1 order by 1, 2, 3; + """ + + qt_mark_join2 """ + select + k1, k2 + , k2 not in (select test_mark_join_t2.k3 from test_mark_join_t2 where test_mark_join_t2.k2 > test_mark_join_t1.k3) vv + from test_mark_join_t1 order by 1, 2, 3; + """ + + qt_mark_join3 """ + select + k1, k2 + , k1 in (select test_mark_join_t2.k1 from test_mark_join_t2 where test_mark_join_t2.k3 < test_mark_join_t1.k3) vv + from test_mark_join_t1 order by 1, 2, 3; + """ + + qt_mark_join4 """ + select + k1, k2 + , k1 not in (select test_mark_join_t2.k2 from test_mark_join_t2 where test_mark_join_t2.k3 = test_mark_join_t1.k3) vv + from test_mark_join_t1 order by 1, 2, 3; + """ + + qt_mark_join5 """ + select + k1, k2 + , k2 not in (select test_mark_join_t2.k3 from test_mark_join_t2 where test_mark_join_t2.k2 = test_mark_join_t1.k3) vv + from test_mark_join_t1 order by 1, 2, 3; + """ + + qt_mark_join6 """ + select + k1, k2 + , k1 in (select test_mark_join_t2.k1 from test_mark_join_t2 where test_mark_join_t2.k3 = test_mark_join_t1.k3) vv + from test_mark_join_t1 order by 1, 2, 3; + """ + + +}