diff --git a/CMakeLists.txt b/CMakeLists.txt index a4675ec2f..82c7d19a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ OPTION(ENABLE_COVERAGE "Enable unittest coverage" OFF) OPTION(ENABLE_NOPIE "Enable no pie" OFF) OPTION(CONCURRENCY "Support concurrency operations" OFF) OPTION(STATIC_STDLIB "Link std library static or dynamic, such as libgcc, libstdc++, libasan" OFF) +OPTION(USE_SIMD "Use SIMD" OFF) MESSAGE(STATUS "HOME dir: $ENV{HOME}") #SET(ENV{变量名} 值) @@ -42,12 +43,19 @@ ELSE() ENDIF(WIN32) # This is for clangd plugin for vscode -SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -Wall -Werror") +# mute sign-compare error in lex/yacc +SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -Wall -Werror -Wno-error=sign-compare") IF (ENABLE_NOPIE) SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -no-pie") ADD_LINK_OPTIONS(-no-pie) ENDIF (ENABLE_NOPIE) +# Requires support for avx2 +IF(USE_SIMD) + SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -mavx2") + ADD_DEFINITIONS(-DUSE_SIMD) +ENDIF(USE_SIMD) + IF(DEBUG) MESSAGE(STATUS "DEBUG has been set as TRUE ${DEBUG}") SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -O0 -g -DDEBUG ") diff --git a/benchmark/aggregate_hash_table_performance_test.cpp b/benchmark/aggregate_hash_table_performance_test.cpp new file mode 100644 index 000000000..c2040bb5a --- /dev/null +++ b/benchmark/aggregate_hash_table_performance_test.cpp @@ -0,0 +1,95 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include + +#include "common/lang/memory.h" +#include "common/lang/vector.h" +#include "sql/expr/aggregate_hash_table.h" + +class AggregateHashTableBenchmark : public benchmark::Fixture +{ +public: + void SetUp(const ::benchmark::State &state) override + { + unique_ptr column1 = make_unique(AttrType::INTS, 4); + unique_ptr column2 = make_unique(AttrType::INTS, 4); + for (int i = 0; i < state.range(0); i++) { + int key = i % 8; + column1->append_one((char *)&key); + column2->append_one((char *)&i); + } + group_chunk_.add_column(std::move(column1), 0); + aggr_chunk_.add_column(std::move(column2), 0); + } + + void TearDown(const ::benchmark::State &state) override + { + group_chunk_.reset(); + aggr_chunk_.reset(); + } + +protected: + Chunk group_chunk_; + Chunk aggr_chunk_; +}; + +class DISABLED_StandardAggregateHashTableBenchmark : public AggregateHashTableBenchmark +{ +public: + void SetUp(const ::benchmark::State &state) override + { + + AggregateHashTableBenchmark::SetUp(state); + AggregateExpr aggregate_expr(AggregateExpr::Type::SUM, nullptr); + vector aggregate_exprs; + aggregate_exprs.push_back(&aggregate_expr); + standard_hash_table_ = make_unique(aggregate_exprs); + } + +protected: + unique_ptr standard_hash_table_; +}; + +BENCHMARK_DEFINE_F(DISABLED_StandardAggregateHashTableBenchmark, Aggregate)(benchmark::State &state) +{ + for (auto _ : state) { + standard_hash_table_->add_chunk(group_chunk_, aggr_chunk_); + } +} + +BENCHMARK_REGISTER_F(DISABLED_StandardAggregateHashTableBenchmark, Aggregate)->Arg(16)->Arg(1024)->Arg(8192); + +#ifdef USE_SIMD +class DISABLED_LinearProbingAggregateHashTableBenchmark : public AggregateHashTableBenchmark +{ +public: + void SetUp(const ::benchmark::State &state) override + { + + AggregateHashTableBenchmark::SetUp(state); + linear_probing_hash_table_ = make_unique>(AggregateExpr::Type::SUM); + } + +protected: + unique_ptr linear_probing_hash_table_; +}; + +BENCHMARK_DEFINE_F(DISABLED_LinearProbingAggregateHashTableBenchmark, Aggregate)(benchmark::State &state) +{ + for (auto _ : state) { + linear_probing_hash_table_->add_chunk(group_chunk_, aggr_chunk_); + } +} + +BENCHMARK_REGISTER_F(DISABLED_LinearProbingAggregateHashTableBenchmark, Aggregate)->Arg(16)->Arg(1024)->Arg(8192); +#endif + +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/benchmark/arithmetic_operator_performance_test.cpp b/benchmark/arithmetic_operator_performance_test.cpp new file mode 100644 index 000000000..a60b84e68 --- /dev/null +++ b/benchmark/arithmetic_operator_performance_test.cpp @@ -0,0 +1,100 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include + +#include "sql/expr/arithmetic_operator.hpp" + +class DISABLED_ArithmeticBenchmark : public benchmark::Fixture +{ +public: + void SetUp(const ::benchmark::State &state) override + { + int size = state.range(0); + left_ = (float *)malloc(size * sizeof(float)); + right_ = (float *)malloc(size * sizeof(float)); + result_ = (float *)malloc(size * sizeof(float)); + + for (int i = 0; i < size; ++i) { + left_[i] = 1.0f; + right_[i] = 0.5f; + result_[i] = 0.0f; + } + } + + void TearDown(const ::benchmark::State &state) override + { + free(left_); + left_ = nullptr; + free(right_); + right_ = nullptr; + free(result_); + result_ = nullptr; + } + +protected: + float *left_ = nullptr; + float *right_ = nullptr; + float *result_ = nullptr; +}; + +BENCHMARK_DEFINE_F(DISABLED_ArithmeticBenchmark, Add)(benchmark::State &state) +{ + for (auto _ : state) { + binary_operator(left_, right_, result_, state.range(0)); + } +} + +BENCHMARK_REGISTER_F(DISABLED_ArithmeticBenchmark, Add)->Arg(10)->Arg(1000)->Arg(10000); + +BENCHMARK_DEFINE_F(DISABLED_ArithmeticBenchmark, Sub)(benchmark::State &state) +{ + for (auto _ : state) { + binary_operator(left_, right_, result_, state.range(0)); + } +} + +BENCHMARK_REGISTER_F(DISABLED_ArithmeticBenchmark, Sub)->Arg(10)->Arg(1000)->Arg(10000); + +#ifdef USE_SIMD +static void DISABLED_benchmark_sum_simd(benchmark::State &state) +{ + int size = state.range(0); + std::vector data(state.range(0), 1); + for (auto _ : state) { + mm256_sum_epi32(data.data(), size); + } +} + +BENCHMARK(DISABLED_benchmark_sum_simd)->RangeMultiplier(2)->Range(1 << 10, 1 << 12); +#endif + +static int sum_scalar(const int *data, int size) +{ + int sum = 0; + for (int i = 0; i < size; i++) { + sum += data[i]; + } + return sum; +} + +static void DISABLED_benchmark_sum_scalar(benchmark::State &state) +{ + int size = state.range(0); + std::vector data(size, 1); + for (auto _ : state) { + int res = sum_scalar(data.data(), size); + benchmark::DoNotOptimize(res); + } +} + +BENCHMARK(DISABLED_benchmark_sum_scalar)->RangeMultiplier(2)->Range(1 << 10, 1 << 12); + +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/benchmark/bplus_tree_concurrency_test.cpp b/benchmark/bplus_tree_concurrency_test.cpp index ab6936262..59714d513 100644 --- a/benchmark/bplus_tree_concurrency_test.cpp +++ b/benchmark/bplus_tree_concurrency_test.cpp @@ -13,8 +13,8 @@ See the Mulan PSL v2 for more details. */ // #include #include -#include +#include "common/lang/stdexcept.h" #include "common/log/log.h" #include "common/math/integer_generator.h" #include "storage/buffer/disk_buffer_pool.h" @@ -71,7 +71,7 @@ class BenchmarkBase : public Fixture const char *filename = btree_filename.c_str(); RC rc = handler_.create( - log_handler_, bpm_, filename, INTS, sizeof(int32_t) /*attr_len*/, internal_max_size, leaf_max_size); + log_handler_, bpm_, filename, AttrType::INTS, sizeof(int32_t) /*attr_len*/, internal_max_size, leaf_max_size); if (rc != RC::SUCCESS) { throw runtime_error("failed to create btree handler"); } diff --git a/benchmark/pax_storage_concurrency_test.cpp b/benchmark/pax_storage_concurrency_test.cpp new file mode 100644 index 000000000..cd15ccea0 --- /dev/null +++ b/benchmark/pax_storage_concurrency_test.cpp @@ -0,0 +1,505 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include +#include + +#include "common/lang/sstream.h" +#define private public +#define protected public +#include "storage/table/table.h" +#undef private +#undef protected + +#include "common/log/log.h" +#include "common/lang/chrono.h" +#include "common/lang/stdexcept.h" +#include "common/lang/memory.h" +#include "common/math/integer_generator.h" +#include "storage/buffer/disk_buffer_pool.h" +#include "storage/common/condition_filter.h" +#include "storage/record/record_manager.h" +#include "storage/trx/vacuous_trx.h" +#include "storage/clog/vacuous_log_handler.h" +#include "storage/buffer/double_write_buffer.h" + +using namespace common; +using namespace benchmark; + +struct Stat +{ + int64_t insert_success_count = 0; + int64_t insert_other_count = 0; + + int64_t delete_success_count = 0; + int64_t not_exist_count = 0; + int64_t delete_other_count = 0; + + int64_t scan_success_count = 0; + int64_t scan_open_failed_count = 0; + int64_t mismatch_count = 0; + int64_t scan_other_count = 0; +}; + +struct TestRecord +{ + char fields[15]; +}; + +class TestConditionFilter : public ConditionFilter +{ +public: + TestConditionFilter(int32_t begin, int32_t end) : begin_(begin), end_(end) {} + + bool filter(const Record &rec) const override + { + const char *data = rec.data(); + int32_t value = *(int32_t *)data; + return value >= begin_ && value <= end_; + } + +private: + int32_t begin_; + int32_t end_; +}; + +class BenchmarkBase : public Fixture +{ +public: + BenchmarkBase() {} + + virtual ~BenchmarkBase() {} + + virtual string Name() const = 0; + + string record_filename() const { return this->Name() + ".record"; } + + virtual void SetUp(const State &state) + { + if (0 != state.thread_index()) { + return; + } + + bpm_.init(make_unique()); + + string log_name = this->Name() + ".log"; + string record_filename = this->record_filename(); + LoggerFactory::init_default(log_name.c_str(), LOG_LEVEL_INFO); + + ::remove(record_filename.c_str()); + + RC rc = bpm_.create_file(record_filename.c_str()); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to create record buffer pool file. filename=%s, rc=%s", record_filename.c_str(), strrc(rc)); + throw runtime_error("failed to create record buffer pool file."); + } + + rc = bpm_.open_file(log_handler_, record_filename.c_str(), buffer_pool_); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to open record file. filename=%s, rc=%s", record_filename.c_str(), strrc(rc)); + throw runtime_error("failed to open record file"); + } + table_meta_ = new TableMeta; + table_meta_->fields_.resize(2); + table_meta_->fields_[0].attr_type_ = AttrType::INTS; + table_meta_->fields_[0].attr_len_ = 4; + table_meta_->fields_[0].field_id_ = 0; + table_meta_->fields_[1].attr_type_ = AttrType::CHARS; + table_meta_->fields_[1].attr_len_ = 11; + table_meta_->fields_[1].field_id_ = 1; + handler_ = new RecordFileHandler(StorageFormat::PAX_FORMAT); + rc = handler_->init(*buffer_pool_, log_handler_, table_meta_); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to init record file handler. rc=%s", strrc(rc)); + throw runtime_error("failed to init record file handler"); + } + LOG_INFO("test %s setup done. threads=%d, thread index=%d", + this->Name().c_str(), state.threads(), state.thread_index()); + } + + virtual void TearDown(const State &state) + { + if (0 != state.thread_index()) { + return; + } + + handler_->close(); + delete handler_; + handler_ = nullptr; + delete table_meta_; + table_meta_ = nullptr; + // TODO 很怪,引入double write buffer后,必须要求先close buffer pool,再执行bpm.close_file。 + // 以后必须修理好bpm、buffer pool、double write buffer之间的关系 + buffer_pool_->close_file(); + bpm_.close_file(this->record_filename().c_str()); + buffer_pool_ = nullptr; + LOG_INFO("test %s teardown done. threads=%d, thread index=%d", + this->Name().c_str(), + state.threads(), + state.thread_index()); + } + + void FillUp(int32_t min, int32_t max, vector &rids) + { + rids.reserve(max - min); + RID rid; + TestRecord record; + vector record_values; + record_values.reserve(max - min); + for (int32_t value = min; value < max; ++value) { + record_values.push_back(value); + } + + random_device rd; + mt19937 random_generator(rd()); + shuffle(record_values.begin(), record_values.end(), random_generator); + + for (int32_t record_value : record_values) { + memcpy(&record.fields[0], &record_value, sizeof(record_value)); + [[maybe_unused]] RC rc = handler_->insert_record(reinterpret_cast(&record), sizeof(record), &rid); + ASSERT(rc == RC::SUCCESS, "failed to insert record into record file. record value=%" PRIu32, record_value); + rids.push_back(rid); + } + + LOG_INFO("fill up done. min=%" PRIu32 ", max=%" PRIu32 ", distance=%" PRIu32, min, max, (max - min)); + } + + uint32_t GetRangeMax(const State &state) const + { + int32_t max = static_cast(state.range(0) * 3); + if (max <= 0) { + max = INT32_MAX - 1; + } + return max; + } + + void Insert(int32_t value, Stat &stat, RID &rid) + { + TestRecord record; + memcpy(&record.fields[0], &value, sizeof(value)); + + RC rc = handler_->insert_record(reinterpret_cast(&record), sizeof(record), &rid); + switch (rc) { + case RC::SUCCESS: { + stat.insert_success_count++; + } break; + default: { + stat.insert_other_count++; + } break; + } + } + + void Delete(const RID &rid, Stat &stat) + { + RC rc = handler_->delete_record(&rid); + switch (rc) { + case RC::SUCCESS: { + stat.delete_success_count++; + } break; + case RC::RECORD_NOT_EXIST: { + stat.not_exist_count++; + } break; + default: { + stat.delete_other_count++; + } break; + } + } + + void Scan(int32_t begin, int32_t end, Stat &stat) + { + TestConditionFilter condition_filter(begin, end); + RecordFileScanner scanner; + VacuousTrx trx; + Table table; + table.table_meta_.storage_format_ = StorageFormat::PAX_FORMAT; + RC rc = scanner.open_scan(&table, *buffer_pool_, &trx, log_handler_, ReadWriteMode::READ_ONLY, &condition_filter); + if (rc != RC::SUCCESS) { + stat.scan_open_failed_count++; + } else { + Record record; + int32_t count = 0; + while (OB_SUCC(rc = scanner.next(record))) { + count++; + } + + if (rc != RC::RECORD_EOF) { + stat.scan_other_count++; + } else if (count != (end - begin + 1)) { + stat.mismatch_count++; + } else { + stat.scan_success_count++; + } + + scanner.close_scan(); + } + } + + void ScanChunk(Stat &stat) + { + ChunkFileScanner scanner; + Table table; + table.table_meta_.storage_format_ = StorageFormat::PAX_FORMAT; + RC rc = scanner.open_scan_chunk(&table, *buffer_pool_, log_handler_, ReadWriteMode::READ_ONLY); + if (rc != RC::SUCCESS) { + stat.scan_open_failed_count++; + } else { + Chunk chunk; + FieldMeta fm; + fm.init("col1", AttrType::INTS, 0, 4, true, 0); + auto col1 = make_unique(fm, 2048); + chunk.add_column(std::move(col1), 0); + while (OB_SUCC(rc = scanner.next_chunk(chunk))) { + chunk.reset_data(); + } + + if (rc != RC::RECORD_EOF) { + stat.scan_other_count++; + } else { + stat.scan_success_count++; + } + + scanner.close_scan(); + } + } + +protected: + BufferPoolManager bpm_{512}; + DiskBufferPool *buffer_pool_ = nullptr; + RecordFileHandler *handler_ = nullptr; + VacuousLogHandler log_handler_; + TableMeta *table_meta_ = nullptr; + ; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct DISABLED_InsertionBenchmark : public BenchmarkBase +{ + string Name() const override { return "insertion"; } +}; + +BENCHMARK_DEFINE_F(DISABLED_InsertionBenchmark, Insertion)(State &state) +{ + IntegerGenerator generator(1, 1 << 31); + Stat stat; + + RID rid; + for (auto _ : state) { + Insert(generator.next(), stat, rid); + } + + state.counters["success"] = Counter(stat.insert_success_count, Counter::kIsRate); + state.counters["other"] = Counter(stat.insert_other_count, Counter::kIsRate); +} + +BENCHMARK_REGISTER_F(DISABLED_InsertionBenchmark, Insertion)->Threads(10); + +//////////////////////////////////////////////////////////////////////////////// + +class DISABLED_DeletionBenchmark : public BenchmarkBase +{ +public: + string Name() const override { return "deletion"; } + + void SetUp(const State &state) override + { + BenchmarkBase::SetUp(state); + + if (0 != state.thread_index()) { + while (!setup_done_) { + this_thread::sleep_for(chrono::milliseconds(100)); + } + return; + } + + uint32_t max = GetRangeMax(state); + ASSERT(max > 0, "invalid argument count. %ld", state.range(0)); + FillUp(0, max, rids_); + ASSERT(rids_.size() > 0, "invalid argument count. %ld", rids_.size()); + setup_done_ = true; + } + +protected: + // 从实际测试情况看,每个线程都会执行setup,但是它们操作的对象都是同一个 + // 但是每个线程set up结束后,就会执行测试了。如果不等待的话,就会导致有些 + // 线程访问的数据不是想要的结果 + volatile bool setup_done_ = false; + vector rids_; +}; + +BENCHMARK_DEFINE_F(DISABLED_DeletionBenchmark, Deletion)(State &state) +{ + IntegerGenerator generator(0, static_cast(rids_.size() - 1)); + Stat stat; + + for (auto _ : state) { + int32_t value = generator.next(); + RID rid = rids_[value]; + Delete(rid, stat); + } + + state.counters["success"] = Counter(stat.delete_success_count, Counter::kIsRate); + state.counters["not_exist"] = Counter(stat.not_exist_count, Counter::kIsRate); + state.counters["other"] = Counter(stat.delete_other_count, Counter::kIsRate); +} + +BENCHMARK_REGISTER_F(DISABLED_DeletionBenchmark, Deletion)->Threads(10)->Arg(4 * 10000); + +//////////////////////////////////////////////////////////////////////////////// + +class DISABLED_ScanBenchmark : public BenchmarkBase +{ +public: + string Name() const override { return "scan"; } + + void SetUp(const State &state) override + { + if (0 != state.thread_index()) { + return; + } + + BenchmarkBase::SetUp(state); + + int32_t max = state.range(0) * 3; + ASSERT(max > 0, "invalid argument count. %ld", state.range(0)); + vector rids; + FillUp(0, max, rids); + } +}; + +BENCHMARK_DEFINE_F(DISABLED_ScanBenchmark, Scan)(State &state) +{ + int max_range_size = 100; + uint32_t max = GetRangeMax(state); + IntegerGenerator begin_generator(1, max - max_range_size); + IntegerGenerator range_generator(1, max_range_size); + Stat stat; + + for (auto _ : state) { + int32_t begin = begin_generator.next(); + int32_t end = begin + range_generator.next(); + Scan(begin, end, stat); + } + + state.counters["success"] = Counter(stat.scan_success_count, Counter::kIsRate); + state.counters["open_failed_count"] = Counter(stat.scan_open_failed_count, Counter::kIsRate); + state.counters["mismatch_number_count"] = Counter(stat.mismatch_count, Counter::kIsRate); + state.counters["other"] = Counter(stat.scan_other_count, Counter::kIsRate); +} + +BENCHMARK_REGISTER_F(DISABLED_ScanBenchmark, Scan)->Threads(10)->Arg(4 * 10000); + +//////////////////////////////////////////////////////////////////////////////// + +class DISABLED_ScanChunkBenchmark : public BenchmarkBase +{ +public: + string Name() const override { return "scan_chunk"; } + + void SetUp(const State &state) override + { + if (0 != state.thread_index()) { + return; + } + + BenchmarkBase::SetUp(state); + + int32_t max = state.range(0) * 3; + ASSERT(max > 0, "invalid argument count. %ld", state.range(0)); + vector rids; + FillUp(0, max, rids); + } +}; + +BENCHMARK_DEFINE_F(DISABLED_ScanChunkBenchmark, ScanChunk)(State &state) +{ + Stat stat; + for (auto _ : state) { + ScanChunk(stat); + } + + state.counters["success"] = Counter(stat.scan_success_count, Counter::kIsRate); + state.counters["open_failed_count"] = Counter(stat.scan_open_failed_count, Counter::kIsRate); + state.counters["mismatch_number_count"] = Counter(stat.mismatch_count, Counter::kIsRate); + state.counters["other"] = Counter(stat.scan_other_count, Counter::kIsRate); +} + +BENCHMARK_REGISTER_F(DISABLED_ScanChunkBenchmark, ScanChunk)->Threads(10)->Arg(4 * 10000); + +//////////////////////////////////////////////////////////////////////////////// + +struct DISABLED_MixtureBenchmark : public BenchmarkBase +{ + string Name() const override { return "mixture"; } +}; + +BENCHMARK_DEFINE_F(DISABLED_MixtureBenchmark, Mixture)(State &state) +{ + pair data_range{0, GetRangeMax(state)}; + pair scan_range{1, 100}; + + IntegerGenerator data_generator(data_range.first, data_range.second); + IntegerGenerator scan_range_generator(scan_range.first, scan_range.second); + IntegerGenerator operation_generator(0, 3); + + Stat stat; + + vector rids; + for (auto _ : state) { + int operation_type = operation_generator.next(); + switch (operation_type) { + case 0: { // insert + int32_t value = data_generator.next(); + RID rid; + Insert(value, stat, rid); + if (rids.size() < 1000000) { + rids.push_back(rid); + } + } break; + case 1: { // delete + int32_t index = data_generator.next(); + if (!rids.empty()) { + index %= rids.size(); + RID rid = rids[index]; + rids.erase(rids.begin() + index); + Delete(rid, stat); + } + } break; + case 2: { // scan + int32_t begin = data_generator.next(); + int32_t end = begin + scan_range_generator.next(); + Scan(begin, end, stat); + } break; + case 3: { // scan chunk + ScanChunk(stat); + } break; + default: { + ASSERT(false, "should not happen. operation=%ld", operation_type); + } + } + } + + state.counters.insert({{"insert_success", Counter(stat.insert_success_count, Counter::kIsRate)}, + {"insert_other", Counter(stat.insert_other_count, Counter::kIsRate)}, + {"delete_success", Counter(stat.delete_success_count, Counter::kIsRate)}, + {"delete_other", Counter(stat.delete_other_count, Counter::kIsRate)}, + {"delete_not_exist", Counter(stat.not_exist_count, Counter::kIsRate)}, + {"scan_success", Counter(stat.scan_success_count, Counter::kIsRate)}, + {"scan_other", Counter(stat.scan_other_count, Counter::kIsRate)}, + {"scan_mismatch", Counter(stat.mismatch_count, Counter::kIsRate)}, + {"scan_open_failed", Counter(stat.scan_open_failed_count, Counter::kIsRate)}}); +} + +BENCHMARK_REGISTER_F(DISABLED_MixtureBenchmark, Mixture)->Threads(10)->Arg(4 * 10000); + +//////////////////////////////////////////////////////////////////////////////// + +BENCHMARK_MAIN(); diff --git a/benchmark/record_manager_concurrency_test.cpp b/benchmark/record_manager_concurrency_test.cpp index 542929ea9..bc96368e9 100644 --- a/benchmark/record_manager_concurrency_test.cpp +++ b/benchmark/record_manager_concurrency_test.cpp @@ -15,8 +15,8 @@ See the Mulan PSL v2 for more details. */ #include #include #include -#include +#include "common/lang/stdexcept.h" #include "common/log/log.h" #include "common/math/integer_generator.h" #include "storage/buffer/disk_buffer_pool.h" @@ -104,7 +104,8 @@ class BenchmarkBase : public Fixture throw runtime_error("failed to open record file"); } - rc = handler_.init(*buffer_pool_, log_handler_); + handler_ = new RecordFileHandler(StorageFormat::ROW_FORMAT); + rc = handler_->init(*buffer_pool_, log_handler_, nullptr); if (rc != RC::SUCCESS) { LOG_WARN("failed to init record file handler. rc=%s", strrc(rc)); throw runtime_error("failed to init record file handler"); @@ -119,7 +120,9 @@ class BenchmarkBase : public Fixture return; } - handler_.close(); + handler_->close(); + delete handler_; + handler_ = nullptr; // TODO 很怪,引入double write buffer后,必须要求先close buffer pool,再执行bpm.close_file。 // 以后必须修理好bpm、buffer pool、double write buffer之间的关系 buffer_pool_->close_file(); @@ -148,7 +151,7 @@ class BenchmarkBase : public Fixture for (int32_t record_value : record_values) { record.int_fields[0] = record_value; - [[maybe_unused]] RC rc = handler_.insert_record(reinterpret_cast(&record), sizeof(record), &rid); + [[maybe_unused]] RC rc = handler_->insert_record(reinterpret_cast(&record), sizeof(record), &rid); ASSERT(rc == RC::SUCCESS, "failed to insert record into record file. record value=%" PRIu32, record_value); rids.push_back(rid); } @@ -170,7 +173,7 @@ class BenchmarkBase : public Fixture TestRecord record; record.int_fields[0] = value; - RC rc = handler_.insert_record(reinterpret_cast(&record), sizeof(record), &rid); + RC rc = handler_->insert_record(reinterpret_cast(&record), sizeof(record), &rid); switch (rc) { case RC::SUCCESS: { stat.insert_success_count++; @@ -183,7 +186,7 @@ class BenchmarkBase : public Fixture void Delete(const RID &rid, Stat &stat) { - RC rc = handler_.delete_record(&rid); + RC rc = handler_->delete_record(&rid); switch (rc) { case RC::SUCCESS: { stat.delete_success_count++; @@ -210,11 +213,10 @@ class BenchmarkBase : public Fixture Record record; int32_t count = 0; while (OB_SUCC(rc = scanner.next(record))) { - ASSERT(rc == RC::SUCCESS, "failed to get record, rc=%s", strrc(rc)); count++; } - if (rc != RC::SUCCESS) { + if (rc != RC::RECORD_EOF) { stat.scan_other_count++; } else if (count != (end - begin + 1)) { stat.mismatch_count++; @@ -227,10 +229,10 @@ class BenchmarkBase : public Fixture } protected: - BufferPoolManager bpm_{512}; - DiskBufferPool *buffer_pool_ = nullptr; - RecordFileHandler handler_; - VacuousLogHandler log_handler_; + BufferPoolManager bpm_{512}; + DiskBufferPool *buffer_pool_ = nullptr; + RecordFileHandler *handler_; + VacuousLogHandler log_handler_; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/benchmark/server_concurrency_test.cpp b/benchmark/server_concurrency_test.cpp index 89bf6b74e..8072aac83 100644 --- a/benchmark/server_concurrency_test.cpp +++ b/benchmark/server_concurrency_test.cpp @@ -20,8 +20,8 @@ See the Mulan PSL v2 for more details. */ #include #include -#include +#include "common/lang/stdexcept.h" #include "common/log/log.h" #include "rc.h" diff --git a/deps/common/conf/ini.cpp b/deps/common/conf/ini.cpp index 63c55269f..76fa4735a 100644 --- a/deps/common/conf/ini.cpp +++ b/deps/common/conf/ini.cpp @@ -19,28 +19,30 @@ See the Mulan PSL v2 for more details. */ #include "common/conf/ini.h" #include "common/defs.h" +#include "common/lang/iostream.h" #include "common/lang/string.h" +#include "common/lang/utility.h" +#include "common/lang/fstream.h" #include "common/log/log.h" namespace common { -const std::string Ini::DEFAULT_SECTION = std::string(""); -const std::map Ini::empty_map_; +const string Ini::DEFAULT_SECTION = string(""); +const map Ini::empty_map_; Ini::Ini() {} Ini::~Ini() {} -void Ini::insert_session(const std::string &session_name) +void Ini::insert_session(const string &session_name) { - std::map session_map; - std::pair> entry = - std::pair>(session_name, session_map); + map session_map; + pair> entry = pair>(session_name, session_map); sections_.insert(entry); } -std::map *Ini::switch_session(const std::string &session_name) +map *Ini::switch_session(const string &session_name) { SessionsMap::iterator it = sections_.find(session_name); if (it != sections_.end()) { @@ -58,7 +60,7 @@ std::map *Ini::switch_session(const std::string &sessi return nullptr; } -const std::map &Ini::get(const std::string §ion) +const map &Ini::get(const string §ion) { SessionsMap::iterator it = sections_.find(section); if (it == sections_.end()) { @@ -68,11 +70,11 @@ const std::map &Ini::get(const std::string §ion) return it->second; } -std::string Ini::get(const std::string &key, const std::string &defaultValue, const std::string §ion) +string Ini::get(const string &key, const string &defaultValue, const string §ion) { - std::map section_map = get(section); + map section_map = get(section); - std::map::iterator it = section_map.find(key); + map::iterator it = section_map.find(key); if (it == section_map.end()) { return defaultValue; } @@ -80,51 +82,51 @@ std::string Ini::get(const std::string &key, const std::string &defaultValue, co return it->second; } -int Ini::put(const std::string &key, const std::string &value, const std::string §ion) +int Ini::put(const string &key, const string &value, const string §ion) { - std::map *section_map = switch_session(section); + map *section_map = switch_session(section); - section_map->insert(std::pair(key, value)); + section_map->insert(pair(key, value)); return 0; } -int Ini::insert_entry(std::map *session_map, const std::string &line) +int Ini::insert_entry(map *session_map, const string &line) { if (session_map == nullptr) { - std::cerr << __FILE__ << __FUNCTION__ << " session map is null" << std::endl; + cerr << __FILE__ << __FUNCTION__ << " session map is null" << endl; return -1; } size_t equal_pos = line.find_first_of('='); - if (equal_pos == std::string::npos) { - std::cerr << __FILE__ << __FUNCTION__ << "Invalid configuration line " << line << std::endl; + if (equal_pos == string::npos) { + cerr << __FILE__ << __FUNCTION__ << "Invalid configuration line " << line << endl; return -1; } - std::string key = line.substr(0, equal_pos); - std::string value = line.substr(equal_pos + 1); + string key = line.substr(0, equal_pos); + string value = line.substr(equal_pos + 1); strip(key); strip(value); - session_map->insert(std::pair(key, value)); + session_map->insert(pair(key, value)); return 0; } -int Ini::load(const std::string &file_name) +int Ini::load(const string &file_name) { - std::ifstream ifs; + ifstream ifs; try { bool continue_last_line = false; - std::map *current_session = switch_session(DEFAULT_SECTION); + map *current_session = switch_session(DEFAULT_SECTION); char line[MAX_CFG_LINE_LEN]; - std::string line_entry; + string line_entry; ifs.open(file_name.c_str()); while (ifs.good()) { @@ -148,7 +150,7 @@ int Ini::load(const std::string &file_name) if (read_buf[0] == CFG_SESSION_START_TAG && read_buf[strlen(read_buf) - 1] == CFG_SESSION_END_TAG) { read_buf[strlen(read_buf) - 1] = '\0'; - std::string session_name = std::string(read_buf + 1); + string session_name = string(read_buf + 1); current_session = switch_session(session_name); @@ -177,19 +179,19 @@ int Ini::load(const std::string &file_name) ifs.close(); file_names_.insert(file_name); - std::cout << "Successfully load " << file_name << std::endl; + cout << "Successfully load " << file_name << endl; } catch (...) { if (ifs.is_open()) { ifs.close(); } - std::cerr << "Failed to load " << file_name << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to load " << file_name << SYS_OUTPUT_ERROR << endl; return -1; } return 0; } -void Ini::to_string(std::string &output_str) +void Ini::to_string(string &output_str) { output_str.clear(); @@ -201,10 +203,9 @@ void Ini::to_string(std::string &output_str) output_str += CFG_SESSION_END_TAG; output_str += "\n"; - std::map §ion_map = it->second; + map §ion_map = it->second; - for (std::map::iterator sub_it = section_map.begin(); sub_it != section_map.end(); - sub_it++) { + for (map::iterator sub_it = section_map.begin(); sub_it != section_map.end(); sub_it++) { output_str += sub_it->first; output_str += "="; output_str += sub_it->second; diff --git a/deps/common/conf/ini.h b/deps/common/conf/ini.h index ec10eb79d..be42a6dac 100644 --- a/deps/common/conf/ini.h +++ b/deps/common/conf/ini.h @@ -18,9 +18,10 @@ See the Mulan PSL v2 for more details. */ #include #include -#include -#include -#include + +#include "common/lang/map.h" +#include "common/lang/set.h" +#include "common/lang/string.h" namespace common { @@ -45,35 +46,34 @@ class Ini * it support load multiple ini configuration files * @return, 0 means success, others means failed */ - int load(const std::string &ini_file); + int load(const string &ini_file); /** * get the map of the section * if the section doesn't exist, return one empty section */ - const std::map &get(const std::string §ion = DEFAULT_SECTION); + const map &get(const string §ion = DEFAULT_SECTION); /** * get the value of the key in the section, * if the key-value doesn't exist, * use the input default_value */ - std::string get( - const std::string &key, const std::string &default_value, const std::string §ion = DEFAULT_SECTION); + string get(const string &key, const string &default_value, const string §ion = DEFAULT_SECTION); /** * put the key-value pair to the section * if the key-value already exist, just replace it * if the section doesn't exist, it will create this section */ - int put(const std::string &key, const std::string &value, const std::string §ion = DEFAULT_SECTION); + int put(const string &key, const string &value, const string §ion = DEFAULT_SECTION); /** * output all configuration to one string */ - void to_string(std::string &output_str); + void to_string(string &output_str); - static const std::string DEFAULT_SECTION; + static const string DEFAULT_SECTION; // one line max length static const int MAX_CFG_LINE_LEN = 1024; @@ -95,28 +95,28 @@ class Ini /** * insert one empty session to sections_ */ - void insert_session(const std::string &session_name); + void insert_session(const string &session_name); /** * switch session according to the session_name * if the section doesn't exist, it will create one */ - std::map *switch_session(const std::string &session_name); + map *switch_session(const string &session_name); /** * insert one entry to session_map * line's format is "key=value" * */ - int insert_entry(std::map *session_map, const std::string &line); + int insert_entry(map *session_map, const string &line); - typedef std::map> SessionsMap; + typedef map> SessionsMap; private: - static const std::map empty_map_; + static const map empty_map_; - std::set file_names_; - SessionsMap sections_; + set file_names_; + SessionsMap sections_; }; /** diff --git a/deps/common/io/io.cpp b/deps/common/io/io.cpp index 9fe483961..ba45c550c 100644 --- a/deps/common/io/io.cpp +++ b/deps/common/io/io.cpp @@ -23,15 +23,14 @@ See the Mulan PSL v2 for more details. */ #include "common/lang/string.h" #include "common/log/log.h" #include "common/math/regex.h" -#include "common/mm/mem.h" namespace common { -int readFromFile(const std::string &fileName, char *&outputData, size_t &fileSize) +int readFromFile(const string &fileName, char *&outputData, size_t &fileSize) { FILE *file = fopen(fileName.c_str(), "rb"); if (file == NULL) { - std::cerr << "Failed to open file " << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to open file " << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; return -1; } @@ -48,19 +47,19 @@ int readFromFile(const std::string &fileName, char *&outputData, size_t &fileSiz memset(buffer, 0, sizeof(buffer)); oneRead = fread(buffer, 1, sizeof(buffer), file); if (ferror(file)) { - std::cerr << "Failed to read data" << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to read data" << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; fclose(file); if (data != NULL) { - lfree(data); + free(data); data = NULL; } return -1; } - data = (char *)lrealloc(data, readSize + oneRead); + data = (char *)realloc(data, readSize + oneRead); if (data == NULL) { - std::cerr << "Failed to alloc memory for " << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; - lfree(data); + cerr << "Failed to alloc memory for " << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; + free(data); fclose(file); return -1; } else { @@ -72,18 +71,18 @@ int readFromFile(const std::string &fileName, char *&outputData, size_t &fileSiz fclose(file); - data = (char *)lrealloc(data, readSize + 1); + data = (char *)realloc(data, readSize + 1); data[readSize] = '\0'; outputData = data; fileSize = readSize; return 0; } -int writeToFile(const std::string &fileName, const char *data, uint32_t dataSize, const char *openMode) +int writeToFile(const string &fileName, const char *data, uint32_t dataSize, const char *openMode) { FILE *file = fopen(fileName.c_str(), openMode); if (file == NULL) { - std::cerr << "Failed to open file " << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to open file " << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; return -1; } @@ -92,7 +91,7 @@ int writeToFile(const std::string &fileName, const char *data, uint32_t dataSize while (leftSize > 0) { int writeCount = fwrite(buffer, 1, leftSize, file); if (writeCount <= 0) { - std::cerr << "Failed to open file " << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to open file " << fileName << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; fclose(file); return -1; } else { @@ -106,13 +105,13 @@ int writeToFile(const std::string &fileName, const char *data, uint32_t dataSize return 0; } -int getFileLines(const std::string &fileName, uint64_t &lineNum) +int getFileLines(const string &fileName, uint64_t &lineNum) { lineNum = 0; char line[4 * ONE_KILO] = {0}; - std::ifstream ifs(fileName.c_str()); + ifstream ifs(fileName.c_str()); if (!ifs) { return -1; } @@ -130,17 +129,17 @@ int getFileLines(const std::string &fileName, uint64_t &lineNum) return 0; } -int getFileNum(int64_t &fileNum, const std::string &path, const std::string &pattern, bool recursive) +int getFileNum(int64_t &fileNum, const string &path, const string &pattern, bool recursive) { try { DIR *dirp = NULL; dirp = opendir(path.c_str()); if (dirp == NULL) { - std::cerr << "Failed to opendir " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to opendir " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; return -1; } - std::string fullPath; + string fullPath; struct dirent *entry = NULL; struct stat fs; while ((entry = readdir(dirp)) != NULL) { @@ -156,7 +155,7 @@ int getFileNum(int64_t &fileNum, const std::string &path, const std::string &pat fullPath += entry->d_name; memset(&fs, 0, sizeof(fs)); if (stat(fullPath.c_str(), &fs) < 0) { - std::cout << "Failed to stat " << fullPath << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cout << "Failed to stat " << fullPath << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; continue; } @@ -188,22 +187,22 @@ int getFileNum(int64_t &fileNum, const std::string &path, const std::string &pat return 0; } catch (...) { - std::cerr << "Failed to get file num " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to get file num " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; } return -1; } -int getFileList(std::vector &fileList, const std::string &path, const std::string &pattern, bool recursive) +int getFileList(vector &fileList, const string &path, const string &pattern, bool recursive) { try { DIR *dirp = NULL; dirp = opendir(path.c_str()); if (dirp == NULL) { - std::cerr << "Failed to opendir " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to opendir " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; return -1; } - std::string fullPath; + string fullPath; struct dirent *entry = NULL; struct stat fs; while ((entry = readdir(dirp)) != NULL) { @@ -219,7 +218,7 @@ int getFileList(std::vector &fileList, const std::string &path, con fullPath += entry->d_name; memset(&fs, 0, sizeof(fs)); if (stat(fullPath.c_str(), &fs) < 0) { - std::cout << "Failed to stat " << fullPath << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cout << "Failed to stat " << fullPath << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; continue; } @@ -250,22 +249,22 @@ int getFileList(std::vector &fileList, const std::string &path, con closedir(dirp); return 0; } catch (...) { - std::cerr << "Failed to get file list " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to get file list " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; } return -1; } -int getDirList(std::vector &dirList, const std::string &path, const std::string &pattern) +int getDirList(vector &dirList, const string &path, const string &pattern) { try { DIR *dirp = NULL; dirp = opendir(path.c_str()); if (dirp == NULL) { - std::cerr << "Failed to opendir " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to opendir " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; return -1; } - std::string fullPath; + string fullPath; struct dirent *entry = NULL; struct stat fs; while ((entry = readdir(dirp)) != NULL) { @@ -281,7 +280,7 @@ int getDirList(std::vector &dirList, const std::string &path, const fullPath += entry->d_name; memset(&fs, 0, sizeof(fs)); if (stat(fullPath.c_str(), &fs) < 0) { - std::cout << "Failed to stat " << fullPath << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cout << "Failed to stat " << fullPath << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; continue; } @@ -300,12 +299,12 @@ int getDirList(std::vector &dirList, const std::string &path, const closedir(dirp); return 0; } catch (...) { - std::cerr << "Failed to get file list " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to get file list " << path << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; } return -1; } -int touch(const std::string &path) +int touch(const string &path) { // CWE367: A check occurs on a file's attributes before // the file is used in a privileged operation, but things @@ -330,7 +329,7 @@ int touch(const std::string &path) int getFileSize(const char *filePath, int64_t &fileLen) { if (filePath == NULL || *filePath == '\0') { - std::cerr << "invalid filepath" << std::endl; + cerr << "invalid filepath" << endl; return -EINVAL; } struct stat statBuf; @@ -338,12 +337,12 @@ int getFileSize(const char *filePath, int64_t &fileLen) int rc = stat(filePath, &statBuf); if (rc) { - std::cerr << "Failed to get stat of " << filePath << "," << errno << ":" << strerror(errno) << std::endl; + cerr << "Failed to get stat of " << filePath << "," << errno << ":" << strerror(errno) << endl; return rc; } if (S_ISDIR(statBuf.st_mode)) { - std::cerr << filePath << " is directory " << std::endl; + cerr << filePath << " is directory " << endl; return -EINVAL; } diff --git a/deps/common/io/io.h b/deps/common/io/io.h index ad9a18655..b68ca268b 100644 --- a/deps/common/io/io.h +++ b/deps/common/io/io.h @@ -14,10 +14,11 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include #include #include "common/defs.h" +#include "common/lang/string.h" +#include "common/lang/vector.h" namespace common { @@ -26,14 +27,14 @@ namespace common { * if success, store file continent to data * if fail, return -1 and don't change data */ -int readFromFile(const std::string &fileName, char *&data, size_t &fileSize); +int readFromFile(const string &fileName, char *&data, size_t &fileSize); -int writeToFile(const std::string &fileName, const char *data, uint32_t dataSize, const char *openMode); +int writeToFile(const string &fileName, const char *data, uint32_t dataSize, const char *openMode); /** * return the line number which line.strip() isn't empty */ -int getFileLines(const std::string &fileName, uint64_t &lineNum); +int getFileLines(const string &fileName, uint64_t &lineNum); /** Get file list from the dir * don't care ".", "..", ".****" hidden files @@ -45,12 +46,11 @@ int getFileLines(const std::string &fileName, uint64_t &lineNum); * @param[in] resursion if this has been set, it will search subdirs * @return 0 if success, error code otherwise */ -int getFileList( - std::vector &fileList, const std::string &path, const std::string &pattern, bool recursive); -int getFileNum(uint64_t &fileNum, const std::string &path, const std::string &pattern, bool recursive); -int getDirList(std::vector &dirList, const std::string &path, const std::string &pattern); +int getFileList(vector &fileList, const string &path, const string &pattern, bool recursive); +int getFileNum(uint64_t &fileNum, const string &path, const string &pattern, bool recursive); +int getDirList(vector &dirList, const string &path, const string &pattern); -int touch(const std::string &fileName); +int touch(const string &fileName); /** * get file size diff --git a/deps/common/io/roll_select_dir.cpp b/deps/common/io/roll_select_dir.cpp deleted file mode 100644 index 1a69d235c..000000000 --- a/deps/common/io/roll_select_dir.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. -miniob is licensed under Mulan PSL v2. -You can use this software according to the terms and conditions of the Mulan PSL v2. -You may obtain a copy of Mulan PSL v2 at: - http://license.coscl.org.cn/MulanPSL2 -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -See the Mulan PSL v2 for more details. */ - -// -// Created by Longda on 2010 -// - -#include "common/io/roll_select_dir.h" -#include "common/io/io.h" -#include "common/log/log.h" -namespace common { - -void RollSelectDir::setBaseDir(std::string baseDir) -{ - mBaseDir = baseDir; - - std::vector dirList; - int rc = getDirList(dirList, mBaseDir, ""); - if (rc) { - LOG_ERROR("Failed to all subdir entry"); - } - - if (dirList.size() == 0) { - MUTEX_LOCK(&mMutex); - - mSubdirs.clear(); - mSubdirs.push_back(mBaseDir); - mPos = 0; - MUTEX_UNLOCK(&mMutex); - - return; - } - - MUTEX_LOCK(&mMutex); - mSubdirs = dirList; - mPos = 0; - MUTEX_UNLOCK(&mMutex); - return; -} - -std::string RollSelectDir::select() -{ - std::string ret; - - MUTEX_LOCK(&mMutex); - ret = mSubdirs[mPos % mSubdirs.size()]; - mPos++; - MUTEX_UNLOCK(&mMutex); - - return ret; -} - -} // namespace common \ No newline at end of file diff --git a/deps/common/io/roll_select_dir.h b/deps/common/io/roll_select_dir.h deleted file mode 100644 index 2ecf6b0d1..000000000 --- a/deps/common/io/roll_select_dir.h +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. -miniob is licensed under Mulan PSL v2. -You can use this software according to the terms and conditions of the Mulan PSL v2. -You may obtain a copy of Mulan PSL v2 at: - http://license.coscl.org.cn/MulanPSL2 -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -See the Mulan PSL v2 for more details. */ - -// -// Created by Longda on 2010 -// - -#ifndef __COMMON_IO_ROLL_SELECT_DIR__ -#define __COMMON_IO_ROLL_SELECT_DIR__ - -#include -#include -#include - -#include "common/defs.h" -#include "common/io/select_dir.h" -#include "common/lang/mutex.h" -namespace common { - -class RollSelectDir : public SelectDir -{ -public: - RollSelectDir() { MUTEX_INIT(&mMutex, NULL); } - ~RollSelectDir() { MUTEX_DESTROY(&mMutex); } - -public: - /** - * inherit from CSelectDir - */ - std::string select(); - void setBaseDir(std::string baseDir); - -public: - std::string mBaseDir; - std::vector mSubdirs; - pthread_mutex_t mMutex; - uint32_t mPos; -}; - -} // namespace common -#endif /* __COMMON_IO_ROLL_SELECT_DIR__ */ diff --git a/deps/common/lang/algorithm.h b/deps/common/lang/algorithm.h new file mode 100644 index 000000000..c8982daa4 --- /dev/null +++ b/deps/common/lang/algorithm.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::max; +using std::min; +using std::transform; \ No newline at end of file diff --git a/deps/common/lang/atomic.h b/deps/common/lang/atomic.h new file mode 100644 index 000000000..f8f12f9fc --- /dev/null +++ b/deps/common/lang/atomic.h @@ -0,0 +1,16 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::atomic; +using std::atomic_bool; \ No newline at end of file diff --git a/deps/common/thread/thread_pool.h b/deps/common/lang/charconv.h similarity index 82% rename from deps/common/thread/thread_pool.h rename to deps/common/lang/charconv.h index ef3716fb0..19e35b3ee 100644 --- a/deps/common/thread/thread_pool.h +++ b/deps/common/lang/charconv.h @@ -8,16 +8,11 @@ EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. */ -// -// Created by Wangyunlai on 2023/01/11. -// - #pragma once -namespace common { -class ThreadPool -{ -public: -}; +#include + +#include "common/lang/system_error.h" -} // namespace common \ No newline at end of file +using std::from_chars; +using std::from_chars_result; diff --git a/deps/common/lang/chrono.h b/deps/common/lang/chrono.h new file mode 100644 index 000000000..c1cfd89fa --- /dev/null +++ b/deps/common/lang/chrono.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +namespace chrono = std::chrono; diff --git a/deps/common/lang/comparator.cpp b/deps/common/lang/comparator.cpp index 40edc64f3..a7c28d7ae 100644 --- a/deps/common/lang/comparator.cpp +++ b/deps/common/lang/comparator.cpp @@ -13,9 +13,10 @@ See the Mulan PSL v2 for more details. */ // #include "common/defs.h" -#include #include +#include "common/lang/algorithm.h" + namespace common { int compare_int(void *arg1, void *arg2) @@ -49,7 +50,7 @@ int compare_string(void *arg1, int arg1_max_length, void *arg2, int arg2_max_len { const char *s1 = (const char *)arg1; const char *s2 = (const char *)arg2; - int maxlen = std::min(arg1_max_length, arg2_max_length); + int maxlen = min(arg1_max_length, arg2_max_length); int result = strncmp(s1, s2, maxlen); if (0 != result) { return result; diff --git a/deps/common/lang/defer.h b/deps/common/lang/defer.h index e8c7672b8..8035f9d16 100644 --- a/deps/common/lang/defer.h +++ b/deps/common/lang/defer.h @@ -14,14 +14,14 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include +#include "common/lang/functional.h" namespace common { class DeferHelper { public: - DeferHelper(const std::function defer) : defer_(std::move(defer)) {} + DeferHelper(const function defer) : defer_(std::move(defer)) {} ~DeferHelper() { @@ -31,7 +31,7 @@ class DeferHelper } private: - const std::function defer_; + const function defer_; }; } // namespace common diff --git a/deps/common/lang/deque.h b/deps/common/lang/deque.h new file mode 100644 index 000000000..5e60b1551 --- /dev/null +++ b/deps/common/lang/deque.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::deque; \ No newline at end of file diff --git a/deps/common/lang/exception.h b/deps/common/lang/exception.h new file mode 100644 index 000000000..d4ce1e260 --- /dev/null +++ b/deps/common/lang/exception.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::exception; \ No newline at end of file diff --git a/deps/common/io/select_dir.h b/deps/common/lang/filesystem.h similarity index 69% rename from deps/common/io/select_dir.h rename to deps/common/lang/filesystem.h index de68d1113..8fcc7d265 100644 --- a/deps/common/io/select_dir.h +++ b/deps/common/lang/filesystem.h @@ -8,20 +8,10 @@ EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. */ -// -// Created by Longda on 2010 -// - #pragma once -#include -namespace common { +#include -class SelectDir -{ -public: - virtual std::string select() { return std::string(""); }; - virtual void setBaseDir(std::string baseDir){}; -}; +#include "common/lang/system_error.h" -} // namespace common +namespace filesystem = std::filesystem; diff --git a/deps/common/lang/fstream.h b/deps/common/lang/fstream.h new file mode 100644 index 000000000..96c7bb643 --- /dev/null +++ b/deps/common/lang/fstream.h @@ -0,0 +1,21 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +#include "common/lang/new.h" +#include "common/lang/ios.h" +#include "common/lang/exception.h" + +using std::fstream; +using std::ifstream; +using std::ofstream; \ No newline at end of file diff --git a/deps/common/lang/functional.h b/deps/common/lang/functional.h new file mode 100644 index 000000000..4cf782c63 --- /dev/null +++ b/deps/common/lang/functional.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::equal_to; +using std::function; +using std::hash; \ No newline at end of file diff --git a/deps/common/lang/iomanip.h b/deps/common/lang/iomanip.h new file mode 100644 index 000000000..26d9e7780 --- /dev/null +++ b/deps/common/lang/iomanip.h @@ -0,0 +1,16 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::setfill; +using std::setw; \ No newline at end of file diff --git a/deps/common/lang/ios.h b/deps/common/lang/ios.h new file mode 100644 index 000000000..e4673e227 --- /dev/null +++ b/deps/common/lang/ios.h @@ -0,0 +1,16 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::ios; +using std::ios_base; \ No newline at end of file diff --git a/deps/common/lang/iostream.h b/deps/common/lang/iostream.h new file mode 100644 index 000000000..33ae5c02f --- /dev/null +++ b/deps/common/lang/iostream.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +#include "common/lang/exception.h" +#include "common/lang/ios.h" +#include "common/lang/new.h" + +using std::cerr; +using std::cin; +using std::cout; +using std::endl; +using std::istream; +using std::ostream; \ No newline at end of file diff --git a/deps/common/lang/iterator.h b/deps/common/lang/iterator.h new file mode 100644 index 000000000..7fcbee15b --- /dev/null +++ b/deps/common/lang/iterator.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::advance; +using std::distance; +using std::random_access_iterator_tag; \ No newline at end of file diff --git a/deps/common/lang/limits.h b/deps/common/lang/limits.h new file mode 100644 index 000000000..b40940b53 --- /dev/null +++ b/deps/common/lang/limits.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::numeric_limits; diff --git a/deps/common/lang/list.h b/deps/common/lang/list.h new file mode 100644 index 000000000..797345446 --- /dev/null +++ b/deps/common/lang/list.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::list; \ No newline at end of file diff --git a/deps/common/lang/lower_bound.h b/deps/common/lang/lower_bound.h index 3aadbf368..7d2253e06 100644 --- a/deps/common/lang/lower_bound.h +++ b/deps/common/lang/lower_bound.h @@ -15,7 +15,7 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include +#include "common/lang/iterator.h" namespace common { /** @@ -34,12 +34,12 @@ ForwardIterator lower_bound( { bool found = false; ForwardIterator iter; - const auto count = std::distance(first, last); + const auto count = distance(first, last); auto last_count = count; while (last_count > 0) { iter = first; auto step = last_count / 2; - std::advance(iter, step); + advance(iter, step); int result = comp(*iter, val); if (0 == result) { first = iter; @@ -91,7 +91,7 @@ template class BinaryIterator { public: - using iterator_category = std::random_access_iterator_tag; + using iterator_category = random_access_iterator_tag; using value_type = T; using difference_type = Distance; using pointer = value_type *; diff --git a/deps/common/lang/lru_cache.h b/deps/common/lang/lru_cache.h index 846d731d4..3e109f194 100644 --- a/deps/common/lang/lru_cache.h +++ b/deps/common/lang/lru_cache.h @@ -14,12 +14,12 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include +#include "common/lang/functional.h" +#include "common/lang/unordered_set.h" namespace common { -template , typename Pred = std::equal_to> +template , typename Pred = equal_to> class LruCache { @@ -134,7 +134,7 @@ class LruCache value = nullptr; } - void foreach (std::function func) + void foreach (function func) { for (ListNode *node = lru_front_; node != nullptr; node = node->next_) { bool ret = func(node->key_, node->value_); @@ -144,7 +144,7 @@ class LruCache } } - void foreach_reverse(std::function func) + void foreach_reverse(function func) { for (ListNode *node = lru_tail_; node != nullptr; node = node->prev_) { bool ret = func(node->key_, node->value_); @@ -217,7 +217,7 @@ class LruCache } private: - using SearchType = std::unordered_set; + using SearchType = unordered_set; SearchType searcher_; ListNode *lru_front_ = nullptr; ListNode *lru_tail_ = nullptr; diff --git a/deps/common/lang/map.h b/deps/common/lang/map.h new file mode 100644 index 000000000..f4c305dfb --- /dev/null +++ b/deps/common/lang/map.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +#include "common/lang/utility.h" + +using std::map; \ No newline at end of file diff --git a/deps/common/lang/memory.h b/deps/common/lang/memory.h new file mode 100644 index 000000000..6fd00fc4a --- /dev/null +++ b/deps/common/lang/memory.h @@ -0,0 +1,16 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::make_unique; +using std::unique_ptr; \ No newline at end of file diff --git a/deps/common/lang/mutex.cpp b/deps/common/lang/mutex.cpp index c8b75d5b7..47523d26b 100644 --- a/deps/common/lang/mutex.cpp +++ b/deps/common/lang/mutex.cpp @@ -12,15 +12,16 @@ See the Mulan PSL v2 for more details. */ // Created by Longda on 2010 // +#include "common/lang/thread.h" #include "common/lang/mutex.h" #include "common/log/log.h" namespace common { -std::map LockTrace::mLocks; -std::map LockTrace::mWaitTimes; -std::map LockTrace::mWaitLocks; -std::map> LockTrace::mOwnLocks; -std::set LockTrace::mEnableRecurisives; +map LockTrace::mLocks; +map LockTrace::mWaitTimes; +map LockTrace::mWaitLocks; +map> LockTrace::mOwnLocks; +set LockTrace::mEnableRecurisives; pthread_rwlock_t LockTrace::mMapMutex = PTHREAD_RWLOCK_INITIALIZER; int LockTrace::mMaxBlockTids = 8; @@ -29,7 +30,7 @@ int LockTrace::mMaxBlockTids = 8; void LockTrace::foundDeadLock(LockID ¤t, LockTrace::LockID &other, pthread_mutex_t *otherWaitMutex) { - std::map::iterator itLocks = mLocks.find(otherWaitMutex); + map::iterator itLocks = mLocks.find(otherWaitMutex); if (itLocks == mLocks.end()) { LOG_ERROR("Thread %ld own mutex %p and try to get mutex %s:%d, " "other thread %ld own mutex %s:%d and try to get %p", @@ -62,13 +63,13 @@ void LockTrace::foundDeadLock(LockID ¤t, LockTrace::LockID &other, pthread } bool LockTrace::deadlockCheck( - LockID ¤t, std::set &ownMutexs, LockTrace::LockID &other, int recusiveNum) + LockID ¤t, set &ownMutexs, LockTrace::LockID &other, int recusiveNum) { if (recusiveNum >= mMaxBlockTids) { return false; } - std::map::iterator otherIt = mWaitLocks.find(other.mThreadId); + map::iterator otherIt = mWaitLocks.find(other.mThreadId); if (otherIt == mWaitLocks.end()) { return false; } @@ -80,7 +81,7 @@ bool LockTrace::deadlockCheck( return true; } - std::map::iterator itLocks = mLocks.find(otherWaitMutex); + map::iterator itLocks = mLocks.find(otherWaitMutex); if (itLocks == mLocks.end()) { return false; } @@ -93,7 +94,7 @@ bool LockTrace::deadlockCheck(pthread_mutex_t *mutex, const long long threadId, { mWaitLocks[threadId] = mutex; - std::map::iterator itLocks = mLocks.find(mutex); + map::iterator itLocks = mLocks.find(mutex); if (itLocks == mLocks.end()) { return false; } @@ -110,11 +111,11 @@ bool LockTrace::deadlockCheck(pthread_mutex_t *mutex, const long long threadId, } } - std::map>::iterator it = mOwnLocks.find(threadId); + map>::iterator it = mOwnLocks.find(threadId); if (it == mOwnLocks.end()) { return false; } - std::set &ownMutexs = it->second; + set &ownMutexs = it->second; if (ownMutexs.empty() == true) { return false; } @@ -125,9 +126,9 @@ bool LockTrace::deadlockCheck(pthread_mutex_t *mutex, const long long threadId, bool LockTrace::checkLockTimes(pthread_mutex_t *mutex, const char *file, const int line) { - std::map::iterator it = mWaitTimes.find(mutex); + map::iterator it = mWaitTimes.find(mutex); if (it == mWaitTimes.end()) { - mWaitTimes.insert(std::pair(mutex, 1)); + mWaitTimes.insert(pair(mutex, 1)); return false; } @@ -136,7 +137,7 @@ bool LockTrace::checkLockTimes(pthread_mutex_t *mutex, const char *file, const i mWaitTimes[mutex] = lockTimes + 1; if (lockTimes >= mMaxBlockTids) { - // std::string lastLockId = lockId.toString(); + // string lastLockId = lockId.toString(); LockTrace::LockID &lockId = mLocks[mutex]; LOG_WARN("mutex %p has been already lock %d times, this time %s:%d, first " "time:%ld:%s:%d", @@ -170,15 +171,15 @@ void LockTrace::insertLock(pthread_mutex_t *mutex, const long long threadId, con { LockID lockID(threadId, file, line); - mLocks.insert(std::pair(mutex, lockID)); + mLocks.insert(pair(mutex, lockID)); mWaitLocks.erase(threadId); // add entry to mOwnLocks - std::set &ownLockSet = mOwnLocks[threadId]; + set &ownLockSet = mOwnLocks[threadId]; ownLockSet.insert(mutex); - std::map::iterator itTimes = mWaitTimes.find(mutex); + map::iterator itTimes = mWaitTimes.find(mutex); if (itTimes == mWaitTimes.end()) { LOG_ERROR("No entry of %p:%s:%d in mWaitTimes", mutex, file, line); @@ -213,19 +214,19 @@ void LockTrace::unlock(pthread_mutex_t *mutex, long long threadId, const char *f mLocks.erase(mutex); - std::set &ownLockSet = mOwnLocks[threadId]; + set &ownLockSet = mOwnLocks[threadId]; ownLockSet.erase(mutex); pthread_rwlock_unlock(&mMapMutex); } -void LockTrace::toString(std::string &result) +void LockTrace::toString(string &result) { const int TEMP_PAIR_LEN = 24; // pthread_mutex_lock(&mMapMutex); result = " mLocks:\n"; - for (std::map::iterator it = mLocks.begin(); it != mLocks.end(); it++) { + for (map::iterator it = mLocks.begin(); it != mLocks.end(); it++) { result += it->second.toString(); char pointerBuf[TEMP_PAIR_LEN] = {0}; @@ -235,14 +236,14 @@ void LockTrace::toString(std::string &result) } result += "mWaitTimes:\n"; - for (std::map::iterator it = mWaitTimes.begin(); it != mWaitTimes.end(); it++) { + for (map::iterator it = mWaitTimes.begin(); it != mWaitTimes.end(); it++) { char pointerBuf[TEMP_PAIR_LEN] = {0}; snprintf(pointerBuf, TEMP_PAIR_LEN, ",mutex:%p, times:%d\n", it->first, it->second); result += pointerBuf; } result += "mWaitLocks:\n"; - for (std::map::iterator it = mWaitLocks.begin(); it != mWaitLocks.end(); it++) { + for (map::iterator it = mWaitLocks.begin(); it != mWaitLocks.end(); it++) { char pointerBuf[TEMP_PAIR_LEN] = {0}; snprintf(pointerBuf, TEMP_PAIR_LEN, diff --git a/deps/common/lang/mutex.h b/deps/common/lang/mutex.h index 29ebbe98f..ad691bd66 100644 --- a/deps/common/lang/mutex.h +++ b/deps/common/lang/mutex.h @@ -24,12 +24,19 @@ See the Mulan PSL v2 for more details. */ #include #include #include -#include #include +#include "common/lang/thread.h" #include "common/log/log.h" -using namespace std; +using std::call_once; +using std::condition_variable; +using std::lock_guard; +using std::mutex; +using std::once_flag; +using std::scoped_lock; +using std::shared_mutex; +using std::unique_lock; namespace common { @@ -43,7 +50,7 @@ class LockTrace static void tryLock(pthread_mutex_t *mutex, const long long threadId, const char *file, const int line); static void unlock(pthread_mutex_t *mutex, const long long threadId, const char *file, const int line); - static void toString(std::string &result); + static void toString(string &result); class LockID { @@ -52,9 +59,9 @@ class LockTrace {} LockID() : mFile(), mThreadId(0), mLine(0) {} - std::string toString() + string toString() { - std::ostringstream oss; + ostringstream oss; oss << "threaId:" << mThreadId << ",file name:" << mFile << ",line:" << mLine; @@ -62,14 +69,14 @@ class LockTrace } public: - std::string mFile; + string mFile; const long long mThreadId; int mLine; }; static void foundDeadLock(LockID ¤t, LockID &other, pthread_mutex_t *otherWaitMutex); - static bool deadlockCheck(LockID ¤t, std::set &ownMutexs, LockID &other, int recusiveNum); + static bool deadlockCheck(LockID ¤t, set &ownMutexs, LockID &other, int recusiveNum); static bool deadlockCheck(pthread_mutex_t *mutex, const long long threadId, const char *file, const int line); @@ -80,13 +87,13 @@ class LockTrace static void setMaxBlockThreads(int blockNum) { mMaxBlockTids = blockNum; } public: - static std::set mEnableRecurisives; + static set mEnableRecurisives; protected: - static std::map mLocks; - static std::map mWaitTimes; - static std::map mWaitLocks; - static std::map> mOwnLocks; + static map mLocks; + static map mWaitTimes; + static map mWaitLocks; + static map> mOwnLocks; static pthread_rwlock_t mMapMutex; static int mMaxBlockTids; @@ -253,7 +260,7 @@ class DebugMutex final private: #ifdef DEBUG - std::mutex lock_; + mutex lock_; #endif }; @@ -269,7 +276,7 @@ class Mutex final private: #ifdef CONCURRENCY - std::mutex lock_; + mutex lock_; #endif }; @@ -289,7 +296,7 @@ class SharedMutex final private: #ifdef CONCURRENCY - std::shared_mutex lock_; + shared_mutex lock_; #endif }; @@ -314,14 +321,14 @@ class RecursiveSharedMutex private: #ifdef CONCURRENCY - std::mutex mutex_; - std::condition_variable shared_lock_cv_; - std::condition_variable exclusive_lock_cv_; - int shared_lock_count_ = 0; - int exclusive_lock_count_ = 0; - std::thread::id recursive_owner_; - int recursive_count_ = 0; // 表示当前线程加写锁加了多少次 -#endif // CONCURRENCY + mutex mutex_; + condition_variable shared_lock_cv_; + condition_variable exclusive_lock_cv_; + int shared_lock_count_ = 0; + int exclusive_lock_count_ = 0; + thread::id recursive_owner_; + int recursive_count_ = 0; // 表示当前线程加写锁加了多少次 +#endif // CONCURRENCY }; } // namespace common diff --git a/deps/common/lang/new.h b/deps/common/lang/new.h new file mode 100644 index 000000000..087479dd9 --- /dev/null +++ b/deps/common/lang/new.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +#include "common/lang/exception.h" + +using std::nothrow; \ No newline at end of file diff --git a/deps/common/lang/queue.h b/deps/common/lang/queue.h new file mode 100644 index 000000000..53368f4a7 --- /dev/null +++ b/deps/common/lang/queue.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::queue; \ No newline at end of file diff --git a/deps/common/lang/random.h b/deps/common/lang/random.h new file mode 100644 index 000000000..1a39e6c86 --- /dev/null +++ b/deps/common/lang/random.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::mt19937; +using std::random_device; +using std::uniform_int_distribution; \ No newline at end of file diff --git a/deps/common/lang/serializable.h b/deps/common/lang/serializable.h index 387d41124..327151c30 100644 --- a/deps/common/lang/serializable.h +++ b/deps/common/lang/serializable.h @@ -12,10 +12,10 @@ See the Mulan PSL v2 for more details. */ // Created by Longda on 2010 // -#ifndef __COMMON_LANG_SERIALIZABLE_H__ -#define __COMMON_LANG_SERIALIZABLE_H__ +#pragma once -#include +#include "common/lang/string.h" +#include "common/lang/iostream.h" namespace common { /** @@ -49,7 +49,7 @@ class Serializable * @param[in] bufferLen, buffer length * @return, used buffer length -- success, -1 means failed */ - virtual int serialize(std::ostream &os) const = 0; + virtual int serialize(ostream &os) const = 0; /* * deserialize bytes to this object @@ -57,7 +57,7 @@ class Serializable * @param[in] bufferLen buffer lenght * @return used buffer length -- success , -1 --failed */ - virtual int deserialize(std::istream &is) = 0; + virtual int deserialize(istream &is) = 0; /** * get serialize size @@ -68,8 +68,7 @@ class Serializable /** * this function will generalize one output string */ - virtual void to_string(std::string &output) const = 0; + virtual void to_string(string &output) const = 0; }; } // namespace common -#endif /* __COMMON_LANG_SERIALIZABLE_H__ */ diff --git a/deps/common/lang/serializer.cpp b/deps/common/lang/serializer.cpp index 996ea6174..312487cd4 100644 --- a/deps/common/lang/serializer.cpp +++ b/deps/common/lang/serializer.cpp @@ -16,8 +16,6 @@ See the Mulan PSL v2 for more details. */ #include "common/lang/serializer.h" -using namespace std; - namespace common { int Serializer::write(span data) { diff --git a/deps/common/lang/serializer.h b/deps/common/lang/serializer.h index beebe6b03..71b04c1b1 100644 --- a/deps/common/lang/serializer.h +++ b/deps/common/lang/serializer.h @@ -14,9 +14,10 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include +#include + +#include "common/lang/vector.h" +#include "common/lang/span.h" namespace common { @@ -27,7 +28,7 @@ namespace common { class Serializer final { public: - using BufferType = std::vector; + using BufferType = vector; public: Serializer() = default; @@ -37,9 +38,9 @@ class Serializer final Serializer &operator=(const Serializer &) = delete; /// @brief 写入指定长度的数据 - int write(std::span data); + int write(span data); /// @brief 写入指定长度的数据 - int write(const char *data, int size) { return write(std::span(data, size)); } + int write(const char *data, int size) { return write(span(data, size)); } /// @brief 当前写入了多少数据 int64_t size() const { return buffer_.size(); } @@ -61,7 +62,7 @@ class Serializer final class Deserializer final { public: - explicit Deserializer(std::span buffer) : buffer_(buffer) {} + explicit Deserializer(span buffer) : buffer_(buffer) {} Deserializer(const char *buffer, int size) : buffer_(buffer, size) {} ~Deserializer() = default; @@ -69,9 +70,9 @@ class Deserializer final Deserializer &operator=(const Deserializer &) = delete; /// @brief 读取指定大小的数据 - int read(std::span data); + int read(span data); /// @brief 读取指定长度的数据 - int read(char *data, int size) { return read(std::span(data, size)); } + int read(char *data, int size) { return read(span(data, size)); } /// @brief buffer的大小 int64_t size() const { return buffer_.size(); } @@ -85,8 +86,8 @@ class Deserializer final int read_int64(int64_t &value); private: - std::span buffer_; ///< 存放数据的buffer - int64_t position_ = 0; ///< 当前读取到的位置 + span buffer_; ///< 存放数据的buffer + int64_t position_ = 0; ///< 当前读取到的位置 }; } // namespace common diff --git a/deps/common/lang/set.h b/deps/common/lang/set.h new file mode 100644 index 000000000..ab4d91ba9 --- /dev/null +++ b/deps/common/lang/set.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::set; \ No newline at end of file diff --git a/deps/common/lang/span.h b/deps/common/lang/span.h new file mode 100644 index 000000000..25dc56bca --- /dev/null +++ b/deps/common/lang/span.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::span; \ No newline at end of file diff --git a/deps/common/lang/sstream.h b/deps/common/lang/sstream.h new file mode 100644 index 000000000..ccba399c0 --- /dev/null +++ b/deps/common/lang/sstream.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +#include "common/lang/exception.h" + +using std::istringstream; +using std::ostringstream; +using std::stringstream; \ No newline at end of file diff --git a/deps/common/lang/stdexcept.h b/deps/common/lang/stdexcept.h new file mode 100644 index 000000000..bf7d51c03 --- /dev/null +++ b/deps/common/lang/stdexcept.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::runtime_error; \ No newline at end of file diff --git a/deps/common/lang/string.cpp b/deps/common/lang/string.cpp index 49029c886..de8c7af1e 100644 --- a/deps/common/lang/string.cpp +++ b/deps/common/lang/string.cpp @@ -18,14 +18,12 @@ See the Mulan PSL v2 for more details. */ #include #include -#include -#include #include -#include -#include -#include #include "common/log/log.h" +#include "common/lang/algorithm.h" +#include "common/lang/iomanip.h" + namespace common { char *strip(char *str_) @@ -46,7 +44,7 @@ char *strip(char *str_) return head; } -void strip(std::string &str) +void strip(string &str) { size_t head = 0; @@ -63,29 +61,29 @@ void strip(std::string &str) } // Translation functions with templates are defined in the header file -std::string size_to_pad_str(int size, int pad) +string size_to_pad_str(int size, int pad) { - std::ostringstream ss; - ss << std::setw(pad) << std::setfill('0') << size; + ostringstream ss; + ss << setw(pad) << setfill('0') << size; return ss.str(); } -std::string &str_to_upper(std::string &s) +string &str_to_upper(string &s) { - std::transform(s.begin(), s.end(), s.begin(), (int (*)(int)) & std::toupper); + transform(s.begin(), s.end(), s.begin(), (int (*)(int)) & toupper); return s; } -std::string &str_to_lower(std::string &s) +string &str_to_lower(string &s) { - std::transform(s.begin(), s.end(), s.begin(), (int (*)(int)) & std::tolower); + transform(s.begin(), s.end(), s.begin(), (int (*)(int)) & tolower); return s; } -void split_string(const std::string &str, std::string delim, std::set &results) +void split_string(const string &str, string delim, set &results) { - int cut_at; - std::string tmp_str(str); + int cut_at; + string tmp_str(str); while ((cut_at = tmp_str.find_first_of(delim)) != (signed)tmp_str.npos) { if (cut_at > 0) { results.insert(tmp_str.substr(0, cut_at)); @@ -98,10 +96,10 @@ void split_string(const std::string &str, std::string delim, std::set &results) +void split_string(const string &str, string delim, vector &results) { - int cut_at; - std::string tmp_str(str); + int cut_at; + string tmp_str(str); while ((cut_at = tmp_str.find_first_of(delim)) != (signed)tmp_str.npos) { if (cut_at > 0) { results.push_back(tmp_str.substr(0, cut_at)); @@ -114,7 +112,7 @@ void split_string(const std::string &str, std::string delim, std::vector &results, bool keep_null) +void split_string(char *str, char dim, vector &results, bool keep_null) { char *p = str; char *l = p; @@ -132,10 +130,9 @@ void split_string(char *str, char dim, std::vector &results, bool keep_n return; } -void merge_string(std::string &str, std::string delim, std::vector &source, size_t result_len) +void merge_string(string &str, string delim, vector &source, size_t result_len) { - - std::ostringstream ss; + ostringstream ss; if (source.empty()) { str = ss.str(); return; @@ -157,7 +154,7 @@ void merge_string(std::string &str, std::string delim, std::vector return; } -void replace(std::string &str, const std::string &old, const std::string &new_str) +void replace(string &str, const string &old, const string &new_str) { if (old.compare(new_str) == 0) { return; @@ -171,12 +168,12 @@ void replace(std::string &str, const std::string &old, const std::string &new_st return; } - std::string result; + string result; size_t index; size_t last_index = 0; - while ((index = str.find(old, last_index)) != std::string::npos) { + while ((index = str.find(old, last_index)) != string::npos) { result += str.substr(last_index, index - last_index); result += new_str; last_index = index + old.length(); @@ -270,7 +267,7 @@ char *substr(const char *s, int n1, int n2) * @param v * @return */ -std::string double_to_str(double v) +string double_to_str(double v) { char buf[256]; snprintf(buf, sizeof(buf), "%.2f", v); @@ -282,6 +279,6 @@ std::string double_to_str(double v) len--; } - return std::string(buf, len); + return string(buf, len); } } // namespace common diff --git a/deps/common/lang/string.h b/deps/common/lang/string.h index f46e1b70a..31683cf5b 100644 --- a/deps/common/lang/string.h +++ b/deps/common/lang/string.h @@ -20,20 +20,23 @@ See the Mulan PSL v2 for more details. */ #include #include -#include -#include -#include #include #include -#include #include "common/defs.h" +#include "common/lang/vector.h" +#include "common/lang/iostream.h" +#include "common/lang/sstream.h" +#include "common/lang/set.h" + +using std::string; + namespace common { /** * remove all white space(like ' ', '\t', '\n') from string */ -void strip(std::string &str); +void strip(string &str); char *strip(char *str); /** @@ -42,21 +45,21 @@ char *strip(char *str); * @param[in] pad decimal digits for the string padding * return output 0-padded size string */ -std::string size_to_pad_str(int size, int pad); +string size_to_pad_str(int size, int pad); /** * Convert a string to upper case. * @param[in,out] s the string to modify * @return a reference to the string that was modified. */ -std::string &str_to_upper(std::string &s); +string &str_to_upper(string &s); /** * Convert a string to lower case. * @param[in,out] s the string to modify * @return a reference to the string that was modified. */ -std::string &str_to_lower(std::string &s); +string &str_to_lower(string &s); /** * Split string str using 'delimiter's @@ -64,15 +67,15 @@ std::string &str_to_lower(std::string &s); * @param[in] delims elimiter characters * @param[in,out] results ector containing the split up string */ -void split_string(const std::string &str, std::string delim, std::set &results); -void split_string(const std::string &str, std::string delim, std::vector &results); -void split_string(char *str, char dim, std::vector &results, bool keep_null = false); +void split_string(const string &str, string delim, set &results); +void split_string(const string &str, string delim, vector &results); +void split_string(char *str, char dim, vector &results, bool keep_null = false); -void merge_string(std::string &str, std::string delim, std::vector &result, size_t result_len = 0); +void merge_string(string &str, string delim, vector &result, size_t result_len = 0); /** * replace old with new in the string */ -void replace(std::string &str, const std::string &old, const std::string &new_str); +void replace(string &str, const string &old, const string &new_str); /** * binary to hexadecimal @@ -96,7 +99,7 @@ char *hex_to_bin(const char *s, char *bin_buff, int *dest_len); * number, \c false otherwise */ template -bool str_to_val(const std::string &str, T &val, std::ios_base &(*radix)(std::ios_base &) = std::dec); +bool str_to_val(const string &str, T &val, ios_base &(*radix)(ios_base &) = std::dec); /** * Convert a numeric value into its string representation @@ -109,14 +112,14 @@ bool str_to_val(const std::string &str, T &val, std::ios_base &(*radix)(std::ios * (hexidecimal). */ template -void val_to_str(const T &val, std::string &str, std::ios_base &(*radix)(std::ios_base &) = std::dec); +void val_to_str(const T &val, string &str, ios_base &(*radix)(ios_base &) = std::dec); /** * Double to string * @param v * @return */ -std::string double_to_str(double v); +string double_to_str(double v); bool is_blank(const char *s); @@ -135,13 +138,13 @@ char *substr(const char *s, int n1, int n2); * get type's name */ template -std::string get_type_name(const T &val); +string get_type_name(const T &val); template -bool str_to_val(const std::string &str, T &val, std::ios_base &(*radix)(std::ios_base &)/* = std::dec */) +bool str_to_val(const string &str, T &val, ios_base &(*radix)(ios_base &)/* = std::dec */) { - bool success = true; - std::istringstream is(str); + bool success = true; + istringstream is(str); if (!(is >> radix >> val)) { val = 0; success = false; @@ -150,22 +153,22 @@ bool str_to_val(const std::string &str, T &val, std::ios_base &(*radix)(std::ios } template -void val_to_str(const T &val, std::string &str, std::ios_base &(*radix)(std::ios_base &)/* = std::dec */) +void val_to_str(const T &val, string &str, ios_base &(*radix)(ios_base &)/* = std::dec */) { - std::stringstream strm; + stringstream strm; strm << radix << val; str = strm.str(); } template -std::string get_type_name(const T &val) +string get_type_name(const T &val) { int status = 0; char *stmp = abi::__cxa_demangle(typeid(val).name(), 0, 0, &status); if (!stmp) return ""; - std::string sret(stmp); + string sret(stmp); ::free(stmp); return sret; diff --git a/deps/common/lang/string_view.h b/deps/common/lang/string_view.h new file mode 100644 index 000000000..73823ab66 --- /dev/null +++ b/deps/common/lang/string_view.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::string_view; diff --git a/deps/common/lang/system_error.h b/deps/common/lang/system_error.h new file mode 100644 index 000000000..6c49080e0 --- /dev/null +++ b/deps/common/lang/system_error.h @@ -0,0 +1,16 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::errc; +using std::error_code; diff --git a/deps/common/lang/thread.h b/deps/common/lang/thread.h new file mode 100644 index 000000000..9dc594646 --- /dev/null +++ b/deps/common/lang/thread.h @@ -0,0 +1,16 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::thread; +namespace this_thread = std::this_thread; \ No newline at end of file diff --git a/deps/common/lang/unordered_map.h b/deps/common/lang/unordered_map.h new file mode 100644 index 000000000..864d4cae1 --- /dev/null +++ b/deps/common/lang/unordered_map.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::unordered_map; \ No newline at end of file diff --git a/deps/common/lang/unordered_set.h b/deps/common/lang/unordered_set.h new file mode 100644 index 000000000..dac6d7146 --- /dev/null +++ b/deps/common/lang/unordered_set.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::unordered_set; \ No newline at end of file diff --git a/deps/common/lang/utility.h b/deps/common/lang/utility.h new file mode 100644 index 000000000..3e05c6486 --- /dev/null +++ b/deps/common/lang/utility.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::pair; \ No newline at end of file diff --git a/deps/common/lang/vector.h b/deps/common/lang/vector.h new file mode 100644 index 000000000..e93b76ec3 --- /dev/null +++ b/deps/common/lang/vector.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +using std::vector; \ No newline at end of file diff --git a/deps/common/log/log.cpp b/deps/common/log/log.cpp index fd4bb86fa..cc1d3c057 100644 --- a/deps/common/log/log.cpp +++ b/deps/common/log/log.cpp @@ -13,18 +13,20 @@ See the Mulan PSL v2 for more details. */ // #include -#include #include #include #include #include "common/lang/string.h" +#include "common/lang/functional.h" +#include "common/lang/iostream.h" +#include "common/lang/new.h" #include "common/log/log.h" namespace common { Log *g_log = nullptr; -Log::Log(const std::string &log_file_name, const LOG_LEVEL log_level, const LOG_LEVEL console_level) +Log::Log(const string &log_file_name, const LOG_LEVEL log_level, const LOG_LEVEL console_level) : log_name_(log_file_name), log_level_(log_level), console_level_(console_level) { prefix_map_[LOG_LEVEL_PANIC] = "PANIC:"; @@ -95,9 +97,9 @@ int Log::output(const LOG_LEVEL level, const char *module, const char *prefix, c va_end(args); if (LOG_LEVEL_PANIC <= level && level <= console_level_) { - std::cout << msg << std::endl; + cout << msg << endl; } else if (default_set_.find(module) != default_set_.end()) { - std::cout << msg << std::endl; + cout << msg << endl; } if (LOG_LEVEL_PANIC <= level && level <= log_level_) { @@ -122,11 +124,11 @@ int Log::output(const LOG_LEVEL level, const char *module, const char *prefix, c locked = false; } - } catch (std::exception &e) { + } catch (exception &e) { if (locked) { pthread_mutex_unlock(&lock_); } - std::cerr << e.what() << std::endl; + cerr << e.what() << endl; return LOG_STATUS_ERR; } @@ -166,7 +168,7 @@ const char *Log::prefix_msg(LOG_LEVEL level) return empty_prefix; } -void Log::set_default_module(const std::string &modules) { split_string(modules, ",", default_set_); } +void Log::set_default_module(const string &modules) { split_string(modules, ",", default_set_); } int Log::set_rotate_type(LOG_ROTATE rotate_type) { @@ -187,12 +189,12 @@ int Log::rotate_by_day(const int year, const int month, const int day) char date[16] = {0}; snprintf(date, sizeof(date), "%04d%02d%02d", year, month, day); - std::string log_file_name = log_name_ + "." + date; + string log_file_name = log_name_ + "." + date; if (ofs_.is_open()) { ofs_.close(); } - ofs_.open(log_file_name.c_str(), std::ios_base::out | std::ios_base::app); + ofs_.open(log_file_name.c_str(), ios_base::out | ios_base::app); if (ofs_.good()) { log_date_.year_ = year; log_date_.mon_ = month; @@ -212,7 +214,7 @@ int Log::rename_old_logs() int max_log_index = 0; while (log_index < MAX_LOG_NUM) { - std::string log_name = log_name_ + "." + size_to_pad_str(log_index, 3); + string log_name = log_name_ + "." + size_to_pad_str(log_index, 3); int result = access(log_name.c_str(), R_OK); if (result) { break; @@ -223,15 +225,15 @@ int Log::rename_old_logs() } if (log_index == MAX_LOG_NUM) { - std::string log_name_rm = log_name_ + "." + size_to_pad_str(log_index, 3); + string log_name_rm = log_name_ + "." + size_to_pad_str(log_index, 3); remove(log_name_rm.c_str()); } log_index = max_log_index; while (log_index > 0) { - std::string log_name_old = log_name_ + "." + size_to_pad_str(log_index, 3); + string log_name_old = log_name_ + "." + size_to_pad_str(log_index, 3); - std::string log_name_new = log_name_ + "." + size_to_pad_str(log_index + 1, 3); + string log_name_new = log_name_ + "." + size_to_pad_str(log_index + 1, 3); int result = rename(log_name_old.c_str(), log_name_new.c_str()); if (result) { @@ -247,7 +249,7 @@ int Log::rotate_by_size() { if (log_line_ < 0) { // The first time open log file - ofs_.open(log_name_.c_str(), std::ios_base::out | std::ios_base::app); + ofs_.open(log_name_.c_str(), ios_base::out | ios_base::app); log_line_ = 0; return LOG_STATUS_OK; } else if (0 <= log_line_ && log_line_ < log_max_line_) { @@ -267,13 +269,13 @@ int Log::rotate_by_size() char log_index_str[4] = {0}; snprintf(log_index_str, sizeof(log_index_str), "%03d", 1); - std::string log_name_new = log_name_ + "." + log_index_str; + string log_name_new = log_name_ + "." + log_index_str; result = rename(log_name_.c_str(), log_name_new.c_str()); if (result) { - std::cerr << "Failed to rename " << log_name_ << " to " << log_name_new << std::endl; + cerr << "Failed to rename " << log_name_ << " to " << log_name_new << endl; } - ofs_.open(log_name_.c_str(), std::ios_base::out | std::ios_base::app); + ofs_.open(log_name_.c_str(), ios_base::out | ios_base::app); if (ofs_.good()) { log_line_ = 0; } else { @@ -301,7 +303,7 @@ int Log::rotate(const int year, const int month, const int day) return result; } -void Log::set_context_getter(std::function context_getter) +void Log::set_context_getter(function context_getter) { if (context_getter) { context_getter_ = context_getter; @@ -323,11 +325,11 @@ LoggerFactory::~LoggerFactory() } int LoggerFactory::init( - const std::string &log_file, Log **logger, LOG_LEVEL log_level, LOG_LEVEL console_level, LOG_ROTATE rotate_type) + const string &log_file, Log **logger, LOG_LEVEL log_level, LOG_LEVEL console_level, LOG_ROTATE rotate_type) { - Log *log = new (std::nothrow) Log(log_file, log_level, console_level); + Log *log = new (nothrow) Log(log_file, log_level, console_level); if (log == nullptr) { - std::cout << "Error: fail to construct a log object!" << std::endl; + cout << "Error: fail to construct a log object!" << endl; return -1; } log->set_rotate_type(rotate_type); @@ -338,7 +340,7 @@ int LoggerFactory::init( } int LoggerFactory::init_default( - const std::string &log_file, LOG_LEVEL log_level, LOG_LEVEL console_level, LOG_ROTATE rotate_type) + const string &log_file, LOG_LEVEL log_level, LOG_LEVEL console_level, LOG_ROTATE rotate_type) { if (g_log != nullptr) { LOG_INFO("Default logger has been initialized"); diff --git a/deps/common/log/log.h b/deps/common/log/log.h index a4e1772f0..f51b798c8 100644 --- a/deps/common/log/log.h +++ b/deps/common/log/log.h @@ -20,14 +20,13 @@ See the Mulan PSL v2 for more details. */ #include #include -#include -#include -#include -#include -#include -#include - #include "common/defs.h" +#include "common/lang/string.h" +#include "common/lang/map.h" +#include "common/lang/set.h" +#include "common/lang/functional.h" +#include "common/lang/iostream.h" +#include "common/lang/fstream.h" namespace common { @@ -59,11 +58,11 @@ typedef enum class Log { public: - Log(const std::string &log_name, const LOG_LEVEL log_level = LOG_LEVEL_INFO, + Log(const string &log_name, const LOG_LEVEL log_level = LOG_LEVEL_INFO, const LOG_LEVEL console_level = LOG_LEVEL_WARN); ~Log(void); - static int init(const std::string &log_file); + static int init(const string &log_file); /** * These functions won't output header information such as __FUNCTION__, @@ -110,7 +109,7 @@ class Log * if one module is default module, * it will output whatever output level is lower than log_level_ or not */ - void set_default_module(const std::string &modules); + void set_default_module(const string &modules); bool check_output(const LOG_LEVEL log_level, const char *module); int rotate(const int year = 0, const int month = 0, const int day = 0); @@ -120,7 +119,7 @@ class Log * @details 比如设置一个获取当前session标识的函数,那么每次在打印日志时都会输出session信息。 * 这个回调函数返回了一个intptr_t类型的数据,可能返回字符串更好,但是现在够用了。 */ - void set_context_getter(std::function context_getter); + void set_context_getter(function context_getter); intptr_t context_id(); private: @@ -135,8 +134,8 @@ class Log private: pthread_mutex_t lock_; - std::ofstream ofs_; - std::string log_name_; + ofstream ofs_; + string log_name_; LOG_LEVEL log_level_; LOG_LEVEL console_level_; @@ -151,13 +150,13 @@ class Log int log_max_line_; LOG_ROTATE rotate_type_; - typedef std::map LogPrefixMap; - LogPrefixMap prefix_map_; + typedef map LogPrefixMap; + LogPrefixMap prefix_map_; - typedef std::set DefaultSet; - DefaultSet default_set_; + typedef set DefaultSet; + DefaultSet default_set_; - std::function context_getter_; + function context_getter_; }; class LoggerFactory @@ -166,10 +165,10 @@ class LoggerFactory LoggerFactory(); virtual ~LoggerFactory(); - static int init(const std::string &log_file, Log **logger, LOG_LEVEL log_level = LOG_LEVEL_INFO, + static int init(const string &log_file, Log **logger, LOG_LEVEL log_level = LOG_LEVEL_INFO, LOG_LEVEL console_level = LOG_LEVEL_WARN, LOG_ROTATE rotate_type = LOG_ROTATE_BYDAY); - static int init_default(const std::string &log_file, LOG_LEVEL log_level = LOG_LEVEL_INFO, + static int init_default(const string &log_file, LOG_LEVEL log_level = LOG_LEVEL_INFO, LOG_LEVEL console_level = LOG_LEVEL_WARN, LOG_ROTATE rotate_type = LOG_ROTATE_BYDAY); }; @@ -290,7 +289,7 @@ int Log::out(const LOG_LEVEL console_level, const LOG_LEVEL log_level, T &msg) char prefix[ONE_KILO] = {0}; LOG_HEAD(prefix, log_level); if (LOG_LEVEL_PANIC <= console_level && console_level <= console_level_) { - std::cout << prefix_map_[console_level] << msg; + cout << prefix_map_[console_level] << msg; } if (LOG_LEVEL_PANIC <= log_level && log_level <= log_level_) { @@ -303,11 +302,11 @@ int Log::out(const LOG_LEVEL console_level, const LOG_LEVEL log_level, T &msg) pthread_mutex_unlock(&lock_); locked = false; } - } catch (std::exception &e) { + } catch (exception &e) { if (locked) { pthread_mutex_unlock(&lock_); } - std::cerr << e.what() << std::endl; + cerr << e.what() << endl; return LOG_STATUS_ERR; } diff --git a/deps/common/math/integer_generator.h b/deps/common/math/integer_generator.h index 2ac182812..637e3982b 100644 --- a/deps/common/math/integer_generator.h +++ b/deps/common/math/integer_generator.h @@ -14,7 +14,7 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include +#include "common/lang/random.h" namespace common { @@ -32,8 +32,8 @@ class IntegerGenerator int max() const { return distrib_.max(); } private: - std::random_device rd_; - std::uniform_int_distribution<> distrib_; + random_device rd_; + uniform_int_distribution<> distrib_; }; } // namespace common diff --git a/deps/common/math/md5.h b/deps/common/math/md5.h index 841afb576..93dfe619d 100644 --- a/deps/common/math/md5.h +++ b/deps/common/math/md5.h @@ -12,8 +12,8 @@ See the Mulan PSL v2 for more details. */ // Created by Longda on 2010 // -#ifndef __COMMON_MATH_MD5_H__ -#define __COMMON_MATH_MD5_H__ +#pragma once + #include namespace common { @@ -68,4 +68,3 @@ void MD5Final(unsigned char[16], MD5_CTX *); #endif } // namespace common -#endif //__COMMON_MATH_MD5_H__ diff --git a/deps/common/math/random_generator.cpp b/deps/common/math/random_generator.cpp index 09dccc498..4a3f48f8e 100644 --- a/deps/common/math/random_generator.cpp +++ b/deps/common/math/random_generator.cpp @@ -12,13 +12,12 @@ See the Mulan PSL v2 for more details. */ // Created by Longda on 2021/4/20. // -#include - #include "common/math/random_generator.h" +#include "common/lang/chrono.h" namespace common { -RandomGenerator::RandomGenerator() : randomData(std::chrono::system_clock::now().time_since_epoch().count()) {} +RandomGenerator::RandomGenerator() : randomData(chrono::system_clock::now().time_since_epoch().count()) {} RandomGenerator::~RandomGenerator() {} diff --git a/deps/common/math/random_generator.h b/deps/common/math/random_generator.h index a3d8a97e2..d90bf080c 100644 --- a/deps/common/math/random_generator.h +++ b/deps/common/math/random_generator.h @@ -11,11 +11,12 @@ See the Mulan PSL v2 for more details. */ // // Created by Longda on 2021/4/20. // -#ifndef __COMMON_MATH_RANDOM_GENERATOR_H_ -#define __COMMON_MATH_RANDOM_GENERATOR_H_ +#pragma once -#include #include + +#include "common/lang/random.h" + namespace common { #define DEFAULT_RANDOM_BUFF_SIZE 512 @@ -33,9 +34,7 @@ class RandomGenerator private: // The GUN Extended TLS Version - std::mt19937 randomData; + mt19937 randomData; }; } // namespace common - -#endif /* __COMMON_MATH_RANDOM_GENERATOR_H_ */ diff --git a/deps/common/math/regex.h b/deps/common/math/regex.h index e1ec13959..f1f8c6c0b 100644 --- a/deps/common/math/regex.h +++ b/deps/common/math/regex.h @@ -12,11 +12,10 @@ See the Mulan PSL v2 for more details. */ // Created by Longda on 2010 // -#ifndef __COMMON_MATH_REGEX_H__ -#define __COMMON_MATH_REGEX_H__ +#pragma once + namespace common { int regex_match(const char *str_, const char *pat_); } // namespace common -#endif /* __COMMON_MATH_REGEX_H__ */ diff --git a/deps/common/math/simd_util.cpp b/deps/common/math/simd_util.cpp new file mode 100644 index 000000000..50eb0e4db --- /dev/null +++ b/deps/common/math/simd_util.cpp @@ -0,0 +1,57 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include "common/math/simd_util.h" + +#if defined(USE_SIMD) + +int mm256_extract_epi32_var_indx(const __m256i vec, const unsigned int i) +{ + __m128i idx = _mm_cvtsi32_si128(i); + __m256i val = _mm256_permutevar8x32_epi32(vec, _mm256_castsi128_si256(idx)); + return _mm_cvtsi128_si32(_mm256_castsi256_si128(val)); +} + +int mm256_sum_epi32(const int *values, int size) +{ + // your code here + int sum = 0; + for (int i = 0; i < size; i++) { + sum += values[i]; + } + return sum; +} + +float mm256_sum_ps(const float *values, int size) +{ + // your code here + float sum = 0; + for (int i = 0; i < size; i++) { + sum += values[i]; + } + return sum; +} + +template +void selective_load(V *memory, int offset, V *vec, __m256i &inv) +{ + int *inv_ptr = reinterpret_cast(&inv); + for (int i = 0; i < SIMD_WIDTH; i++) { + if (inv_ptr[i] == -1) { + vec[i] = memory[offset++]; + } + } +} +template void selective_load(uint32_t *memory, int offset, uint32_t *vec, __m256i &inv); +template void selective_load(int *memory, int offset, int *vec, __m256i &inv); +template void selective_load(float *memory, int offset, float *vec, __m256i &inv); + +#endif \ No newline at end of file diff --git a/deps/common/math/simd_util.h b/deps/common/math/simd_util.h new file mode 100644 index 000000000..ccb7f451c --- /dev/null +++ b/deps/common/math/simd_util.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#if defined(USE_SIMD) +#include + +static constexpr int SIMD_WIDTH = 8; // AVX2 (256bit) + +/// @brief 从 vec 中提取下标为 i 的 int 类型的值。 +int mm256_extract_epi32_var_indx(const __m256i vec, const unsigned int i); + +/// @brief 数组求和 +int mm256_sum_epi32(const int *values, int size); +float mm256_sum_ps(const float *values, int size); + +/// @brief selective load 的标量实现 +template +void selective_load(V *memory, int offset, V *vec, __m256i &inv); +#endif \ No newline at end of file diff --git a/deps/common/metrics/histogram_snapshot.cpp b/deps/common/metrics/histogram_snapshot.cpp index 025e68548..c512b3411 100644 --- a/deps/common/metrics/histogram_snapshot.cpp +++ b/deps/common/metrics/histogram_snapshot.cpp @@ -20,7 +20,6 @@ See the Mulan PSL v2 for more details. */ #include #include #include -#include namespace common { diff --git a/deps/common/mm/mem.h b/deps/common/mm/mem.h deleted file mode 100644 index c4f96e3c4..000000000 --- a/deps/common/mm/mem.h +++ /dev/null @@ -1,132 +0,0 @@ -/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. -miniob is licensed under Mulan PSL v2. -You can use this software according to the terms and conditions of the Mulan PSL v2. -You may obtain a copy of Mulan PSL v2 at: - http://license.coscl.org.cn/MulanPSL2 -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -See the Mulan PSL v2 for more details. */ - -// -// Created by Longda on 2010 -// - -#pragma once - -#include -#include - -#include - -#include "common/defs.h" -namespace common { - -#ifndef MEM_DEBUG -#define lcalloc calloc -#define lmalloc malloc -#define lfree free -#define lrealloc realloc - -#else - -typedef struct MemID_t -{ -public: - const static int MEM_FILENAME_LEN = 32; - struct MemID_t *mNext; - char mFile[MEM_FILENAME_LEN]; - uint64_t mSize; - uint32_t mLine; -} MemID; - -class CLMemTrace -{ -public: - static void *malloc(size_t size, const char *file, const int line, bool retry = false) throw(std::bad_alloc); - - // just use for realloc, same functionality as realloc - static void *realloc(void *ptr, size_t size, const char *file, const int line); - - static void free(void *ptr); - - static void output(); - - /** - * set whether show every details - */ - static void setVerbose(bool verbose) { mVerbose = verbose; } - -protected: - static std::new_handler getNewHandler(); - - /** - * Don't use stl due to map or string will use new, - * so it will leading to dead loop - */ - // static std::map mMemIds; - const static int MEM_HASHTABLE_SIZE = 16384; - - static MemID *mMemIDs[MEM_HASHTABLE_SIZE]; - static uint64_t mUsedSize; - static pthread_mutex_t mMutex; - static bool mVerbose; -}; - -#define lcalloc(nmemb, size) Lcalloc(nmemb, size, __FILE__, __LINE__) - -#define lmalloc(size) Lmalloc(size, __FILE__, __LINE__) - -#define lrealloc(ptr, size) Lrealloc(ptr, size, __FILE__, __LINE__) - -#define lfree(ptr) Lfree(ptr) - -void *Lcalloc(size_t nmemb, size_t size, const char *file, const int line); -void *Lmalloc(size_t size, const char *file, const int line); -void Lfree(void *ptr); -void *Lrealloc(void *ptr, size_t size, const char *file, const int line); - -/** - * My own operator new function, it will record alloc information - * default alloc memory or DEBUG_NEW will go to this function - */ -static void *operator new(std::size_t size, const char *file, int line); -static void *operator new[](std::size_t size, const char *file, int line); - -/** - * when user use debug_new, then it will go to this function - */ -static void *operator new(std::size_t size) throw(std::bad_alloc); -static void *operator new[](std::size_t size) throw(std::bad_alloc); - -/** - * when user use nothrow 's new function, it will go this function - */ -static void *operator new(std::size_t size, const std::nothrow_t &) throw(); -static void *operator new[](std::size_t size, const std::nothrow_t &) throw(); - -/** - * my own operator function, low level function - */ -static void operator delete(void *pointer, const char *file, int line); -static void operator delete[](void *pointer, const char *file, int line); - -/** - * the default delete function - */ -static void operator delete(void *pointer); -static void operator delete[](void *pointer); - -///** -// * nothrow 's delete function, it will be the pair of nothrow's new function -// */ -// void operator delete(void* pointer, const std::nothrow_t&); -// void operator delete[](void* pointer, const std::nothrow_t&); - -#define new DEBUG_NEW -#define DEBUG_NEW new (__FILE__, __LINE__) -#define debug_new new - -#endif /* MEM_DEBUG */ - -} // namespace common diff --git a/deps/common/mm/mem_pool.cpp b/deps/common/mm/mem_pool.cpp index b7e2dc291..3692f84b5 100644 --- a/deps/common/mm/mem_pool.cpp +++ b/deps/common/mm/mem_pool.cpp @@ -57,7 +57,7 @@ void MemPoolItem::cleanup() frees.clear(); this->size = 0; - for (std::list::iterator iter = pools.begin(); iter != pools.end(); iter++) { + for (list::iterator iter = pools.begin(); iter != pools.end(); iter++) { void *pool = *iter; ::free(pool); @@ -126,11 +126,11 @@ void *MemPoolItem::alloc() return buffer; } -MemPoolItem::unique_ptr MemPoolItem::alloc_unique_ptr() +MemPoolItem::item_unique_ptr MemPoolItem::alloc_unique_ptr() { void *item = this->alloc(); auto deleter = [this](void *p) { this->free(p); }; - return MemPoolItem::unique_ptr(item, deleter); + return MemPoolItem::item_unique_ptr(item, deleter); } void MemPoolItem::free(void *buf) diff --git a/deps/common/mm/mem_pool.h b/deps/common/mm/mem_pool.h index 39a345878..a76c0b9fa 100644 --- a/deps/common/mm/mem_pool.h +++ b/deps/common/mm/mem_pool.h @@ -14,15 +14,14 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include -#include -#include #include -#include #include "common/lang/mutex.h" +#include "common/lang/string.h" +#include "common/lang/set.h" +#include "common/lang/list.h" +#include "common/lang/memory.h" +#include "common/lang/sstream.h" #include "common/log/log.h" #include "common/os/os.h" @@ -85,17 +84,17 @@ class MemPool * Print the MemPool status * @return */ - virtual std::string to_string() = 0; + virtual string to_string() = 0; - const std::string get_name() const { return name; } - bool is_dynamic() const { return dynamic; } - int get_size() const { return size; } + const string get_name() const { return name; } + bool is_dynamic() const { return dynamic; } + int get_size() const { return size; } protected: pthread_mutex_t mutex; int size; bool dynamic; - std::string name; + string name; }; /** @@ -145,7 +144,7 @@ class MemPoolSimple : public MemPool * Print the MemPool status * @return */ - std::string to_string(); + string to_string(); int get_item_num_per_pool() const { return item_num_per_pool; } @@ -158,10 +157,10 @@ class MemPoolSimple : public MemPool } protected: - std::list pools; - std::set used; - std::list frees; - int item_num_per_pool; + list pools; + set used; + list frees; + int item_num_per_pool; }; template @@ -208,7 +207,7 @@ void MemPoolSimple::cleanup() frees.clear(); this->size = 0; - for (typename std::list::iterator iter = pools.begin(); iter != pools.end(); iter++) { + for (typename list::iterator iter = pools.begin(); iter != pools.end(); iter++) { T *pool = *iter; delete[] pool; @@ -294,9 +293,9 @@ void MemPoolSimple::free(T *buf) } template -std::string MemPoolSimple::to_string() +string MemPoolSimple::to_string() { - std::stringstream ss; + stringstream ss; ss << "name:" << this->name << "," << "dyanmic:" << this->dynamic << "," @@ -310,7 +309,7 @@ std::string MemPoolSimple::to_string() class MemPoolItem { public: - using unique_ptr = std::unique_ptr>; + using item_unique_ptr = unique_ptr>; public: MemPoolItem(const char *tag) : name(tag) @@ -353,8 +352,8 @@ class MemPoolItem * Alloc one frame from memory Pool * @return */ - void *alloc(); - unique_ptr alloc_unique_ptr(); + void *alloc(); + item_unique_ptr alloc_unique_ptr(); /** * Free one item, the resouce will return to memory Pool @@ -375,10 +374,10 @@ class MemPoolItem return it != used.end(); } - std::string to_string() + string to_string() { - std::stringstream ss; + stringstream ss; ss << "name:" << this->name << "," << "dyanmic:" << this->dynamic << "," @@ -389,11 +388,11 @@ class MemPoolItem return ss.str(); } - const std::string get_name() const { return name; } - bool is_dynamic() const { return dynamic; } - int get_size() const { return size; } - int get_item_size() const { return item_size; } - int get_item_num_per_pool() const { return item_num_per_pool; } + const string get_name() const { return name; } + bool is_dynamic() const { return dynamic; } + int get_size() const { return size; } + int get_item_size() const { return item_size; } + int get_item_num_per_pool() const { return item_num_per_pool; } int get_used_num() { @@ -405,15 +404,15 @@ class MemPoolItem protected: pthread_mutex_t mutex; - std::string name; + string name; bool dynamic; int size; int item_size; int item_num_per_pool; - std::list pools; - std::set used; - std::list frees; + list pools; + set used; + list frees; }; } // namespace common diff --git a/deps/common/os/os.cpp b/deps/common/os/os.cpp index 88eef3654..c866bb669 100644 --- a/deps/common/os/os.cpp +++ b/deps/common/os/os.cpp @@ -13,15 +13,15 @@ See the Mulan PSL v2 for more details. */ // #include -#include #include "common/defs.h" #include "common/log/log.h" #include "common/os/os.h" +#include "common/lang/thread.h" namespace common { // Don't care windows -uint32_t getCpuNum() { return std::thread::hardware_concurrency(); } +uint32_t getCpuNum() { return thread::hardware_concurrency(); } #define MAX_STACK_SIZE 32 diff --git a/deps/common/os/path.cpp b/deps/common/os/path.cpp index ada5651ef..4b6fbde3f 100644 --- a/deps/common/os/path.cpp +++ b/deps/common/os/path.cpp @@ -16,22 +16,23 @@ See the Mulan PSL v2 for more details. */ #include #include -#include - #include "common/defs.h" #include "common/log/log.h" #include "common/os/path.h" +#include "common/lang/string.h" +#include "common/lang/vector.h" + namespace common { -std::string getFileName(const std::string &fullPath) +string getFileName(const string &fullPath) { - std::string szRt; + string szRt; size_t pos; try { pos = fullPath.rfind(FILE_PATH_SPLIT); - if (pos != std::string::npos && pos < fullPath.size() - 1) { + if (pos != string::npos && pos < fullPath.size() - 1) { szRt = fullPath.substr(pos + 1, fullPath.size() - pos - 1); - } else if (pos == std::string::npos) { + } else if (pos == string::npos) { szRt = fullPath; } else { szRt = ""; @@ -41,7 +42,7 @@ std::string getFileName(const std::string &fullPath) return szRt; } -void getFileName(const char *path, std::string &fileName) +void getFileName(const char *path, string &fileName) { // Don't care the last character as FILE_PATH_SPLIT const char *endPos = strrchr(path, FILE_PATH_SPLIT); @@ -59,15 +60,15 @@ void getFileName(const char *path, std::string &fileName) return; } -std::string getDirName(const std::string &fullPath) +string getDirName(const string &fullPath) { - std::string szRt; + string szRt; size_t pos; try { pos = fullPath.rfind(FILE_PATH_SPLIT); - if (pos != std::string::npos && pos > 0) { + if (pos != string::npos && pos > 0) { szRt = fullPath.substr(0, pos); - } else if (pos == std::string::npos) { + } else if (pos == string::npos) { szRt = fullPath; } else { // pos == 0 @@ -77,7 +78,7 @@ std::string getDirName(const std::string &fullPath) } catch (...) {} return szRt; } -void getDirName(const char *path, std::string &parent) +void getDirName(const char *path, string &parent) { // Don't care the last character as FILE_PATH_SPLIT const char *endPos = strrchr(path, FILE_PATH_SPLIT); @@ -95,15 +96,15 @@ void getDirName(const char *path, std::string &parent) return; } -std::string getFilePath(const std::string &fullPath) +string getFilePath(const string &fullPath) { - std::string szRt; + string szRt; size_t pos; try { pos = fullPath.rfind("/"); - if (pos != std::string::npos) { + if (pos != string::npos) { szRt = fullPath.substr(0, pos); - } else if (pos == std::string::npos) { + } else if (pos == string::npos) { szRt = fullPath; } else { szRt = ""; @@ -113,9 +114,9 @@ std::string getFilePath(const std::string &fullPath) return szRt; } -std::string getAboslutPath(const char *path) +string getAboslutPath(const char *path) { - std::string aPath(path); + string aPath(path); if (path[0] != '/') { const int MAX_SIZE = 256; char current_absolute_path[MAX_SIZE]; @@ -132,7 +133,7 @@ bool is_directory(const char *path) return (0 == stat(path, &st)) && (st.st_mode & S_IFDIR); } -bool check_directory(std::string &path) +bool check_directory(string &path) { while (!path.empty() && path.back() == '/') path.erase(path.size() - 1, 1); @@ -166,7 +167,7 @@ bool check_directory(std::string &path) return true; } -int list_file(const char *path, const char *filter_pattern, std::vector &files) +int list_file(const char *path, const char *filter_pattern, vector &files) { regex_t reg; if (filter_pattern) { diff --git a/deps/common/os/path.h b/deps/common/os/path.h index 5ff6fcb81..8307ca4c9 100644 --- a/deps/common/os/path.h +++ b/deps/common/os/path.h @@ -14,7 +14,8 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include +#include "common/lang/string.h" + namespace common { /** @@ -26,8 +27,8 @@ namespace common { * input: happy --> return : happy * input: "" --> return : "" */ -std::string getFileName(const std::string &fullPath); -void getFileName(const char *path, std::string &fileName); +string getFileName(const string &fullPath); +void getFileName(const char *path, string &fileName); /** * get file path from full path @@ -37,15 +38,15 @@ void getFileName(const char *path, std::string &fileName); * input: happy --> return : happy * input: "" --> return : "" */ -std::string getFilePath(const std::string &fullPath); -void getDirName(const char *path, std::string &parent); +string getFilePath(const string &fullPath); +void getDirName(const char *path, string &parent); /** * Get absolute path * input: path * reutrn absolutely path */ -std::string getAboslutPath(const char *path); +string getAboslutPath(const char *path); /** * 判断给定目录是否是文件夹 @@ -57,13 +58,13 @@ bool is_directory(const char *path); * 如果不存在将会逐级创建 * @return 创建失败,或者不是文件夹将会返回失败 */ -bool check_directory(std::string &path); +bool check_directory(string &path); /** * 列出指定文件夹下符合指定模式的所有文件 * @param filter_pattern 示例 ^miniob.*bin$ * @return 成功返回找到的文件个数,否则返回-1 */ -int list_file(const char *path, const char *filter_pattern, std::vector &files); // io/io.h::getFileList +int list_file(const char *path, const char *filter_pattern, vector &files); // io/io.h::getFileList } // namespace common diff --git a/deps/common/os/pidfile.cpp b/deps/common/os/pidfile.cpp index 3ec8443de..91df67864 100644 --- a/deps/common/os/pidfile.cpp +++ b/deps/common/os/pidfile.cpp @@ -15,7 +15,6 @@ See the Mulan PSL v2 for more details. */ #include #include #include -#include #include #include #include @@ -24,21 +23,24 @@ See the Mulan PSL v2 for more details. */ #include "common/log/log.h" #include "common/os/pidfile.h" +#include "common/lang/iostream.h" +#include "common/lang/fstream.h" + namespace common { -std::string &getPidPath() +string &getPidPath() { - static std::string path; + static string path; return path; } void setPidPath(const char *progName) { - std::string &path = getPidPath(); + string &path = getPidPath(); if (progName != NULL) { - path = std::string(_PATH_TMP) + progName + ".pid"; + path = string(_PATH_TMP) + progName + ".pid"; } else { path = ""; } @@ -47,19 +49,19 @@ void setPidPath(const char *progName) int writePidFile(const char *progName) { assert(progName); - std::ofstream ostr; + ofstream ostr; int rv = 1; setPidPath(progName); - std::string path = getPidPath(); - ostr.open(path.c_str(), std::ios::trunc); + string path = getPidPath(); + ostr.open(path.c_str(), ios::trunc); if (ostr.good()) { - ostr << getpid() << std::endl; + ostr << getpid() << endl; ostr.close(); rv = 0; } else { rv = errno; - std::cerr << "error opening PID file " << path.c_str() << SYS_OUTPUT_ERROR << std::endl; + cerr << "error opening PID file " << path.c_str() << SYS_OUTPUT_ERROR << endl; } return rv; @@ -67,7 +69,7 @@ int writePidFile(const char *progName) void removePidFile(void) { - std::string path = getPidPath(); + string path = getPidPath(); if (!path.empty()) { unlink(path.c_str()); setPidPath(NULL); diff --git a/deps/common/os/pidfile.h b/deps/common/os/pidfile.h index bc3cbe5b1..c34c4c491 100644 --- a/deps/common/os/pidfile.h +++ b/deps/common/os/pidfile.h @@ -14,6 +14,8 @@ See the Mulan PSL v2 for more details. */ #pragma once +#include "common/lang/string.h" + namespace common { //! Generates a PID file for the current component @@ -34,6 +36,6 @@ int writePidFile(const char *progName); */ void removePidFile(void); -std::string &getPidPath(); +string &getPidPath(); } // namespace common diff --git a/deps/common/os/process.cpp b/deps/common/os/process.cpp index a18292614..039e5e1f5 100644 --- a/deps/common/os/process.cpp +++ b/deps/common/os/process.cpp @@ -18,12 +18,12 @@ See the Mulan PSL v2 for more details. */ #include #include -#include - #include "common/io/io.h" #include "common/log/log.h" #include "common/os/path.h" #include "common/os/process.h" +#include "common/lang/iostream.h" + namespace common { #ifdef __MACH__ @@ -35,9 +35,9 @@ namespace common { #define RWRR (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) -std::string get_process_name(const char *prog_name) +string get_process_name(const char *prog_name) { - std::string process_name; + string process_name; int buf_len = strlen(prog_name); @@ -45,7 +45,7 @@ std::string get_process_name(const char *prog_name) char *buf = new char[buf_len + 1]; if (buf == NULL) { - std::cerr << "Failed to alloc memory for program name." << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to alloc memory for program name." << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; return ""; } snprintf(buf, buf_len + 1, "%s", prog_name); // 第二个参数需为buf_len + 1 @@ -72,7 +72,7 @@ int daemonize_service(bool close_std_streams) #endif // Here after the fork; the parent is dead and setsid() is called if (rc != 0) { - std::cerr << "Error: unable to daemonize: " << strerror(errno) << "\n"; + cerr << "Error: unable to daemonize: " << strerror(errno) << "\n"; } return rc; } @@ -82,7 +82,7 @@ int daemonize_service(const char *std_out_file, const char *std_err_file) int rc = daemonize_service(false); if (rc != 0) { - std::cerr << "Error: \n"; + cerr << "Error: \n"; return rc; } @@ -106,7 +106,7 @@ void sys_log_redirect(const char *std_out_file, const char *std_err_file) struct timeval tv; rc = gettimeofday(&tv, NULL); if (rc != 0) { - std::cerr << "Fail to get current time" << std::endl; + cerr << "Fail to get current time" << endl; tv.tv_sec = 0; } @@ -114,7 +114,7 @@ void sys_log_redirect(const char *std_out_file, const char *std_err_file) // Always use append-write. And if not exist, create it. std_err_flag = std_out_flag = O_CREAT | O_APPEND | O_WRONLY; - std::string err_file = getAboslutPath(std_err_file); + string err_file = getAboslutPath(std_err_file); // CWE367: A check occurs on a file's attributes before the file is // used in a privileged operation, but things may have changed @@ -134,9 +134,9 @@ void sys_log_redirect(const char *std_out_file, const char *std_err_file) close(errfd); } setvbuf(stderr, NULL, _IONBF, 0); // Make sure stderr is not buffering - std::cerr << "Process " << getpid() << " built error output at " << tv.tv_sec << std::endl; + cerr << "Process " << getpid() << " built error output at " << tv.tv_sec << endl; - std::string outFile = getAboslutPath(std_out_file); + string outFile = getAboslutPath(std_out_file); // Redirect stdout to outFile.c_str() // rc = stat(outFile.c_str(), &st); @@ -152,7 +152,7 @@ void sys_log_redirect(const char *std_out_file, const char *std_err_file) close(outfd); } setvbuf(stdout, NULL, _IONBF, 0); // Make sure stdout not buffering - std::cout << "Process " << getpid() << " built standard output at " << tv.tv_sec << std::endl; + cout << "Process " << getpid() << " built standard output at " << tv.tv_sec << endl; return; } diff --git a/deps/common/os/process.h b/deps/common/os/process.h index e992182d9..2281c26cf 100644 --- a/deps/common/os/process.h +++ b/deps/common/os/process.h @@ -14,6 +14,8 @@ See the Mulan PSL v2 for more details. */ #pragma once +#include "common/lang/string.h" + namespace common { //! Get process Name @@ -21,7 +23,7 @@ namespace common { * @param[in] prog_full_name process full name with full path * @return process_name_ process name without directory path */ -std::string get_process_name(const char *prog_full_name); +string get_process_name(const char *prog_full_name); //! Runs the service as a daemon /** * Backgrounds the calling service as a system daemon by detaching it from diff --git a/deps/common/os/process_param.cpp b/deps/common/os/process_param.cpp index e70d28712..218e3d3ac 100644 --- a/deps/common/os/process_param.cpp +++ b/deps/common/os/process_param.cpp @@ -24,7 +24,7 @@ ProcessParam *&the_process_param() return process_cfg; } -void ProcessParam::init_default(std::string &process_name) +void ProcessParam::init_default(string &process_name) { assert(process_name.empty() == false); this->process_name_ = process_name; diff --git a/deps/common/os/process_param.h b/deps/common/os/process_param.h index 59e7bca1f..521ae6926 100644 --- a/deps/common/os/process_param.h +++ b/deps/common/os/process_param.h @@ -14,8 +14,9 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include +#include "common/lang/string.h" +#include "common/lang/vector.h" + namespace common { class ProcessParam @@ -26,31 +27,31 @@ class ProcessParam virtual ~ProcessParam() {} - void init_default(std::string &process_name); + void init_default(string &process_name); - const std::string &get_std_out() const { return std_out_; } + const string &get_std_out() const { return std_out_; } - void set_std_out(const std::string &std_out) { ProcessParam::std_out_ = std_out; } + void set_std_out(const string &std_out) { ProcessParam::std_out_ = std_out; } - const std::string &get_std_err() const { return std_err_; } + const string &get_std_err() const { return std_err_; } - void set_std_err(const std::string &std_err) { ProcessParam::std_err_ = std_err; } + void set_std_err(const string &std_err) { ProcessParam::std_err_ = std_err; } - const std::string &get_conf() const { return conf; } + const string &get_conf() const { return conf; } - void set_conf(const std::string &conf) { ProcessParam::conf = conf; } + void set_conf(const string &conf) { ProcessParam::conf = conf; } - const std::string &get_process_name() const { return process_name_; } + const string &get_process_name() const { return process_name_; } - void set_process_name(const std::string &processName) { ProcessParam::process_name_ = processName; } + void set_process_name(const string &processName) { ProcessParam::process_name_ = processName; } bool is_demon() const { return demon; } void set_demon(bool demon) { ProcessParam::demon = demon; } - const std::vector &get_args() const { return args; } + const vector &get_args() const { return args; } - void set_args(const std::vector &args) { ProcessParam::args = args; } + void set_args(const vector &args) { ProcessParam::args = args; } void set_server_port(int port) { server_port_ = port; } @@ -58,11 +59,11 @@ class ProcessParam void set_unix_socket_path(const char *unix_socket_path) { unix_socket_path_ = unix_socket_path; } - const std::string &get_unix_socket_path() const { return unix_socket_path_; } + const string &get_unix_socket_path() const { return unix_socket_path_; } void set_protocol(const char *protocol) { protocol_ = protocol; } - const std::string &get_protocol() const { return protocol_; } + const string &get_protocol() const { return protocol_; } void set_trx_kit_name(const char *kit_name) { @@ -70,7 +71,7 @@ class ProcessParam trx_kit_name_ = kit_name; } } - const std::string &trx_kit_name() const { return trx_kit_name_; } + const string &trx_kit_name() const { return trx_kit_name_; } void set_thread_handling_name(const char *thread_handling_name) { @@ -79,29 +80,29 @@ class ProcessParam } } - const std::string &thread_handling_name() const { return thread_handling_name_; } + const string &thread_handling_name() const { return thread_handling_name_; } void set_buffer_pool_memory_size(int bytes) { buffer_pool_memory_size_ = bytes; } int buffer_pool_memory_size() const { return buffer_pool_memory_size_; } - void set_durability_mode(const char *mode) { durability_mode_ = mode; } - const std::string &durability_mode() const { return durability_mode_; } + void set_durability_mode(const char *mode) { durability_mode_ = mode; } + const string &durability_mode() const { return durability_mode_; } private: - std::string std_out_; // The output file - std::string std_err_; // The err output file - std::string conf; // The configuration file - std::string process_name_; // The process name - bool demon = false; // whether demon or not - std::vector args; // arguments - int server_port_ = -1; // server port(if valid, will overwrite the port in the config file) - std::string unix_socket_path_; - std::string protocol_; - std::string trx_kit_name_; - std::string thread_handling_name_; - int buffer_pool_memory_size_ = -1; - std::string durability_mode_; + string std_out_; // The output file + string std_err_; // The err output file + string conf; // The configuration file + string process_name_; // The process name + bool demon = false; // whether demon or not + vector args; // arguments + int server_port_ = -1; // server port(if valid, will overwrite the port in the config file) + string unix_socket_path_; + string protocol_; + string trx_kit_name_; + string thread_handling_name_; + int buffer_pool_memory_size_ = -1; + string durability_mode_; }; ProcessParam *&the_process_param(); diff --git a/deps/common/os/signal.cpp b/deps/common/os/signal.cpp index 2753dd999..a10bf0abe 100644 --- a/deps/common/os/signal.cpp +++ b/deps/common/os/signal.cpp @@ -15,6 +15,8 @@ See the Mulan PSL v2 for more details. */ #include "common/os/signal.h" #include "common/log/log.h" #include "pthread.h" +#include "common/lang/iostream.h" + namespace common { void set_signal_handler(int sig, sighandler_t func) @@ -26,7 +28,7 @@ void set_signal_handler(int sig, sighandler_t func) int rc = sigaction(sig, &newsa, &oldsa); if (rc) { - std::cerr << "Failed to set signal " << sig << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << std::endl; + cerr << "Failed to set signal " << sig << SYS_OUTPUT_FILE_POS << SYS_OUTPUT_ERROR << endl; } } diff --git a/deps/common/queue/simple_queue.h b/deps/common/queue/simple_queue.h index 7cb976876..f517901cc 100644 --- a/deps/common/queue/simple_queue.h +++ b/deps/common/queue/simple_queue.h @@ -14,9 +14,9 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include #include "common/queue/queue.h" +#include "common/lang/mutex.h" +#include "common/lang/queue.h" namespace common { @@ -46,8 +46,8 @@ class SimpleQueue : public Queue int size() const override; private: - std::mutex mutex_; - std::queue queue_; + mutex mutex_; + queue queue_; }; } // namespace common diff --git a/deps/common/queue/simple_queue.ipp b/deps/common/queue/simple_queue.ipp index 4a5a40b2f..5e72651b0 100644 --- a/deps/common/queue/simple_queue.ipp +++ b/deps/common/queue/simple_queue.ipp @@ -17,7 +17,7 @@ namespace common { template int SimpleQueue::push(T &&value) { - std::lock_guard lock(mutex_); + lock_guard lock(mutex_); queue_.push(std::move(value)); return 0; } @@ -25,7 +25,7 @@ int SimpleQueue::push(T &&value) template int SimpleQueue::pop(T &value) { - std::lock_guard lock(mutex_); + lock_guard lock(mutex_); if (queue_.empty()) { return -1; } diff --git a/deps/common/thread/runnable.h b/deps/common/thread/runnable.h index fe9ae1c22..2bc0fbd2d 100644 --- a/deps/common/thread/runnable.h +++ b/deps/common/thread/runnable.h @@ -14,7 +14,7 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include +#include "common/lang/functional.h" namespace common { @@ -38,12 +38,12 @@ class Runnable class RunnableAdaptor : public Runnable { public: - RunnableAdaptor(std::function callable) : callable_(callable) {} + RunnableAdaptor(function callable) : callable_(callable) {} void run() override { callable_(); } private: - std::function callable_; + function callable_; }; } // namespace common diff --git a/deps/common/thread/thread_pool_executor.h b/deps/common/thread/thread_pool_executor.h index 274081598..a5789f887 100644 --- a/deps/common/thread/thread_pool_executor.h +++ b/deps/common/thread/thread_pool_executor.h @@ -15,15 +15,15 @@ See the Mulan PSL v2 for more details. */ #pragma once #include -#include -#include -#include -#include -#include -#include #include "common/queue/queue.h" #include "common/thread/runnable.h" +#include "common/lang/mutex.h" +#include "common/lang/atomic.h" +#include "common/lang/memory.h" +#include "common/lang/map.h" +#include "common/lang/chrono.h" +#include "common/lang/thread.h" namespace common { @@ -66,7 +66,7 @@ class ThreadPoolExecutor * @param work_queue 任务队列 */ int init(const char *name, int core_pool_size, int max_pool_size, long keep_alive_time_ms, - std::unique_ptr>> &&work_queue); + unique_ptr>> &&work_queue); /** * @brief 提交一个任务,不一定可以立即执行 @@ -74,7 +74,7 @@ class ThreadPoolExecutor * @param task 任务 * @return int 成功放入队列返回0 */ - int execute(std::unique_ptr &&task); + int execute(unique_ptr &&task); /** * @brief 提交一个任务,不一定可以立即执行 @@ -82,7 +82,7 @@ class ThreadPoolExecutor * @param callable 任务 * @return int 成功放入队列返回0 */ - int execute(const std::function &callable); + int execute(const function &callable); /** * @brief 关闭线程池 @@ -158,28 +158,28 @@ class ThreadPoolExecutor struct ThreadData { - bool core_thread = false; /// 是否是核心线程 - bool idle = false; /// 是否空闲 - bool terminated = false; /// 是否已经退出 - std::thread *thread_ptr = nullptr; /// 线程指针 + bool core_thread = false; /// 是否是核心线程 + bool idle = false; /// 是否空闲 + bool terminated = false; /// 是否已经退出 + thread *thread_ptr = nullptr; /// 线程指针 }; private: State state_ = State::NEW; /// 线程池状态 - int core_pool_size_ = 0; /// 核心线程个数 - int max_pool_size_ = 0; /// 最大线程个数 - std::chrono::milliseconds keep_alive_time_ms_; /// 非核心线程空闲多久后退出 + int core_pool_size_ = 0; /// 核心线程个数 + int max_pool_size_ = 0; /// 最大线程个数 + chrono::milliseconds keep_alive_time_ms_; /// 非核心线程空闲多久后退出 - std::unique_ptr>> work_queue_; /// 任务队列 + unique_ptr>> work_queue_; /// 任务队列 - mutable std::mutex lock_; /// 保护线程池内部数据的锁 - std::map threads_; /// 线程列表 + mutable mutex lock_; /// 保护线程池内部数据的锁 + map threads_; /// 线程列表 - int largest_pool_size_ = 0; /// 历史上达到的最大的线程个数 - std::atomic task_count_ = 0; /// 处理过的任务个数 - std::atomic active_count_ = 0; /// 活跃线程个数 - std::string pool_name_; /// 线程池名称 + int largest_pool_size_ = 0; /// 历史上达到的最大的线程个数 + atomic task_count_ = 0; /// 处理过的任务个数 + atomic active_count_ = 0; /// 活跃线程个数 + string pool_name_; /// 线程池名称 }; } // namespace common diff --git a/deps/common/time/datetime.cpp b/deps/common/time/datetime.cpp index 05f4af645..9e1c06620 100644 --- a/deps/common/time/datetime.cpp +++ b/deps/common/time/datetime.cpp @@ -14,15 +14,16 @@ See the Mulan PSL v2 for more details. */ #include "common/time/datetime.h" -#include "pthread.h" -#include "stdio.h" -#include "string.h" -#include -#include -#include +#include +#include +#include + +#include "common/lang/iomanip.h" +#include "common/lang/sstream.h" +#include "common/lang/string.h" namespace common { -DateTime::DateTime(std::string &xml_str) +DateTime::DateTime(string &xml_str) { tm tmp; sscanf(xml_str.c_str(), @@ -37,7 +38,7 @@ DateTime::DateTime(std::string &xml_str) m_time = make_hms(tmp.tm_hour, tmp.tm_min, tmp.tm_sec, 0); } -time_t DateTime::str_to_time_t(std::string &xml_str) +time_t DateTime::str_to_time_t(string &xml_str) { tm tmp; sscanf(xml_str.c_str(), @@ -53,17 +54,17 @@ time_t DateTime::str_to_time_t(std::string &xml_str) return to_time_t(); } -std::string DateTime::time_t_to_str(int timet) +string DateTime::time_t_to_str(int timet) { - std::ostringstream oss; - oss << std::dec << std::setw(10) << timet; + ostringstream oss; + oss << std::dec << setw(10) << timet; return oss.str(); } -std::string DateTime::time_t_to_xml_str(time_t timet) +string DateTime::time_t_to_xml_str(time_t timet) { - std::string ret_val; - std::ostringstream oss; + string ret_val; + ostringstream oss; struct tm tmbuf; tm *tm_info = gmtime_r(&timet, &tmbuf); oss << tm_info->tm_year + 1900 << "-"; @@ -86,10 +87,10 @@ std::string DateTime::time_t_to_xml_str(time_t timet) return ret_val; } -std::string DateTime::str_to_time_t_str(std::string &xml_str) +string DateTime::str_to_time_t_str(string &xml_str) { tm tmp; - std::ostringstream oss; + ostringstream oss; sscanf(xml_str.c_str(), "%04d-%02d-%02dT%02d:%02d:%02dZ", &tmp.tm_year, @@ -101,7 +102,7 @@ std::string DateTime::str_to_time_t_str(std::string &xml_str) m_date = julian_date(tmp.tm_year, tmp.tm_mon, tmp.tm_mday); m_time = make_hms(tmp.tm_hour, tmp.tm_min, tmp.tm_sec, 0); time_t timestamp = to_time_t(); - oss << std::dec << std::setw(10) << timestamp; + oss << std::dec << setw(10) << timestamp; return oss.str(); } @@ -121,12 +122,12 @@ DateTime DateTime::now() } //! Return date and time as a string in Xml Schema date-time format -std::string DateTime::to_xml_date_time() +string DateTime::to_xml_date_time() { - std::string ret_val; + string ret_val; tm tm_info; - std::ostringstream oss; + ostringstream oss; tm_info = to_tm(); oss << tm_info.tm_year + 1900 << "-"; @@ -149,13 +150,13 @@ std::string DateTime::to_xml_date_time() return ret_val; } -time_t DateTime::add_duration(std::string xml_duration) +time_t DateTime::add_duration(string xml_duration) { add_duration_date_time(xml_duration); return to_time_t(); } -void DateTime::add_duration_date_time(std::string xml_duration) +void DateTime::add_duration_date_time(string xml_duration) { // start datetime values int s_year, s_month, s_day; @@ -249,9 +250,9 @@ int DateTime::max_day_in_month_for(int yr, int month) } } -void DateTime::parse_duration(std::string dur_str, struct tm &tm_t) +void DateTime::parse_duration(string dur_str, struct tm &tm_t) { - std::string::size_type index = 0; + string::size_type index = 0; bzero(&tm_t, sizeof(tm_t)); if (dur_str[index] != 'P') { return; @@ -266,7 +267,7 @@ void DateTime::parse_duration(std::string dur_str, struct tm &tm_t) sign = -1; index++; } - std::string sY = dur_str.substr(index, ind_y); + string sY = dur_str.substr(index, ind_y); sscanf(sY.c_str(), "%d", &tm_t.tm_year); tm_t.tm_year *= sign; index = ind_y + 1; @@ -342,7 +343,7 @@ void DateTime::parse_duration(std::string dur_str, struct tm &tm_t) // generate OBJ_ID_TIMESTMP_DIGITS types unique timestamp string // caller doesn't need get any lock #define OBJ_ID_TIMESTMP_DIGITS 14 -std::string Now::unique() +string Now::unique() { struct timeval tv; uint64_t temp; @@ -371,12 +372,12 @@ std::string Now::unique() // further refine below code, the common time unique function // should not cover OBJ_ID_TIMESTMP_DIGITS, which is only // related with the object id. - std::ostringstream oss; - oss << std::hex << std::setw(OBJ_ID_TIMESTMP_DIGITS) << std::setfill('0') << temp; + ostringstream oss; + oss << std::hex << setw(OBJ_ID_TIMESTMP_DIGITS) << setfill('0') << temp; return oss.str(); } -bool DateTime::is_valid_xml_datetime(const std::string &str) +bool DateTime::is_valid_xml_datetime(const string &str) { // check length. 20 is the length of a xml date if (str.length() != 20) diff --git a/deps/common/time/datetime.h b/deps/common/time/datetime.h index 2be7476ae..e4d4bf953 100644 --- a/deps/common/time/datetime.h +++ b/deps/common/time/datetime.h @@ -12,8 +12,7 @@ See the Mulan PSL v2 for more details. */ // Created by Longda on 2010 // -#ifndef __COMMON_TIME_DATETIME_H__ -#define __COMMON_TIME_DATETIME_H__ +#pragma once #include #include @@ -24,6 +23,7 @@ See the Mulan PSL v2 for more details. */ #include #include "common/defs.h" +#include "common/lang/string.h" namespace common { @@ -95,10 +95,10 @@ struct DateTime } // Construct from the xml datetime format - DateTime(std::string &xml_time); + DateTime(string &xml_time); // check whether a string is valid with a xml datetime format - static bool is_valid_xml_datetime(const std::string &str); + static bool is_valid_xml_datetime(const string &str); // Load the referenced values with the year, month and day // portions of the date in a single operation @@ -185,19 +185,19 @@ struct DateTime } // Return date and time as a string in XML Schema Date-Time format - std::string to_xml_date_time(); + string to_xml_date_time(); // Return time_t from XML schema date-time format. - time_t str_to_time_t(std::string &xml_str); + time_t str_to_time_t(string &xml_str); // Return xml time str from time_t. - std::string time_t_to_xml_str(time_t timet); + string time_t_to_xml_str(time_t timet); // Return time_t str from time_t. - std::string time_t_to_str(int timet); + string time_t_to_str(int timet); // Return time_t string from XML schema date-time format. - std::string str_to_time_t_str(std::string &xml_str); + string str_to_time_t_str(string &xml_str); // Helper method to convert a broken down time to a number of // milliseconds since midnight @@ -253,41 +253,41 @@ struct DateTime // Return a human-friendly string representation of the timestamp, // expressed in terms of the local timezone - std::string to_string_local() + string to_string_local() { const time_t tt = to_time_t(); // 'man asctime' specifies that buffer must be at least 26 bytes char buffer[32]; struct tm tm; asctime_r(localtime_r(&tt, &tm), &(buffer[0])); - std::string s(buffer); + string s(buffer); return s; } // Return a human-friendly string representation of the timestamp, // expressed in terms of Coordinated Universal Time (UTC) - std::string to_string_utc() + string to_string_utc() { const time_t tt = to_time_t(); // 'man asctime' specifies that buffer must be at least 26 bytes char buffer[32]; struct tm tm; asctime_r(gmtime_r(&tt, &tm), &(buffer[0])); - std::string s(buffer); + string s(buffer); return s; } // add duration to this time - time_t add_duration(std::string xml_dur); + time_t add_duration(string xml_dur); // add duration to this time - void add_duration_date_time(std::string xml_dur); + void add_duration_date_time(string xml_dur); // add duration to this time int max_day_in_month_for(int year, int month); // parse the duration string and convert it to struct tm - void parse_duration(std::string dur_str, struct tm &tm_t); + void parse_duration(string dur_str, struct tm &tm_t); }; inline bool operator==(const DateTime &lhs, const DateTime &rhs) @@ -428,8 +428,7 @@ class Now return msec; } - static std::string unique(); + static string unique(); }; } // namespace common -#endif //__COMMON_TIME_DATETIME_H__ diff --git a/deps/memtracer/allocator.cpp b/deps/memtracer/allocator.cpp index ba06da297..e83eccdb5 100644 --- a/deps/memtracer/allocator.cpp +++ b/deps/memtracer/allocator.cpp @@ -8,9 +8,11 @@ EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. */ -#include "memtracer/allocator.h" +#include // mmap/munmap #include +#include "memtracer/allocator.h" + // `dlsym` calls `calloc` internally, so here use a dummy buffer // to avoid infinite loop when hook functions initialized. // ref: @@ -157,6 +159,7 @@ mt_visible int posix_memalign(void **memptr, size_t alignment, size_t size) exit(-1); } +#ifdef LINUX mt_visible int brk(void *addr) { MEMTRACER_LOG("brk not supported\n"); @@ -174,6 +177,23 @@ mt_visible long int syscall(long int __sysno, ...) MEMTRACER_LOG("syscall not supported\n"); exit(-1); } +#elif defined(__MACH__) +mt_visible void *brk(const void *addr) +{ + MEMTRACER_LOG("brk not supported\n"); + exit(-1); +} +mt_visible void *sbrk(int increment) +{ + MEMTRACER_LOG("sbrk not supported\n"); + exit(-1); +} +mt_visible int syscall(int __sysno, ...) +{ + MEMTRACER_LOG("syscall not supported\n"); + exit(-1); +} +#endif mt_visible void *operator new(std::size_t size) { return malloc(size); } diff --git a/deps/memtracer/allocator.h b/deps/memtracer/allocator.h index ff772c8bb..33359ac52 100644 --- a/deps/memtracer/allocator.h +++ b/deps/memtracer/allocator.h @@ -9,7 +9,7 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. */ #pragma once -#include // mmap/munmap +//#include // mmap/munmap #include "memtracer/common.h" #include "memtracer/memtracer.h" @@ -48,14 +48,21 @@ mt_visible void operator delete(void *ptr, std::size_t size) noexcept; mt_visible void operator delete[](void *ptr, std::size_t size) noexcept; // unsupported libc functions, for simpler memory tracking. -extern "C" mt_visible char *realpath(const char *fname, char *resolved_name); -extern "C" mt_visible void *memalign(size_t alignment, size_t size); -extern "C" mt_visible void *valloc(size_t size); -extern "C" mt_visible void *pvalloc(size_t size); -extern "C" mt_visible int posix_memalign(void **memptr, size_t alignment, size_t size); +extern "C" mt_visible char *realpath(const char *fname, char *resolved_name); +extern "C" mt_visible void *memalign(size_t alignment, size_t size); +extern "C" mt_visible void *valloc(size_t size); +extern "C" mt_visible void *pvalloc(size_t size); +extern "C" mt_visible int posix_memalign(void **memptr, size_t alignment, size_t size); + +#ifdef LINUX extern "C" mt_visible int brk(void *addr); extern "C" mt_visible void *sbrk(intptr_t increment); extern "C" mt_visible long int syscall(long int __sysno, ...); +#elif defined(__MACH__) +extern "C" mt_visible void *brk(const void *addr); +extern "C" mt_visible void *sbrk(int increment); +extern "C" mt_visible int syscall(int __sysno, ...); +#endif // forword libc interface #if defined(__GLIBC__) && defined(__linux__) diff --git a/deps/memtracer/memtracer.h b/deps/memtracer/memtracer.h index 5aad74169..ba393698e 100644 --- a/deps/memtracer/memtracer.h +++ b/deps/memtracer/memtracer.h @@ -11,10 +11,11 @@ See the Mulan PSL v2 for more details. */ #pragma once #include -#include -#include -#include + #include "memtracer/common.h" +#include "common/lang/thread.h" +#include "common/lang/atomic.h" +#include "common/lang/mutex.h" namespace memtracer { @@ -52,14 +53,14 @@ class MemTracer void set_memory_limit(size_t memory_limit) { - std::call_once(memory_limit_once_, [&]() { memory_limit_ = memory_limit; }); + call_once(memory_limit_once_, [&]() { memory_limit_ = memory_limit; }); } void alloc(size_t size); void free(size_t size); - inline void init_hook_funcs() { std::call_once(init_hook_funcs_once_, init_hook_funcs_impl); } + inline void init_hook_funcs() { call_once(init_hook_funcs_once_, init_hook_funcs_impl); } private: static void init_hook_funcs_impl(); @@ -71,14 +72,14 @@ class MemTracer static void stat(); private: - bool is_stop_ = false; - std::atomic allocated_memory_{}; - std::atomic alloc_cnt_{}; - std::atomic free_cnt_{}; - std::once_flag init_hook_funcs_once_; - std::once_flag memory_limit_once_; - size_t memory_limit_ = UINT64_MAX; - size_t print_interval_ms_ = 0; - std::thread t_; + bool is_stop_ = false; + atomic allocated_memory_{}; + atomic alloc_cnt_{}; + atomic free_cnt_{}; + once_flag init_hook_funcs_once_; + once_flag memory_limit_once_; + size_t memory_limit_ = UINT64_MAX; + size_t print_interval_ms_ = 0; + thread t_; }; } // namespace memtracer \ No newline at end of file diff --git a/docs/docs/design/miniob-aggregation-and-group-by.md b/docs/docs/design/miniob-aggregation-and-group-by.md new file mode 100644 index 000000000..183be1ad8 --- /dev/null +++ b/docs/docs/design/miniob-aggregation-and-group-by.md @@ -0,0 +1,140 @@ +--- +title: MiniOB 向量化执行 aggregation 和 group by 实现 +--- + +# MiniOB 向量化执行 aggregation 和 group by 实现 + +本篇文档介绍 MiniOB 向量化执行中的简单聚合(ungrouped aggregation)和分组聚合(grouped aggregation)实现。 + +## 执行模型(processing model) + +首先,介绍下执行模型的概念,数据库执行模型定义了在数据库中如何执行一个查询计划。这里简单介绍下两种执行模型:火山模型(Volcano iteration model) 和向量化模型(Vectorization model)。 + +``` + Volcano iteration model Vectorization model + + ┌───────────┐ ┌───────────┐ + │ │ │ │ + │ operator1 │ │ operator1 │ + │ │ │ │ + └─┬─────▲───┘ └─┬─────▲───┘ + │ │ │ │ +next()│ │ tuple next_chunk()│ │ chunk/batch + ┌─▼─────┴───┐ ┌─▼─────┴───┐ + │ │ │ │ + │ operator2 │ │ operator2 │ + │ │ │ │ + └─┬─────▲───┘ └─┬─────▲───┘ + │ │ │ │ +next()│ │ tuple next_chunk()│ │ chunk/batch + ┌─▼─────┴───┐ ┌─▼─────┴───┐ + │ │ │ │ + │ operator3 │ │ operator3 │ + │ │ │ │ + └───────────┘ └───────────┘ +``` + +### 火山模型 + +在火山模型中,每个查询计划操作符(operator)都实现了一个 `next()` 函数。在每次调用时,操作符要么返回一个元组(tuple),要么在没有更多元组时返回一个空结果。每个操作符在它的子运算符调用 `next()` 来获取元组,然后对它们进行处理,并向上返回结果。 + +### 向量化执行模型 + +向量化执行模型与火山模型类似,其中每个运算符都有一个生成元组的 `next_chunk()` 方法。然而不同的是,`next_chunk()`方法的每次调用都会获取一组元组而不是仅仅一个元组,这会分摊迭代调用开销。 + +### MiniOB 中的实现 + +在 MiniOB 中已经实现了上述的两种执行模型,可以通过配置项 `execution_mode` 用于区分这两种执行模型。默认使用火山模型,按 `tuple` 迭代。可通过下述 SQL 语句设置执行模型。 + +将执行模型设置为按 `chunk` 迭代的向量化模型。 + +```sql +SET execution_mode = 'chunk_iterator'; +``` + +将执行模型设置为按 `tuple` 迭代的火山模型。 + +```sql +SET execution_mode = 'tuple_iterator'; +``` + +## 向量化执行模型中 aggregation 和 group by 实现 + +### aggregation 实现 + +aggregation 实现主要位于`src/sql/operator/aggregate_vec_physical_operator.cpp` 中,这里需要实现不带分组的聚合,即类似 `select sum(a),sum(b) from t` 的SQL语句。 +在 aggregation 实现中,主要涉及到以下几个步骤: +1. 在 `open()` 函数中,通过调用下层算子的 `next(Chunk &chunk)` 来不断获得下层算子的输出结果(Chunk)。对从下层算子获得的 Chunk 进行表达式计算,聚合计算,并将聚合结果暂存在聚合算子中。 +2. 在 `next(Chunk &chunk)` 函数中,将暂存的聚合结果按 Chunk 格式向上返回。 + +### group by 实现 + +group by 实现主要位于`src/sql/operator/group_by_vec_physical_operator.cpp`中,这里需要实现分组聚合,即类似`select a, sum(b) from t group by a`的SQL语句。 +group by 实现采用了基于哈希的分组方式,主要涉及到以下几个步骤: +1. 在 `open()` 函数中,通过调用下层算子的`next(Chunk &chunk)` 来不断获得下层算子的输出结果(Chunk)。对从下层算子获得的 Chunk 进行表达式计算,根据分组列计算出分组位置,并将聚合结果暂存在相应的分组位置中。 +2. 在 `next(Chunk &chunk)` 函数中,将暂存在哈希表中的聚合结果按 Chunk 格式向上返回。 + +### 实验 + +1. 需要补充 `src/observer/sql/parser/yacc_sql.y` 中 标注`// your code here` 位置的 `aggregation` 和 `group by` 相关的语法。并通过 `src/sql/parser/gen_parser.sh` 生成正确的语法分析代码。 +2. 需要实现 `src/observer/sql/operator/aggregate_vec_physical_operator.cpp` 中标注 `// your code here` 位置的代码。 +3. 需要完整实现位于 `src/observer/sql/operator/group_by_vec_physical_operator.h` 的group by 算子。 +4. 需要实现 `src/observer/sql/expr/aggregate_hash_table.cpp::StandardAggregateHashTable` 中标注 `// your code here` 位置的代码。 + +### 测试 + +需通过 `unittest/observer/parser_test.cpp` 和 `test/case/test/vectorized-aggregation-and-group-by.test`。 + +注意1:训练营中的测试采用对MiniOB/MySQL 输入相同的测试 SQL(MiniOB 中建表语句包含 storage format 选项,MySQL 中不包含),对比 MiniOB 执行结果与 MySQL 执行结果的方式进行。训练营中的测试 case 相比 `test/case/test/vectorized-aggregation-and-group-by.test` 更加多样(包含一些随机生成的 SQL),但不存在更加复杂的 SQL(不需要实现除 `sum` 之外的聚合函数)。 + +注意2:如果需要运行 `parser_test` 请移除`DISABLED_` 前缀。 + +## 实现 SIMD 指令优化的 aggregation 和 group by + +### SIMD 介绍 + +如下图所示,Single Instruction Multiple Data(SIMD),单指令多数据流,可以使用一条指令同时完成多个数据的运算操作。传统的指令架构是SISD就是单指令单数据流,每条指令只能对一个数据执行操作。因此,通过合理使用 SIMD 指令,可以减少运算执行过程中的 CPU 指令数,从而优化计算开销。[参考资料](#参考资料)中包含了 SIMD 介绍文章及指令手册。 + +``` + SISD SIMD +┌────┐ ┌────┐ ┌────┐ ┌────┐ ┌────┐ ┌────┐ +│ a0 │ + │ b0 │ = │ c0 │ │ a0 │ + │ b0 │ = │ c0 │ +└────┘ └────┘ └────┘ │ │ │ │ │ │ + │ │ │ │ │ │ +┌────┐ ┌────┐ ┌────┐ │ │ │ │ │ │ +│ a1 │ + │ b1 │ = │ c1 │ │ a1 │ + │ b1 │ = │ c1 │ +└────┘ └────┘ └────┘ │ │ │ │ │ │ + │ │ │ │ │ │ +┌────┐ ┌────┐ ┌────┐ │ │ │ │ │ │ +│ a2 │ + │ b2 │ = │ c2 │ │ a2 │ + │ b2 │ = │ c2 │ +└────┘ └────┘ └────┘ │ │ │ │ │ │ + │ │ │ │ │ │ +┌────┐ ┌────┐ ┌────┐ │ │ │ │ │ │ +│ a3 │ + │ b3 │ = │ c3 │ │ a3 │ + │ b3 │ = │ c3 │ +└────┘ └────┘ └────┘ └────┘ └────┘ └────┘ +``` + +### SIMD 指令在 MiniOB 中的应用 + +通过 SIMD 指令,我们可以优化 MiniOB 向量化执行引擎中的部分批量运算操作,如表达式计算,聚合计算,hash group by等。在 `src/observer/sql/expr/arithmetic_operator.hpp` 中需要实现基于 SIMD 指令的算术运算;在`deps/common/math/simd_util.cpp` 中需要实现基于 SIMD 指令的数组求和函数 `mm256_sum_epi32` 和`_mm256_storeu_si256`; 使用 SIMD 指令优化 hash group by 的关键在于实按批操作的哈希表,MiniOB 的实现参考了论文:`Rethinking SIMD Vectorization for In-Memory Databases` 中的线性探测哈希表(Algorithm 5),更多细节可以参考`src/observer/sql/expr/aggregate_hash_table.cpp`中的注释。 + +注意:在编译时,只有使用 `-DUSE_SIMD=ON` 时(默认关闭),才会编译 SIMD 指令相关代码。 + +### 实验 + +1. 需要使用 SIMD 指令实现 `src/observer/sql/expr/arithmetic_operator.hpp` 中标注 `// your code here` 位置的代码。 +2. 需要使用 SIMD 指令实现 `deps/common/math/simd_util.cpp` 中标注 `// your code here` 位置的代码。 +3. 需要使用 SIMD 指令实现 `src/observer/sql/expr/aggregate_hash_table.cpp::LinearProbingAggregateHashTable` 中标注 `// your code here` 位置的代码。 + +### 测试 + +需要通过 `arithmetic_operator_test` 和 `aggregate_hash_table_test` 单元测试以及 `arithmetic_operator_performance_test` 和 `aggregate_hash_table_performance_test` 性能测试。 + +注意:如果需要运行 `arithmetic_operator_test`、`aggregate_hash_table_test`、`arithmetic_operator_performance_test`、`aggregate_hash_table_performance_test` ,请移除`DISABLED_` 前缀。 + +### 参考资料 + +1. [Rethinking SIMD Vectorization for In-Memory Databases](https://15721.courses.cs.cmu.edu/spring2016/papers/p1493-polychroniou.pdf) +2. [vectorized hash table](https://15721.courses.cs.cmu.edu/spring2024/slides/06-vectorization.pdf) +3. [SIMD 入门文章](https://zhuanlan.zhihu.com/p/94649418) +4. [SIMD 指令手册](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html) \ No newline at end of file diff --git a/docs/docs/design/miniob-buffer-pool.md b/docs/docs/design/miniob-buffer-pool.md index e6ce7bdbd..72fd9e8ab 100644 --- a/docs/docs/design/miniob-buffer-pool.md +++ b/docs/docs/design/miniob-buffer-pool.md @@ -82,4 +82,4 @@ MiniOB 的 Record Manager 做了简化,有一些假设,记录通常都是比 上面的图片展示了 MiniOB 的 Record Manager 是怎么实现的,以及 Record 在文件中是如何组织的。 -Record Manage 是在 Buffer Pool 的基础上实现的,比如 page0 是 Buffer Pool 里面使用的元数据,Record Manage 利用了其他的一些页面。每个页面有一个头信息 Page Header,一个 Bitmap,Bitmap 为 0 表示最近的记录是不是已经有有效数据;1 表示有有效数据。Page Header 中记录了当前页面一共有多少记录、最多可以容纳多少记录、每个记录的实际长度与对齐后的长度等信息。 +Record Manager 是在 Buffer Pool 的基础上实现的,比如 page0 是 Buffer Pool 里面使用的元数据,Record Manager 利用了其他的一些页面。每个页面有一个头信息 Page Header,一个 Bitmap,Bitmap 为 0 表示最近的记录是不是已经有有效数据;1 表示有有效数据。Page Header 中记录了当前页面一共有多少记录、最多可以容纳多少记录、每个记录的实际长度与对齐后的长度等信息。 diff --git a/docs/docs/design/miniob-pax-storage.md b/docs/docs/design/miniob-pax-storage.md new file mode 100644 index 000000000..6eb9cad94 --- /dev/null +++ b/docs/docs/design/miniob-pax-storage.md @@ -0,0 +1,130 @@ +--- +title: PAX 存储格式 +--- + +# MiniOB PAX 存储格式 + +本篇文档介绍 MiniOB 中 PAX 存储格式。 + +## 存储模型(Storage Models) + +数据库的存储模型规定了它如何在磁盘和内存中组织数据。首先,我们来介绍下三种经典的存储模型。 + +### N-ARY Storage Model (NSM) + +在 NSM 存储模型中,一行记录的所有属性连续存储在数据库页面(Page)中,这也被称为行式存储。NSM 适合 OLTP 工作负载。因为在 OLTP 负载中,查询更有可能访问整个记录(对整个记录进行增删改查)。 + +``` + Col1 Col2 Col3 Page + ┌─────────────┬─┐ ┌──────────┬──────────┐ +Row1│ a1 b1 c1│ │ │PageHeader│ a1 b1 c1 │ + ├─────────────┤ │ ├────────┬─┴──────┬───┤ +Row2│ a2 b2 c2│ │ │a2 b2 c2│a3 b3 c3│.. │ + ├─────────────┤ │ ├────────┴────────┴───┤ +Row3│ a3 b3 c3│ │ │... │ + ├─────────────┘ │ ├─────────────────────┤ +... │ .. .. .. │ │... │ +... │ .. .. .. │ └─────────────────────┘ + │ │ +RowN│ an bn cn │ + └───────────────┘ +``` + +### Decomposition Storage Model (DSM) + +在 DSM 存储模型中,所有记录的单个属性被连续存储在数据块/文件中。这也被称为列式存储。DSM 适合 OLAP 工作负载,因为 OLAP 负载中往往会对表属性的一个子集执行扫描和计算。 + +``` + File/Block + Col1 Col2 Col3 ┌─────────────────────┐ + ┌────┬────┬────┬┐ │ Header │ +Row1│ a1 │ b1 │ c1 ││ ├─────────────────────┤ + │ │ │ ││ │a1 a2 a3 ......... an│ +Row2│ a2 │ b2 │ c2 ││ └─────────────────────┘ + │ │ │ ││ +Row3│ a3 │ b3 │ c3 ││ ┌─────────────────────┐ + │ │ │ ││ │ Header │ + │ . │....│. ││ ├─────────────────────┤ +... │ │ │ ││ │b1 b2 b3 ......... bn│ +... │ .. │ .. │ .. ││ └─────────────────────┘ +RowN│ an │ bn │ cn ││ + └────┴────┴────┴┘ ┌─────────────────────┐ + │ Header │ + ├─────────────────────┤ + │c1 c2 c3 ......... cn│ + └─────────────────────┘ +``` + +### Partition Attributes Across (PAX) + +PAX (Partition Attributes Across) 是一种混合存储格式,它在数据库页面(Page)内对属性进行垂直分区。 + +``` + Col1 Col2 Col3 Page + ┌─────────────┬─┐ ┌──────────┬──────────┐ +Row1│ a1 b1 c1│ │ │PageHeader│ a1 a2 a3 │ + │ │ │ ├──────────┼──────────┤ +Row2│ a2 b2 c2│ │ │b1 b2 b3 │ c1 c2 c3 │ + │ │ │ └──────────┴──────────┘ +Row3│ a3 b3 c3│ │ .... + ├─────────────┘ │ ┌──────────┬──────────┐ + │ ......... │ │PageHeader│ ..... an │ +... ├─────────────┐ │ ├──────────┼──────────┤ +... │ .. .. ..│ │ │...... bn │ ..... cn │ +RowN│ an bn cn│ │ └──────────┴──────────┘ + └─────────────┴─┘ +``` + +## MiniOB 中 PAX 存储格式 + +### 实现 + +在 MiniOB 中,RecordManager 负责一个文件中表记录(Record)的组织/管理。在没有实现 PAX 存储格式之前,MiniOB 只支持行存格式,每个记录连续存储在页面(Page)中,通过`RowRecordPageHandler` 对单个页面中的记录进行管理。需要通过实现 `PaxRecordPageHandler` 来支持页面内 PAX 存储格式的管理。 +Page 内的 PAX 存储格式如下: +``` +| PageHeader | record allocate bitmap | column index | +|------------|------------------------| ------------- | +| column1 | column2 | ..................... | columnN | +``` +其中 `PageHeader` 与 `bitmap` 和行式存储中的作用一致,`column index` 用于定位列数据在页面内的偏移量,每列数据连续存储。 + +`column index` 结构如下,为一个连续的数组。假设某个页面共有 `n + 1` 列,分别为`col_0, col_1, ..., col_n`,`col_i` 表示列 ID(column id)为 `i + 1`的列在页面内的起始地址(`i < n`)。当 `i = n`时,`col_n` 表示列 ID 为 `n` 的列在页面内的结束地址 + 1。 +``` +| col_0 | col_1 | col_2 | ...... | col_n | +|-------|-------|-------|---------|-------| +``` + + +MiniOB 支持了创建 PAX 表的语法。当不指定存储格式时,默认创建行存格式的表。 +``` +CREATE TABLE table_name + (table_definition_list) [storage_format_option] + +storage_format_option: + storage format=row + | storage format=pax +``` +示例: + +创建行存格式的表: + +```sql +create table t(a int,b int) storage format=row; +create table t(a int,b int); +``` + +创建列存格式的表: +```sql +create table t(a int,b int) storage format=pax; +``` + +### 实验 + +实现 PAX 存储格式,需要完成 `src/observer/storage/record/record_manager.cpp` 中 `PaxRecordPageHandler::insert_record`, `PaxRecordPageHandler::get_chunk`, `PaxRecordPageHandler::get_record` 三个函数(标注 `// your code here` 的位置),详情可参考这三个函数的注释。行存格式存储是已经在MiniOB 中完整实现的,实现 PAX 存储格式的过程中可以参考 `RowRecordPageHandler`。 + +### 测试 + +通过 `unittest/pax_storage_test.cpp` 中所有测试用例,通过`benchmark/pax_storage_concurrency_test.cpp` 性能测试。 + +注意:如果需要运行 `pax_storage_test` 、`pax_storage_concurrency_test`,请移除`DISABLED_` 前缀。 + diff --git a/docs/docs/design/miniob-sql-parser.md b/docs/docs/design/miniob-sql-parser.md index 59efb8797..b67936d90 100644 --- a/docs/docs/design/miniob-sql-parser.md +++ b/docs/docs/design/miniob-sql-parser.md @@ -121,7 +121,7 @@ create_table_stmt: /*create table 语句的语法解析树*/ 将会生成词法分析代码 lex_sql.h 和 lex_sql.cpp,语法分析代码 yacc_sql.hpp 和 yacc_sql.cpp。 -注意:flex 使用 2.5.35 版本测试通过,bison使用**3.7**版本测试通过(请不要使用旧版本,比如macos自带的bision)。 +注意:flex 使用 2.5.35 版本测试通过,bison使用**3.7**版本测试通过(请不要使用旧版本,比如macos自带的bision)。 注意:当前没有把lex_sql.l和yacc_sql.y加入CMakefile.txt中,所以修改这两个文件后,需要手动生成c代码,然后再执行编译。 diff --git a/docs/docs/how_to_build.md b/docs/docs/how_to_build.md index acddb4ffc..18750178b 100644 --- a/docs/docs/how_to_build.md +++ b/docs/docs/how_to_build.md @@ -11,7 +11,7 @@ title: 如何编译 MiniOB 需要使用: - cmake 版本 >= 3.13 -- gcc/clang gcc建议8.3以上,编译器需要支持c++20新标准 +- gcc/clang gcc 11 以上,clang 14以上,编译器需要支持c++20新标准 - flex (2.5+), bison (3.7+) 用于生成词法语法分析代码 ## 1. 环境初始化 diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 6fa8c8234..ac2bbd5f9 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -34,6 +34,8 @@ nav: - design/miniob-bplus-tree-concurrency.md - design/miniob-thread-model.md - design/miniob-mysql-protocol.md + - design/miniob-pax-storage.md + - design/miniob-aggregation-and-group-by.md - Doxy 代码文档: design/doxy/html/index.html - OceanBase 数据库大赛: - game/introduction.md diff --git a/src/obclient/client.cpp b/src/obclient/client.cpp index 3a1b33d0a..66c2f7e43 100644 --- a/src/obclient/client.cpp +++ b/src/obclient/client.cpp @@ -38,11 +38,12 @@ See the Mulan PSL v2 for more details. */ #define MAX_MEM_BUFFER_SIZE 8192 #define PORT_DEFAULT 6789 +using namespace std; using namespace common; #ifdef USE_READLINE -const std::string HISTORY_FILE = std::string(getenv("HOME")) + "/.miniob.history"; -time_t last_history_write_time = 0; +const string HISTORY_FILE = string(getenv("HOME")) + "/.miniob.history"; +time_t last_history_write_time = 0; char *my_readline(const char *prompt) { diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index c62ac3c36..1e8e5e927 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -3,16 +3,15 @@ MESSAGE(STATUS "This is CMAKE_CURRENT_SOURCE_DIR dir " ${CMAKE_CURRENT_SOURCE_DI INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) FILE(GLOB_RECURSE ALL_SRC *.cpp *.c) -SET(MAIN_SRC main.cpp) +SET(MAIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) MESSAGE("MAIN SRC: " ${MAIN_SRC}) FOREACH (F ${ALL_SRC}) IF (NOT ${F} STREQUAL ${MAIN_SRC}) SET(LIB_SRC ${LIB_SRC} ${F}) + MESSAGE("Use " ${F}) ENDIF() - MESSAGE("Use " ${F}) - ENDFOREACH (F) SET(LIBEVENT_STATIC_LINK TRUE) diff --git a/src/observer/common/init.cpp b/src/observer/common/init.cpp index 36283fa9a..94d0df965 100644 --- a/src/observer/common/init.cpp +++ b/src/observer/common/init.cpp @@ -16,6 +16,7 @@ See the Mulan PSL v2 for more details. */ #include "common/conf/ini.h" #include "common/lang/string.h" +#include "common/lang/iostream.h" #include "common/log/log.h" #include "common/os/path.h" #include "common/os/pidfile.h" @@ -29,7 +30,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/default/default_handler.h" #include "storage/trx/trx.h" -using namespace std; using namespace common; bool *&_get_init() diff --git a/src/observer/common/rc.cpp b/src/observer/common/rc.cpp index 313a92842..9c2ad88e8 100644 --- a/src/observer/common/rc.cpp +++ b/src/observer/common/rc.cpp @@ -24,7 +24,7 @@ const char *strrc(RC rc) switch (rc) { DEFINE_RCS; default: { - return "unkown"; + return "unknown"; } } #undef DEFINE_RC diff --git a/src/observer/common/types.h b/src/observer/common/types.h index fcd4749e5..88a9f7fc1 100644 --- a/src/observer/common/types.h +++ b/src/observer/common/types.h @@ -40,5 +40,27 @@ enum class ReadWriteMode READ_WRITE }; +/** + * @brief 存储格式 + * @details 当前仅支持行存格式(ROW_FORMAT)以及 PAX 存储格式(PAX_FORMAT)。 + */ +enum class StorageFormat +{ + UNKNOWN_FORMAT = 0, + ROW_FORMAT, + PAX_FORMAT +}; + +/** + * @brief 执行引擎模式 + * @details 当前支持按行处理(TUPLE_ITERATOR)以及按批处理(CHUNK_ITERATOR)两种模式。 + */ +enum class ExecutionMode +{ + UNKNOWN_MODE = 0, + TUPLE_ITERATOR, + CHUNK_ITERATOR +}; + /// page的CRC校验和 using CheckSum = unsigned int; diff --git a/src/observer/event/session_event.h b/src/observer/event/session_event.h index 323a37b54..7a279ca36 100644 --- a/src/observer/event/session_event.h +++ b/src/observer/event/session_event.h @@ -14,8 +14,7 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include - +#include "common/lang/string.h" #include "event/sql_debug.h" #include "sql/executor/sql_result.h" @@ -35,15 +34,15 @@ class SessionEvent Communicator *get_communicator() const; Session *session() const; - void set_query(const std::string &query) { query_ = query; } + void set_query(const string &query) { query_ = query; } - const std::string &query() const { return query_; } - SqlResult *sql_result() { return &sql_result_; } - SqlDebug &sql_debug() { return sql_debug_; } + const string &query() const { return query_; } + SqlResult *sql_result() { return &sql_result_; } + SqlDebug &sql_debug() { return sql_debug_; } private: Communicator *communicator_ = nullptr; ///< 与客户端通讯的对象 SqlResult sql_result_; ///< SQL执行结果 SqlDebug sql_debug_; ///< SQL调试信息 - std::string query_; ///< SQL语句 + string query_; ///< SQL语句 }; diff --git a/src/observer/event/sql_debug.cpp b/src/observer/event/sql_debug.cpp index 4f34951cd..ab68dc17e 100644 --- a/src/observer/event/sql_debug.cpp +++ b/src/observer/event/sql_debug.cpp @@ -18,7 +18,6 @@ See the Mulan PSL v2 for more details. */ #include "event/sql_debug.h" #include "session/session.h" -using namespace std; void SqlDebug::add_debug_info(const string &debug_info) { debug_infos_.push_back(debug_info); } diff --git a/src/observer/event/sql_debug.h b/src/observer/event/sql_debug.h index 3f3f5c132..66d182466 100644 --- a/src/observer/event/sql_debug.h +++ b/src/observer/event/sql_debug.h @@ -14,8 +14,8 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include +#include "common/lang/string.h" +#include "common/lang/list.h" /** * @brief SQL调试信息 @@ -30,13 +30,13 @@ class SqlDebug SqlDebug() = default; virtual ~SqlDebug() = default; - void add_debug_info(const std::string &debug_info); + void add_debug_info(const string &debug_info); void clear_debug_info(); - const std::list &get_debug_infos() const; + const list &get_debug_infos() const; private: - std::list debug_infos_; + list debug_infos_; }; /** diff --git a/src/observer/event/sql_event.cpp b/src/observer/event/sql_event.cpp index a6e98e53c..6df5fc255 100644 --- a/src/observer/event/sql_event.cpp +++ b/src/observer/event/sql_event.cpp @@ -17,7 +17,7 @@ See the Mulan PSL v2 for more details. */ #include "event/session_event.h" #include "sql/stmt/stmt.h" -SQLStageEvent::SQLStageEvent(SessionEvent *event, const std::string &sql) : session_event_(event), sql_(sql) {} +SQLStageEvent::SQLStageEvent(SessionEvent *event, const string &sql) : session_event_(event), sql_(sql) {} SQLStageEvent::~SQLStageEvent() noexcept { diff --git a/src/observer/event/sql_event.h b/src/observer/event/sql_event.h index be3009f66..cd7db0659 100644 --- a/src/observer/event/sql_event.h +++ b/src/observer/event/sql_event.h @@ -14,9 +14,9 @@ See the Mulan PSL v2 for more details. */ #pragma once +#include "common/lang/string.h" +#include "common/lang/memory.h" #include "sql/operator/physical_operator.h" -#include -#include class SessionEvent; class Stmt; @@ -28,26 +28,26 @@ class ParsedSqlNode; class SQLStageEvent { public: - SQLStageEvent(SessionEvent *event, const std::string &sql); + SQLStageEvent(SessionEvent *event, const string &sql); virtual ~SQLStageEvent() noexcept; SessionEvent *session_event() const { return session_event_; } - const std::string &sql() const { return sql_; } - const std::unique_ptr &sql_node() const { return sql_node_; } - Stmt *stmt() const { return stmt_; } - std::unique_ptr &physical_operator() { return operator_; } - const std::unique_ptr &physical_operator() const { return operator_; } + const string &sql() const { return sql_; } + const unique_ptr &sql_node() const { return sql_node_; } + Stmt *stmt() const { return stmt_; } + unique_ptr &physical_operator() { return operator_; } + const unique_ptr &physical_operator() const { return operator_; } void set_sql(const char *sql) { sql_ = sql; } - void set_sql_node(std::unique_ptr sql_node) { sql_node_ = std::move(sql_node); } + void set_sql_node(unique_ptr sql_node) { sql_node_ = std::move(sql_node); } void set_stmt(Stmt *stmt) { stmt_ = stmt; } - void set_operator(std::unique_ptr oper) { operator_ = std::move(oper); } + void set_operator(unique_ptr oper) { operator_ = std::move(oper); } private: - SessionEvent *session_event_ = nullptr; - std::string sql_; ///< 处理的SQL语句 - std::unique_ptr sql_node_; ///< 语法解析后的SQL命令 - Stmt *stmt_ = nullptr; ///< Resolver之后生成的数据结构 - std::unique_ptr operator_; ///< 生成的执行计划,也可能没有 + SessionEvent *session_event_ = nullptr; + string sql_; ///< 处理的SQL语句 + unique_ptr sql_node_; ///< 语法解析后的SQL命令 + Stmt *stmt_ = nullptr; ///< Resolver之后生成的数据结构 + unique_ptr operator_; ///< 生成的执行计划,也可能没有 }; diff --git a/src/observer/main.cpp b/src/observer/main.cpp index 1f385fa3a..94f48d4c3 100644 --- a/src/observer/main.cpp +++ b/src/observer/main.cpp @@ -15,20 +15,20 @@ See the Mulan PSL v2 for more details. */ * Author: Longda Feng */ -#include #include #include #include "common/ini_setting.h" #include "common/init.h" +#include "common/lang/iostream.h" #include "common/lang/string.h" +#include "common/lang/map.h" #include "common/os/process.h" #include "common/os/signal.h" #include "common/log/log.h" #include "net/server.h" #include "net/server_param.h" -using namespace std; using namespace common; #define NET "NET" @@ -37,7 +37,7 @@ static Server *g_server = nullptr; void usage() { - cout << "Useage " << endl; + cout << "Usage " << endl; cout << "-p: server port. if not specified, the item in the config file will be used" << endl; cout << "-f: path of config file." << endl; cout << "-s: use unix socket and the argument is socket address" << endl; @@ -82,7 +82,7 @@ void parse_parameter(int argc, char **argv) Server *init_server() { - std::map net_section = get_properties()->get(NET); + map net_section = get_properties()->get(NET); ProcessParam *process_param = the_process_param(); @@ -90,15 +90,15 @@ Server *init_server() long max_connection_num = MAX_CONNECTION_NUM_DEFAULT; int port = PORT_DEFAULT; - std::map::iterator it = net_section.find(CLIENT_ADDRESS); + map::iterator it = net_section.find(CLIENT_ADDRESS); if (it != net_section.end()) { - std::string str = it->second; + string str = it->second; str_to_val(str, listen_addr); } it = net_section.find(MAX_CONNECTION_NUM); if (it != net_section.end()) { - std::string str = it->second; + string str = it->second; str_to_val(str, max_connection_num); } @@ -108,7 +108,7 @@ Server *init_server() } else { it = net_section.find(PORT); if (it != net_section.end()) { - std::string str = it->second; + string str = it->second; str_to_val(str, port); } } @@ -189,7 +189,7 @@ int main(int argc, char **argv) rc = init(the_process_param()); if (rc != STATUS_SUCCESS) { - std::cerr << "Shutdown due to failed to init!" << endl; + cerr << "Shutdown due to failed to init!" << endl; cleanup(); return rc; } diff --git a/src/observer/net/buffered_writer.cpp b/src/observer/net/buffered_writer.cpp index be9ee3379..de33799a8 100644 --- a/src/observer/net/buffered_writer.cpp +++ b/src/observer/net/buffered_writer.cpp @@ -18,8 +18,6 @@ See the Mulan PSL v2 for more details. */ #include "net/buffered_writer.h" -using namespace std; - BufferedWriter::BufferedWriter(int fd) : fd_(fd), buffer_() {} BufferedWriter::BufferedWriter(int fd, int32_t size) : fd_(fd), buffer_(size) {} diff --git a/src/observer/net/cli_communicator.cpp b/src/observer/net/cli_communicator.cpp index 1e962ed30..396dc280c 100644 --- a/src/observer/net/cli_communicator.cpp +++ b/src/observer/net/cli_communicator.cpp @@ -31,7 +31,6 @@ See the Mulan PSL v2 for more details. */ #define MAX_MEM_BUFFER_SIZE 8192 #define PORT_DEFAULT 6789 -using namespace std; using namespace common; #ifdef USE_READLINE diff --git a/src/observer/net/cli_communicator.h b/src/observer/net/cli_communicator.h index 995d36c16..68c5a2eaa 100644 --- a/src/observer/net/cli_communicator.h +++ b/src/observer/net/cli_communicator.h @@ -28,7 +28,7 @@ class CliCommunicator : public PlainCommunicator CliCommunicator() = default; virtual ~CliCommunicator() = default; - RC init(int fd, std::unique_ptr session, const std::string &addr) override; + RC init(int fd, unique_ptr session, const string &addr) override; RC read_event(SessionEvent *&event) override; RC write_result(SessionEvent *event, bool &need_disconnect) override; diff --git a/src/observer/net/communicator.cpp b/src/observer/net/communicator.cpp index 8b81c179d..a190187a1 100644 --- a/src/observer/net/communicator.cpp +++ b/src/observer/net/communicator.cpp @@ -21,8 +21,6 @@ See the Mulan PSL v2 for more details. */ #include "common/lang/mutex.h" -using namespace std; - RC Communicator::init(int fd, unique_ptr session, const std::string &addr) { fd_ = fd; diff --git a/src/observer/net/communicator.h b/src/observer/net/communicator.h index 437919491..f88665e9d 100644 --- a/src/observer/net/communicator.h +++ b/src/observer/net/communicator.h @@ -14,10 +14,9 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include - #include "common/rc.h" +#include "common/lang/string.h" +#include "common/lang/memory.h" struct ConnectionContext; class SessionEvent; @@ -47,7 +46,7 @@ class Communicator /** * @brief 接收到一个新的连接时,进行初始化 */ - virtual RC init(int fd, std::unique_ptr session, const std::string &addr); + virtual RC init(int fd, unique_ptr session, const string &addr); /** * @brief 监听到有新的数据到达,调用此函数进行接收消息 @@ -81,10 +80,10 @@ class Communicator int fd() const { return fd_; } protected: - std::unique_ptr session_; - std::string addr_; - BufferedWriter *writer_ = nullptr; - int fd_ = -1; + unique_ptr session_; + string addr_; + BufferedWriter *writer_ = nullptr; + int fd_ = -1; }; /** diff --git a/src/observer/net/java_thread_pool_thread_handler.cpp b/src/observer/net/java_thread_pool_thread_handler.cpp index 9dd1957e8..476fa916e 100644 --- a/src/observer/net/java_thread_pool_thread_handler.cpp +++ b/src/observer/net/java_thread_pool_thread_handler.cpp @@ -21,7 +21,6 @@ See the Mulan PSL v2 for more details. */ #include "common/thread/runnable.h" #include "common/queue/simple_queue.h" -using namespace std; using namespace common; /** diff --git a/src/observer/net/java_thread_pool_thread_handler.h b/src/observer/net/java_thread_pool_thread_handler.h index 6a14f550b..98f68dbb9 100644 --- a/src/observer/net/java_thread_pool_thread_handler.h +++ b/src/observer/net/java_thread_pool_thread_handler.h @@ -14,11 +14,10 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include - #include "net/thread_handler.h" #include "net/sql_task_handler.h" #include "common/thread/thread_pool_executor.h" +#include "common/lang/mutex.h" struct EventCallbackAg; @@ -62,10 +61,10 @@ class JavaThreadPoolThreadHandler : public ThreadHandler void event_loop_thread(); private: - std::mutex lock_; - struct event_base *event_base_ = nullptr; /// libevent 的event_base - common::ThreadPoolExecutor executor_; /// 线程池 - std::map event_map_; /// 每个连接与它关联的数据 + mutex lock_; + struct event_base *event_base_ = nullptr; /// libevent 的event_base + common::ThreadPoolExecutor executor_; /// 线程池 + map event_map_; /// 每个连接与它关联的数据 SqlTaskHandler sql_task_handler_; /// SQL请求处理器 }; \ No newline at end of file diff --git a/src/observer/net/mysql_communicator.cpp b/src/observer/net/mysql_communicator.cpp index 8abb7b4f1..ce51a3c31 100644 --- a/src/observer/net/mysql_communicator.cpp +++ b/src/observer/net/mysql_communicator.cpp @@ -23,8 +23,6 @@ See the Mulan PSL v2 for more details. */ #include "net/mysql_communicator.h" #include "sql/operator/string_list_physical_operator.h" -using namespace std; - /** * @brief MySQL协议相关实现 * @defgroup MySQLProtocol @@ -313,7 +311,7 @@ class BasePacket * @param[in] capabilities MySQL协议中的capability标志 * @param[out] net_packet 编码后的网络包 */ - virtual RC encode(uint32_t capabilities, std::vector &net_packet) const = 0; + virtual RC encode(uint32_t capabilities, vector &net_packet) const = 0; }; /** @@ -345,7 +343,7 @@ struct HandshakeV10 : public BasePacket /** * https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_v10.html */ - virtual RC encode(uint32_t capabilities, std::vector &net_packet) const override + virtual RC encode(uint32_t capabilities, vector &net_packet) const override { net_packet.resize(100); @@ -387,7 +385,7 @@ struct OkPacket : public BasePacket int32_t last_insert_id = 0; int16_t status_flags = 0x22; int16_t warnings = 0; - std::string info; // human readable status information + string info; // human readable status information OkPacket(int8_t sequence = 0) : BasePacket(sequence) {} virtual ~OkPacket() = default; @@ -395,7 +393,7 @@ struct OkPacket : public BasePacket /** * https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_ok_packet.html */ - RC encode(uint32_t capabilities, std::vector &net_packet) const override + RC encode(uint32_t capabilities, vector &net_packet) const override { net_packet.resize(100); char *buf = net_packet.data(); @@ -442,7 +440,7 @@ struct EofPacket : public BasePacket EofPacket(int8_t sequence = 0) : BasePacket(sequence) {} virtual ~EofPacket() = default; - RC encode(uint32_t capabilities, std::vector &net_packet) const override + RC encode(uint32_t capabilities, vector &net_packet) const override { net_packet.resize(10); char *buf = net_packet.data(); @@ -479,13 +477,13 @@ struct ErrPacket : public BasePacket int8_t header = 0xFF; int16_t error_code = 0; char sql_state_marker[1] = {'#'}; - std::string sql_state{"HY000"}; - std::string error_message; + string sql_state{"HY000"}; + string error_message; ErrPacket(int8_t sequence = 0) : BasePacket(sequence) {} virtual ~ErrPacket() = default; - virtual RC encode(uint32_t capabilities, std::vector &net_packet) const override + virtual RC encode(uint32_t capabilities, vector &net_packet) const override { net_packet.resize(1000); char *buf = net_packet.data(); @@ -524,7 +522,7 @@ struct QueryPacket { PacketHeader packet_header; int8_t command; // 0x03: COM_QUERY - std::string query; // the text of the SQL query to execute + string query; // the text of the SQL query to execute }; /** @@ -532,7 +530,7 @@ struct QueryPacket * @details packet_header is not included in net_packet * [MySQL Protocol COM_QUERY](https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query.html) */ -RC decode_query_packet(std::vector &net_packet, QueryPacket &query_packet) +RC decode_query_packet(vector &net_packet, QueryPacket &query_packet) { // query field is a null terminated string query_packet.query.assign(net_packet.data() + 1, net_packet.size() - 1); @@ -558,7 +556,7 @@ RC create_version_comment_sql_result(SqlResult *sql_result) StringListPhysicalOperator *oper = new StringListPhysicalOperator(); oper->append(version_comments); - sql_result->set_operator(std::unique_ptr(oper)); + sql_result->set_operator(unique_ptr(oper)); return RC::SUCCESS; } @@ -569,7 +567,7 @@ RC create_version_comment_sql_result(SqlResult *sql_result) * @param session 当前的会话 * @param addr 对端地址 */ -RC MysqlCommunicator::init(int fd, unique_ptr session, const std::string &addr) +RC MysqlCommunicator::init(int fd, unique_ptr session, const string &addr) { // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase.html // 按照协议描述,服务端在连接建立后需要先向客户端发送握手信息 @@ -633,7 +631,7 @@ RC MysqlCommunicator::read_event(SessionEvent *&event) sizeof(packet_header), packet_header.sequence_id, packet_header.payload_length, fd_); sequence_id_ = packet_header.sequence_id + 1; - std::vector buf(packet_header.payload_length); + vector buf(packet_header.payload_length); ret = common::readn(fd_, buf.data(), packet_header.payload_length); if (ret != 0) { LOG_WARN("failed to read packet payload. length=%d, addr=%s, error=%s", @@ -676,7 +674,7 @@ RC MysqlCommunicator::read_event(SessionEvent *&event) } LOG_TRACE("query command: %s", query_packet.query.c_str()); - if (query_packet.query.find("select @@version_comment") != std::string::npos) { + if (query_packet.query.find("select @@version_comment") != string::npos) { bool need_disconnect; return handle_version_comment(need_disconnect); } @@ -702,7 +700,7 @@ RC MysqlCommunicator::write_state(SessionEvent *event, bool &need_disconnect) const int buf_size = 2048; char *buf = new char[buf_size]; - const std::string &state_string = sql_result->state_string(); + const string &state_string = sql_result->state_string(); if (state_string.empty()) { const char *result = strrc(sql_result->return_code()); snprintf(buf, buf_size, "%s", result); @@ -782,7 +780,7 @@ RC MysqlCommunicator::write_result(SessionEvent *event, bool &need_disconnect) } } - rc = send_result_rows(sql_result, cell_num == 0, need_disconnect); + rc = send_result_rows(event, sql_result, cell_num == 0, need_disconnect); } RC close_rc = sql_result->close(); @@ -795,7 +793,7 @@ RC MysqlCommunicator::write_result(SessionEvent *event, bool &need_disconnect) RC MysqlCommunicator::send_packet(const BasePacket &packet) { - std::vector net_packet; + vector net_packet; RC rc = packet.encode(client_capabilities_flag_, net_packet); if (rc != RC::SUCCESS) { @@ -832,7 +830,7 @@ RC MysqlCommunicator::send_column_definition(SqlResult *sql_result, bool &need_d return rc; } - std::vector net_packet; + vector net_packet; net_packet.resize(1024); char *buf = net_packet.data(); int pos = 0; @@ -942,15 +940,44 @@ RC MysqlCommunicator::send_column_definition(SqlResult *sql_result, bool &need_d * @param no_column_def 为了特殊处理没有返回值的语句,比如insert/delete,需要做特殊处理。 * 这种语句只需要返回一个ok packet即可 */ -RC MysqlCommunicator::send_result_rows(SqlResult *sql_result, bool no_column_def, bool &need_disconnect) +RC MysqlCommunicator::send_result_rows(SessionEvent *event, SqlResult *sql_result, bool no_column_def, bool &need_disconnect) { RC rc = RC::SUCCESS; - std::vector packet; + vector packet; packet.resize(4 * 1024 * 1024); // TODO warning: length cannot be fix int affected_rows = 0; + if (event->session()->get_execution_mode() == ExecutionMode::CHUNK_ITERATOR + && event->session()->used_chunk_mode()) { + rc = write_chunk_result(sql_result, packet, affected_rows, need_disconnect); + } else { + rc = write_tuple_result(sql_result, packet, affected_rows, need_disconnect); + } + + // 所有行发送完成后,发送一个EOF或OK包 + if ((client_capabilities_flag_ & CLIENT_DEPRECATE_EOF) || no_column_def) { + LOG_TRACE("client has CLIENT_DEPRECATE_EOF or has empty column, send ok packet"); + OkPacket ok_packet; + ok_packet.packet_header.sequence_id = sequence_id_++; + ok_packet.affected_rows = affected_rows; + rc = send_packet(ok_packet); + } else { + LOG_TRACE("send eof packet to client"); + EofPacket eof_packet; + eof_packet.packet_header.sequence_id = sequence_id_++; + rc = send_packet(eof_packet); + } + + LOG_TRACE("send rows to client done"); + need_disconnect = false; + return rc; +} + +RC MysqlCommunicator::write_tuple_result(SqlResult *sql_result, vector &packet, int &affected_rows, bool &need_disconnect) +{ Tuple *tuple = nullptr; + RC rc = RC::SUCCESS; while (RC::SUCCESS == (rc = sql_result->next_tuple(tuple))) { assert(tuple != nullptr); @@ -990,22 +1017,42 @@ RC MysqlCommunicator::send_result_rows(SqlResult *sql_result, bool no_column_def return rc; } } + return rc; +} +RC MysqlCommunicator::write_chunk_result(SqlResult *sql_result, vector &packet, int &affected_rows, bool &need_disconnect) +{ + Chunk chunk; + RC rc = RC::SUCCESS; + while (RC::SUCCESS == (rc = sql_result->next_chunk(chunk))) { + int column_num = chunk.column_num(); + if (column_num == 0) { + continue; + } + for (int i = 0; i < chunk.rows(); i++) { + affected_rows++; + // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset.html + // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset_row.html + // note: if some field is null, send a 0xFB + char *buf = packet.data(); + int pos = 0; + + pos += 3; + pos += store_int1(buf + pos, sequence_id_++); + + for (int col_idx = 0; col_idx < column_num; col_idx++) { + Value value = chunk.get_value(col_idx, i); + pos += store_lenenc_string(buf + pos, value.to_string().c_str()); + } - // 所有行发送完成后,发送一个EOF或OK包 - if ((client_capabilities_flag_ & CLIENT_DEPRECATE_EOF) || no_column_def) { - LOG_TRACE("client has CLIENT_DEPRECATE_EOF or has empty column, send ok packet"); - OkPacket ok_packet; - ok_packet.packet_header.sequence_id = sequence_id_++; - ok_packet.affected_rows = affected_rows; - rc = send_packet(ok_packet); - } else { - LOG_TRACE("send eof packet to client"); - EofPacket eof_packet; - eof_packet.packet_header.sequence_id = sequence_id_++; - rc = send_packet(eof_packet); + int payload_length = pos - 4; + store_int3(buf, payload_length); + rc = writer_->writen(buf, pos); + if (OB_FAIL(rc)) { + LOG_WARN("failed to send row packet to client. addr=%s, error=%s", addr(), strerror(errno)); + need_disconnect = true; + return rc; + } + } } - - LOG_TRACE("send rows to client done"); - need_disconnect = false; return rc; } diff --git a/src/observer/net/mysql_communicator.h b/src/observer/net/mysql_communicator.h index 26e3b4078..2f69794c0 100644 --- a/src/observer/net/mysql_communicator.h +++ b/src/observer/net/mysql_communicator.h @@ -15,6 +15,7 @@ See the Mulan PSL v2 for more details. */ #pragma once #include "net/communicator.h" +#include "common/lang/string.h" class SqlResult; class BasePacket; @@ -35,7 +36,7 @@ class MysqlCommunicator : public Communicator * @details 参考MySQL或MariaDB的手册,服务端要首先向客户端发送一个握手包,等客户端回复后, * 再回复一个OkPacket或ErrPacket */ - virtual RC init(int fd, std::unique_ptr session, const std::string &addr) override; + virtual RC init(int fd, unique_ptr session, const string &addr) override; /** * @brief 有新的消息到达时,接收消息 @@ -76,12 +77,13 @@ class MysqlCommunicator : public Communicator /** * @brief 返回客户端行数据 * + * @param[in] event 任务数据 * @param[in] sql_result 返回的结果 * @param no_column_def 是否没有列描述信息 * @param[out] need_disconnect 是否需要断开连接 * @return RC */ - RC send_result_rows(SqlResult *sql_result, bool no_column_def, bool &need_disconnect); + RC send_result_rows(SessionEvent *event, SqlResult *sql_result, bool no_column_def, bool &need_disconnect); /** * @brief 根据实际测试,客户端在连接上来时,会发起一个 version_comment的查询 @@ -89,6 +91,9 @@ class MysqlCommunicator : public Communicator */ RC handle_version_comment(bool &need_disconnect); + RC write_tuple_result(SqlResult *sql_result, vector &packet, int &affected_rows, bool &need_disconnect); + RC write_chunk_result(SqlResult *sql_result, vector &packet, int &affected_rows, bool &need_disconnect); + private: //! 握手阶段(鉴权),需要做一些特殊处理,所以加个字段单独标记 bool authed_ = false; diff --git a/src/observer/net/one_thread_per_connection_thread_handler.cpp b/src/observer/net/one_thread_per_connection_thread_handler.cpp index 1cc5e7a41..91ff69043 100644 --- a/src/observer/net/one_thread_per_connection_thread_handler.cpp +++ b/src/observer/net/one_thread_per_connection_thread_handler.cpp @@ -13,15 +13,16 @@ See the Mulan PSL v2 for more details. */ // #include -#include #include "net/one_thread_per_connection_thread_handler.h" #include "common/log/log.h" +#include "common/lang/thread.h" +#include "common/lang/mutex.h" +#include "common/lang/chrono.h" #include "common/thread/thread_util.h" #include "net/communicator.h" #include "net/sql_task_handler.h" -using namespace std; using namespace common; class Worker @@ -40,7 +41,7 @@ class Worker RC start() { - thread_ = new std::thread(std::ref(*this)); + thread_ = new thread(std::ref(*this)); return RC::SUCCESS; } @@ -53,7 +54,7 @@ class Worker RC join() { if (thread_) { - if (thread_->get_id() == std::this_thread::get_id()) { + if (thread_->get_id() == this_thread::get_id()) { thread_->detach(); // 如果当前线程join当前线程,就会卡死 } else { thread_->join(); @@ -107,7 +108,7 @@ class Worker ThreadHandler &host_; SqlTaskHandler task_handler_; Communicator *communicator_ = nullptr; - std::thread *thread_ = nullptr; + thread *thread_ = nullptr; volatile bool running_ = true; }; diff --git a/src/observer/net/one_thread_per_connection_thread_handler.h b/src/observer/net/one_thread_per_connection_thread_handler.h index 9e27a3219..12c9f1c4d 100644 --- a/src/observer/net/one_thread_per_connection_thread_handler.h +++ b/src/observer/net/one_thread_per_connection_thread_handler.h @@ -12,10 +12,9 @@ See the Mulan PSL v2 for more details. */ // Created by Wangyunlai on 2024/01/10. // -#include -#include - #include "net/thread_handler.h" +#include "common/lang/mutex.h" +#include "common/lang/unordered_map.h" class Worker; @@ -44,7 +43,7 @@ class OneThreadPerConnectionThreadHandler : public ThreadHandler private: /// 记录一个连接Communicator关联的线程数据 - std::unordered_map thread_map_; // 当前编译器没有支持jthread + unordered_map thread_map_; // 当前编译器没有支持jthread /// 保护线程安全的锁 - std::mutex lock_; + mutex lock_; }; \ No newline at end of file diff --git a/src/observer/net/plain_communicator.cpp b/src/observer/net/plain_communicator.cpp index 6c5e1604f..e16746716 100644 --- a/src/observer/net/plain_communicator.cpp +++ b/src/observer/net/plain_communicator.cpp @@ -20,8 +20,6 @@ See the Mulan PSL v2 for more details. */ #include "session/session.h" #include "sql/expr/tuple.h" -using namespace std; - PlainCommunicator::PlainCommunicator() { send_message_delimiter_.assign(1, '\0'); @@ -239,7 +237,42 @@ RC PlainCommunicator::write_result_internal(SessionEvent *event, bool &need_disc } rc = RC::SUCCESS; + if (event->session()->get_execution_mode() == ExecutionMode::CHUNK_ITERATOR + && event->session()->used_chunk_mode()) { + rc = write_chunk_result(sql_result); + } else { + rc = write_tuple_result(sql_result); + } + + if (OB_FAIL(rc)) { + return rc; + } + + if (cell_num == 0) { + // 除了select之外,其它的消息通常不会通过operator来返回结果,表头和行数据都是空的 + // 这里针对这种情况做特殊处理,当表头和行数据都是空的时候,就返回处理的结果 + // 可能是insert/delete等操作,不直接返回给客户端数据,这里把处理结果返回给客户端 + RC rc_close = sql_result->close(); + if (rc == RC::SUCCESS) { + rc = rc_close; + } + sql_result->set_return_code(rc); + return write_state(event, need_disconnect); + } else { + need_disconnect = false; + } + RC rc_close = sql_result->close(); + if (OB_SUCC(rc)) { + rc = rc_close; + } + + return rc; +} + +RC PlainCommunicator::write_tuple_result(SqlResult *sql_result) +{ + RC rc = RC::SUCCESS; Tuple *tuple = nullptr; while (RC::SUCCESS == (rc = sql_result->next_tuple(tuple))) { assert(tuple != nullptr); @@ -260,6 +293,7 @@ RC PlainCommunicator::write_result_internal(SessionEvent *event, bool &need_disc Value value; rc = tuple->cell_at(i, value); if (rc != RC::SUCCESS) { + LOG_WARN("failed to get tuple cell value. rc=%s", strrc(rc)); sql_result->close(); return rc; } @@ -287,25 +321,53 @@ RC PlainCommunicator::write_result_internal(SessionEvent *event, bool &need_disc if (rc == RC::RECORD_EOF) { rc = RC::SUCCESS; } + return rc; +} - if (cell_num == 0) { - // 除了select之外,其它的消息通常不会通过operator来返回结果,表头和行数据都是空的 - // 这里针对这种情况做特殊处理,当表头和行数据都是空的时候,就返回处理的结果 - // 可能是insert/delete等操作,不直接返回给客户端数据,这里把处理结果返回给客户端 - RC rc_close = sql_result->close(); - if (rc == RC::SUCCESS) { - rc = rc_close; +RC PlainCommunicator::write_chunk_result(SqlResult *sql_result) +{ + RC rc = RC::SUCCESS; + Chunk chunk; + while (RC::SUCCESS == (rc = sql_result->next_chunk(chunk))) { + int col_num = chunk.column_num(); + for (int row_idx = 0; row_idx < chunk.rows(); row_idx++) { + for (int col_idx = 0; col_idx < col_num; col_idx++) { + if (col_idx != 0) { + const char *delim = " | "; + + rc = writer_->writen(delim, strlen(delim)); + if (OB_FAIL(rc)) { + LOG_WARN("failed to send data to client. err=%s", strerror(errno)); + sql_result->close(); + return rc; + } + } + + Value value = chunk.get_value(col_idx, row_idx); + + string cell_str = value.to_string(); + + rc = writer_->writen(cell_str.data(), cell_str.size()); + if (OB_FAIL(rc)) { + LOG_WARN("failed to send data to client. err=%s", strerror(errno)); + sql_result->close(); + return rc; + } + } + char newline = '\n'; + + rc = writer_->writen(&newline, 1); + if (OB_FAIL(rc)) { + LOG_WARN("failed to send data to client. err=%s", strerror(errno)); + sql_result->close(); + return rc; + } } - sql_result->set_return_code(rc); - return write_state(event, need_disconnect); - } else { - need_disconnect = false; + chunk.reset(); } - RC rc_close = sql_result->close(); - if (OB_SUCC(rc)) { - rc = rc_close; + if (rc == RC::RECORD_EOF) { + rc = RC::SUCCESS; } - return rc; } \ No newline at end of file diff --git a/src/observer/net/plain_communicator.h b/src/observer/net/plain_communicator.h index 01db59b09..0dfabf9a2 100644 --- a/src/observer/net/plain_communicator.h +++ b/src/observer/net/plain_communicator.h @@ -14,9 +14,10 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include - #include "net/communicator.h" +#include "common/lang/vector.h" + +class SqlResult; /** * @brief 与客户端进行通讯 @@ -36,8 +37,10 @@ class PlainCommunicator : public Communicator RC write_state(SessionEvent *event, bool &need_disconnect); RC write_debug(SessionEvent *event, bool &need_disconnect); RC write_result_internal(SessionEvent *event, bool &need_disconnect); + RC write_tuple_result(SqlResult *sql_result); + RC write_chunk_result(SqlResult *sql_result); protected: - std::vector send_message_delimiter_; ///< 发送消息分隔符 - std::vector debug_message_prefix_; ///< 调试信息前缀 + vector send_message_delimiter_; ///< 发送消息分隔符 + vector debug_message_prefix_; ///< 调试信息前缀 }; diff --git a/src/observer/net/ring_buffer.cpp b/src/observer/net/ring_buffer.cpp index 7468cbf8f..c3b5f74e0 100644 --- a/src/observer/net/ring_buffer.cpp +++ b/src/observer/net/ring_buffer.cpp @@ -12,13 +12,10 @@ See the Mulan PSL v2 for more details. */ // Created by Wangyunlai on 2023/06/16. // -#include - +#include "common/lang/algorithm.h" #include "common/log/log.h" #include "net/ring_buffer.h" -using namespace std; - const int32_t DEFAULT_BUFFER_SIZE = 16 * 1024; RingBuffer::RingBuffer() : RingBuffer(DEFAULT_BUFFER_SIZE) {} diff --git a/src/observer/net/ring_buffer.h b/src/observer/net/ring_buffer.h index 5e4bf84dc..5968e8858 100644 --- a/src/observer/net/ring_buffer.h +++ b/src/observer/net/ring_buffer.h @@ -14,9 +14,8 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include - #include "common/rc.h" +#include "common/lang/vector.h" /** * @brief 环形缓存,当前用于通讯写入数据时的缓存 @@ -88,7 +87,7 @@ class RingBuffer int32_t read_pos() const { return (write_pos_ - this->size() + capacity()) % capacity(); } private: - std::vector buffer_; ///< 缓存使用的内存,使用vector方便管理 - int32_t data_size_ = 0; ///< 已经写入的数据量 - int32_t write_pos_ = 0; ///< 当前写指针的位置,范围不会超出[0, capacity) + vector buffer_; ///< 缓存使用的内存,使用vector方便管理 + int32_t data_size_ = 0; ///< 已经写入的数据量 + int32_t write_pos_ = 0; ///< 当前写指针的位置,范围不会超出[0, capacity) }; \ No newline at end of file diff --git a/src/observer/net/server.cpp b/src/observer/net/server.cpp index 0b4013840..f946de013 100644 --- a/src/observer/net/server.cpp +++ b/src/observer/net/server.cpp @@ -42,7 +42,6 @@ See the Mulan PSL v2 for more details. */ #include "net/thread_handler.h" #include "net/sql_task_handler.h" -using namespace std; using namespace common; ServerParam::ServerParam() @@ -96,9 +95,9 @@ void NetServer::accept(int fd) ::close(client_fd); return; } - std::stringstream address; + stringstream address; address << ip_addr << ":" << addr.sin_port; - std::string addr_str = address.str(); + string addr_str = address.str(); ret = set_non_block(client_fd); if (ret < 0) { diff --git a/src/observer/net/server_param.h b/src/observer/net/server_param.h index c4eaddeb2..2a498a79a 100644 --- a/src/observer/net/server_param.h +++ b/src/observer/net/server_param.h @@ -15,7 +15,7 @@ See the Mulan PSL v2 for more details. */ #pragma once #include "net/communicator.h" -#include +#include "common/lang/string.h" /** * @brief 服务端启动参数 @@ -37,7 +37,7 @@ class ServerParam int port; ///< 监听的端口号 - std::string unix_socket_path; ///< unix socket的路径 + string unix_socket_path; ///< unix socket的路径 bool use_std_io = false; ///< 是否使用标准输入输出作为通信条件 @@ -47,5 +47,5 @@ class ServerParam CommunicateProtocol protocol; ///< 通讯协议,目前支持文本协议和mysql协议 - std::string thread_handling; ///< 线程池模型 + string thread_handling; ///< 线程池模型 }; diff --git a/src/observer/net/sql_task_handler.cpp b/src/observer/net/sql_task_handler.cpp index d4d61f5af..1086c02db 100644 --- a/src/observer/net/sql_task_handler.cpp +++ b/src/observer/net/sql_task_handler.cpp @@ -34,7 +34,11 @@ RC SqlTaskHandler::handle_event(Communicator *communicator) SQLStageEvent sql_event(event, event->query()); - (void)handle_sql(&sql_event); + rc = handle_sql(&sql_event); + if (OB_FAIL(rc)) { + LOG_TRACE("failed to handle sql. rc=%s", strrc(rc)); + event->sql_result()->set_return_code(rc); + } bool need_disconnect = false; diff --git a/src/observer/session/session.cpp b/src/observer/session/session.cpp index 7de8b1736..eaa5e10c6 100644 --- a/src/observer/session/session.cpp +++ b/src/observer/session/session.cpp @@ -44,7 +44,7 @@ const char *Session::get_current_db_name() const Db *Session::get_current_db() const { return db_; } -void Session::set_current_db(const std::string &dbname) +void Session::set_current_db(const string &dbname) { DefaultHandler &handler = *GCTX.handler_; Db *db = handler.find_db(dbname.c_str()); diff --git a/src/observer/session/session.h b/src/observer/session/session.h index a50050234..16b816011 100644 --- a/src/observer/session/session.h +++ b/src/observer/session/session.h @@ -14,7 +14,8 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include +#include "common/types.h" +#include "common/lang/string.h" class Trx; class Db; @@ -48,7 +49,7 @@ class Session * * @param dbname 数据库名字 */ - void set_current_db(const std::string &dbname); + void set_current_db(const string &dbname); /** * @brief 设置当前事务为多语句模式,需要明确的指出提交或回滚 @@ -79,6 +80,13 @@ class Session void set_sql_debug(bool sql_debug) { sql_debug_ = sql_debug; } bool sql_debug_on() const { return sql_debug_; } + void set_execution_mode(const ExecutionMode mode) { execution_mode_ = mode; } + ExecutionMode get_execution_mode() const { return execution_mode_; } + + bool used_chunk_mode() { return used_chunk_mode_; } + + void set_used_chunk_mode(bool used_chunk_mode) { used_chunk_mode_ = used_chunk_mode; } + /** * @brief 将指定会话设置到线程变量中 * @@ -99,4 +107,10 @@ class Session bool trx_multi_operation_mode_ = false; ///< 当前事务的模式,是否多语句模式. 单语句模式自动提交 bool sql_debug_ = false; ///< 是否输出SQL调试信息 + + // 是否使用了 `chunk_iterator` 模式。 只有在设置了 `chunk_iterator` + // 并且可以生成相关物理执行计划时才会使用 `chunk_iterator` 模式。 + bool used_chunk_mode_ = false; + + ExecutionMode execution_mode_ = ExecutionMode::TUPLE_ITERATOR; }; diff --git a/src/observer/session/session_stage.cpp b/src/observer/session/session_stage.cpp index 69a89ec8d..8ca555e53 100644 --- a/src/observer/session/session_stage.cpp +++ b/src/observer/session/session_stage.cpp @@ -15,7 +15,6 @@ See the Mulan PSL v2 for more details. */ #include "session_stage.h" #include -#include #include "common/conf/ini.h" #include "common/lang/mutex.h" @@ -35,7 +34,7 @@ SessionStage::~SessionStage() {} // TODO remove me void SessionStage::handle_request(SessionEvent *sev) { - std::string sql = sev->query(); + string sql = sev->query(); if (common::is_blank(sql.c_str())) { return; } @@ -58,7 +57,7 @@ void SessionStage::handle_request(SessionEvent *sev) void SessionStage::handle_request2(SessionEvent *event) { - const std::string &sql = event->query(); + const string &sql = event->query(); if (common::is_blank(sql.c_str())) { return; } diff --git a/src/observer/sql/executor/create_table_executor.cpp b/src/observer/sql/executor/create_table_executor.cpp index cb13dc771..3ef9cac9a 100644 --- a/src/observer/sql/executor/create_table_executor.cpp +++ b/src/observer/sql/executor/create_table_executor.cpp @@ -32,7 +32,7 @@ RC CreateTableExecutor::execute(SQLStageEvent *sql_event) CreateTableStmt *create_table_stmt = static_cast(stmt); const char *table_name = create_table_stmt->table_name().c_str(); - RC rc = session->get_current_db()->create_table(table_name, create_table_stmt->attr_infos()); + RC rc = session->get_current_db()->create_table(table_name, create_table_stmt->attr_infos(), create_table_stmt->storage_format()); return rc; } \ No newline at end of file diff --git a/src/observer/sql/executor/execute_stage.cpp b/src/observer/sql/executor/execute_stage.cpp index 28b417f40..4e61b02a7 100644 --- a/src/observer/sql/executor/execute_stage.cpp +++ b/src/observer/sql/executor/execute_stage.cpp @@ -55,45 +55,10 @@ RC ExecuteStage::handle_request_with_physical_operator(SQLStageEvent *sql_event) { RC rc = RC::SUCCESS; - Stmt *stmt = sql_event->stmt(); - ASSERT(stmt != nullptr, "SQL Statement shouldn't be empty!"); - unique_ptr &physical_operator = sql_event->physical_operator(); ASSERT(physical_operator != nullptr, "physical operator should not be null"); - // TODO 这里也可以优化一下,是否可以让physical operator自己设置tuple schema - TupleSchema schema; - switch (stmt->type()) { - case StmtType::SELECT: { - SelectStmt *select_stmt = static_cast(stmt); - bool with_table_name = select_stmt->tables().size() > 1; - - for (const Field &field : select_stmt->query_fields()) { - if (with_table_name) { - schema.append_cell(field.table_name(), field.field_name()); - } else { - schema.append_cell(field.field_name()); - } - } - } break; - - case StmtType::CALC: { - CalcPhysicalOperator *calc_operator = static_cast(physical_operator.get()); - for (const unique_ptr &expr : calc_operator->expressions()) { - schema.append_cell(expr->name().c_str()); - } - } break; - - case StmtType::EXPLAIN: { - schema.append_cell("Query Plan"); - } break; - default: { - // 只有select返回结果 - } break; - } - SqlResult *sql_result = sql_event->session_event()->sql_result(); - sql_result->set_tuple_schema(schema); sql_result->set_operator(std::move(physical_operator)); return rc; } diff --git a/src/observer/sql/executor/load_data_executor.cpp b/src/observer/sql/executor/load_data_executor.cpp index bdd1bca5a..bdb685572 100644 --- a/src/observer/sql/executor/load_data_executor.cpp +++ b/src/observer/sql/executor/load_data_executor.cpp @@ -58,12 +58,12 @@ RC insert_record_from_file( const FieldMeta *field = table->table_meta().field(i + sys_field_num); std::string &file_value = file_values[i]; - if (field->type() != CHARS) { + if (field->type() != AttrType::CHARS) { common::strip(file_value); } switch (field->type()) { - case INTS: { + case AttrType::INTS: { deserialize_stream.clear(); // 清理stream的状态,防止多次解析出现异常 deserialize_stream.str(file_value); @@ -79,7 +79,7 @@ RC insert_record_from_file( } break; - case FLOATS: { + case AttrType::FLOATS: { deserialize_stream.clear(); deserialize_stream.str(file_value); @@ -92,11 +92,11 @@ RC insert_record_from_file( record_values[i].set_float(float_value); } } break; - case CHARS: { + case AttrType::CHARS: { record_values[i].set_string(file_value.c_str()); } break; default: { - errmsg << "Unsupported field type to loading: " << field->type(); + errmsg << "Unsupported field type to loading: " << attr_type_to_string(field->type()); rc = RC::SCHEMA_FIELD_TYPE_MISMATCH; } break; } diff --git a/src/observer/sql/executor/set_variable_executor.cpp b/src/observer/sql/executor/set_variable_executor.cpp new file mode 100644 index 000000000..b5518a18e --- /dev/null +++ b/src/observer/sql/executor/set_variable_executor.cpp @@ -0,0 +1,98 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "sql/executor/set_variable_executor.h" + +RC SetVariableExecutor::execute(SQLStageEvent *sql_event) +{ + RC rc = RC::SUCCESS; + + Session *session = sql_event->session_event()->session(); + SetVariableStmt *stmt = (SetVariableStmt *)sql_event->stmt(); + + const char *var_name = stmt->var_name(); + const Value &var_value = stmt->var_value(); + if (strcasecmp(var_name, "sql_debug") == 0) { + bool bool_value = false; + rc = var_value_to_boolean(var_value, bool_value); + if (rc == RC::SUCCESS) { + session->set_sql_debug(bool_value); + LOG_TRACE("set sql_debug to %d", bool_value); + } + } else if (strcasecmp(var_name, "execution_mode") == 0) { + ExecutionMode execution_mode = ExecutionMode::UNKNOWN_MODE; + rc = get_execution_mode(var_value, execution_mode); + if (rc == RC::SUCCESS && execution_mode != ExecutionMode::UNKNOWN_MODE) { + session->set_execution_mode(execution_mode); + } else { + rc = RC::INVALID_ARGUMENT; + } + } else { + rc = RC::VARIABLE_NOT_EXISTS; + } + + return rc; +} + +RC SetVariableExecutor::var_value_to_boolean(const Value &var_value, bool &bool_value) const +{ + RC rc = RC::SUCCESS; + + if (var_value.attr_type() == AttrType::BOOLEANS) { + bool_value = var_value.get_boolean(); + } else if (var_value.attr_type() == AttrType::INTS) { + bool_value = var_value.get_int() != 0; + } else if (var_value.attr_type() == AttrType::FLOATS) { + bool_value = var_value.get_float() != 0.0; + } else if (var_value.attr_type() == AttrType::CHARS) { + + std::string true_strings[] = {"true", "on", "yes", "t", "1"}; + + std::string false_strings[] = {"false", "off", "no", "f", "0"}; + + for (size_t i = 0; i < sizeof(true_strings) / sizeof(true_strings[0]); i++) { + if (strcasecmp(var_value.get_string().c_str(), true_strings[i].c_str()) == 0) { + bool_value = true; + return rc; + } + } + + for (size_t i = 0; i < sizeof(false_strings) / sizeof(false_strings[0]); i++) { + if (strcasecmp(var_value.get_string().c_str(), false_strings[i].c_str()) == 0) { + bool_value = false; + return rc; + } + } + rc = RC::VARIABLE_NOT_VALID; + } + + return rc; +} + +RC SetVariableExecutor::get_execution_mode(const Value &var_value, ExecutionMode &execution_mode) const +{ + RC rc = RC::SUCCESS; + + if (var_value.attr_type() == AttrType::CHARS) { + if (strcasecmp(var_value.get_string().c_str(), "TUPLE_ITERATOR") == 0) { + execution_mode = ExecutionMode::TUPLE_ITERATOR; + } else if (strcasecmp(var_value.get_string().c_str(), "CHUNK_ITERATOR") == 0) { + execution_mode = ExecutionMode::CHUNK_ITERATOR; + } else { + execution_mode = ExecutionMode::UNKNOWN_MODE; + rc = RC::VARIABLE_NOT_VALID; + } + } else { + execution_mode = ExecutionMode::UNKNOWN_MODE; + rc = RC::VARIABLE_NOT_VALID; + } + + return rc; +} \ No newline at end of file diff --git a/src/observer/sql/executor/set_variable_executor.h b/src/observer/sql/executor/set_variable_executor.h index b3fe959f9..8af0d2b69 100644 --- a/src/observer/sql/executor/set_variable_executor.h +++ b/src/observer/sql/executor/set_variable_executor.h @@ -30,64 +30,10 @@ class SetVariableExecutor SetVariableExecutor() = default; virtual ~SetVariableExecutor() = default; - RC execute(SQLStageEvent *sql_event) - { - RC rc = RC::SUCCESS; - - Session *session = sql_event->session_event()->session(); - SetVariableStmt *stmt = (SetVariableStmt *)sql_event->stmt(); - - const char *var_name = stmt->var_name(); - const Value &var_value = stmt->var_value(); - if (strcasecmp(var_name, "sql_debug") == 0) { - bool bool_value = false; - rc = var_value_to_boolean(var_value, bool_value); - if (rc != RC::SUCCESS) { - return rc; - } - - session->set_sql_debug(bool_value); - LOG_TRACE("set sql_debug to %d", bool_value); - } else { - rc = RC::VARIABLE_NOT_EXISTS; - } - - return RC::SUCCESS; - } + RC execute(SQLStageEvent *sql_event); private: - RC var_value_to_boolean(const Value &var_value, bool &bool_value) const - { - RC rc = RC::SUCCESS; - - if (var_value.attr_type() == AttrType::BOOLEANS) { - bool_value = var_value.get_boolean(); - } else if (var_value.attr_type() == AttrType::INTS) { - bool_value = var_value.get_int() != 0; - } else if (var_value.attr_type() == AttrType::FLOATS) { - bool_value = var_value.get_float() != 0.0; - } else if (var_value.attr_type() == AttrType::CHARS) { - - std::string true_strings[] = {"true", "on", "yes", "t", "1"}; - - std::string false_strings[] = {"false", "off", "no", "f", "0"}; - - for (size_t i = 0; i < sizeof(true_strings) / sizeof(true_strings[0]); i++) { - if (strcasecmp(var_value.get_string().c_str(), true_strings[i].c_str()) == 0) { - bool_value = true; - return rc; - } - } - - for (size_t i = 0; i < sizeof(false_strings) / sizeof(false_strings[0]); i++) { - if (strcasecmp(var_value.get_string().c_str(), false_strings[i].c_str()) == 0) { - bool_value = false; - return rc; - } - } - rc = RC::VARIABLE_NOT_VALID; - } + RC var_value_to_boolean(const Value &var_value, bool &bool_value) const; - return rc; - } + RC get_execution_mode(const Value &var_value, ExecutionMode &execution_mode) const; }; \ No newline at end of file diff --git a/src/observer/sql/executor/sql_result.cpp b/src/observer/sql/executor/sql_result.cpp index 2d90ed6de..a8b1b8551 100644 --- a/src/observer/sql/executor/sql_result.cpp +++ b/src/observer/sql/executor/sql_result.cpp @@ -69,8 +69,15 @@ RC SqlResult::next_tuple(Tuple *&tuple) return rc; } +RC SqlResult::next_chunk(Chunk &chunk) +{ + RC rc = operator_->next(chunk); + return rc; +} + void SqlResult::set_operator(std::unique_ptr oper) { ASSERT(operator_ == nullptr, "current operator is not null. Result is not closed?"); operator_ = std::move(oper); + operator_->tuple_schema(tuple_schema_); } diff --git a/src/observer/sql/executor/sql_result.h b/src/observer/sql/executor/sql_result.h index 68a930e47..016cfd179 100644 --- a/src/observer/sql/executor/sql_result.h +++ b/src/observer/sql/executor/sql_result.h @@ -49,6 +49,7 @@ class SqlResult RC open(); RC close(); RC next_tuple(Tuple *&tuple); + RC next_chunk(Chunk &chunk); private: Session *session_ = nullptr; ///< 当前所属会话 diff --git a/src/observer/sql/expr/aggregate_hash_table.cpp b/src/observer/sql/expr/aggregate_hash_table.cpp new file mode 100644 index 000000000..cde07fd41 --- /dev/null +++ b/src/observer/sql/expr/aggregate_hash_table.cpp @@ -0,0 +1,246 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "sql/expr/aggregate_hash_table.h" + +// ----------------------------------StandardAggregateHashTable------------------ + +RC StandardAggregateHashTable::add_chunk(Chunk &groups_chunk, Chunk &aggrs_chunk) +{ + // your code here + exit(-1); +} + +void StandardAggregateHashTable::Scanner::open_scan() +{ + it_ = static_cast(hash_table_)->begin(); + end_ = static_cast(hash_table_)->end(); +} + +RC StandardAggregateHashTable::Scanner::next(Chunk &output_chunk) +{ + if (it_ == end_) { + return RC::RECORD_EOF; + } + while (it_ != end_ && output_chunk.rows() <= output_chunk.capacity()) { + auto &group_by_values = it_->first; + auto &aggrs = it_->second; + for (int i = 0; i < output_chunk.column_num(); i++) { + auto col_idx = output_chunk.column_ids(i); + if (col_idx >= static_cast(group_by_values.size())) { + output_chunk.column(i).append_one((char *)aggrs[col_idx - group_by_values.size()].data()); + } else { + output_chunk.column(i).append_one((char *)group_by_values[col_idx].data()); + } + } + it_++; + } + if (it_ == end_) { + return RC::SUCCESS; + } + + return RC::SUCCESS; +} + +size_t StandardAggregateHashTable::VectorHash::operator()(const vector &vec) const +{ + size_t hash = 0; + for (const auto &elem : vec) { + hash ^= std::hash()(elem.to_string()); + } + return hash; +} + +bool StandardAggregateHashTable::VectorEqual::operator()(const vector &lhs, const vector &rhs) const +{ + if (lhs.size() != rhs.size()) { + return false; + } + for (size_t i = 0; i < lhs.size(); ++i) { + if (rhs[i].compare(lhs[i]) != 0) { + return false; + } + } + return true; +} + +// ----------------------------------LinearProbingAggregateHashTable------------------ +#ifdef USE_SIMD +template +RC LinearProbingAggregateHashTable::add_chunk(Chunk &group_chunk, Chunk &aggr_chunk) +{ + if (group_chunk.column_num() != 1 || aggr_chunk.column_num() != 1) { + LOG_WARN("group_chunk and aggr_chunk size must be 1."); + return RC::INVALID_ARGUMENT; + } + if (group_chunk.rows() != aggr_chunk.rows()) { + LOG_WARN("group_chunk and aggr _chunk rows must be equal."); + return RC::INVALID_ARGUMENT; + } + add_batch((int *)group_chunk.column(0).data(), (V *)aggr_chunk.column(0).data(), group_chunk.rows()); + return RC::SUCCESS; +} + +template +void LinearProbingAggregateHashTable::Scanner::open_scan() +{ + capacity_ = static_cast(hash_table_)->capacity(); + size_ = static_cast(hash_table_)->size(); + scan_pos_ = 0; + scan_count_ = 0; +} + +template +RC LinearProbingAggregateHashTable::Scanner::next(Chunk &output_chunk) +{ + if (scan_pos_ >= capacity_ || scan_count_ >= size_) { + return RC::RECORD_EOF; + } + auto linear_probing_hash_table = static_cast(hash_table_); + while (scan_pos_ < capacity_ && scan_count_ < size_ && output_chunk.rows() <= output_chunk.capacity()) { + int key; + V value; + RC rc = linear_probing_hash_table->iter_get(scan_pos_, key, value); + if (rc == RC::SUCCESS) { + output_chunk.column(0).append_one((char *)&key); + output_chunk.column(1).append_one((char *)&value); + scan_count_++; + } + scan_pos_++; + } + return RC::SUCCESS; +} + +template +void LinearProbingAggregateHashTable::Scanner::close_scan() +{ + capacity_ = -1; + size_ = -1; + scan_pos_ = -1; + scan_count_ = 0; +} + +template +RC LinearProbingAggregateHashTable::get(int key, V &value) +{ + RC rc = RC::SUCCESS; + int index = (key % capacity_ + capacity_) % capacity_; + int iterate_cnt = 0; + while (true) { + if (keys_[index] == EMPTY_KEY) { + rc = RC::NOT_EXIST; + break; + } else if (keys_[index] == key) { + value = values_[index]; + break; + } else { + index += 1; + index %= capacity_; + iterate_cnt++; + if (iterate_cnt > capacity_) { + rc = RC::NOT_EXIST; + break; + } + } + } + return rc; +} + +template +RC LinearProbingAggregateHashTable::iter_get(int pos, int &key, V &value) +{ + RC rc = RC::SUCCESS; + if (keys_[pos] == LinearProbingAggregateHashTable::EMPTY_KEY) { + rc = RC::NOT_EXIST; + } else { + key = keys_[pos]; + value = values_[pos]; + } + return rc; +} + +template +void LinearProbingAggregateHashTable::aggregate(V *value, V value_to_aggregate) +{ + if (aggregate_type_ == AggregateExpr::Type::SUM) { + *value += value_to_aggregate; + } else { + ASSERT(false, "unsupported aggregate type"); + } +} + +template +void LinearProbingAggregateHashTable::resize() +{ + capacity_ *= 2; + std::vector new_keys(capacity_); + std::vector new_values(capacity_); + + for (size_t i = 0; i < keys_.size(); i++) { + auto &key = keys_[i]; + auto &value = values_[i]; + if (key != EMPTY_KEY) { + int index = (key % capacity_ + capacity_) % capacity_; + while (new_keys[index] != EMPTY_KEY) { + index = (index + 1) % capacity_; + } + new_keys[index] = key; + new_values[index] = value; + } + } + + keys_ = std::move(new_keys); + values_ = std::move(new_values); +} + +template +void LinearProbingAggregateHashTable::resize_if_need() +{ + if (size_ >= capacity_ / 2) { + resize(); + } +} + +template +void LinearProbingAggregateHashTable::add_batch(int *input_keys, V *input_values, int len) +{ + // your code here + exit(-1); + + // inv (invalid) 表示是否有效,inv[i] = -1 表示有效,inv[i] = 0 表示无效。 + // key[SIMD_WIDTH],value[SIMD_WIDTH] 表示当前循环中处理的键值对。 + // off (offset) 表示线性探测冲突时的偏移量,key[i] 每次遇到冲突键,则off[i]++,如果key[i] 已经完成聚合,则off[i] = 0, + // i = 0 表示selective load 的起始位置。 + // inv 全部初始化为 -1 + // off 全部初始化为 0 + + // for (; i + SIMD_WIDTH <= len;) { + // 1: 根据 `inv` 变量的值,从 `input_keys` 中 `selective load` `SIMD_WIDTH` 个不同的输入键值对。 + // 2. 计算 i += |inv|, `|inv|` 表示 `inv` 中有效的个数 + // 3. 计算 hash 值, + // 4. 根据聚合类型(目前只支持 sum),在哈希表中更新聚合结果。如果本次循环,没有找到key[i] 在哈希表中的位置,则不更新聚合结果。 + // 5. gather 操作,根据 hash 值将 keys_ 的 gather 结果写入 table_key 中。 + // 6. 更新 inv 和 off。如果本次循环key[i] 聚合完成,则inv[i]=-1,表示该位置在下次循环中读取新的键值对。 + // 如果本次循环 key[i] 未在哈希表中聚合完成(table_key[i] != key[i]),则inv[i] = 0,表示该位置在下次循环中不需要读取新的键值对。 + // 如果本次循环中,key[i]聚合完成,则off[i] 更新为 0,表示线性探测偏移量为 0,key[i] 未完成聚合,则off[i]++,表示线性探测偏移量加 1。 + // } + //7. 通过标量线性探测,处理剩余键值对 + + // resize_if_need(); +} + +template +const int LinearProbingAggregateHashTable::EMPTY_KEY = 0xffffffff; +template +const int LinearProbingAggregateHashTable::DEFAULT_CAPACITY = 16384; + +template class LinearProbingAggregateHashTable; +template class LinearProbingAggregateHashTable; +#endif \ No newline at end of file diff --git a/src/observer/sql/expr/aggregate_hash_table.h b/src/observer/sql/expr/aggregate_hash_table.h new file mode 100644 index 000000000..89719c9bc --- /dev/null +++ b/src/observer/sql/expr/aggregate_hash_table.h @@ -0,0 +1,171 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include +#include + +#include "common/math/simd_util.h" +#include "common/rc.h" +#include "sql/expr/expression.h" + +/** + * @brief 用于hash group by 的哈希表实现,不支持并发访问。 + */ +class AggregateHashTable +{ +public: + class Scanner + { + public: + explicit Scanner(AggregateHashTable *hash_table) : hash_table_(hash_table) {} + virtual ~Scanner() = default; + + virtual void open_scan() = 0; + + /** + * 通过扫描哈希表,将哈希表中的聚合结果写入 chunk 中。 + */ + virtual RC next(Chunk &chunk) = 0; + + virtual void close_scan(){}; + + protected: + AggregateHashTable *hash_table_; + }; + + /** + * @brief 将 groups_chunk 和 aggrs_chunk 写入到哈希表中。哈希表中记录了聚合结果。 + */ + virtual RC add_chunk(Chunk &groups_chunk, Chunk &aggrs_chunk) = 0; + + virtual ~AggregateHashTable() = default; +}; + +class StandardAggregateHashTable : public AggregateHashTable +{ +private: + struct VectorHash + { + std::size_t operator()(const std::vector &vec) const; + }; + + struct VectorEqual + { + bool operator()(const std::vector &lhs, const std::vector &rhs) const; + }; + +public: + using StandardHashTable = std::unordered_map, std::vector, VectorHash, VectorEqual>; + class Scanner : public AggregateHashTable::Scanner + { + public: + explicit Scanner(AggregateHashTable *hash_table) : AggregateHashTable::Scanner(hash_table) {} + ~Scanner() = default; + + void open_scan() override; + + RC next(Chunk &chunk) override; + + private: + StandardHashTable::iterator end_; + StandardHashTable::iterator it_; + }; + StandardAggregateHashTable(const std::vector aggregations) + { + for (auto &expr : aggregations) { + ASSERT(expr->type() == ExprType::AGGREGATION, "expect aggregate expression"); + auto *aggregation_expr = static_cast(expr); + aggr_types_.push_back(aggregation_expr->aggregate_type()); + } + } + + virtual ~StandardAggregateHashTable() {} + + RC add_chunk(Chunk &groups_chunk, Chunk &aggrs_chunk) override; + + StandardHashTable::iterator begin() { return aggr_values_.begin(); } + StandardHashTable::iterator end() { return aggr_values_.end(); } + +private: + /// group by values -> aggregate values + StandardHashTable aggr_values_; + std::vector aggr_types_; +}; + +/** + * @brief 线性探测哈希表实现 + * @note 当前只支持group by 列为 char/char(4) 类型,且聚合列为单列。 + */ +#ifdef USE_SIMD +template +class LinearProbingAggregateHashTable : public AggregateHashTable +{ +public: + class Scanner : public AggregateHashTable::Scanner + { + public: + explicit Scanner(AggregateHashTable *hash_table) : AggregateHashTable::Scanner(hash_table) {} + ~Scanner() = default; + + void open_scan() override; + + RC next(Chunk &chunk) override; + + void close_scan() override; + + private: + int capacity_ = -1; + int size_ = -1; + int scan_pos_ = -1; + int scan_count_ = 0; + }; + + LinearProbingAggregateHashTable(AggregateExpr::Type aggregate_type, int capacity = DEFAULT_CAPACITY) + : keys_(capacity, EMPTY_KEY), values_(capacity, 0), capacity_(capacity), aggregate_type_(aggregate_type) + {} + virtual ~LinearProbingAggregateHashTable() {} + + RC get(int key, V &value); + + RC iter_get(int pos, int &key, V &value); + + RC add_chunk(Chunk &group_chunk, Chunk &aggr_chunk) override; + + int capacity() { return capacity_; } + int size() { return size_; } + +private: + /** + * @brief 将键值对以批量的形式写入哈希表中,这里参考了论文 + * `Rethinking SIMD Vectorization for In-Memory Databases` 中的 `Algorithm 5`。 + * @param input_keys 输入的键数组 + * @param input_values 输入的值数组,与键数组一一对应。 + * @param len 键值对数组的长度 + */ + void add_batch(int *input_keys, V *input_values, int len); + + void aggregate(V *value, V value_to_aggregate); + + void resize(); + + void resize_if_need(); + +private: + static const int EMPTY_KEY; + static const int DEFAULT_CAPACITY; + + std::vector keys_; + std::vector values_; + int size_ = 0; + int capacity_ = 0; + AggregateExpr::Type aggregate_type_; +}; +#endif \ No newline at end of file diff --git a/src/observer/sql/expr/aggregate_state.cpp b/src/observer/sql/expr/aggregate_state.cpp new file mode 100644 index 000000000..124cb277d --- /dev/null +++ b/src/observer/sql/expr/aggregate_state.cpp @@ -0,0 +1,35 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include + +#include "sql/expr/aggregate_state.h" + +#ifdef USE_SIMD +#include "common/math/simd_util.h" +#endif +template +void SumState::update(const T *values, int size) +{ +#ifdef USE_SIMD + if constexpr (std::is_same::value) { + value += mm256_sum_ps(values, size); + } else if constexpr (std::is_same::value) { + value += mm256_sum_epi32(values, size); + } +#else + for (int i = 0; i < size; ++i) { + value += values[i]; + } +#endif +} + +template class SumState; +template class SumState; \ No newline at end of file diff --git a/src/observer/sql/expr/aggregate_state.h b/src/observer/sql/expr/aggregate_state.h new file mode 100644 index 000000000..785ae3379 --- /dev/null +++ b/src/observer/sql/expr/aggregate_state.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +template +class SumState +{ +public: + SumState() : value(0) {} + T value; + void update(const T *values, int size); +}; \ No newline at end of file diff --git a/src/observer/sql/expr/aggregator.cpp b/src/observer/sql/expr/aggregator.cpp new file mode 100644 index 000000000..35d8573ca --- /dev/null +++ b/src/observer/sql/expr/aggregator.cpp @@ -0,0 +1,47 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/05/29. +// + +#include "sql/expr/aggregator.h" +#include "common/log/log.h" + +RC SumAggregator::accumulate(const Value &value) +{ + if (value_.attr_type() == AttrType::UNDEFINED) { + value_ = value; + return RC::SUCCESS; + } + + ASSERT(value.attr_type() == value_.attr_type(), "type mismatch. value type: %s, value_.type: %s", + attr_type_to_string(value.attr_type()), attr_type_to_string(value_.attr_type())); + + switch (value.attr_type()) + { + case AttrType::INTS: { + value_.set_int(value.get_int() + value_.get_int()); + } break; + case AttrType::FLOATS: { + value_.set_float(value.get_float() + value_.get_float()); + } break; + default: { + return RC::INTERNAL; + } + } + return RC::SUCCESS; +} + +RC SumAggregator::evaluate(Value& result) +{ + result = value_; + return RC::SUCCESS; +} diff --git a/src/observer/sql/expr/aggregator.h b/src/observer/sql/expr/aggregator.h new file mode 100644 index 000000000..3b3ac74c1 --- /dev/null +++ b/src/observer/sql/expr/aggregator.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/05/29. +// + +#pragma once + +#include "sql/parser/value.h" +#include "common/rc.h" + +class Aggregator +{ +public: + virtual ~Aggregator() = default; + + virtual RC accumulate(const Value &value) = 0; + virtual RC evaluate(Value &result) = 0; + +protected: + Value value_; +}; + +class SumAggregator : public Aggregator +{ +public: + RC accumulate(const Value &value) override; + RC evaluate(Value &result) override; +}; \ No newline at end of file diff --git a/src/observer/sql/expr/arithmetic_operator.hpp b/src/observer/sql/expr/arithmetic_operator.hpp new file mode 100644 index 000000000..2b15e2ef2 --- /dev/null +++ b/src/observer/sql/expr/arithmetic_operator.hpp @@ -0,0 +1,374 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#if defined(USE_SIMD) +#include "common/math/simd_util.h" +#endif + +#include "storage/common/column.h" + +struct Equal +{ + template + static inline bool operation(const T &left, const T &right) + { + return left == right; + } +#if defined(USE_SIMD) + static inline __m256 operation(const __m256 &left, const __m256 &right) + { + return _mm256_cmp_ps(left, right, _CMP_EQ_OS); + } + + static inline __m256i operation(const __m256i &left, const __m256i &right) { return _mm256_cmpeq_epi32(left, right); } +#endif +}; +struct NotEqual +{ + template + static inline bool operation(const T &left, const T &right) + { + return left != right; + } +#if defined(USE_SIMD) + static inline __m256 operation(const __m256 &left, const __m256 &right) + { + return _mm256_cmp_ps(left, right, _CMP_NEQ_OS); + } + + static inline __m256i operation(const __m256i &left, const __m256i &right) + { + return _mm256_xor_si256(_mm256_set1_epi32(-1), _mm256_cmpeq_epi32(left, right)); + } +#endif +}; + +struct GreatThan +{ + template + static inline bool operation(const T &left, const T &right) + { + return left > right; + } +#if defined(USE_SIMD) + static inline __m256 operation(const __m256 &left, const __m256 &right) + { + return _mm256_cmp_ps(left, right, _CMP_GT_OS); + } + + static inline __m256i operation(const __m256i &left, const __m256i &right) { return _mm256_cmpgt_epi32(left, right); } +#endif +}; + +struct GreatEqual +{ + template + static inline bool operation(const T &left, const T &right) + { + return left >= right; + } + +#if defined(USE_SIMD) + static inline __m256 operation(const __m256 &left, const __m256 &right) + { + return _mm256_cmp_ps(left, right, _CMP_GE_OS); + } + + static inline __m256i operation(const __m256i &left, const __m256i &right) + { + return _mm256_cmpgt_epi32(left, right) | _mm256_cmpeq_epi32(left, right); + } +#endif +}; + +struct LessThan +{ + template + static inline bool operation(const T &left, const T &right) + { + return left < right; + } +#if defined(USE_SIMD) + static inline __m256 operation(const __m256 &left, const __m256 &right) + { + return _mm256_cmp_ps(left, right, _CMP_LT_OS); + } + + static inline __m256i operation(const __m256i &left, const __m256i &right) { return _mm256_cmpgt_epi32(right, left); } +#endif +}; + +struct LessEqual +{ + template + static inline bool operation(const T &left, const T &right) + { + return left <= right; + } +#if defined(USE_SIMD) + static inline __m256 operation(const __m256 &left, const __m256 &right) + { + return _mm256_cmp_ps(left, right, _CMP_LE_OS); + } + + static inline __m256i operation(const __m256i &left, const __m256i &right) + { + return _mm256_or_si256(_mm256_cmpgt_epi32(right, left), _mm256_cmpeq_epi32(left, right)); + } +#endif +}; + +struct AddOperator +{ + template + static inline T operation(T left, T right) + { + return left + right; + } + +#if defined(USE_SIMD) + static inline __m256 operation(__m256 left, __m256 right) { return _mm256_add_ps(left, right); } + + static inline __m256i operation(__m256i left, __m256i right) { return _mm256_add_epi32(left, right); } +#endif +}; + +struct SubtractOperator +{ + template + static inline T operation(T left, T right) + { + return left - right; + } + // your code here +#if defined(USE_SIMD) + static inline __m256 operation(__m256 left, __m256 right) { exit(-1); } + + static inline __m256i operation(__m256i left, __m256i right) { exit(-1); } +#endif +}; + +struct MultiplyOperator +{ + template + static inline T operation(T left, T right) + { + return left * right; + } +// your code here +#if defined(USE_SIMD) + static inline __m256 operation(__m256 left, __m256 right) { exit(-1); } + + static inline __m256i operation(__m256i left, __m256i right) { exit(-1); } +#endif +}; + +struct DivideOperator +{ + template + static inline T operation(T left, T right) + { + // TODO: `right = 0` is invalid + return left / right; + } + +#if defined(USE_SIMD) + static inline __m256 operation(__m256 left, __m256 right) { return _mm256_div_ps(left, right); } + static inline __m256i operation(__m256i left, __m256i right) + { + + __m256 left_float = _mm256_cvtepi32_ps(left); + __m256 right_float = _mm256_cvtepi32_ps(right); + __m256 result_float = _mm256_div_ps(left_float, right_float); + return _mm256_cvttps_epi32(result_float); + ; + } +#endif +}; + +struct NegateOperator +{ + template + static inline T operation(T input) + { + return -input; + } +}; + +template +void compare_operation(T *left, T *right, int n, std::vector &result) +{ +#if defined(USE_SIMD) + int i = 0; + if constexpr (std::is_same::value) { + for (; i <= n - SIMD_WIDTH; i += SIMD_WIDTH) { + __m256 left_value, right_value; + + if constexpr (LEFT_CONSTANT) { + left_value = _mm256_set1_ps(left[0]); + } else { + left_value = _mm256_loadu_ps(&left[i]); + } + + if constexpr (RIGHT_CONSTANT) { + right_value = _mm256_set1_ps(right[0]); + } else { + right_value = _mm256_loadu_ps(&right[i]); + } + + __m256 result_values = OP::operation(left_value, right_value); + __m256i mask = _mm256_castps_si256(result_values); + + for (int j = 0; j < SIMD_WIDTH; j++) { + result[i + j] &= mm256_extract_epi32_var_indx(mask, j) ? 1 : 0; + } + } + } else if constexpr (std::is_same::value) { + for (; i <= n - SIMD_WIDTH; i += SIMD_WIDTH) { + __m256i left_value, right_value; + + if (LEFT_CONSTANT) { + left_value = _mm256_set1_epi32(left[0]); + } else { + left_value = _mm256_loadu_si256((__m256i *)&left[i]); + } + + if (RIGHT_CONSTANT) { + right_value = _mm256_set1_epi32(right[0]); + } else { + right_value = _mm256_loadu_si256((__m256i *)&right[i]); + } + + __m256i result_values = OP::operation(left_value, right_value); + + for (int j = 0; j < SIMD_WIDTH; j++) { + result[i + j] &= mm256_extract_epi32_var_indx(result_values, j) ? 1 : 0; + } + } + } + + for (; i < n; i++) { + auto &left_value = left[LEFT_CONSTANT ? 0 : i]; + auto &right_value = right[RIGHT_CONSTANT ? 0 : i]; + result[i] &= OP::operation(left_value, right_value) ? 1 : 0; + } +#else + for (int i = 0; i < n; i++) { + auto &left_value = left[LEFT_CONSTANT ? 0 : i]; + auto &right_value = right[RIGHT_CONSTANT ? 0 : i]; + result[i] &= OP::operation(left_value, right_value) ? 1 : 0; + } +#endif +} + +template +void binary_operator(T *left_data, T *right_data, T *result_data, int size) +{ +#if defined(USE_SIMD) + int i = 0; + + if constexpr (std::is_same::value) { + for (; i <= size - SIMD_WIDTH; i += SIMD_WIDTH) { + __m256 left_value, right_value; + + if (LEFT_CONSTANT) { + left_value = _mm256_set1_ps(left_data[0]); + } else { + left_value = _mm256_loadu_ps(&left_data[i]); + } + + if (RIGHT_CONSTANT) { + right_value = _mm256_set1_ps(right_data[0]); + } else { + right_value = _mm256_loadu_ps(&right_data[i]); + } + + __m256 result_value = OP::operation(left_value, right_value); + _mm256_storeu_ps(&result_data[i], result_value); + } + } else if constexpr (std::is_same::value) { + for (; i <= size - SIMD_WIDTH; i += SIMD_WIDTH) { + __m256i left_value, right_value; + + if (LEFT_CONSTANT) { + left_value = _mm256_set1_epi32(left_data[0]); + } else { + left_value = _mm256_loadu_si256((const __m256i *)&left_data[i]); + } + + if (RIGHT_CONSTANT) { + right_value = _mm256_set1_epi32(right_data[0]); + } else { + right_value = _mm256_loadu_si256((const __m256i *)&right_data[i]); + } + + __m256i result_value = OP::operation(left_value, right_value); + _mm256_storeu_si256((__m256i *)&result_data[i], result_value); + } + } + + // 处理剩余未对齐的数据 + for (; i < size; i++) { + auto &left_value = left_data[LEFT_CONSTANT ? 0 : i]; + auto &right_value = right_data[RIGHT_CONSTANT ? 0 : i]; + result_data[i] = OP::template operation(left_value, right_value); + } +#else + for (int i = 0; i < size; i++) { + auto &left_value = left_data[LEFT_CONSTANT ? 0 : i]; + auto &right_value = right_data[RIGHT_CONSTANT ? 0 : i]; + result_data[i] = OP::template operation(left_value, right_value); + } +#endif +} + +template +void unary_operator(T *input, T *result_data, int size) +{ + for (int i = 0; i < size; i++) { + auto &value = input[CONSTANT ? 0 : i]; + result_data[i] = OP::template operation(value); + } +} + +// TODO: optimized with simd +template +void compare_result(T *left, T *right, int n, std::vector &result, CompOp op) +{ + switch (op) { + case CompOp::EQUAL_TO: { + compare_operation(left, right, n, result); + break; + } + case CompOp::NOT_EQUAL: { + compare_operation(left, right, n, result); + break; + } + case CompOp::GREAT_EQUAL: { + compare_operation(left, right, n, result); + break; + } + case CompOp::GREAT_THAN: { + compare_operation(left, right, n, result); + break; + } + case CompOp::LESS_EQUAL: { + compare_operation(left, right, n, result); + break; + } + case CompOp::LESS_THAN: { + compare_operation(left, right, n, result); + break; + } + default: break; + } +} \ No newline at end of file diff --git a/src/observer/sql/expr/composite_tuple.cpp b/src/observer/sql/expr/composite_tuple.cpp new file mode 100644 index 000000000..40ff81284 --- /dev/null +++ b/src/observer/sql/expr/composite_tuple.cpp @@ -0,0 +1,71 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/5/31. +// + +#include "sql/expr/composite_tuple.h" +#include "common/log/log.h" + +using namespace std; + +int CompositeTuple::cell_num() const +{ + int num = 0; + for (const auto &tuple : tuples_) { + num += tuple->cell_num(); + } + return num; +} + +RC CompositeTuple::cell_at(int index, Value &cell) const +{ + for (const auto &tuple : tuples_) { + if (index < tuple->cell_num()) { + return tuple->cell_at(index, cell); + } else { + index -= tuple->cell_num(); + } + } + return RC::NOTFOUND; +} + +RC CompositeTuple::spec_at(int index, TupleCellSpec &spec) const +{ + for (const auto &tuple : tuples_) { + if (index < tuple->cell_num()) { + return tuple->spec_at(index, spec); + } else { + index -= tuple->cell_num(); + } + } + return RC::NOTFOUND; +} + +RC CompositeTuple::find_cell(const TupleCellSpec &spec, Value &cell) const +{ + RC rc = RC::NOTFOUND; + for (const auto &tuple : tuples_) { + rc = tuple->find_cell(spec, cell); + if (OB_SUCC(rc)) { + break; + } + } + return rc; +} + +void CompositeTuple::add_tuple(unique_ptr tuple) { tuples_.push_back(std::move(tuple)); } + +Tuple &CompositeTuple::tuple_at(size_t index) +{ + ASSERT(index < tuples_.size(), "index=%d, tuples_size=%d", index, tuples_.size()); + return *tuples_[index]; +} \ No newline at end of file diff --git a/src/observer/sql/expr/composite_tuple.h b/src/observer/sql/expr/composite_tuple.h new file mode 100644 index 000000000..d3e655776 --- /dev/null +++ b/src/observer/sql/expr/composite_tuple.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/5/31. +// + +#pragma once + +#include "sql/expr/tuple.h" + +/** + * @brief 组合的Tuple + * @ingroup Tuple + * TODO 单元测试 + */ +class CompositeTuple : public Tuple +{ +public: + CompositeTuple() = default; + virtual ~CompositeTuple() = default; + + /// @brief 删除默认构造函数 + CompositeTuple(const CompositeTuple &) = delete; + /// @brief 删除默认赋值函数 + CompositeTuple &operator=(const CompositeTuple &) = delete; + + /// @brief 保留移动构造函数 + CompositeTuple(CompositeTuple &&) = default; + /// @brief 保留移动赋值函数 + CompositeTuple &operator=(CompositeTuple &&) = default; + + int cell_num() const override; + RC cell_at(int index, Value &cell) const override; + RC spec_at(int index, TupleCellSpec &spec) const override; + RC find_cell(const TupleCellSpec &spec, Value &cell) const override; + + void add_tuple(std::unique_ptr tuple); + Tuple &tuple_at(size_t index); + +private: + std::vector> tuples_; +}; \ No newline at end of file diff --git a/src/observer/sql/expr/expression.cpp b/src/observer/sql/expr/expression.cpp index aa13116ff..0d5b45333 100644 --- a/src/observer/sql/expr/expression.cpp +++ b/src/observer/sql/expr/expression.cpp @@ -14,6 +14,7 @@ See the Mulan PSL v2 for more details. */ #include "sql/expr/expression.h" #include "sql/expr/tuple.h" +#include "sql/expr/arithmetic_operator.hpp" using namespace std; @@ -22,12 +23,54 @@ RC FieldExpr::get_value(const Tuple &tuple, Value &value) const return tuple.find_cell(TupleCellSpec(table_name(), field_name()), value); } +bool FieldExpr::equal(const Expression &other) const +{ + if (this == &other) { + return true; + } + if (other.type() != ExprType::FIELD) { + return false; + } + const auto &other_field_expr = static_cast(other); + return table_name() == other_field_expr.table_name() && field_name() == other_field_expr.field_name(); +} + +// TODO: 在进行表达式计算时,`chunk` 包含了所有列,因此可以通过 `field_id` 获取到对应列。 +// 后续可以优化成在 `FieldExpr` 中存储 `chunk` 中某列的位置信息。 +RC FieldExpr::get_column(Chunk &chunk, Column &column) +{ + if (pos_ != -1) { + column.reference(chunk.column(pos_)); + } else { + column.reference(chunk.column(field().meta()->field_id())); + } + return RC::SUCCESS; +} + +bool ValueExpr::equal(const Expression &other) const +{ + if (this == &other) { + return true; + } + if (other.type() != ExprType::VALUE) { + return false; + } + const auto &other_value_expr = static_cast(other); + return value_.compare(other_value_expr.get_value()) == 0; +} + RC ValueExpr::get_value(const Tuple &tuple, Value &value) const { value = value_; return RC::SUCCESS; } +RC ValueExpr::get_column(Chunk &chunk, Column &column) +{ + column.init(value_); + return RC::SUCCESS; +} + ///////////////////////////////////////////////////////////////////////////////// CastExpr::CastExpr(unique_ptr child, AttrType cast_type) : child_(std::move(child)), cast_type_(cast_type) {} @@ -43,7 +86,7 @@ RC CastExpr::cast(const Value &value, Value &cast_value) const } switch (cast_type_) { - case BOOLEANS: { + case AttrType::BOOLEANS: { bool val = value.get_boolean(); cast_value.set_boolean(val); } break; @@ -119,8 +162,8 @@ RC ComparisonExpr::compare_value(const Value &left, const Value &right, bool &re RC ComparisonExpr::try_get_value(Value &cell) const { if (left_->type() == ExprType::VALUE && right_->type() == ExprType::VALUE) { - ValueExpr *left_value_expr = static_cast(left_.get()); - ValueExpr *right_value_expr = static_cast(right_.get()); + ValueExpr * left_value_expr = static_cast(left_.get()); + ValueExpr * right_value_expr = static_cast(right_.get()); const Value &left_cell = left_value_expr->get_value(); const Value &right_cell = right_value_expr->get_value(); @@ -162,6 +205,57 @@ RC ComparisonExpr::get_value(const Tuple &tuple, Value &value) const return rc; } +RC ComparisonExpr::eval(Chunk &chunk, std::vector &select) +{ + RC rc = RC::SUCCESS; + Column left_column; + Column right_column; + + rc = left_->get_column(chunk, left_column); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to get value of left expression. rc=%s", strrc(rc)); + return rc; + } + rc = right_->get_column(chunk, right_column); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to get value of right expression. rc=%s", strrc(rc)); + return rc; + } + if (left_column.attr_type() != right_column.attr_type()) { + LOG_WARN("cannot compare columns with different types"); + return RC::INTERNAL; + } + if (left_column.attr_type() == AttrType::INTS) { + rc = compare_column(left_column, right_column, select); + } else if (left_column.attr_type() == AttrType::FLOATS) { + rc = compare_column(left_column, right_column, select); + } else { + // TODO: support string compare + LOG_WARN("unsupported data type %d", left_column.attr_type()); + return RC::INTERNAL; + } + return rc; +} + +template +RC ComparisonExpr::compare_column(const Column &left, const Column &right, std::vector &result) const +{ + RC rc = RC::SUCCESS; + + bool left_const = left.column_type() == Column::Type::CONSTANT_COLUMN; + bool right_const = right.column_type() == Column::Type::CONSTANT_COLUMN; + if (left_const && right_const) { + compare_result((T *)left.data(), (T *)right.data(), left.count(), result, comp_); + } else if (left_const && !right_const) { + compare_result((T *)left.data(), (T *)right.data(), right.count(), result, comp_); + } else if (!left_const && right_const) { + compare_result((T *)left.data(), (T *)right.data(), left.count(), result, comp_); + } else { + compare_result((T *)left.data(), (T *)right.data(), left.count(), result, comp_); + } + return rc; +} + //////////////////////////////////////////////////////////////////////////////// ConjunctionExpr::ConjunctionExpr(Type type, vector> &children) : conjunction_type_(type), children_(std::move(children)) @@ -203,6 +297,18 @@ ArithmeticExpr::ArithmeticExpr(ArithmeticExpr::Type type, unique_ptr : arithmetic_type_(type), left_(std::move(left)), right_(std::move(right)) {} +bool ArithmeticExpr::equal(const Expression &other) const +{ + if (this == &other) { + return true; + } + if (type() != other.type()) { + return false; + } + auto &other_arith_expr = static_cast(other); + return arithmetic_type_ == other_arith_expr.arithmetic_type() && left_->equal(*other_arith_expr.left_) && + right_->equal(*other_arith_expr.right_); +} AttrType ArithmeticExpr::value_type() const { if (!right_) { @@ -284,6 +390,74 @@ RC ArithmeticExpr::calc_value(const Value &left_value, const Value &right_value, return rc; } +template +RC ArithmeticExpr::execute_calc( + const Column &left, const Column &right, Column &result, Type type, AttrType attr_type) const +{ + RC rc = RC::SUCCESS; + switch (type) { + case Type::ADD: { + if (attr_type == AttrType::INTS) { + binary_operator( + (int *)left.data(), (int *)right.data(), (int *)result.data(), result.capacity()); + } else if (attr_type == AttrType::FLOATS) { + binary_operator( + (float *)left.data(), (float *)right.data(), (float *)result.data(), result.capacity()); + } else { + rc = RC::UNIMPLENMENT; + } + } break; + case Type::SUB: + if (attr_type == AttrType::INTS) { + binary_operator( + (int *)left.data(), (int *)right.data(), (int *)result.data(), result.capacity()); + } else if (attr_type == AttrType::FLOATS) { + binary_operator( + (float *)left.data(), (float *)right.data(), (float *)result.data(), result.capacity()); + } else { + rc = RC::UNIMPLENMENT; + } + break; + case Type::MUL: + if (attr_type == AttrType::INTS) { + binary_operator( + (int *)left.data(), (int *)right.data(), (int *)result.data(), result.capacity()); + } else if (attr_type == AttrType::FLOATS) { + binary_operator( + (float *)left.data(), (float *)right.data(), (float *)result.data(), result.capacity()); + } else { + rc = RC::UNIMPLENMENT; + } + break; + case Type::DIV: + if (attr_type == AttrType::INTS) { + binary_operator( + (int *)left.data(), (int *)right.data(), (int *)result.data(), result.capacity()); + } else if (attr_type == AttrType::FLOATS) { + binary_operator( + (float *)left.data(), (float *)right.data(), (float *)result.data(), result.capacity()); + } else { + rc = RC::UNIMPLENMENT; + } + break; + case Type::NEGATIVE: + if (attr_type == AttrType::INTS) { + unary_operator((int *)left.data(), (int *)result.data(), result.capacity()); + } else if (attr_type == AttrType::FLOATS) { + unary_operator( + (float *)left.data(), (float *)result.data(), result.capacity()); + } else { + rc = RC::UNIMPLENMENT; + } + break; + default: rc = RC::UNIMPLENMENT; break; + } + if (rc == RC::SUCCESS) { + result.set_count(result.capacity()); + } + return rc; +} + RC ArithmeticExpr::get_value(const Tuple &tuple, Value &value) const { RC rc = RC::SUCCESS; @@ -304,6 +478,53 @@ RC ArithmeticExpr::get_value(const Tuple &tuple, Value &value) const return calc_value(left_value, right_value, value); } +RC ArithmeticExpr::get_column(Chunk &chunk, Column &column) +{ + RC rc = RC::SUCCESS; + if (pos_ != -1) { + column.reference(chunk.column(pos_)); + return rc; + } + Column left_column; + Column right_column; + + rc = left_->get_column(chunk, left_column); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to get column of left expression. rc=%s", strrc(rc)); + return rc; + } + rc = right_->get_column(chunk, right_column); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to get column of right expression. rc=%s", strrc(rc)); + return rc; + } + return calc_column(left_column, right_column, column); +} + +RC ArithmeticExpr::calc_column(const Column &left_column, const Column &right_column, Column &column) const +{ + RC rc = RC::SUCCESS; + + const AttrType target_type = value_type(); + column.init(target_type, left_column.attr_len(), std::max(left_column.count(), right_column.count())); + bool left_const = left_column.column_type() == Column::Type::CONSTANT_COLUMN; + bool right_const = right_column.column_type() == Column::Type::CONSTANT_COLUMN; + if (left_const && right_const) { + column.set_column_type(Column::Type::CONSTANT_COLUMN); + rc = execute_calc(left_column, right_column, column, arithmetic_type_, target_type); + } else if (left_const && !right_const) { + column.set_column_type(Column::Type::NORMAL_COLUMN); + rc = execute_calc(left_column, right_column, column, arithmetic_type_, target_type); + } else if (!left_const && right_const) { + column.set_column_type(Column::Type::NORMAL_COLUMN); + rc = execute_calc(left_column, right_column, column, arithmetic_type_, target_type); + } else { + column.set_column_type(Column::Type::NORMAL_COLUMN); + rc = execute_calc(left_column, right_column, column, arithmetic_type_, target_type); + } + return rc; +} + RC ArithmeticExpr::try_get_value(Value &value) const { RC rc = RC::SUCCESS; @@ -326,4 +547,79 @@ RC ArithmeticExpr::try_get_value(Value &value) const } return calc_value(left_value, right_value, value); +} + +//////////////////////////////////////////////////////////////////////////////// + +UnboundAggregateExpr::UnboundAggregateExpr(const char *aggregate_name, Expression *child) + : aggregate_name_(aggregate_name), child_(child) +{} + +//////////////////////////////////////////////////////////////////////////////// +AggregateExpr::AggregateExpr(Type type, Expression *child) : aggregate_type_(type), child_(child) {} + +AggregateExpr::AggregateExpr(Type type, unique_ptr child) : aggregate_type_(type), child_(std::move(child)) +{} + +RC AggregateExpr::get_column(Chunk &chunk, Column &column) +{ + RC rc = RC::SUCCESS; + if (pos_ != -1) { + column.reference(chunk.column(pos_)); + } else { + rc = RC::INTERNAL; + } + return rc; +} + +bool AggregateExpr::equal(const Expression &other) const +{ + if (this == &other) { + return true; + } + if (other.type() != type()) { + return false; + } + const AggregateExpr &other_aggr_expr = static_cast(other); + return aggregate_type_ == other_aggr_expr.aggregate_type() && child_->equal(*other_aggr_expr.child()); +} + +unique_ptr AggregateExpr::create_aggregator() const +{ + unique_ptr aggregator; + switch (aggregate_type_) { + case Type::SUM: { + aggregator = make_unique(); + break; + } + default: { + ASSERT(false, "unsupported aggregate type"); + break; + } + } + return aggregator; +} + +RC AggregateExpr::get_value(const Tuple &tuple, Value &value) const +{ + return tuple.find_cell(TupleCellSpec(name()), value); +} + +RC AggregateExpr::type_from_string(const char *type_str, AggregateExpr::Type &type) +{ + RC rc = RC::SUCCESS; + if (0 == strcasecmp(type_str, "count")) { + type = Type::COUNT; + } else if (0 == strcasecmp(type_str, "sum")) { + type = Type::SUM; + } else if (0 == strcasecmp(type_str, "avg")) { + type = Type::AVG; + } else if (0 == strcasecmp(type_str, "max")) { + type = Type::MAX; + } else if (0 == strcasecmp(type_str, "min")) { + type = Type::MIN; + } else { + rc = RC::INVALID_ARGUMENT; + } + return rc; } \ No newline at end of file diff --git a/src/observer/sql/expr/expression.h b/src/observer/sql/expr/expression.h index ace5e0a67..278b1a78b 100644 --- a/src/observer/sql/expr/expression.h +++ b/src/observer/sql/expr/expression.h @@ -19,6 +19,8 @@ See the Mulan PSL v2 for more details. */ #include "sql/parser/value.h" #include "storage/field/field.h" +#include "sql/expr/aggregator.h" +#include "storage/common/chunk.h" class Tuple; @@ -34,13 +36,17 @@ class Tuple; enum class ExprType { NONE, - STAR, ///< 星号,表示所有字段 + STAR, ///< 星号,表示所有字段 + UNBOUND_FIELD, ///< 未绑定的字段,需要在resolver阶段解析为FieldExpr + UNBOUND_AGGREGATION, ///< 未绑定的聚合函数,需要在resolver阶段解析为AggregateExpr + FIELD, ///< 字段。在实际执行时,根据行数据内容提取对应字段的值 VALUE, ///< 常量值 CAST, ///< 需要做类型转换的表达式 COMPARISON, ///< 需要做比较的表达式 CONJUNCTION, ///< 多个表达式使用同一种关系(AND或OR)来联结 ARITHMETIC, ///< 算术运算 + AGGREGATION, ///< 聚合运算 }; /** @@ -53,6 +59,8 @@ enum class ExprType * 通常表达式的值,是在真实的算子运算过程中,拿到具体的tuple后 * 才能计算出来真实的值。但是有些表达式可能就表示某一个固定的 * 值,比如ValueExpr。 + * + * TODO 区分unbound和bound的表达式 */ class Expression { @@ -60,6 +68,10 @@ class Expression Expression() = default; virtual ~Expression() = default; + /** + * @brief 判断两个表达式是否相等 + */ + virtual bool equal(const Expression &other) const { return false; } /** * @brief 根据具体的tuple,来计算当前表达式的值。tuple有可能是一个具体某个表的行数据 */ @@ -71,6 +83,11 @@ class Expression */ virtual RC try_get_value(Value &value) const { return RC::UNIMPLENMENT; } + /** + * @brief 从 `chunk` 中获取表达式的计算结果 `column` + */ + virtual RC get_column(Chunk &chunk, Column &column) { return RC::UNIMPLENMENT; } + /** * @brief 表达式的类型 * 可以根据表达式类型来转换为具体的子类 @@ -83,16 +100,81 @@ class Expression */ virtual AttrType value_type() const = 0; + /** + * @brief 表达式值的长度 + */ + virtual int value_length() const { return -1; } + /** * @brief 表达式的名字,比如是字段名称,或者用户在执行SQL语句时输入的内容 */ - virtual std::string name() const { return name_; } + virtual const char *name() const { return name_.c_str(); } virtual void set_name(std::string name) { name_ = name; } + /** + * @brief 表达式在下层算子返回的 chunk 中的位置 + */ + virtual int pos() const { return pos_; } + virtual void set_pos(int pos) { pos_ = pos; } + + /** + * @brief 用于 ComparisonExpr 获得比较结果 `select`。 + */ + virtual RC eval(Chunk &chunk, std::vector &select) { return RC::UNIMPLENMENT; } + +protected: + /** + * @brief 表达式在下层算子返回的 chunk 中的位置 + * @details 当 pos_ = -1 时表示下层算子没有在返回的 chunk 中计算出该表达式的计算结果, + * 当 pos_ >= 0时表示在下层算子中已经计算出该表达式的值(比如聚合表达式),且该表达式对应的结果位于 + * chunk 中 下标为 pos_ 的列中。 + */ + int pos_ = -1; + private: std::string name_; }; +class StarExpr : public Expression +{ +public: + StarExpr() : table_name_() {} + StarExpr(const char *table_name) : table_name_(table_name) {} + virtual ~StarExpr() = default; + + ExprType type() const override { return ExprType::STAR; } + AttrType value_type() const override { return AttrType::UNDEFINED; } + + RC get_value(const Tuple &tuple, Value &value) const override { return RC::UNIMPLENMENT; } // 不需要实现 + + const char *table_name() const { return table_name_.c_str(); } + +private: + std::string table_name_; +}; + +class UnboundFieldExpr : public Expression +{ +public: + UnboundFieldExpr(const std::string &table_name, const std::string &field_name) + : table_name_(table_name), field_name_(field_name) + {} + + virtual ~UnboundFieldExpr() = default; + + ExprType type() const override { return ExprType::UNBOUND_FIELD; } + AttrType value_type() const override { return AttrType::UNDEFINED; } + + RC get_value(const Tuple &tuple, Value &value) const override { return RC::INTERNAL; } + + const char *table_name() const { return table_name_.c_str(); } + const char *field_name() const { return field_name_.c_str(); } + +private: + std::string table_name_; + std::string field_name_; +}; + /** * @brief 字段表达式 * @ingroup Expression @@ -106,8 +188,11 @@ class FieldExpr : public Expression virtual ~FieldExpr() = default; + bool equal(const Expression &other) const override; + ExprType type() const override { return ExprType::FIELD; } AttrType value_type() const override { return field_.attr_type(); } + int value_length() const override { return field_.meta()->len(); } Field &field() { return field_; } @@ -116,6 +201,8 @@ class FieldExpr : public Expression const char *table_name() const { return field_.table_name(); } const char *field_name() const { return field_.field_name(); } + RC get_column(Chunk &chunk, Column &column) override; + RC get_value(const Tuple &tuple, Value &value) const override; private: @@ -134,7 +221,10 @@ class ValueExpr : public Expression virtual ~ValueExpr() = default; + bool equal(const Expression &other) const override; + RC get_value(const Tuple &tuple, Value &value) const override; + RC get_column(Chunk &chunk, Column &column) override; RC try_get_value(Value &value) const override { value = value_; @@ -143,6 +233,7 @@ class ValueExpr : public Expression ExprType type() const override { return ExprType::VALUE; } AttrType value_type() const override { return value_.attr_type(); } + int value_length() const override { return value_.length(); } void get_value(Value &value) const { value = value_; } const Value &get_value() const { return value_; } @@ -162,7 +253,8 @@ class CastExpr : public Expression virtual ~CastExpr(); ExprType type() const override { return ExprType::CAST; } - RC get_value(const Tuple &tuple, Value &value) const override; + + RC get_value(const Tuple &tuple, Value &value) const override; RC try_get_value(Value &value) const override; @@ -190,9 +282,15 @@ class ComparisonExpr : public Expression ExprType type() const override { return ExprType::COMPARISON; } RC get_value(const Tuple &tuple, Value &value) const override; - AttrType value_type() const override { return BOOLEANS; } + AttrType value_type() const override { return AttrType::BOOLEANS; } CompOp comp() const { return comp_; } + /** + * @brief 根据 ComparisonExpr 获得 `select` 结果。 + * select 的长度与chunk 的行数相同,表示每一行在ComparisonExpr 计算后是否会被输出。 + */ + RC eval(Chunk &chunk, std::vector &select) override; + std::unique_ptr &left() { return left_; } std::unique_ptr &right() { return right_; } @@ -208,6 +306,9 @@ class ComparisonExpr : public Expression */ RC compare_value(const Value &left, const Value &right, bool &value) const; + template + RC compare_column(const Column &left, const Column &right, std::vector &result) const; + private: CompOp comp_; std::unique_ptr left_; @@ -234,7 +335,7 @@ class ConjunctionExpr : public Expression virtual ~ConjunctionExpr() = default; ExprType type() const override { return ExprType::CONJUNCTION; } - AttrType value_type() const override { return BOOLEANS; } + AttrType value_type() const override { return AttrType::BOOLEANS; } RC get_value(const Tuple &tuple, Value &value) const override; Type conjunction_type() const { return conjunction_type_; } @@ -267,11 +368,22 @@ class ArithmeticExpr : public Expression ArithmeticExpr(Type type, std::unique_ptr left, std::unique_ptr right); virtual ~ArithmeticExpr() = default; + bool equal(const Expression &other) const override; ExprType type() const override { return ExprType::ARITHMETIC; } AttrType value_type() const override; + int value_length() const override + { + if (!right_) { + return left_->value_length(); + } + return 4; // sizeof(float) or sizeof(int) + }; RC get_value(const Tuple &tuple, Value &value) const override; + + RC get_column(Chunk &chunk, Column &column) override; + RC try_get_value(Value &value) const override; Type arithmetic_type() const { return arithmetic_type_; } @@ -282,8 +394,77 @@ class ArithmeticExpr : public Expression private: RC calc_value(const Value &left_value, const Value &right_value, Value &value) const; + RC calc_column(const Column &left_column, const Column &right_column, Column &column) const; + + template + RC execute_calc(const Column &left, const Column &right, Column &result, Type type, AttrType attr_type) const; + private: Type arithmetic_type_; std::unique_ptr left_; std::unique_ptr right_; +}; + +class UnboundAggregateExpr : public Expression +{ +public: + UnboundAggregateExpr(const char *aggregate_name, Expression *child); + virtual ~UnboundAggregateExpr() = default; + + ExprType type() const override { return ExprType::UNBOUND_AGGREGATION; } + + const char *aggregate_name() const { return aggregate_name_.c_str(); } + + std::unique_ptr &child() { return child_; } + + RC get_value(const Tuple &tuple, Value &value) const override { return RC::INTERNAL; } + AttrType value_type() const override { return child_->value_type(); } + +private: + std::string aggregate_name_; + std::unique_ptr child_; +}; + +class AggregateExpr : public Expression +{ +public: + enum class Type + { + COUNT, + SUM, + AVG, + MAX, + MIN, + }; + +public: + AggregateExpr(Type type, Expression *child); + AggregateExpr(Type type, std::unique_ptr child); + virtual ~AggregateExpr() = default; + + bool equal(const Expression &other) const override; + + ExprType type() const override { return ExprType::AGGREGATION; } + + AttrType value_type() const override { return child_->value_type(); } + int value_length() const override { return child_->value_length(); } + + RC get_value(const Tuple &tuple, Value &value) const override; + + RC get_column(Chunk &chunk, Column &column) override; + + Type aggregate_type() const { return aggregate_type_; } + + std::unique_ptr &child() { return child_; } + + const std::unique_ptr &child() const { return child_; } + + std::unique_ptr create_aggregator() const; + +public: + static RC type_from_string(const char *type_str, Type &type); + +private: + Type aggregate_type_; + std::unique_ptr child_; }; \ No newline at end of file diff --git a/src/observer/sql/expr/expression_iterator.cpp b/src/observer/sql/expr/expression_iterator.cpp new file mode 100644 index 000000000..ca8cd6d5e --- /dev/null +++ b/src/observer/sql/expr/expression_iterator.cpp @@ -0,0 +1,80 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/05/30. +// + +#include "sql/expr/expression_iterator.h" +#include "sql/expr/expression.h" +#include "common/log/log.h" + +using namespace std; + +RC ExpressionIterator::iterate_child_expr(Expression &expr, function &)> callback) +{ + RC rc = RC::SUCCESS; + + switch (expr.type()) { + case ExprType::CAST: { + auto &cast_expr = static_cast(expr); + rc = callback(cast_expr.child()); + } break; + + case ExprType::COMPARISON: { + + auto &comparison_expr = static_cast(expr); + rc = callback(comparison_expr.left()); + + if (OB_SUCC(rc)) { + rc = callback(comparison_expr.right()); + } + + } break; + + case ExprType::CONJUNCTION: { + auto &conjunction_expr = static_cast(expr); + for (auto &child : conjunction_expr.children()) { + rc = callback(child); + if (OB_FAIL(rc)) { + break; + } + } + } break; + + case ExprType::ARITHMETIC: { + + auto &arithmetic_expr = static_cast(expr); + rc = callback(arithmetic_expr.left()); + if (OB_SUCC(rc)) { + rc = callback(arithmetic_expr.right()); + } + } break; + + case ExprType::AGGREGATION: { + auto &aggregate_expr = static_cast(expr); + rc = callback(aggregate_expr.child()); + } break; + + case ExprType::NONE: + case ExprType::STAR: + case ExprType::UNBOUND_FIELD: + case ExprType::FIELD: + case ExprType::VALUE: { + // Do nothing + } break; + + default: { + ASSERT(false, "Unknown expression type"); + } + } + + return rc; +} \ No newline at end of file diff --git a/src/observer/sql/expr/expression_iterator.h b/src/observer/sql/expr/expression_iterator.h new file mode 100644 index 000000000..2ad0298ca --- /dev/null +++ b/src/observer/sql/expr/expression_iterator.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/05/30. +// + +#pragma once + +#include +#include + +#include "common/rc.h" + +class Expression; + +class ExpressionIterator +{ +public: + static RC iterate_child_expr(Expression &expr, std::function &)> callback); +}; \ No newline at end of file diff --git a/src/observer/sql/expr/expression_tuple.h b/src/observer/sql/expr/expression_tuple.h new file mode 100644 index 000000000..36a26aeda --- /dev/null +++ b/src/observer/sql/expr/expression_tuple.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/5/31. +// + +#pragma once + +#include + +#include "sql/expr/tuple.h" +#include "sql/parser/value.h" +#include "common/rc.h" + +template +class ExpressionTuple : public Tuple +{ +public: + ExpressionTuple(const std::vector &expressions) : expressions_(expressions) {} + virtual ~ExpressionTuple() = default; + + void set_tuple(const Tuple *tuple) { child_tuple_ = tuple; } + + int cell_num() const override { return static_cast(expressions_.size()); } + + RC cell_at(int index, Value &cell) const override + { + if (index < 0 || index >= cell_num()) { + return RC::INVALID_ARGUMENT; + } + + const ExprPointerType &expression = expressions_[index]; + return get_value(expression, cell); + } + + RC spec_at(int index, TupleCellSpec &spec) const override + { + if (index < 0 || index >= cell_num()) { + return RC::INVALID_ARGUMENT; + } + + const ExprPointerType &expression = expressions_[index]; + spec = TupleCellSpec(expression->name()); + return RC::SUCCESS; + } + + RC find_cell(const TupleCellSpec &spec, Value &cell) const override + { + RC rc = RC::SUCCESS; + if (child_tuple_ != nullptr) { + rc = child_tuple_->find_cell(spec, cell); + if (OB_SUCC(rc)) { + return rc; + } + } + + rc = RC::NOTFOUND; + for (const ExprPointerType &expression : expressions_) { + if (0 == strcmp(spec.alias(), expression->name())) { + rc = get_value(expression, cell); + break; + } + } + + return rc; + } + +private: + RC get_value(const ExprPointerType &expression, Value &value) const + { + RC rc = RC::SUCCESS; + if (child_tuple_ != nullptr) { + rc = expression->get_value(*child_tuple_, value); + } else { + rc = expression->try_get_value(value); + } + return rc; + } + +private: + const std::vector &expressions_; + const Tuple *child_tuple_ = nullptr; +}; \ No newline at end of file diff --git a/src/observer/sql/expr/tuple.h b/src/observer/sql/expr/tuple.h index 260b80425..7290073ec 100644 --- a/src/observer/sql/expr/tuple.h +++ b/src/observer/sql/expr/tuple.h @@ -41,7 +41,7 @@ class Table; * / \ * Row(t1) Row(t2) * @endcode - * + * TODO 一个类拆分成一个文件,并放到单独的目录中 */ /** @@ -86,6 +86,8 @@ class Tuple */ virtual RC cell_at(int index, Value &cell) const = 0; + virtual RC spec_at(int index, TupleCellSpec &spec) const = 0; + /** * @brief 根据cell的描述,获取cell的值 * @@ -112,6 +114,44 @@ class Tuple } return str; } + + virtual RC compare(const Tuple &other, int &result) const + { + RC rc = RC::SUCCESS; + + const int this_cell_num = this->cell_num(); + const int other_cell_num = other.cell_num(); + if (this_cell_num < other_cell_num) { + result = -1; + return rc; + } + if (this_cell_num > other_cell_num) { + result = 1; + return rc; + } + + Value this_value; + Value other_value; + for (int i = 0; i < this_cell_num; i++) { + rc = this->cell_at(i, this_value); + if (OB_FAIL(rc)) { + return rc; + } + + rc = other.cell_at(i, other_value); + if (OB_FAIL(rc)) { + return rc; + } + + result = this_value.compare(other_value); + if (0 != result) { + return rc; + } + } + + result = 0; + return rc; + } }; /** @@ -161,6 +201,13 @@ class RowTuple : public Tuple return RC::SUCCESS; } + RC spec_at(int index, TupleCellSpec &spec) const override + { + const Field &field = speces_[index]->field(); + spec = TupleCellSpec(table_->name(), field.field_name()); + return RC::SUCCESS; + } + RC find_cell(const TupleCellSpec &spec, Value &cell) const override { const char *table_name = spec.table_name(); @@ -206,36 +253,41 @@ class RowTuple : public Tuple * @ingroup Tuple * @details 一般在select语句中使用。 * 投影也可以是很复杂的操作,比如某些字段需要做类型转换、重命名、表达式运算、函数计算等。 - * 当前的实现是比较简单的,只是选择部分字段,不做任何其他操作。 */ class ProjectTuple : public Tuple { public: - ProjectTuple() = default; - virtual ~ProjectTuple() + ProjectTuple() = default; + virtual ~ProjectTuple() = default; + + void set_expressions(std::vector> &&expressions) { - for (TupleCellSpec *spec : speces_) { - delete spec; - } - speces_.clear(); + expressions_ = std::move(expressions); } + auto get_expressions() const -> const std::vector> & { return expressions_; } + void set_tuple(Tuple *tuple) { this->tuple_ = tuple; } - void add_cell_spec(TupleCellSpec *spec) { speces_.push_back(spec); } - int cell_num() const override { return speces_.size(); } + int cell_num() const override { return static_cast(expressions_.size()); } RC cell_at(int index, Value &cell) const override { - if (index < 0 || index >= static_cast(speces_.size())) { + if (index < 0 || index >= cell_num()) { return RC::INTERNAL; } if (tuple_ == nullptr) { return RC::INTERNAL; } - const TupleCellSpec *spec = speces_[index]; - return tuple_->find_cell(*spec, cell); + Expression *expr = expressions_[index].get(); + return expr->get_value(*tuple_, cell); + } + + RC spec_at(int index, TupleCellSpec &spec) const override + { + spec = TupleCellSpec(expressions_[index]->name()); + return RC::SUCCESS; } RC find_cell(const TupleCellSpec &spec, Value &cell) const override { return tuple_->find_cell(spec, cell); } @@ -251,46 +303,14 @@ class ProjectTuple : public Tuple } #endif private: - std::vector speces_; - Tuple *tuple_ = nullptr; -}; - -class ExpressionTuple : public Tuple -{ -public: - ExpressionTuple(std::vector> &expressions) : expressions_(expressions) {} - - virtual ~ExpressionTuple() {} - - int cell_num() const override { return expressions_.size(); } - - RC cell_at(int index, Value &cell) const override - { - if (index < 0 || index >= static_cast(expressions_.size())) { - return RC::INTERNAL; - } - - const Expression *expr = expressions_[index].get(); - return expr->try_get_value(cell); - } - - RC find_cell(const TupleCellSpec &spec, Value &cell) const override - { - for (const std::unique_ptr &expr : expressions_) { - if (0 == strcmp(spec.alias(), expr->name().c_str())) { - return expr->try_get_value(cell); - } - } - return RC::NOTFOUND; - } - -private: - const std::vector> &expressions_; + std::vector> expressions_; + Tuple *tuple_ = nullptr; }; /** * @brief 一些常量值组成的Tuple * @ingroup Tuple + * TODO 使用单独文件 */ class ValueListTuple : public Tuple { @@ -298,6 +318,7 @@ class ValueListTuple : public Tuple ValueListTuple() = default; virtual ~ValueListTuple() = default; + void set_names(const std::vector &specs) { specs_ = specs; } void set_cells(const std::vector &cells) { cells_ = cells; } virtual int cell_num() const override { return static_cast(cells_.size()); } @@ -312,16 +333,62 @@ class ValueListTuple : public Tuple return RC::SUCCESS; } - virtual RC find_cell(const TupleCellSpec &spec, Value &cell) const override { return RC::INTERNAL; } + RC spec_at(int index, TupleCellSpec &spec) const override + { + if (index < 0 || index >= cell_num()) { + return RC::NOTFOUND; + } + + spec = specs_[index]; + return RC::SUCCESS; + } + + virtual RC find_cell(const TupleCellSpec &spec, Value &cell) const override + { + ASSERT(cells_.size() == specs_.size(), "cells_.size()=%d, specs_.size()=%d", cells_.size(), specs_.size()); + + const int size = static_cast(specs_.size()); + for (int i = 0; i < size; i++) { + if (specs_[i].equals(spec)) { + cell = cells_[i]; + return RC::SUCCESS; + } + } + return RC::NOTFOUND; + } + + static RC make(const Tuple &tuple, ValueListTuple &value_list) + { + const int cell_num = tuple.cell_num(); + for (int i = 0; i < cell_num; i++) { + Value cell; + RC rc = tuple.cell_at(i, cell); + if (OB_FAIL(rc)) { + return rc; + } + + TupleCellSpec spec; + rc = tuple.spec_at(i, spec); + if (OB_FAIL(rc)) { + return rc; + } + + value_list.cells_.push_back(cell); + value_list.specs_.push_back(spec); + } + return RC::SUCCESS; + } private: - std::vector cells_; + std::vector cells_; + std::vector specs_; }; /** * @brief 将两个tuple合并为一个tuple * @ingroup Tuple * @details 在join算子中使用 + * TODO replace with composite tuple */ class JoinedTuple : public Tuple { @@ -337,7 +404,7 @@ class JoinedTuple : public Tuple RC cell_at(int index, Value &value) const override { const int left_cell_num = left_->cell_num(); - if (index > 0 && index < left_cell_num) { + if (index >= 0 && index < left_cell_num) { return left_->cell_at(index, value); } @@ -348,6 +415,20 @@ class JoinedTuple : public Tuple return RC::NOTFOUND; } + RC spec_at(int index, TupleCellSpec &spec) const override + { + const int left_cell_num = left_->cell_num(); + if (index >= 0 && index < left_cell_num) { + return left_->spec_at(index, spec); + } + + if (index >= left_cell_num && index < left_cell_num + right_->cell_num()) { + return right_->spec_at(index - left_cell_num, spec); + } + + return RC::NOTFOUND; + } + RC find_cell(const TupleCellSpec &spec, Value &value) const override { RC rc = left_->find_cell(spec, value); diff --git a/src/observer/sql/expr/tuple_cell.cpp b/src/observer/sql/expr/tuple_cell.cpp index 05a46a9f5..4646643dd 100644 --- a/src/observer/sql/expr/tuple_cell.cpp +++ b/src/observer/sql/expr/tuple_cell.cpp @@ -15,6 +15,8 @@ See the Mulan PSL v2 for more details. */ #include "sql/expr/tuple_cell.h" #include "common/lang/string.h" +using namespace std; + TupleCellSpec::TupleCellSpec(const char *table_name, const char *field_name, const char *alias) { if (table_name) { @@ -40,3 +42,6 @@ TupleCellSpec::TupleCellSpec(const char *alias) alias_ = alias; } } + +TupleCellSpec::TupleCellSpec(const string &alias) : alias_(alias) +{} diff --git a/src/observer/sql/expr/tuple_cell.h b/src/observer/sql/expr/tuple_cell.h index 6143b37c2..b9ba210e9 100644 --- a/src/observer/sql/expr/tuple_cell.h +++ b/src/observer/sql/expr/tuple_cell.h @@ -17,16 +17,23 @@ See the Mulan PSL v2 for more details. */ #include "storage/field/field_meta.h" #include -class TupleCellSpec +class TupleCellSpec final { public: + TupleCellSpec() = default; TupleCellSpec(const char *table_name, const char *field_name, const char *alias = nullptr); - TupleCellSpec(const char *alias); + explicit TupleCellSpec(const char *alias); + explicit TupleCellSpec(const std::string &alias); const char *table_name() const { return table_name_.c_str(); } const char *field_name() const { return field_name_.c_str(); } const char *alias() const { return alias_.c_str(); } + bool equals(const TupleCellSpec &other) const + { + return table_name_ == other.table_name_ && field_name_ == other.field_name_ && alias_ == other.alias_; + } + private: std::string table_name_; std::string field_name_; diff --git a/src/observer/sql/operator/aggregate_vec_physical_operator.cpp b/src/observer/sql/operator/aggregate_vec_physical_operator.cpp new file mode 100644 index 000000000..710c64721 --- /dev/null +++ b/src/observer/sql/operator/aggregate_vec_physical_operator.cpp @@ -0,0 +1,112 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include "common/log/log.h" +#include "sql/operator/aggregate_vec_physical_operator.h" +#include "sql/expr/aggregate_state.h" +#include "sql/expr/expression_tuple.h" +#include "sql/expr/composite_tuple.h" + +using namespace std; +using namespace common; + +AggregateVecPhysicalOperator::AggregateVecPhysicalOperator(vector &&expressions) +{ + aggregate_expressions_ = std::move(expressions); + value_expressions_.reserve(aggregate_expressions_.size()); + + ranges::for_each(aggregate_expressions_, [this](Expression *expr) { + auto * aggregate_expr = static_cast(expr); + Expression *child_expr = aggregate_expr->child().get(); + ASSERT(child_expr != nullptr, "aggregation expression must have a child expression"); + value_expressions_.emplace_back(child_expr); + }); + + for (size_t i = 0; i < aggregate_expressions_.size(); i++) { + auto &expr = aggregate_expressions_[i]; + ASSERT(expr->type() == ExprType::AGGREGATION, "expected an aggregation expression"); + auto *aggregate_expr = static_cast(expr); + + if (aggregate_expr->aggregate_type() == AggregateExpr::Type::SUM) { + if (aggregate_expr->value_type() == AttrType::INTS) { + void *aggr_value = malloc(sizeof(SumState)); + ((SumState *)aggr_value)->value = 0; + aggr_values_.insert(aggr_value); + output_chunk_.add_column(make_unique(AttrType::INTS, sizeof(int)), i); + } else if (aggregate_expr->value_type() == AttrType::FLOATS) { + void *aggr_value = malloc(sizeof(SumState)); + ((SumState *)aggr_value)->value = 0; + aggr_values_.insert(aggr_value); + output_chunk_.add_column(make_unique(AttrType::FLOATS, sizeof(float)), i); + } + } else { + ASSERT(false, "not supported aggregation type"); + } + } +} + +RC AggregateVecPhysicalOperator::open(Trx *trx) +{ + ASSERT(children_.size() == 1, "group by operator only support one child, but got %d", children_.size()); + + PhysicalOperator &child = *children_[0]; + RC rc = child.open(trx); + if (OB_FAIL(rc)) { + LOG_INFO("failed to open child operator. rc=%s", strrc(rc)); + return rc; + } + + while (OB_SUCC(rc = child.next(chunk_))) { + for (size_t aggr_idx = 0; aggr_idx < aggregate_expressions_.size(); aggr_idx++) { + Column column; + value_expressions_[aggr_idx]->get_column(chunk_, column); + ASSERT(aggregate_expressions_[aggr_idx]->type() == ExprType::AGGREGATION, "expect aggregate expression"); + auto *aggregate_expr = static_cast(aggregate_expressions_[aggr_idx]); + if (aggregate_expr->aggregate_type() == AggregateExpr::Type::SUM) { + if (aggregate_expr->value_type() == AttrType::INTS) { + update_aggregate_state, int>(aggr_values_.at(aggr_idx), column); + } else if (aggregate_expr->value_type() == AttrType::FLOATS) { + update_aggregate_state, float>(aggr_values_.at(aggr_idx), column); + } else { + ASSERT(false, "not supported value type"); + } + } else { + ASSERT(false, "not supported aggregation type"); + } + } + } + + if (rc == RC::RECORD_EOF) { + rc = RC::SUCCESS; + } + + return rc; +} +template +void AggregateVecPhysicalOperator::update_aggregate_state(void *state, const Column &column) +{ + STATE *state_ptr = reinterpret_cast(state); + T * data = (T *)column.data(); + state_ptr->update(data, column.count()); +} + +RC AggregateVecPhysicalOperator::next(Chunk &chunk) +{ + // your code here + exit(-1); +} + +RC AggregateVecPhysicalOperator::close() +{ + children_[0]->close(); + LOG_INFO("close group by operator"); + return RC::SUCCESS; +} \ No newline at end of file diff --git a/src/observer/sql/operator/aggregate_vec_physical_operator.h b/src/observer/sql/operator/aggregate_vec_physical_operator.h new file mode 100644 index 000000000..143710f94 --- /dev/null +++ b/src/observer/sql/operator/aggregate_vec_physical_operator.h @@ -0,0 +1,74 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include "sql/operator/physical_operator.h" + +/** + * @brief 聚合物理算子 (Vectorized) + * @ingroup PhysicalOperator + */ +class AggregateVecPhysicalOperator : public PhysicalOperator +{ +public: + AggregateVecPhysicalOperator(std::vector &&expressions); + + virtual ~AggregateVecPhysicalOperator() = default; + + PhysicalOperatorType type() const override { return PhysicalOperatorType::AGGREGATE_VEC; } + + RC open(Trx *trx) override; + RC next(Chunk &chunk) override; + RC close() override; + +private: + template + void update_aggregate_state(void *state, const Column &column); + + template + void append_to_column(void *state, Column &column) + { + STATE *state_ptr = reinterpret_cast(state); + column.append_one((char *)&state_ptr->value); + } + +private: + class AggregateValues + { + public: + AggregateValues() = default; + + void insert(void *aggr_value) { data_.push_back(aggr_value); } + + void *at(size_t index) + { + ASSERT(index <= data_.size(), "index out of range"); + return data_[index]; + } + + size_t size() { return data_.size(); } + ~AggregateValues() + { + for (auto &aggr_value : data_) { + free(aggr_value); + aggr_value = nullptr; + } + } + + private: + std::vector data_; + }; + std::vector aggregate_expressions_; /// 聚合表达式 + std::vector value_expressions_; + Chunk chunk_; + Chunk output_chunk_; + AggregateValues aggr_values_; +}; \ No newline at end of file diff --git a/src/observer/sql/operator/calc_physical_operator.h b/src/observer/sql/operator/calc_physical_operator.h index 4ac282814..d7002c823 100644 --- a/src/observer/sql/operator/calc_physical_operator.h +++ b/src/observer/sql/operator/calc_physical_operator.h @@ -14,7 +14,7 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include "sql/expr/tuple.h" +#include "sql/expr/expression_tuple.h" #include "sql/operator/physical_operator.h" class CalcPhysicalOperator : public PhysicalOperator @@ -59,8 +59,16 @@ class CalcPhysicalOperator : public PhysicalOperator const std::vector> &expressions() const { return expressions_; } + RC tuple_schema(TupleSchema &schema) const override + { + for (const std::unique_ptr &expression : expressions_) { + schema.append_cell(expression->name()); + } + return RC::SUCCESS; + } + private: - std::vector> expressions_; - ExpressionTuple tuple_; - bool emitted_ = false; + std::vector> expressions_; + ExpressionTuple> tuple_; + bool emitted_ = false; }; diff --git a/src/observer/sql/operator/explain_physical_operator.cpp b/src/observer/sql/operator/explain_physical_operator.cpp index 01adec7ae..bda6ef329 100644 --- a/src/observer/sql/operator/explain_physical_operator.cpp +++ b/src/observer/sql/operator/explain_physical_operator.cpp @@ -26,12 +26,8 @@ RC ExplainPhysicalOperator::open(Trx *) RC ExplainPhysicalOperator::close() { return RC::SUCCESS; } -RC ExplainPhysicalOperator::next() +void ExplainPhysicalOperator::generate_physical_plan() { - if (!physical_plan_.empty()) { - return RC::RECORD_EOF; - } - stringstream ss; ss << "OPERATOR(NAME)\n"; @@ -47,6 +43,14 @@ RC ExplainPhysicalOperator::next() } physical_plan_ = ss.str(); +} + +RC ExplainPhysicalOperator::next() +{ + if (!physical_plan_.empty()) { + return RC::RECORD_EOF; + } + generate_physical_plan(); vector cells; Value cell; @@ -56,6 +60,21 @@ RC ExplainPhysicalOperator::next() return RC::SUCCESS; } +RC ExplainPhysicalOperator::next(Chunk &chunk) +{ + if (!physical_plan_.empty()) { + return RC::RECORD_EOF; + } + generate_physical_plan(); + + Value cell; + cell.set_string(physical_plan_.c_str()); + auto column = make_unique(); + column->init(cell); + chunk.add_column(std::move(column), 0); + return RC::SUCCESS; +} + Tuple *ExplainPhysicalOperator::current_tuple() { return &tuple_; } /** diff --git a/src/observer/sql/operator/explain_physical_operator.h b/src/observer/sql/operator/explain_physical_operator.h index d0fbff427..73431ddc8 100644 --- a/src/observer/sql/operator/explain_physical_operator.h +++ b/src/observer/sql/operator/explain_physical_operator.h @@ -30,12 +30,21 @@ class ExplainPhysicalOperator : public PhysicalOperator RC open(Trx *trx) override; RC next() override; + RC next(Chunk &chunk) override; RC close() override; Tuple *current_tuple() override; + RC tuple_schema(TupleSchema &schema) const override + { + schema.append_cell("Query Plan"); + return RC::SUCCESS; + } + private: void to_string(std::ostream &os, PhysicalOperator *oper, int level, bool last_child, std::vector &ends); + void generate_physical_plan(); + private: std::string physical_plan_; ValueListTuple tuple_; diff --git a/src/observer/sql/operator/expr_vec_physical_operator.cpp b/src/observer/sql/operator/expr_vec_physical_operator.cpp new file mode 100644 index 000000000..d1babab00 --- /dev/null +++ b/src/observer/sql/operator/expr_vec_physical_operator.cpp @@ -0,0 +1,61 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "common/log/log.h" +#include "sql/operator/expr_vec_physical_operator.h" +#include "sql/expr/expression_tuple.h" +#include "sql/expr/composite_tuple.h" + +using namespace std; +using namespace common; + +ExprVecPhysicalOperator::ExprVecPhysicalOperator(std::vector &&expressions) +{ + expressions_ = std::move(expressions); +} + +RC ExprVecPhysicalOperator::open(Trx *trx) +{ + ASSERT(children_.size() == 1, "group by operator only support one child, but got %d", children_.size()); + + PhysicalOperator &child = *children_[0]; + RC rc = child.open(trx); + if (OB_FAIL(rc)) { + LOG_INFO("failed to open child operator. rc=%s", strrc(rc)); + return rc; + } + return rc; +} + +RC ExprVecPhysicalOperator::next(Chunk &chunk) +{ + RC rc = RC::SUCCESS; + ASSERT(children_.size() == 1, "group by operator only support one child, but got %d", children_.size()); + + PhysicalOperator &child = *children_[0]; + chunk.reset(); + evaled_chunk_.reset(); + if (OB_SUCC(rc = child.next(chunk_))) { + for (size_t i = 0; i < expressions_.size(); i++) { + auto column = std::make_unique(); + expressions_[i]->get_column(chunk_, *column); + evaled_chunk_.add_column(std::move(column), i); + } + chunk.reference(evaled_chunk_); + } + return rc; +} + +RC ExprVecPhysicalOperator::close() +{ + children_[0]->close(); + LOG_INFO("close group by operator"); + return RC::SUCCESS; +} \ No newline at end of file diff --git a/src/observer/sql/operator/expr_vec_physical_operator.h b/src/observer/sql/operator/expr_vec_physical_operator.h new file mode 100644 index 000000000..f48db7d56 --- /dev/null +++ b/src/observer/sql/operator/expr_vec_physical_operator.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include "sql/operator/physical_operator.h" + +/** + * @brief 表达式物理算子(Vectorized) + * @ingroup PhysicalOperator + */ +class ExprVecPhysicalOperator : public PhysicalOperator +{ +public: + ExprVecPhysicalOperator(std::vector &&expressions); + + virtual ~ExprVecPhysicalOperator() = default; + + PhysicalOperatorType type() const override { return PhysicalOperatorType::EXPR_VEC; } + + RC open(Trx *trx) override; + RC next(Chunk &chunk) override; + RC close() override; + +private: + std::vector expressions_; /// 表达式 + Chunk chunk_; + Chunk evaled_chunk_; +}; \ No newline at end of file diff --git a/src/observer/sql/operator/group_by_logical_operator.cpp b/src/observer/sql/operator/group_by_logical_operator.cpp new file mode 100644 index 000000000..c35bbf917 --- /dev/null +++ b/src/observer/sql/operator/group_by_logical_operator.cpp @@ -0,0 +1,28 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by WangYunlai on 2024/05/30. +// + +#include + +#include "common/log/log.h" +#include "sql/operator/group_by_logical_operator.h" +#include "sql/expr/expression.h" + +using namespace std; + +GroupByLogicalOperator::GroupByLogicalOperator(vector> &&group_by_exprs, + vector &&expressions) +{ + group_by_expressions_ = std::move(group_by_exprs); + aggregate_expressions_ = std::move(expressions); +} \ No newline at end of file diff --git a/src/observer/sql/operator/group_by_logical_operator.h b/src/observer/sql/operator/group_by_logical_operator.h new file mode 100644 index 000000000..888e94788 --- /dev/null +++ b/src/observer/sql/operator/group_by_logical_operator.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by WangYunlai on 2024/05/29. +// + +#pragma once + +#include "sql/operator/logical_operator.h" + +class GroupByLogicalOperator : public LogicalOperator +{ +public: + GroupByLogicalOperator( + std::vector> &&group_by_exprs, std::vector &&expressions); + + virtual ~GroupByLogicalOperator() = default; + + LogicalOperatorType type() const override { return LogicalOperatorType::GROUP_BY; } + + auto &group_by_expressions() { return group_by_expressions_; } + auto &aggregate_expressions() { return aggregate_expressions_; } + +private: + std::vector> group_by_expressions_; + std::vector aggregate_expressions_; ///< 输出的表达式,可能包含聚合函数 +}; \ No newline at end of file diff --git a/src/observer/sql/operator/group_by_physical_operator.cpp b/src/observer/sql/operator/group_by_physical_operator.cpp new file mode 100644 index 000000000..e28aea55c --- /dev/null +++ b/src/observer/sql/operator/group_by_physical_operator.cpp @@ -0,0 +1,104 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by WangYunlai on 2024/06/11. +// +#include + +#include "common/log/log.h" +#include "sql/operator/group_by_physical_operator.h" +#include "sql/expr/expression_tuple.h" +#include "sql/expr/composite_tuple.h" + +using namespace std; +using namespace common; + +GroupByPhysicalOperator::GroupByPhysicalOperator(vector &&expressions) +{ + aggregate_expressions_ = std::move(expressions); + value_expressions_.reserve(aggregate_expressions_.size()); + ranges::for_each(aggregate_expressions_, [this](Expression *expr) { + auto *aggregate_expr = static_cast(expr); + Expression *child_expr = aggregate_expr->child().get(); + ASSERT(child_expr != nullptr, "aggregate expression must have a child expression"); + value_expressions_.emplace_back(child_expr); + }); +} + +void GroupByPhysicalOperator::create_aggregator_list(AggregatorList &aggregator_list) +{ + aggregator_list.clear(); + aggregator_list.reserve(aggregate_expressions_.size()); + ranges::for_each(aggregate_expressions_, [&aggregator_list](Expression *expr) { + auto *aggregate_expr = static_cast(expr); + aggregator_list.emplace_back(aggregate_expr->create_aggregator()); + }); +} + +RC GroupByPhysicalOperator::aggregate(AggregatorList &aggregator_list, const Tuple &tuple) +{ + ASSERT(static_cast(aggregator_list.size()) == tuple.cell_num(), + "aggregator list size must be equal to tuple size. aggregator num: %d, tuple num: %d", + aggregator_list.size(), tuple.cell_num()); + + RC rc = RC::SUCCESS; + Value value; + const int size = static_cast(aggregator_list.size()); + for (int i = 0; i < size; i++) { + Aggregator *aggregator = aggregator_list[i].get(); + + rc = tuple.cell_at(i, value); + if (OB_FAIL(rc)) { + LOG_WARN("failed to get value from expression. rc=%s", strrc(rc)); + return rc; + } + + rc = aggregator->accumulate(value); + if (OB_FAIL(rc)) { + LOG_WARN("failed to accumulate value. rc=%s", strrc(rc)); + return rc; + } + } + + return rc; +} + +RC GroupByPhysicalOperator::evaluate(GroupValueType &group_value) +{ + RC rc = RC::SUCCESS; + + vector aggregator_names; + for (Expression *expr : aggregate_expressions_) { + aggregator_names.emplace_back(expr->name()); + } + + AggregatorList &aggregators = get<0>(group_value); + CompositeTuple &composite_value_tuple = get<1>(group_value); + + ValueListTuple evaluated_tuple; + vector values; + for (unique_ptr &aggregator : aggregators) { + Value value; + rc = aggregator->evaluate(value); + if (OB_FAIL(rc)) { + LOG_WARN("failed to evaluate aggregator. rc=%s", strrc(rc)); + return rc; + } + values.emplace_back(value); + } + + evaluated_tuple.set_cells(values); + evaluated_tuple.set_names(aggregator_names); + + composite_value_tuple.add_tuple(make_unique(std::move(evaluated_tuple))); + + return rc; +} \ No newline at end of file diff --git a/src/observer/sql/operator/group_by_physical_operator.h b/src/observer/sql/operator/group_by_physical_operator.h new file mode 100644 index 000000000..5c2da57e4 --- /dev/null +++ b/src/observer/sql/operator/group_by_physical_operator.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by WangYunlai on 2024/05/29. +// + +#pragma once + +#include "sql/operator/physical_operator.h" +#include "sql/expr/composite_tuple.h" + +/** + * @brief Group By 物理算子基类 + * @ingroup PhysicalOperator + */ +class GroupByPhysicalOperator : public PhysicalOperator +{ +public: + GroupByPhysicalOperator(std::vector &&expressions); + virtual ~GroupByPhysicalOperator() = default; + +protected: + using AggregatorList = std::vector>; + /** + * @brief 聚合出来的一组数据 + * @details + * 第一个参数是聚合函数列表,比如需要计算 sum(a), avg(b), count(c)。 + * 第二个参数是聚合的最终结果,它也包含两个元素,第一个是缓存下来的元组,第二个是聚合函数计算的结果。 + * 第二个参数中,之所以要缓存下来一个元组,是要解决这个问题: + * select a, b, sum(a) from t group by a; + * 我们需要知道b的值是什么,虽然它不确定。 + */ + using GroupValueType = std::tuple; + +protected: + void create_aggregator_list(AggregatorList &aggregator_list); + + /// @brief 聚合一条记录 + /// @param aggregator_list 需要执行聚合运算的列表 + /// @param tuple 执行聚合运算的一条记录 + RC aggregate(AggregatorList &aggregator_list, const Tuple &tuple); + + /// @brief 所有tuple聚合结束后,运算最终结果 + RC evaluate(GroupValueType &group_value); + +protected: + std::vector aggregate_expressions_; /// 聚合表达式 + std::vector value_expressions_; /// 计算聚合时的表达式 +}; \ No newline at end of file diff --git a/src/observer/sql/operator/group_by_vec_physical_operator.cpp b/src/observer/sql/operator/group_by_vec_physical_operator.cpp new file mode 100644 index 000000000..726035899 --- /dev/null +++ b/src/observer/sql/operator/group_by_vec_physical_operator.cpp @@ -0,0 +1,11 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "sql/operator/group_by_vec_physical_operator.h" \ No newline at end of file diff --git a/src/observer/sql/operator/group_by_vec_physical_operator.h b/src/observer/sql/operator/group_by_vec_physical_operator.h new file mode 100644 index 000000000..be8eef8e1 --- /dev/null +++ b/src/observer/sql/operator/group_by_vec_physical_operator.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include "sql/expr/aggregate_hash_table.h" +#include "sql/operator/physical_operator.h" + +/** + * @brief Group By 物理算子(vectorized) + * @ingroup PhysicalOperator + */ +class GroupByVecPhysicalOperator : public PhysicalOperator +{ +public: + GroupByVecPhysicalOperator( + std::vector> &&group_by_exprs, std::vector &&expressions){}; + + virtual ~GroupByVecPhysicalOperator() = default; + + PhysicalOperatorType type() const override { return PhysicalOperatorType::GROUP_BY_VEC; } + + RC open(Trx *trx) override { return RC::UNIMPLENMENT; } + RC next(Chunk &chunk) override { return RC::UNIMPLENMENT; } + RC close() override { return RC::UNIMPLENMENT; } + +private: +}; \ No newline at end of file diff --git a/src/observer/sql/operator/hash_group_by_physical_operator.cpp b/src/observer/sql/operator/hash_group_by_physical_operator.cpp new file mode 100644 index 000000000..f408c0ec9 --- /dev/null +++ b/src/observer/sql/operator/hash_group_by_physical_operator.cpp @@ -0,0 +1,179 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by WangYunlai on 2024/05/30. +// + +#include "common/log/log.h" +#include "sql/operator/hash_group_by_physical_operator.h" +#include "sql/expr/expression_tuple.h" +#include "sql/expr/composite_tuple.h" + +using namespace std; +using namespace common; + +HashGroupByPhysicalOperator::HashGroupByPhysicalOperator( + vector> &&group_by_exprs, vector &&expressions) + : GroupByPhysicalOperator(std::move(expressions)), group_by_exprs_(std::move(group_by_exprs)) +{ +} + +RC HashGroupByPhysicalOperator::open(Trx *trx) +{ + ASSERT(children_.size() == 1, "group by operator only support one child, but got %d", children_.size()); + + PhysicalOperator &child = *children_[0]; + RC rc = child.open(trx); + if (OB_FAIL(rc)) { + LOG_INFO("failed to open child operator. rc=%s", strrc(rc)); + return rc; + } + + ExpressionTuple group_value_expression_tuple(value_expressions_); + + ValueListTuple group_by_evaluated_tuple; + + while (OB_SUCC(rc = child.next())) { + Tuple *child_tuple = child.current_tuple(); + if (nullptr == child_tuple) { + LOG_WARN("failed to get tuple from child operator. rc=%s", strrc(rc)); + return RC::INTERNAL; + } + + // 找到对应的group + GroupType *found_group = nullptr; + rc = find_group(*child_tuple, found_group); + if (OB_FAIL(rc)) { + LOG_WARN("failed to find group. rc=%s", strrc(rc)); + return rc; + } + + // 计算需要做聚合的值 + group_value_expression_tuple.set_tuple(child_tuple); + + // 计算聚合值 + GroupValueType &group_value = get<1>(*found_group); + rc = aggregate(get<0>(group_value), group_value_expression_tuple); + if (OB_FAIL(rc)) { + LOG_WARN("failed to aggregate values. rc=%s", strrc(rc)); + return rc; + } + } + + if (RC::RECORD_EOF == rc) { + rc = RC::SUCCESS; + } + + if (OB_FAIL(rc)) { + LOG_WARN("failed to get next tuple. rc=%s", strrc(rc)); + return rc; + } + + // 得到最终聚合后的值 + for (GroupType &group : groups_) { + GroupValueType &group_value = get<1>(group); + rc = evaluate(group_value); + if (OB_FAIL(rc)) { + LOG_WARN("failed to evaluate group value. rc=%s", strrc(rc)); + return rc; + } + } + + current_group_ = groups_.begin(); + first_emited_ = false; + return rc; +} + +RC HashGroupByPhysicalOperator::next() +{ + if (current_group_ == groups_.end()) { + return RC::RECORD_EOF; + } + + if (first_emited_) { + ++current_group_; + } else { + first_emited_ = true; + } + if (current_group_ == groups_.end()) { + return RC::RECORD_EOF; + } + + return RC::SUCCESS; +} + +RC HashGroupByPhysicalOperator::close() +{ + children_[0]->close(); + LOG_INFO("close group by operator"); + return RC::SUCCESS; +} + +Tuple *HashGroupByPhysicalOperator::current_tuple() +{ + if (current_group_ != groups_.end()) { + GroupValueType &group_value = get<1>(*current_group_); + return &get<1>(group_value); + } + return nullptr; +} + +RC HashGroupByPhysicalOperator::find_group(const Tuple &child_tuple, GroupType *&found_group) +{ + found_group = nullptr; + + RC rc = RC::SUCCESS; + + ExpressionTuple> group_by_expression_tuple(group_by_exprs_); + ValueListTuple group_by_evaluated_tuple; + group_by_expression_tuple.set_tuple(&child_tuple); + rc = ValueListTuple::make(group_by_expression_tuple, group_by_evaluated_tuple); + if (OB_FAIL(rc)) { + LOG_WARN("failed to get values from expression tuple. rc=%s", strrc(rc)); + return rc; + } + + // 找到对应的group + for (GroupType &group : groups_) { + int compare_result = 0; + rc = group_by_evaluated_tuple.compare(get<0>(group), compare_result); + if (OB_FAIL(rc)) { + LOG_WARN("failed to compare group by values. rc=%s", strrc(rc)); + return rc; + } + + if (compare_result == 0) { + found_group = &group; + break; + } + } + + // 如果没有找到对应的group,创建一个新的group + if (nullptr == found_group) { + AggregatorList aggregator_list; + create_aggregator_list(aggregator_list); + + ValueListTuple child_tuple_to_value; + rc = ValueListTuple::make(child_tuple, child_tuple_to_value); + if (OB_FAIL(rc)) { + LOG_WARN("failed to make tuple to value list. rc=%s", strrc(rc)); + return rc; + } + + CompositeTuple composite_tuple; + composite_tuple.add_tuple(make_unique(std::move(child_tuple_to_value))); + groups_.emplace_back(std::move(group_by_evaluated_tuple), + GroupValueType(std::move(aggregator_list), std::move(composite_tuple))); + found_group = &groups_.back(); + } + + return rc; +} \ No newline at end of file diff --git a/src/observer/sql/operator/hash_group_by_physical_operator.h b/src/observer/sql/operator/hash_group_by_physical_operator.h new file mode 100644 index 000000000..c38ae39da --- /dev/null +++ b/src/observer/sql/operator/hash_group_by_physical_operator.h @@ -0,0 +1,62 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by WangYunlai on 2024/05/29. +// + +#pragma once + +#include "sql/operator/group_by_physical_operator.h" +#include "sql/expr/composite_tuple.h" + +/** + * @brief Group By Hash 方式物理算子 + * @ingroup PhysicalOperator + * @details 通过 hash 的方式进行 group by 操作。当聚合函数存在 group by + * 表达式时,默认采用这个物理算子(当前也只有这个物理算子)。 NOTE: + * 当前并没有使用hash方式实现,而是每次使用线性查找的方式。 + */ +class HashGroupByPhysicalOperator : public GroupByPhysicalOperator +{ +public: + HashGroupByPhysicalOperator( + std::vector> &&group_by_exprs, std::vector &&expressions); + + virtual ~HashGroupByPhysicalOperator() = default; + + PhysicalOperatorType type() const override { return PhysicalOperatorType::HASH_GROUP_BY; } + + RC open(Trx *trx) override; + RC next() override; + RC close() override; + + Tuple *current_tuple() override; + +private: + using AggregatorList = GroupByPhysicalOperator::AggregatorList; + using GroupValueType = GroupByPhysicalOperator::GroupValueType; + /// 聚合出来的一组数据 + using GroupType = std::tuple; + +private: + RC find_group(const Tuple &child_tuple, GroupType *&found_group); + +private: + std::vector> group_by_exprs_; + + /// 一组一条数据 + /// pair的first是group by 的值列表,second是计算出来的表达式值列表 + /// TODO 改成hash/unordered_map + std::vector groups_; + + std::vector::iterator current_group_; + bool first_emited_ = false; /// 第一条数据是否已经输出 +}; \ No newline at end of file diff --git a/src/observer/sql/operator/index_scan_physical_operator.cpp b/src/observer/sql/operator/index_scan_physical_operator.cpp index 14ea84eb0..51bed88f1 100644 --- a/src/observer/sql/operator/index_scan_physical_operator.cpp +++ b/src/observer/sql/operator/index_scan_physical_operator.cpp @@ -68,8 +68,6 @@ RC IndexScanPhysicalOperator::next() RID rid; RC rc = RC::SUCCESS; - record_page_handler_.cleanup(); - bool filter_result = false; while (RC::SUCCESS == (rc = index_scanner_->next_entry(&rid))) { rc = record_handler_->get_record(rid, current_record_); diff --git a/src/observer/sql/operator/index_scan_physical_operator.h b/src/observer/sql/operator/index_scan_physical_operator.h index 45f93806f..902c49f61 100644 --- a/src/observer/sql/operator/index_scan_physical_operator.h +++ b/src/observer/sql/operator/index_scan_physical_operator.h @@ -54,9 +54,8 @@ class IndexScanPhysicalOperator : public PhysicalOperator IndexScanner *index_scanner_ = nullptr; RecordFileHandler *record_handler_ = nullptr; - RecordPageHandler record_page_handler_; - Record current_record_; - RowTuple tuple_; + Record current_record_; + RowTuple tuple_; Value left_value_; Value right_value_; diff --git a/src/observer/sql/operator/logical_operator.cpp b/src/observer/sql/operator/logical_operator.cpp index b36e117e2..d11d6fd1e 100644 --- a/src/observer/sql/operator/logical_operator.cpp +++ b/src/observer/sql/operator/logical_operator.cpp @@ -17,3 +17,21 @@ See the Mulan PSL v2 for more details. */ LogicalOperator::~LogicalOperator() {} void LogicalOperator::add_child(std::unique_ptr oper) { children_.emplace_back(std::move(oper)); } +bool LogicalOperator::can_generate_vectorized_operator(const LogicalOperatorType &type) +{ + bool bool_ret = false; + switch (type) + { + case LogicalOperatorType::CALC: + case LogicalOperatorType::DELETE: + case LogicalOperatorType::INSERT: + bool_ret = false; + break; + + default: + bool_ret = true; + break; + } + return bool_ret; +} + diff --git a/src/observer/sql/operator/logical_operator.h b/src/observer/sql/operator/logical_operator.h index fb89bca02..32fb13cd8 100644 --- a/src/observer/sql/operator/logical_operator.h +++ b/src/observer/sql/operator/logical_operator.h @@ -40,6 +40,7 @@ enum class LogicalOperatorType INSERT, ///< 插入 DELETE, ///< 删除,删除可能会有子查询 EXPLAIN, ///< 查看执行计划 + GROUP_BY, ///< 分组 }; /** @@ -54,9 +55,10 @@ class LogicalOperator virtual LogicalOperatorType type() const = 0; - void add_child(std::unique_ptr oper); - std::vector> &children() { return children_; } - std::vector> &expressions() { return expressions_; } + void add_child(std::unique_ptr oper); + auto children() -> std::vector> &{ return children_; } + auto expressions() -> std::vector> &{ return expressions_; } + static bool can_generate_vectorized_operator(const LogicalOperatorType &type); protected: std::vector> children_; ///< 子算子 diff --git a/src/observer/sql/operator/physical_operator.cpp b/src/observer/sql/operator/physical_operator.cpp index d1268fcaf..fa81906d0 100644 --- a/src/observer/sql/operator/physical_operator.cpp +++ b/src/observer/sql/operator/physical_operator.cpp @@ -26,12 +26,17 @@ std::string physical_operator_type_name(PhysicalOperatorType type) case PhysicalOperatorType::DELETE: return "DELETE"; case PhysicalOperatorType::PROJECT: return "PROJECT"; case PhysicalOperatorType::STRING_LIST: return "STRING_LIST"; + case PhysicalOperatorType::HASH_GROUP_BY: return "HASH_GROUP_BY"; + case PhysicalOperatorType::SCALAR_GROUP_BY: return "SCALAR_GROUP_BY"; + case PhysicalOperatorType::AGGREGATE_VEC: return "AGGREGATE_VEC"; + case PhysicalOperatorType::GROUP_BY_VEC: return "GROUP_BY_VEC"; + case PhysicalOperatorType::PROJECT_VEC: return "PROJECT_VEC"; + case PhysicalOperatorType::TABLE_SCAN_VEC: return "TABLE_SCAN_VEC"; + case PhysicalOperatorType::EXPR_VEC: return "EXPR_VEC"; default: return "UNKNOWN"; } } -PhysicalOperator::~PhysicalOperator() {} - std::string PhysicalOperator::name() const { return physical_operator_type_name(type()); } std::string PhysicalOperator::param() const { return ""; } diff --git a/src/observer/sql/operator/physical_operator.h b/src/observer/sql/operator/physical_operator.h index 39f94ecfd..6ed13a20d 100644 --- a/src/observer/sql/operator/physical_operator.h +++ b/src/observer/sql/operator/physical_operator.h @@ -38,15 +38,23 @@ class Trx; enum class PhysicalOperatorType { TABLE_SCAN, + TABLE_SCAN_VEC, INDEX_SCAN, NESTED_LOOP_JOIN, EXPLAIN, PREDICATE, + PREDICATE_VEC, PROJECT, + PROJECT_VEC, CALC, STRING_LIST, DELETE, INSERT, + SCALAR_GROUP_BY, + HASH_GROUP_BY, + GROUP_BY_VEC, + AGGREGATE_VEC, + EXPR_VEC, }; /** @@ -58,7 +66,7 @@ class PhysicalOperator public: PhysicalOperator() = default; - virtual ~PhysicalOperator(); + virtual ~PhysicalOperator() = default; /** * 这两个函数是为了打印时使用的,比如在explain中 @@ -69,10 +77,13 @@ class PhysicalOperator virtual PhysicalOperatorType type() const = 0; virtual RC open(Trx *trx) = 0; - virtual RC next() = 0; - virtual RC close() = 0; + virtual RC next() { return RC::UNIMPLENMENT; } + virtual RC next(Chunk &chunk) { return RC::UNIMPLENMENT; } + virtual RC close() = 0; - virtual Tuple *current_tuple() = 0; + virtual Tuple *current_tuple() { return nullptr; } + + virtual RC tuple_schema(TupleSchema &schema) const { return RC::UNIMPLENMENT; } void add_child(std::unique_ptr oper) { children_.emplace_back(std::move(oper)); } diff --git a/src/observer/sql/operator/predicate_physical_operator.cpp b/src/observer/sql/operator/predicate_physical_operator.cpp index d2f767ed4..61b564451 100644 --- a/src/observer/sql/operator/predicate_physical_operator.cpp +++ b/src/observer/sql/operator/predicate_physical_operator.cpp @@ -20,7 +20,7 @@ See the Mulan PSL v2 for more details. */ PredicatePhysicalOperator::PredicatePhysicalOperator(std::unique_ptr expr) : expression_(std::move(expr)) { - ASSERT(expression_->value_type() == BOOLEANS, "predicate's expression should be BOOLEAN type"); + ASSERT(expression_->value_type() == AttrType::BOOLEANS, "predicate's expression should be BOOLEAN type"); } RC PredicatePhysicalOperator::open(Trx *trx) @@ -66,3 +66,8 @@ RC PredicatePhysicalOperator::close() } Tuple *PredicatePhysicalOperator::current_tuple() { return children_[0]->current_tuple(); } + +RC PredicatePhysicalOperator::tuple_schema(TupleSchema &schema) const +{ + return children_[0]->tuple_schema(schema); +} diff --git a/src/observer/sql/operator/predicate_physical_operator.h b/src/observer/sql/operator/predicate_physical_operator.h index 3ef325935..fbd5b1042 100644 --- a/src/observer/sql/operator/predicate_physical_operator.h +++ b/src/observer/sql/operator/predicate_physical_operator.h @@ -38,6 +38,8 @@ class PredicatePhysicalOperator : public PhysicalOperator Tuple *current_tuple() override; + RC tuple_schema(TupleSchema &schema) const override; + private: std::unique_ptr expression_; }; diff --git a/src/observer/sql/operator/project_logical_operator.cpp b/src/observer/sql/operator/project_logical_operator.cpp index e6bcbc882..08d576b80 100644 --- a/src/observer/sql/operator/project_logical_operator.cpp +++ b/src/observer/sql/operator/project_logical_operator.cpp @@ -14,4 +14,9 @@ See the Mulan PSL v2 for more details. */ #include "sql/operator/project_logical_operator.h" -ProjectLogicalOperator::ProjectLogicalOperator(const std::vector &fields) : fields_(fields) {} +using namespace std; + +ProjectLogicalOperator::ProjectLogicalOperator(vector> &&expressions) +{ + expressions_ = std::move(expressions); +} diff --git a/src/observer/sql/operator/project_logical_operator.h b/src/observer/sql/operator/project_logical_operator.h index 0a67f6699..0d4a2d500 100644 --- a/src/observer/sql/operator/project_logical_operator.h +++ b/src/observer/sql/operator/project_logical_operator.h @@ -29,19 +29,11 @@ See the Mulan PSL v2 for more details. */ class ProjectLogicalOperator : public LogicalOperator { public: - ProjectLogicalOperator(const std::vector &fields); + ProjectLogicalOperator(std::vector> &&expressions); virtual ~ProjectLogicalOperator() = default; LogicalOperatorType type() const override { return LogicalOperatorType::PROJECTION; } std::vector> &expressions() { return expressions_; } const std::vector> &expressions() const { return expressions_; } - const std::vector &fields() const { return fields_; } - -private: - //! 投影映射的字段名称 - //! 并不是所有的select都会查看表字段,也可能是常量数字、字符串, - //! 或者是执行某个函数。所以这里应该是表达式Expression。 - //! 不过现在简单处理,就使用字段来描述 - std::vector fields_; }; diff --git a/src/observer/sql/operator/project_physical_operator.cpp b/src/observer/sql/operator/project_physical_operator.cpp index 55014379b..9c06125b6 100644 --- a/src/observer/sql/operator/project_physical_operator.cpp +++ b/src/observer/sql/operator/project_physical_operator.cpp @@ -17,6 +17,13 @@ See the Mulan PSL v2 for more details. */ #include "storage/record/record.h" #include "storage/table/table.h" +using namespace std; + +ProjectPhysicalOperator::ProjectPhysicalOperator(vector> &&expressions) + : expressions_(std::move(expressions)), tuple_(expressions_) +{ +} + RC ProjectPhysicalOperator::open(Trx *trx) { if (children_.empty()) { @@ -54,10 +61,10 @@ Tuple *ProjectPhysicalOperator::current_tuple() return &tuple_; } -void ProjectPhysicalOperator::add_projection(const Table *table, const FieldMeta *field_meta) +RC ProjectPhysicalOperator::tuple_schema(TupleSchema &schema) const { - // 对单表来说,展示的(alias) 字段总是字段名称, - // 对多表查询来说,展示的alias 需要带表名字 - TupleCellSpec *spec = new TupleCellSpec(table->name(), field_meta->name(), field_meta->name()); - tuple_.add_cell_spec(spec); -} + for (const unique_ptr &expression : expressions_) { + schema.append_cell(expression->name()); + } + return RC::SUCCESS; +} \ No newline at end of file diff --git a/src/observer/sql/operator/project_physical_operator.h b/src/observer/sql/operator/project_physical_operator.h index de7f9cf3a..603e76572 100644 --- a/src/observer/sql/operator/project_physical_operator.h +++ b/src/observer/sql/operator/project_physical_operator.h @@ -15,6 +15,7 @@ See the Mulan PSL v2 for more details. */ #pragma once #include "sql/operator/physical_operator.h" +#include "sql/expr/expression_tuple.h" /** * @brief 选择/投影物理算子 @@ -23,13 +24,10 @@ See the Mulan PSL v2 for more details. */ class ProjectPhysicalOperator : public PhysicalOperator { public: - ProjectPhysicalOperator() {} + ProjectPhysicalOperator(std::vector> &&expressions); virtual ~ProjectPhysicalOperator() = default; - void add_expressions(std::vector> &&expressions) {} - void add_projection(const Table *table, const FieldMeta *field); - PhysicalOperatorType type() const override { return PhysicalOperatorType::PROJECT; } RC open(Trx *trx) override; @@ -40,6 +38,9 @@ class ProjectPhysicalOperator : public PhysicalOperator Tuple *current_tuple() override; + RC tuple_schema(TupleSchema &schema) const override; + private: - ProjectTuple tuple_; + std::vector> expressions_; + ExpressionTuple> tuple_; }; diff --git a/src/observer/sql/operator/project_vec_physical_operator.cpp b/src/observer/sql/operator/project_vec_physical_operator.cpp new file mode 100644 index 000000000..6decf2c8b --- /dev/null +++ b/src/observer/sql/operator/project_vec_physical_operator.cpp @@ -0,0 +1,73 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "sql/operator/project_vec_physical_operator.h" +#include "common/log/log.h" +#include "storage/record/record.h" +#include "storage/table/table.h" + +using namespace std; + +ProjectVecPhysicalOperator::ProjectVecPhysicalOperator(vector> &&expressions) + : expressions_(std::move(expressions)) +{ + int expr_pos = 0; + for (auto &expr : expressions_) { + chunk_.add_column(make_unique(expr->value_type(), expr->value_length()), expr_pos); + expr_pos++; + } +} +RC ProjectVecPhysicalOperator::open(Trx *trx) +{ + if (children_.empty()) { + return RC::SUCCESS; + } + RC rc = children_[0]->open(trx); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to open child operator: %s", strrc(rc)); + return rc; + } + + return RC::SUCCESS; +} + +RC ProjectVecPhysicalOperator::next(Chunk &chunk) +{ + if (children_.empty()) { + return RC::RECORD_EOF; + } + chunk_.reset_data(); + RC rc = children_[0]->next(chunk_); + if (rc == RC::RECORD_EOF) { + return rc; + } else if (rc == RC::SUCCESS) { + rc = chunk.reference(chunk_); + } else { + LOG_WARN("failed to get next tuple: %s", strrc(rc)); + return rc; + } + return rc; +} + +RC ProjectVecPhysicalOperator::close() +{ + if (!children_.empty()) { + children_[0]->close(); + } + return RC::SUCCESS; +} + +RC ProjectVecPhysicalOperator::tuple_schema(TupleSchema &schema) const +{ + for (const unique_ptr &expression : expressions_) { + schema.append_cell(expression->name()); + } + return RC::SUCCESS; +} diff --git a/src/observer/sql/operator/project_vec_physical_operator.h b/src/observer/sql/operator/project_vec_physical_operator.h new file mode 100644 index 000000000..c63a1154d --- /dev/null +++ b/src/observer/sql/operator/project_vec_physical_operator.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include "sql/operator/physical_operator.h" +#include "sql/expr/expression_tuple.h" + +/** + * @brief 选择/投影物理算子(vectorized) + * @ingroup PhysicalOperator + */ +class ProjectVecPhysicalOperator : public PhysicalOperator +{ +public: + ProjectVecPhysicalOperator() {} + ProjectVecPhysicalOperator(std::vector> &&expressions); + + virtual ~ProjectVecPhysicalOperator() = default; + + PhysicalOperatorType type() const override { return PhysicalOperatorType::PROJECT_VEC; } + + RC open(Trx *trx) override; + RC next(Chunk &chunk) override; + RC close() override; + + RC tuple_schema(TupleSchema &schema) const override; + + std::vector> &expressions() { return expressions_; } + +private: + std::vector> expressions_; + Chunk chunk_; +}; diff --git a/src/observer/sql/operator/scalar_group_by_physical_operator.cpp b/src/observer/sql/operator/scalar_group_by_physical_operator.cpp new file mode 100644 index 000000000..b67e9f163 --- /dev/null +++ b/src/observer/sql/operator/scalar_group_by_physical_operator.cpp @@ -0,0 +1,119 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by WangYunlai on 2024/05/30. +// + +#include "common/log/log.h" +#include "sql/operator/scalar_group_by_physical_operator.h" +#include "sql/expr/expression_tuple.h" +#include "sql/expr/composite_tuple.h" + +using namespace std; +using namespace common; + +ScalarGroupByPhysicalOperator::ScalarGroupByPhysicalOperator(vector &&expressions) + : GroupByPhysicalOperator(std::move(expressions)) +{} + +RC ScalarGroupByPhysicalOperator::open(Trx *trx) +{ + ASSERT(children_.size() == 1, "group by operator only support one child, but got %d", children_.size()); + + PhysicalOperator &child = *children_[0]; + RC rc = child.open(trx); + if (OB_FAIL(rc)) { + LOG_INFO("failed to open child operator. rc=%s", strrc(rc)); + return rc; + } + + ExpressionTuple group_value_expression_tuple(value_expressions_); + + ValueListTuple group_by_evaluated_tuple; + + while (OB_SUCC(rc = child.next())) { + Tuple *child_tuple = child.current_tuple(); + if (nullptr == child_tuple) { + LOG_WARN("failed to get tuple from child operator. rc=%s", strrc(rc)); + return RC::INTERNAL; + } + + // 计算需要做聚合的值 + group_value_expression_tuple.set_tuple(child_tuple); + + // 计算聚合值 + if (group_value_ == nullptr) { + AggregatorList aggregator_list; + create_aggregator_list(aggregator_list); + + ValueListTuple child_tuple_to_value; + rc = ValueListTuple::make(*child_tuple, child_tuple_to_value); + if (OB_FAIL(rc)) { + LOG_WARN("failed to make tuple to value list. rc=%s", strrc(rc)); + return rc; + } + + CompositeTuple composite_tuple; + composite_tuple.add_tuple(make_unique(std::move(child_tuple_to_value))); + group_value_ = make_unique(std::move(aggregator_list), std::move(composite_tuple)); + } + + rc = aggregate(get<0>(*group_value_), group_value_expression_tuple); + if (OB_FAIL(rc)) { + LOG_WARN("failed to aggregate values. rc=%s", strrc(rc)); + return rc; + } + } + + if (RC::RECORD_EOF == rc) { + rc = RC::SUCCESS; + } + + if (OB_FAIL(rc)) { + LOG_WARN("failed to get next tuple. rc=%s", strrc(rc)); + return rc; + } + + // 得到最终聚合后的值 + if (group_value_) { + rc = evaluate(*group_value_); + } + + emitted_ = false; + return rc; +} + +RC ScalarGroupByPhysicalOperator::next() +{ + if (group_value_ == nullptr || emitted_) { + return RC::RECORD_EOF; + } + + emitted_ = true; + + return RC::SUCCESS; +} + +RC ScalarGroupByPhysicalOperator::close() +{ + group_value_.reset(); + emitted_ = false; + return RC::SUCCESS; +} + +Tuple *ScalarGroupByPhysicalOperator::current_tuple() +{ + if (group_value_ == nullptr) { + return nullptr; + } + + return &get<1>(*group_value_); +} \ No newline at end of file diff --git a/src/observer/sql/operator/scalar_group_by_physical_operator.h b/src/observer/sql/operator/scalar_group_by_physical_operator.h new file mode 100644 index 000000000..110d965a8 --- /dev/null +++ b/src/observer/sql/operator/scalar_group_by_physical_operator.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by WangYunlai on 2024/06/11. +// + +#pragma once + +#include "sql/operator/group_by_physical_operator.h" + +/** + * @brief 没有 group by 表达式的 group by 物理算子 + * @ingroup PhysicalOperator + */ +class ScalarGroupByPhysicalOperator : public GroupByPhysicalOperator +{ +public: + ScalarGroupByPhysicalOperator(std::vector &&expressions); + virtual ~ScalarGroupByPhysicalOperator() = default; + + PhysicalOperatorType type() const override { return PhysicalOperatorType::SCALAR_GROUP_BY; } + + RC open(Trx *trx) override; + RC next() override; + RC close() override; + + Tuple *current_tuple() override; + +private: + std::unique_ptr group_value_; + bool emitted_ = false; /// 标识是否已经输出过 +}; \ No newline at end of file diff --git a/src/observer/sql/operator/table_get_logical_operator.cpp b/src/observer/sql/operator/table_get_logical_operator.cpp index 96f86ac8e..6933c0cd3 100644 --- a/src/observer/sql/operator/table_get_logical_operator.cpp +++ b/src/observer/sql/operator/table_get_logical_operator.cpp @@ -14,8 +14,8 @@ See the Mulan PSL v2 for more details. */ #include "sql/operator/table_get_logical_operator.h" -TableGetLogicalOperator::TableGetLogicalOperator(Table *table, const std::vector &fields, ReadWriteMode mode) - : table_(table), fields_(fields), mode_(mode) +TableGetLogicalOperator::TableGetLogicalOperator(Table *table, ReadWriteMode mode) + : table_(table), mode_(mode) {} void TableGetLogicalOperator::set_predicates(std::vector> &&exprs) diff --git a/src/observer/sql/operator/table_get_logical_operator.h b/src/observer/sql/operator/table_get_logical_operator.h index c934fc013..4551a3717 100644 --- a/src/observer/sql/operator/table_get_logical_operator.h +++ b/src/observer/sql/operator/table_get_logical_operator.h @@ -25,7 +25,7 @@ See the Mulan PSL v2 for more details. */ class TableGetLogicalOperator : public LogicalOperator { public: - TableGetLogicalOperator(Table *table, const std::vector &fields, ReadWriteMode mode); + TableGetLogicalOperator(Table *table, ReadWriteMode mode); virtual ~TableGetLogicalOperator() = default; LogicalOperatorType type() const override { return LogicalOperatorType::TABLE_GET; } @@ -37,9 +37,8 @@ class TableGetLogicalOperator : public LogicalOperator auto predicates() -> std::vector> & { return predicates_; } private: - Table *table_ = nullptr; - std::vector fields_; - ReadWriteMode mode_ = ReadWriteMode::READ_WRITE; + Table *table_ = nullptr; + ReadWriteMode mode_ = ReadWriteMode::READ_WRITE; // 与当前表相关的过滤操作,可以尝试在遍历数据时执行 // 这里的表达式都是比较简单的比较运算,并且左右两边都是取字段表达式或值表达式 diff --git a/src/observer/sql/operator/table_scan_physical_operator.cpp b/src/observer/sql/operator/table_scan_physical_operator.cpp index 4f7b1e8fa..c7eabaaa1 100644 --- a/src/observer/sql/operator/table_scan_physical_operator.cpp +++ b/src/observer/sql/operator/table_scan_physical_operator.cpp @@ -48,7 +48,6 @@ RC TableScanPhysicalOperator::next() break; } else { sql_debug("a tuple is filtered: %s", tuple_.to_string().c_str()); - rc = RC::RECORD_EOF; } } return rc; diff --git a/src/observer/sql/operator/table_scan_vec_physical_operator.cpp b/src/observer/sql/operator/table_scan_vec_physical_operator.cpp new file mode 100644 index 000000000..9f7ace99f --- /dev/null +++ b/src/observer/sql/operator/table_scan_vec_physical_operator.cpp @@ -0,0 +1,85 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "sql/operator/table_scan_vec_physical_operator.h" +#include "event/sql_debug.h" +#include "storage/table/table.h" + +using namespace std; + +RC TableScanVecPhysicalOperator::open(Trx *trx) +{ + RC rc = table_->get_chunk_scanner(chunk_scanner_, trx, mode_); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to get chunk scanner", strrc(rc)); + return rc; + } + // TODO: don't need to fetch all columns from record manager + for (int i = 0; i < table_->table_meta().field_num(); ++i) { + all_columns_.add_column( + make_unique(*table_->table_meta().field(i)), table_->table_meta().field(i)->field_id()); + filterd_columns_.add_column( + make_unique(*table_->table_meta().field(i)), table_->table_meta().field(i)->field_id()); + } + return rc; +} + +RC TableScanVecPhysicalOperator::next(Chunk &chunk) +{ + RC rc = RC::SUCCESS; + + all_columns_.reset_data(); + filterd_columns_.reset_data(); + if (OB_SUCC(rc = chunk_scanner_.next_chunk(all_columns_))) { + select_.assign(all_columns_.rows(), 1); + if (predicates_.empty()) { + chunk.reference(all_columns_); + } else { + rc = filter(all_columns_); + if (rc != RC::SUCCESS) { + LOG_TRACE("filtered failed=%s", strrc(rc)); + return rc; + } + // TODO: if all setted, it doesn't need to set one by one + for (int i = 0; i < all_columns_.rows(); i++) { + if (select_[i] == 0) { + continue; + } + for (int j = 0; j < all_columns_.column_num(); j++) { + filterd_columns_.column(j).append_one( + (char *)all_columns_.column(filterd_columns_.column_ids(j)).get_value(i).data()); + } + } + chunk.reference(filterd_columns_); + } + } + return rc; +} + +RC TableScanVecPhysicalOperator::close() { return chunk_scanner_.close_scan(); } + +string TableScanVecPhysicalOperator::param() const { return table_->name(); } + +void TableScanVecPhysicalOperator::set_predicates(vector> &&exprs) +{ + predicates_ = std::move(exprs); +} + +RC TableScanVecPhysicalOperator::filter(Chunk &chunk) +{ + RC rc = RC::SUCCESS; + for (unique_ptr &expr : predicates_) { + rc = expr->eval(chunk, select_); + if (rc != RC::SUCCESS) { + return rc; + } + } + return rc; +} diff --git a/src/observer/sql/operator/table_scan_vec_physical_operator.h b/src/observer/sql/operator/table_scan_vec_physical_operator.h new file mode 100644 index 000000000..4f10f3d1b --- /dev/null +++ b/src/observer/sql/operator/table_scan_vec_physical_operator.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include "common/rc.h" +#include "sql/operator/physical_operator.h" +#include "storage/record/record_manager.h" +#include "common/types.h" + +class Table; + +/** + * @brief 表扫描物理算子(vectorized) + * @ingroup PhysicalOperator + */ +class TableScanVecPhysicalOperator : public PhysicalOperator +{ +public: + TableScanVecPhysicalOperator(Table *table, ReadWriteMode mode) : table_(table), mode_(mode) {} + + virtual ~TableScanVecPhysicalOperator() = default; + + std::string param() const override; + + PhysicalOperatorType type() const override { return PhysicalOperatorType::TABLE_SCAN_VEC; } + + RC open(Trx *trx) override; + RC next(Chunk &chunk) override; + RC close() override; + + void set_predicates(std::vector> &&exprs); + +private: + RC filter(Chunk &chunk); + +private: + Table *table_ = nullptr; + ReadWriteMode mode_ = ReadWriteMode::READ_WRITE; + ChunkFileScanner chunk_scanner_; + Chunk all_columns_; + Chunk filterd_columns_; + std::vector select_; + std::vector> predicates_; +}; diff --git a/src/observer/sql/optimizer/conjunction_simplification_rule.cpp b/src/observer/sql/optimizer/conjunction_simplification_rule.cpp index 1495285d6..d60a1b55d 100644 --- a/src/observer/sql/optimizer/conjunction_simplification_rule.cpp +++ b/src/observer/sql/optimizer/conjunction_simplification_rule.cpp @@ -18,7 +18,7 @@ See the Mulan PSL v2 for more details. */ RC try_to_get_bool_constant(std::unique_ptr &expr, bool &constant_value) { - if (expr->type() == ExprType::VALUE && expr->value_type() == BOOLEANS) { + if (expr->type() == ExprType::VALUE && expr->value_type() == AttrType::BOOLEANS) { auto value_expr = static_cast(expr.get()); constant_value = value_expr->get_value().get_boolean(); return RC::SUCCESS; diff --git a/src/observer/sql/optimizer/expression_rewriter.cpp b/src/observer/sql/optimizer/expression_rewriter.cpp index 7d4f5c83f..b0c6d2455 100644 --- a/src/observer/sql/optimizer/expression_rewriter.cpp +++ b/src/observer/sql/optimizer/expression_rewriter.cpp @@ -17,20 +17,22 @@ See the Mulan PSL v2 for more details. */ #include "sql/optimizer/comparison_simplification_rule.h" #include "sql/optimizer/conjunction_simplification_rule.h" +using namespace std; + ExpressionRewriter::ExpressionRewriter() { expr_rewrite_rules_.emplace_back(new ComparisonSimplificationRule); expr_rewrite_rules_.emplace_back(new ConjunctionSimplificationRule); } -RC ExpressionRewriter::rewrite(std::unique_ptr &oper, bool &change_made) +RC ExpressionRewriter::rewrite(unique_ptr &oper, bool &change_made) { RC rc = RC::SUCCESS; - bool sub_change_made = false; + bool sub_change_made = false; - std::vector> &expressions = oper->expressions(); - for (std::unique_ptr &expr : expressions) { + vector> &expressions = oper->expressions(); + for (unique_ptr &expr : expressions) { rc = rewrite_expression(expr, sub_change_made); if (rc != RC::SUCCESS) { break; @@ -45,8 +47,8 @@ RC ExpressionRewriter::rewrite(std::unique_ptr &oper, bool &cha return rc; } - std::vector> &child_opers = oper->children(); - for (std::unique_ptr &child_oper : child_opers) { + vector> &child_opers = oper->children(); + for (unique_ptr &child_oper : child_opers) { bool sub_change_made = false; rc = rewrite(child_oper, sub_change_made); if (sub_change_made && !change_made) { @@ -59,12 +61,12 @@ RC ExpressionRewriter::rewrite(std::unique_ptr &oper, bool &cha return rc; } -RC ExpressionRewriter::rewrite_expression(std::unique_ptr &expr, bool &change_made) +RC ExpressionRewriter::rewrite_expression(unique_ptr &expr, bool &change_made) { RC rc = RC::SUCCESS; change_made = false; - for (std::unique_ptr &rule : expr_rewrite_rules_) { + for (unique_ptr &rule : expr_rewrite_rules_) { bool sub_change_made = false; rc = rule->rewrite(expr, sub_change_made); @@ -87,7 +89,7 @@ RC ExpressionRewriter::rewrite_expression(std::unique_ptr &expr, boo } break; case ExprType::CAST: { - std::unique_ptr &child_expr = (static_cast(expr.get()))->child(); + unique_ptr &child_expr = (static_cast(expr.get()))->child(); rc = rewrite_expression(child_expr, change_made); } break; @@ -95,8 +97,8 @@ RC ExpressionRewriter::rewrite_expression(std::unique_ptr &expr, boo case ExprType::COMPARISON: { auto comparison_expr = static_cast(expr.get()); - std::unique_ptr &left_expr = comparison_expr->left(); - std::unique_ptr &right_expr = comparison_expr->right(); + unique_ptr &left_expr = comparison_expr->left(); + unique_ptr &right_expr = comparison_expr->right(); bool left_change_made = false; @@ -120,8 +122,8 @@ RC ExpressionRewriter::rewrite_expression(std::unique_ptr &expr, boo case ExprType::CONJUNCTION: { auto conjunction_expr = static_cast(expr.get()); - std::vector> &children = conjunction_expr->children(); - for (std::unique_ptr &child_expr : children) { + vector> &children = conjunction_expr->children(); + for (unique_ptr &child_expr : children) { bool sub_change_made = false; rc = rewrite_expression(child_expr, sub_change_made); diff --git a/src/observer/sql/optimizer/logical_plan_generator.cpp b/src/observer/sql/optimizer/logical_plan_generator.cpp index 7d14c5690..af6c33bb6 100644 --- a/src/observer/sql/optimizer/logical_plan_generator.cpp +++ b/src/observer/sql/optimizer/logical_plan_generator.cpp @@ -25,6 +25,7 @@ See the Mulan PSL v2 for more details. */ #include "sql/operator/predicate_logical_operator.h" #include "sql/operator/project_logical_operator.h" #include "sql/operator/table_get_logical_operator.h" +#include "sql/operator/group_by_logical_operator.h" #include "sql/stmt/calc_stmt.h" #include "sql/stmt/delete_stmt.h" @@ -34,7 +35,10 @@ See the Mulan PSL v2 for more details. */ #include "sql/stmt/select_stmt.h" #include "sql/stmt/stmt.h" +#include "sql/expr/expression_iterator.h" + using namespace std; +using namespace common; RC LogicalPlanGenerator::create(Stmt *stmt, unique_ptr &logical_operator) { @@ -43,31 +47,31 @@ RC LogicalPlanGenerator::create(Stmt *stmt, unique_ptr &logical case StmtType::CALC: { CalcStmt *calc_stmt = static_cast(stmt); - rc = create_plan(calc_stmt, logical_operator); + rc = create_plan(calc_stmt, logical_operator); } break; case StmtType::SELECT: { SelectStmt *select_stmt = static_cast(stmt); - rc = create_plan(select_stmt, logical_operator); + rc = create_plan(select_stmt, logical_operator); } break; case StmtType::INSERT: { InsertStmt *insert_stmt = static_cast(stmt); - rc = create_plan(insert_stmt, logical_operator); + rc = create_plan(insert_stmt, logical_operator); } break; case StmtType::DELETE: { DeleteStmt *delete_stmt = static_cast(stmt); - rc = create_plan(delete_stmt, logical_operator); + rc = create_plan(delete_stmt, logical_operator); } break; case StmtType::EXPLAIN: { ExplainStmt *explain_stmt = static_cast(stmt); - rc = create_plan(explain_stmt, logical_operator); + rc = create_plan(explain_stmt, logical_operator); } break; default: { rc = RC::UNIMPLENMENT; @@ -84,19 +88,15 @@ RC LogicalPlanGenerator::create_plan(CalcStmt *calc_stmt, std::unique_ptr &logical_operator) { + unique_ptr *last_oper = nullptr; + unique_ptr table_oper(nullptr); + last_oper = &table_oper; - const std::vector &tables = select_stmt->tables(); - const std::vector &all_fields = select_stmt->query_fields(); + const std::vector
&tables = select_stmt->tables(); for (Table *table : tables) { - std::vector fields; - for (const Field &field : all_fields) { - if (0 == strcmp(field.table_name(), table->name())) { - fields.push_back(field); - } - } - unique_ptr table_get_oper(new TableGetLogicalOperator(table, fields, ReadWriteMode::READ_ONLY)); + unique_ptr table_get_oper(new TableGetLogicalOperator(table, ReadWriteMode::READ_ONLY)); if (table_oper == nullptr) { table_oper = std::move(table_get_oper); } else { @@ -109,25 +109,41 @@ RC LogicalPlanGenerator::create_plan(SelectStmt *select_stmt, unique_ptr predicate_oper; - RC rc = create_plan(select_stmt->filter_stmt(), predicate_oper); - if (rc != RC::SUCCESS) { + RC rc = create_plan(select_stmt->filter_stmt(), predicate_oper); + if (OB_FAIL(rc)) { LOG_WARN("failed to create predicate logical plan. rc=%s", strrc(rc)); return rc; } - unique_ptr project_oper(new ProjectLogicalOperator(all_fields)); if (predicate_oper) { - if (table_oper) { - predicate_oper->add_child(std::move(table_oper)); + if (*last_oper) { + predicate_oper->add_child(std::move(*last_oper)); } - project_oper->add_child(std::move(predicate_oper)); - } else { - if (table_oper) { - project_oper->add_child(std::move(table_oper)); + + last_oper = &predicate_oper; + } + + unique_ptr group_by_oper; + rc = create_group_by_plan(select_stmt, group_by_oper); + if (OB_FAIL(rc)) { + LOG_WARN("failed to create group by logical plan. rc=%s", strrc(rc)); + return rc; + } + + if (group_by_oper) { + if (*last_oper) { + group_by_oper->add_child(std::move(*last_oper)); } + + last_oper = &group_by_oper; + } + + auto project_oper = make_unique(std::move(select_stmt->query_expressions())); + if (*last_oper) { + project_oper->add_child(std::move(*last_oper)); } - logical_operator.swap(project_oper); + logical_operator = std::move(project_oper); return RC::SUCCESS; } @@ -173,17 +189,13 @@ RC LogicalPlanGenerator::create_plan(InsertStmt *insert_stmt, unique_ptr &logical_operator) { - Table *table = delete_stmt->table(); - FilterStmt *filter_stmt = delete_stmt->filter_stmt(); - std::vector fields; - for (int i = table->table_meta().sys_field_num(); i < table->table_meta().field_num(); i++) { - const FieldMeta *field_meta = table->table_meta().field(i); - fields.push_back(Field(table, field_meta)); - } - unique_ptr table_get_oper(new TableGetLogicalOperator(table, fields, ReadWriteMode::READ_WRITE)); + Table *table = delete_stmt->table(); + FilterStmt *filter_stmt = delete_stmt->filter_stmt(); + unique_ptr table_get_oper(new TableGetLogicalOperator(table, ReadWriteMode::READ_WRITE)); unique_ptr predicate_oper; - RC rc = create_plan(filter_stmt, predicate_oper); + + RC rc = create_plan(filter_stmt, predicate_oper); if (rc != RC::SUCCESS) { return rc; } @@ -205,9 +217,9 @@ RC LogicalPlanGenerator::create_plan(ExplainStmt *explain_stmt, unique_ptr child_oper; - Stmt *child_stmt = explain_stmt->child(); + Stmt *child_stmt = explain_stmt->child(); - RC rc = create(child_stmt, child_oper); + RC rc = create(child_stmt, child_oper); if (rc != RC::SUCCESS) { LOG_WARN("failed to create explain's child operator. rc=%s", strrc(rc)); return rc; @@ -217,3 +229,81 @@ RC LogicalPlanGenerator::create_plan(ExplainStmt *explain_stmt, unique_ptradd_child(std::move(child_oper)); return rc; } + +RC LogicalPlanGenerator::create_group_by_plan(SelectStmt *select_stmt, unique_ptr &logical_operator) +{ + vector> &group_by_expressions = select_stmt->group_by(); + vector aggregate_expressions; + vector> &query_expressions = select_stmt->query_expressions(); + function&)> collector = [&](unique_ptr &expr) -> RC { + RC rc = RC::SUCCESS; + if (expr->type() == ExprType::AGGREGATION) { + expr->set_pos(aggregate_expressions.size() + group_by_expressions.size()); + aggregate_expressions.push_back(expr.get()); + } + rc = ExpressionIterator::iterate_child_expr(*expr, collector); + return rc; + }; + + function&)> bind_group_by_expr = [&](unique_ptr &expr) -> RC { + RC rc = RC::SUCCESS; + for (size_t i = 0; i < group_by_expressions.size(); i++) { + auto &group_by = group_by_expressions[i]; + if (expr->type() == ExprType::AGGREGATION) { + break; + } else if (expr->equal(*group_by)) { + expr->set_pos(i); + continue; + } else { + rc = ExpressionIterator::iterate_child_expr(*expr, bind_group_by_expr); + } + } + return rc; + }; + + bool found_unbound_column = false; + function&)> find_unbound_column = [&](unique_ptr &expr) -> RC { + RC rc = RC::SUCCESS; + if (expr->type() == ExprType::AGGREGATION) { + // do nothing + } else if (expr->pos() != -1) { + // do nothing + } else if (expr->type() == ExprType::FIELD) { + found_unbound_column = true; + }else { + rc = ExpressionIterator::iterate_child_expr(*expr, find_unbound_column); + } + return rc; + }; + + + for (unique_ptr &expression : query_expressions) { + bind_group_by_expr(expression); + } + + for (unique_ptr &expression : query_expressions) { + find_unbound_column(expression); + } + + // collect all aggregate expressions + for (unique_ptr &expression : query_expressions) { + collector(expression); + } + + if (group_by_expressions.empty() && aggregate_expressions.empty()) { + // 既没有group by也没有聚合函数,不需要group by + return RC::SUCCESS; + } + + if (found_unbound_column) { + LOG_WARN("column must appear in the GROUP BY clause or must be part of an aggregate function"); + return RC::INVALID_ARGUMENT; + } + + // 如果只需要聚合,但是没有group by 语句,需要生成一个空的group by 语句 + + auto group_by_oper = make_unique(std::move(group_by_expressions), + std::move(aggregate_expressions)); + logical_operator = std::move(group_by_oper); + return RC::SUCCESS; +} \ No newline at end of file diff --git a/src/observer/sql/optimizer/logical_plan_generator.h b/src/observer/sql/optimizer/logical_plan_generator.h index bf56d896f..951ae8f47 100644 --- a/src/observer/sql/optimizer/logical_plan_generator.h +++ b/src/observer/sql/optimizer/logical_plan_generator.h @@ -42,4 +42,6 @@ class LogicalPlanGenerator RC create_plan(InsertStmt *insert_stmt, std::unique_ptr &logical_operator); RC create_plan(DeleteStmt *delete_stmt, std::unique_ptr &logical_operator); RC create_plan(ExplainStmt *explain_stmt, std::unique_ptr &logical_operator); + + RC create_group_by_plan(SelectStmt *select_stmt, std::unique_ptr &logical_operator); }; \ No newline at end of file diff --git a/src/observer/sql/optimizer/optimize_stage.cpp b/src/observer/sql/optimizer/optimize_stage.cpp index 756af61a4..19847fcb5 100644 --- a/src/observer/sql/optimizer/optimize_stage.cpp +++ b/src/observer/sql/optimizer/optimize_stage.cpp @@ -33,7 +33,7 @@ RC OptimizeStage::handle_request(SQLStageEvent *sql_event) { unique_ptr logical_operator; - RC rc = create_logical_plan(sql_event, logical_operator); + RC rc = create_logical_plan(sql_event, logical_operator); if (rc != RC::SUCCESS) { if (rc != RC::UNIMPLENMENT) { LOG_WARN("failed to create logical plan. rc=%s", strrc(rc)); @@ -41,6 +41,8 @@ RC OptimizeStage::handle_request(SQLStageEvent *sql_event) return rc; } + ASSERT(logical_operator, "logical operator is null"); + rc = rewrite(logical_operator); if (rc != RC::SUCCESS) { LOG_WARN("failed to rewrite plan. rc=%s", strrc(rc)); @@ -54,7 +56,7 @@ RC OptimizeStage::handle_request(SQLStageEvent *sql_event) } unique_ptr physical_operator; - rc = generate_physical_plan(logical_operator, physical_operator); + rc = generate_physical_plan(logical_operator, physical_operator, sql_event->session_event()->session()); if (rc != RC::SUCCESS) { LOG_WARN("failed to generate physical plan. rc=%s", strrc(rc)); return rc; @@ -72,10 +74,18 @@ RC OptimizeStage::optimize(unique_ptr &oper) } RC OptimizeStage::generate_physical_plan( - unique_ptr &logical_operator, unique_ptr &physical_operator) + unique_ptr &logical_operator, unique_ptr &physical_operator, Session *session) { RC rc = RC::SUCCESS; - rc = physical_plan_generator_.create(*logical_operator, physical_operator); + if (session->get_execution_mode() == ExecutionMode::CHUNK_ITERATOR && LogicalOperator::can_generate_vectorized_operator(logical_operator->type())) { + LOG_INFO("use chunk iterator"); + session->set_used_chunk_mode(true); + rc = physical_plan_generator_.create_vec(*logical_operator, physical_operator); + } else { + LOG_INFO("use tuple iterator"); + session->set_used_chunk_mode(false); + rc = physical_plan_generator_.create(*logical_operator, physical_operator); + } if (rc != RC::SUCCESS) { LOG_WARN("failed to create physical operator. rc=%s", strrc(rc)); } diff --git a/src/observer/sql/optimizer/optimize_stage.h b/src/observer/sql/optimizer/optimize_stage.h index 1c2b9dba2..a1b24ecf7 100644 --- a/src/observer/sql/optimizer/optimize_stage.h +++ b/src/observer/sql/optimizer/optimize_stage.h @@ -15,6 +15,7 @@ See the Mulan PSL v2 for more details. */ #pragma once #include "common/rc.h" +#include "session/session.h" #include "sql/operator/logical_operator.h" #include "sql/operator/physical_operator.h" #include "sql/optimizer/logical_plan_generator.h" @@ -70,8 +71,8 @@ class OptimizeStage * 而物理计划描述怎么做,比如如何从某张表按照什么条件获取什么数据,是否使用索引,使用哪个索引等。 * @param physical_operator 生成的物理计划。通常是一个多叉树的形状,这里就拿着根节点就可以了。 */ - RC generate_physical_plan( - std::unique_ptr &logical_operator, std::unique_ptr &physical_operator); + RC generate_physical_plan(std::unique_ptr &logical_operator, + std::unique_ptr &physical_operator, Session *session); private: LogicalPlanGenerator logical_plan_generator_; ///< 根据SQL生成逻辑计划 diff --git a/src/observer/sql/optimizer/physical_plan_generator.cpp b/src/observer/sql/optimizer/physical_plan_generator.cpp index afedc8539..9e946c2d3 100644 --- a/src/observer/sql/optimizer/physical_plan_generator.cpp +++ b/src/observer/sql/optimizer/physical_plan_generator.cpp @@ -16,12 +16,15 @@ See the Mulan PSL v2 for more details. */ #include "common/log/log.h" #include "sql/expr/expression.h" +#include "sql/operator/aggregate_vec_physical_operator.h" #include "sql/operator/calc_logical_operator.h" #include "sql/operator/calc_physical_operator.h" #include "sql/operator/delete_logical_operator.h" #include "sql/operator/delete_physical_operator.h" #include "sql/operator/explain_logical_operator.h" #include "sql/operator/explain_physical_operator.h" +#include "sql/operator/expr_vec_physical_operator.h" +#include "sql/operator/group_by_vec_physical_operator.h" #include "sql/operator/index_scan_physical_operator.h" #include "sql/operator/insert_logical_operator.h" #include "sql/operator/insert_physical_operator.h" @@ -31,8 +34,14 @@ See the Mulan PSL v2 for more details. */ #include "sql/operator/predicate_physical_operator.h" #include "sql/operator/project_logical_operator.h" #include "sql/operator/project_physical_operator.h" +#include "sql/operator/project_vec_physical_operator.h" #include "sql/operator/table_get_logical_operator.h" #include "sql/operator/table_scan_physical_operator.h" +#include "sql/operator/group_by_logical_operator.h" +#include "sql/operator/group_by_physical_operator.h" +#include "sql/operator/hash_group_by_physical_operator.h" +#include "sql/operator/scalar_group_by_physical_operator.h" +#include "sql/operator/table_scan_vec_physical_operator.h" #include "sql/optimizer/physical_plan_generator.h" using namespace std; @@ -74,13 +83,44 @@ RC PhysicalPlanGenerator::create(LogicalOperator &logical_operator, unique_ptr

(logical_operator), oper); } break; + case LogicalOperatorType::GROUP_BY: { + return create_plan(static_cast(logical_operator), oper); + } break; + default: { + ASSERT(false, "unknown logical operator type"); return RC::INVALID_ARGUMENT; } } return rc; } +RC PhysicalPlanGenerator::create_vec(LogicalOperator &logical_operator, unique_ptr &oper) +{ + RC rc = RC::SUCCESS; + + switch (logical_operator.type()) { + case LogicalOperatorType::TABLE_GET: { + return create_vec_plan(static_cast(logical_operator), oper); + } break; + case LogicalOperatorType::PROJECTION: { + return create_vec_plan(static_cast(logical_operator), oper); + } break; + case LogicalOperatorType::GROUP_BY: { + return create_vec_plan(static_cast(logical_operator), oper); + } break; + case LogicalOperatorType::EXPLAIN: { + return create_vec_plan(static_cast(logical_operator), oper); + } break; + default: { + return RC::INVALID_ARGUMENT; + } + } + return rc; +} + + + RC PhysicalPlanGenerator::create_plan(TableGetLogicalOperator &table_get_oper, unique_ptr &oper) { vector> &predicates = table_get_oper.predicates(); @@ -131,8 +171,13 @@ RC PhysicalPlanGenerator::create_plan(TableGetLogicalOperator &table_get_oper, u ASSERT(value_expr != nullptr, "got an index but value expr is null ?"); const Value &value = value_expr->get_value(); - IndexScanPhysicalOperator *index_scan_oper = new IndexScanPhysicalOperator( - table, index, table_get_oper.read_write_mode(), &value, true /*left_inclusive*/, &value, true /*right_inclusive*/); + IndexScanPhysicalOperator *index_scan_oper = new IndexScanPhysicalOperator(table, + index, + table_get_oper.read_write_mode(), + &value, + true /*left_inclusive*/, + &value, + true /*right_inclusive*/); index_scan_oper->set_predicates(std::move(predicates)); oper = unique_ptr(index_scan_oper); @@ -179,24 +224,20 @@ RC PhysicalPlanGenerator::create_plan(ProjectLogicalOperator &project_oper, uniq RC rc = RC::SUCCESS; if (!child_opers.empty()) { LogicalOperator *child_oper = child_opers.front().get(); - rc = create(*child_oper, child_phy_oper); - if (rc != RC::SUCCESS) { + + rc = create(*child_oper, child_phy_oper); + if (OB_FAIL(rc)) { LOG_WARN("failed to create project logical operator's child physical operator. rc=%s", strrc(rc)); return rc; } } - ProjectPhysicalOperator *project_operator = new ProjectPhysicalOperator; - const vector &project_fields = project_oper.fields(); - for (const Field &field : project_fields) { - project_operator->add_projection(field.table(), field.meta()); - } - + auto project_operator = make_unique(std::move(project_oper.expressions())); if (child_phy_oper) { project_operator->add_child(std::move(child_phy_oper)); } - oper = unique_ptr(project_operator); + oper = std::move(project_operator); LOG_TRACE("create a project physical operator"); return rc; @@ -292,3 +333,131 @@ RC PhysicalPlanGenerator::create_plan(CalcLogicalOperator &logical_oper, std::un oper.reset(calc_oper); return rc; } + +RC PhysicalPlanGenerator::create_plan(GroupByLogicalOperator &logical_oper, std::unique_ptr &oper) +{ + RC rc = RC::SUCCESS; + + vector> &group_by_expressions = logical_oper.group_by_expressions(); + unique_ptr group_by_oper; + if (group_by_expressions.empty()) { + group_by_oper = make_unique(std::move(logical_oper.aggregate_expressions())); + } else { + group_by_oper = make_unique(std::move(logical_oper.group_by_expressions()), + std::move(logical_oper.aggregate_expressions())); + } + + ASSERT(logical_oper.children().size() == 1, "group by operator should have 1 child"); + + LogicalOperator &child_oper = *logical_oper.children().front(); + unique_ptr child_physical_oper; + rc = create(child_oper, child_physical_oper); + if (OB_FAIL(rc)) { + LOG_WARN("failed to create child physical operator of group by operator. rc=%s", strrc(rc)); + return rc; + } + + group_by_oper->add_child(std::move(child_physical_oper)); + + oper = std::move(group_by_oper); + return rc; +} + +RC PhysicalPlanGenerator::create_vec_plan(TableGetLogicalOperator &table_get_oper, unique_ptr &oper) +{ + vector> &predicates = table_get_oper.predicates(); + Table *table = table_get_oper.table(); + TableScanVecPhysicalOperator *table_scan_oper = new TableScanVecPhysicalOperator(table, table_get_oper.read_write_mode()); + table_scan_oper->set_predicates(std::move(predicates)); + oper = unique_ptr(table_scan_oper); + LOG_TRACE("use vectorized table scan"); + + return RC::SUCCESS; +} + +RC PhysicalPlanGenerator::create_vec_plan(GroupByLogicalOperator &logical_oper, unique_ptr &oper) +{ + RC rc = RC::SUCCESS; + unique_ptr physical_oper = nullptr; + if (logical_oper.group_by_expressions().empty()) { + physical_oper = make_unique(std::move(logical_oper.aggregate_expressions())); + } else { + physical_oper = make_unique( + std::move(logical_oper.group_by_expressions()), std::move(logical_oper.aggregate_expressions())); + + } + + ASSERT(logical_oper.children().size() == 1, "group by operator should have 1 child"); + + LogicalOperator &child_oper = *logical_oper.children().front(); + unique_ptr child_physical_oper; + rc = create_vec(child_oper, child_physical_oper); + if (OB_FAIL(rc)) { + LOG_WARN("failed to create child physical operator of group by(vec) operator. rc=%s", strrc(rc)); + return rc; + } + + physical_oper->add_child(std::move(child_physical_oper)); + + oper = std::move(physical_oper); + return rc; + + return RC::SUCCESS; +} + +RC PhysicalPlanGenerator::create_vec_plan(ProjectLogicalOperator &project_oper, unique_ptr &oper) +{ + vector> &child_opers = project_oper.children(); + + unique_ptr child_phy_oper; + + RC rc = RC::SUCCESS; + if (!child_opers.empty()) { + LogicalOperator *child_oper = child_opers.front().get(); + rc = create_vec(*child_oper, child_phy_oper); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to create project logical operator's child physical operator. rc=%s", strrc(rc)); + return rc; + } + } + + auto project_operator = make_unique(std::move(project_oper.expressions())); + + if (child_phy_oper != nullptr) { + std::vector expressions; + for (auto &expr : project_operator->expressions()) { + expressions.push_back(expr.get()); + } + auto expr_operator = make_unique(std::move(expressions)); + expr_operator->add_child(std::move(child_phy_oper)); + project_operator->add_child(std::move(expr_operator)); + } + + oper = std::move(project_operator); + + LOG_TRACE("create a project physical operator"); + return rc; +} + + +RC PhysicalPlanGenerator::create_vec_plan(ExplainLogicalOperator &explain_oper, unique_ptr &oper) +{ + vector> &child_opers = explain_oper.children(); + + RC rc = RC::SUCCESS; + // reuse `ExplainPhysicalOperator` in explain vectorized physical plan + unique_ptr explain_physical_oper(new ExplainPhysicalOperator); + for (unique_ptr &child_oper : child_opers) { + unique_ptr child_physical_oper; + rc = create_vec(*child_oper, child_physical_oper); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to create child physical operator. rc=%s", strrc(rc)); + return rc; + } + + explain_physical_oper->add_child(std::move(child_physical_oper)); + } + + oper = std::move(explain_physical_oper); + return rc; +} \ No newline at end of file diff --git a/src/observer/sql/optimizer/physical_plan_generator.h b/src/observer/sql/optimizer/physical_plan_generator.h index 99a7b946c..bac779a04 100644 --- a/src/observer/sql/optimizer/physical_plan_generator.h +++ b/src/observer/sql/optimizer/physical_plan_generator.h @@ -26,6 +26,7 @@ class DeleteLogicalOperator; class ExplainLogicalOperator; class JoinLogicalOperator; class CalcLogicalOperator; +class GroupByLogicalOperator; /** * @brief 物理计划生成器 @@ -40,6 +41,7 @@ class PhysicalPlanGenerator virtual ~PhysicalPlanGenerator() = default; RC create(LogicalOperator &logical_operator, std::unique_ptr &oper); + RC create_vec(LogicalOperator &logical_operator, std::unique_ptr &oper); private: RC create_plan(TableGetLogicalOperator &logical_oper, std::unique_ptr &oper); @@ -50,4 +52,9 @@ class PhysicalPlanGenerator RC create_plan(ExplainLogicalOperator &logical_oper, std::unique_ptr &oper); RC create_plan(JoinLogicalOperator &logical_oper, std::unique_ptr &oper); RC create_plan(CalcLogicalOperator &logical_oper, std::unique_ptr &oper); + RC create_plan(GroupByLogicalOperator &logical_oper, std::unique_ptr &oper); + RC create_vec_plan(ProjectLogicalOperator &logical_oper, std::unique_ptr &oper); + RC create_vec_plan(TableGetLogicalOperator &logical_oper, std::unique_ptr &oper); + RC create_vec_plan(GroupByLogicalOperator &logical_oper, std::unique_ptr &oper); + RC create_vec_plan(ExplainLogicalOperator &logical_oper, std::unique_ptr &oper); }; diff --git a/src/observer/sql/optimizer/predicate_pushdown_rewriter.cpp b/src/observer/sql/optimizer/predicate_pushdown_rewriter.cpp index 38df9254a..f206342fc 100644 --- a/src/observer/sql/optimizer/predicate_pushdown_rewriter.cpp +++ b/src/observer/sql/optimizer/predicate_pushdown_rewriter.cpp @@ -49,7 +49,7 @@ RC PredicatePushdownRewriter::rewrite(std::unique_ptr &oper, bo return rc; } - if (!predicate_expr) { + if (!predicate_expr || is_empty_predicate(predicate_expr)) { // 所有的表达式都下推到了下层算子 // 这个predicate operator其实就可以不要了。但是这里没办法删除,弄一个空的表达式吧 LOG_TRACE("all expressions of predicate operator were pushdown to table get operator, then make a fake one"); @@ -65,6 +65,23 @@ RC PredicatePushdownRewriter::rewrite(std::unique_ptr &oper, bo return rc; } +bool PredicatePushdownRewriter::is_empty_predicate(std::unique_ptr &expr) +{ + bool bool_ret = false; + if (!expr) { + return true; + } + + if (expr->type() == ExprType::CONJUNCTION) { + ConjunctionExpr *conjunction_expr = static_cast(expr.get()); + if (conjunction_expr->children().empty()) { + bool_ret = true; + } + } + + return bool_ret; +} + /** * 查看表达式是否可以直接下放到table get算子的filter * @param expr 是当前的表达式。如果可以下放给table get 算子,执行完成后expr就失效了 @@ -79,6 +96,8 @@ RC PredicatePushdownRewriter::get_exprs_can_pushdown( ConjunctionExpr *conjunction_expr = static_cast(expr.get()); // 或 操作的比较,太复杂,现在不考虑 if (conjunction_expr->conjunction_type() == ConjunctionExpr::Type::OR) { + LOG_WARN("unsupported or operation"); + rc = RC::UNIMPLENMENT; return rc; } @@ -101,12 +120,6 @@ RC PredicatePushdownRewriter::get_exprs_can_pushdown( } else if (expr->type() == ExprType::COMPARISON) { // 如果是比较操作,并且比较的左边或右边是表某个列值,那么就下推下去 auto comparison_expr = static_cast(expr.get()); - CompOp comp = comparison_expr->comp(); - if (comp != EQUAL_TO) { - // 简单处理,仅取等值比较。当然还可以取一些范围比较,还有 like % 等操作 - // 其它的还有 is null 等 - return rc; - } std::unique_ptr &left_expr = comparison_expr->left(); std::unique_ptr &right_expr = comparison_expr->right(); diff --git a/src/observer/sql/optimizer/predicate_pushdown_rewriter.h b/src/observer/sql/optimizer/predicate_pushdown_rewriter.h index ac7eec3f3..027262309 100644 --- a/src/observer/sql/optimizer/predicate_pushdown_rewriter.h +++ b/src/observer/sql/optimizer/predicate_pushdown_rewriter.h @@ -33,4 +33,5 @@ class PredicatePushdownRewriter : public RewriteRule private: RC get_exprs_can_pushdown( std::unique_ptr &expr, std::vector> &pushdown_exprs); + bool is_empty_predicate(std::unique_ptr &expr); }; diff --git a/src/observer/sql/parser/expression_binder.cpp b/src/observer/sql/parser/expression_binder.cpp new file mode 100644 index 000000000..2daf85cf5 --- /dev/null +++ b/src/observer/sql/parser/expression_binder.cpp @@ -0,0 +1,451 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/05/29. +// + +#include + +#include "common/log/log.h" +#include "common/lang/string.h" +#include "sql/parser/expression_binder.h" +#include "sql/expr/expression_iterator.h" + +using namespace std; +using namespace common; + +Table *BinderContext::find_table(const char *table_name) const +{ + auto pred = [table_name](Table *table) { return 0 == strcasecmp(table_name, table->name()); }; + auto iter = ranges::find_if(query_tables_, pred); + if (iter == query_tables_.end()) { + return nullptr; + } + return *iter; +} + +//////////////////////////////////////////////////////////////////////////////// +static void wildcard_fields(Table *table, vector> &expressions) +{ + const TableMeta &table_meta = table->table_meta(); + const int field_num = table_meta.field_num(); + for (int i = table_meta.sys_field_num(); i < field_num; i++) { + Field field(table, table_meta.field(i)); + FieldExpr *field_expr = new FieldExpr(field); + field_expr->set_name(field.field_name()); + expressions.emplace_back(field_expr); + } +} + +RC ExpressionBinder::bind_expression(unique_ptr &expr, vector> &bound_expressions) +{ + if (nullptr == expr) { + return RC::SUCCESS; + } + + switch (expr->type()) { + case ExprType::STAR: { + return bind_star_expression(expr, bound_expressions); + } break; + + case ExprType::UNBOUND_FIELD: { + return bind_unbound_field_expression(expr, bound_expressions); + } break; + + case ExprType::UNBOUND_AGGREGATION: { + return bind_aggregate_expression(expr, bound_expressions); + } break; + + case ExprType::FIELD: { + return bind_field_expression(expr, bound_expressions); + } break; + + case ExprType::VALUE: { + return bind_value_expression(expr, bound_expressions); + } break; + + case ExprType::CAST: { + return bind_cast_expression(expr, bound_expressions); + } break; + + case ExprType::COMPARISON: { + return bind_comparison_expression(expr, bound_expressions); + } break; + + case ExprType::CONJUNCTION: { + return bind_conjunction_expression(expr, bound_expressions); + } break; + + case ExprType::ARITHMETIC: { + return bind_arithmetic_expression(expr, bound_expressions); + } break; + + case ExprType::AGGREGATION: { + ASSERT(false, "shouldn't be here"); + } break; + + default: { + LOG_WARN("unknown expression type: %d", static_cast(expr->type())); + return RC::INTERNAL; + } + } + return RC::INTERNAL; +} + +RC ExpressionBinder::bind_star_expression( + unique_ptr &expr, vector> &bound_expressions) +{ + if (nullptr == expr) { + return RC::SUCCESS; + } + + auto star_expr = static_cast(expr.get()); + + vector

tables_to_wildcard; + + const char *table_name = star_expr->table_name(); + if (!is_blank(table_name) && 0 != strcmp(table_name, "*")) { + Table *table = context_.find_table(table_name); + if (nullptr == table) { + LOG_INFO("no such table in from list: %s", table_name); + return RC::SCHEMA_TABLE_NOT_EXIST; + } + + tables_to_wildcard.push_back(table); + } else { + const vector
&all_tables = context_.query_tables(); + tables_to_wildcard.insert(tables_to_wildcard.end(), all_tables.begin(), all_tables.end()); + } + + for (Table *table : tables_to_wildcard) { + wildcard_fields(table, bound_expressions); + } + + return RC::SUCCESS; +} + +RC ExpressionBinder::bind_unbound_field_expression( + unique_ptr &expr, vector> &bound_expressions) +{ + if (nullptr == expr) { + return RC::SUCCESS; + } + + auto unbound_field_expr = static_cast(expr.get()); + + const char *table_name = unbound_field_expr->table_name(); + const char *field_name = unbound_field_expr->field_name(); + + Table *table = nullptr; + if (is_blank(table_name)) { + if (context_.query_tables().size() != 1) { + LOG_INFO("cannot determine table for field: %s", field_name); + return RC::SCHEMA_TABLE_NOT_EXIST; + } + + table = context_.query_tables()[0]; + } else { + table = context_.find_table(table_name); + if (nullptr == table) { + LOG_INFO("no such table in from list: %s", table_name); + return RC::SCHEMA_TABLE_NOT_EXIST; + } + } + + if (0 == strcmp(field_name, "*")) { + wildcard_fields(table, bound_expressions); + } else { + const FieldMeta *field_meta = table->table_meta().field(field_name); + if (nullptr == field_meta) { + LOG_INFO("no such field in table: %s.%s", table_name, field_name); + return RC::SCHEMA_FIELD_MISSING; + } + + Field field(table, field_meta); + FieldExpr *field_expr = new FieldExpr(field); + field_expr->set_name(field_name); + bound_expressions.emplace_back(field_expr); + } + + return RC::SUCCESS; +} + +RC ExpressionBinder::bind_field_expression( + unique_ptr &field_expr, vector> &bound_expressions) +{ + bound_expressions.emplace_back(std::move(field_expr)); + return RC::SUCCESS; +} + +RC ExpressionBinder::bind_value_expression( + unique_ptr &value_expr, vector> &bound_expressions) +{ + bound_expressions.emplace_back(std::move(value_expr)); + return RC::SUCCESS; +} + +RC ExpressionBinder::bind_cast_expression( + unique_ptr &expr, vector> &bound_expressions) +{ + if (nullptr == expr) { + return RC::SUCCESS; + } + + auto cast_expr = static_cast(expr.get()); + + vector> child_bound_expressions; + unique_ptr &child_expr = cast_expr->child(); + + RC rc = bind_expression(child_expr, child_bound_expressions); + if (rc != RC::SUCCESS) { + return rc; + } + + if (child_bound_expressions.size() != 1) { + LOG_WARN("invalid children number of cast expression: %d", child_bound_expressions.size()); + return RC::INVALID_ARGUMENT; + } + + unique_ptr &child = child_bound_expressions[0]; + if (child.get() == child_expr.get()) { + return RC::SUCCESS; + } + + child_expr.reset(child.release()); + bound_expressions.emplace_back(std::move(expr)); + return RC::SUCCESS; +} + +RC ExpressionBinder::bind_comparison_expression( + unique_ptr &expr, vector> &bound_expressions) +{ + if (nullptr == expr) { + return RC::SUCCESS; + } + + auto comparison_expr = static_cast(expr.get()); + + vector> child_bound_expressions; + unique_ptr &left_expr = comparison_expr->left(); + unique_ptr &right_expr = comparison_expr->right(); + + RC rc = bind_expression(left_expr, child_bound_expressions); + if (rc != RC::SUCCESS) { + return rc; + } + + if (child_bound_expressions.size() != 1) { + LOG_WARN("invalid left children number of comparison expression: %d", child_bound_expressions.size()); + return RC::INVALID_ARGUMENT; + } + + unique_ptr &left = child_bound_expressions[0]; + if (left.get() != left_expr.get()) { + left_expr.reset(left.release()); + } + + child_bound_expressions.clear(); + rc = bind_expression(right_expr, child_bound_expressions); + if (rc != RC::SUCCESS) { + return rc; + } + + if (child_bound_expressions.size() != 1) { + LOG_WARN("invalid right children number of comparison expression: %d", child_bound_expressions.size()); + return RC::INVALID_ARGUMENT; + } + + unique_ptr &right = child_bound_expressions[0]; + if (right.get() != right_expr.get()) { + right_expr.reset(right.release()); + } + + bound_expressions.emplace_back(std::move(expr)); + return RC::SUCCESS; +} + +RC ExpressionBinder::bind_conjunction_expression( + unique_ptr &expr, vector> &bound_expressions) +{ + if (nullptr == expr) { + return RC::SUCCESS; + } + + auto conjunction_expr = static_cast(expr.get()); + + vector> child_bound_expressions; + vector> &children = conjunction_expr->children(); + + for (unique_ptr &child_expr : children) { + child_bound_expressions.clear(); + + RC rc = bind_expression(child_expr, child_bound_expressions); + if (rc != RC::SUCCESS) { + return rc; + } + + if (child_bound_expressions.size() != 1) { + LOG_WARN("invalid children number of conjunction expression: %d", child_bound_expressions.size()); + return RC::INVALID_ARGUMENT; + } + + unique_ptr &child = child_bound_expressions[0]; + if (child.get() != child_expr.get()) { + child_expr.reset(child.release()); + } + } + + bound_expressions.emplace_back(std::move(expr)); + + return RC::SUCCESS; +} + +RC ExpressionBinder::bind_arithmetic_expression( + unique_ptr &expr, vector> &bound_expressions) +{ + if (nullptr == expr) { + return RC::SUCCESS; + } + + auto arithmetic_expr = static_cast(expr.get()); + + vector> child_bound_expressions; + unique_ptr &left_expr = arithmetic_expr->left(); + unique_ptr &right_expr = arithmetic_expr->right(); + + RC rc = bind_expression(left_expr, child_bound_expressions); + if (OB_FAIL(rc)) { + return rc; + } + + if (child_bound_expressions.size() != 1) { + LOG_WARN("invalid left children number of comparison expression: %d", child_bound_expressions.size()); + return RC::INVALID_ARGUMENT; + } + + unique_ptr &left = child_bound_expressions[0]; + if (left.get() != left_expr.get()) { + left_expr.reset(left.release()); + } + + child_bound_expressions.clear(); + rc = bind_expression(right_expr, child_bound_expressions); + if (OB_FAIL(rc)) { + return rc; + } + + if (child_bound_expressions.size() != 1) { + LOG_WARN("invalid right children number of comparison expression: %d", child_bound_expressions.size()); + return RC::INVALID_ARGUMENT; + } + + unique_ptr &right = child_bound_expressions[0]; + if (right.get() != right_expr.get()) { + right_expr.reset(right.release()); + } + + bound_expressions.emplace_back(std::move(expr)); + return RC::SUCCESS; +} + +RC check_aggregate_expression(AggregateExpr &expression) +{ + // 必须有一个子表达式 + Expression *child_expression = expression.child().get(); + if (nullptr == child_expression) { + LOG_WARN("child expression of aggregate expression is null"); + return RC::INVALID_ARGUMENT; + } + + // 校验数据类型与聚合类型是否匹配 + AggregateExpr::Type aggregate_type = expression.aggregate_type(); + AttrType child_value_type = child_expression->value_type(); + switch (aggregate_type) { + case AggregateExpr::Type::SUM: + case AggregateExpr::Type::AVG: { + // 仅支持数值类型 + if (child_value_type != AttrType::INTS && child_value_type != AttrType::FLOATS) { + LOG_WARN("invalid child value type for aggregate expression: %d", static_cast(child_value_type)); + return RC::INVALID_ARGUMENT; + } + } break; + + case AggregateExpr::Type::COUNT: + case AggregateExpr::Type::MAX: + case AggregateExpr::Type::MIN: { + // 任何类型都支持 + } break; + } + + // 子表达式中不能再包含聚合表达式 + function&)> check_aggregate_expr = [&](unique_ptr &expr) -> RC { + RC rc = RC::SUCCESS; + if (expr->type() == ExprType::AGGREGATION) { + LOG_WARN("aggregate expression cannot be nested"); + return RC::INVALID_ARGUMENT; + } + rc = ExpressionIterator::iterate_child_expr(*expr, check_aggregate_expr); + return rc; + }; + + RC rc = ExpressionIterator::iterate_child_expr(expression, check_aggregate_expr); + + return rc; +} + +RC ExpressionBinder::bind_aggregate_expression( + unique_ptr &expr, vector> &bound_expressions) +{ + if (nullptr == expr) { + return RC::SUCCESS; + } + + auto unbound_aggregate_expr = static_cast(expr.get()); + const char *aggregate_name = unbound_aggregate_expr->aggregate_name(); + AggregateExpr::Type aggregate_type; + RC rc = AggregateExpr::type_from_string(aggregate_name, aggregate_type); + if (OB_FAIL(rc)) { + LOG_WARN("invalid aggregate name: %s", aggregate_name); + return rc; + } + + unique_ptr &child_expr = unbound_aggregate_expr->child(); + vector> child_bound_expressions; + + if (child_expr->type() == ExprType::STAR && aggregate_type == AggregateExpr::Type::COUNT) { + ValueExpr *value_expr = new ValueExpr(Value(1)); + child_expr.reset(value_expr); + } else { + rc = bind_expression(child_expr, child_bound_expressions); + if (OB_FAIL(rc)) { + return rc; + } + + if (child_bound_expressions.size() != 1) { + LOG_WARN("invalid children number of aggregate expression: %d", child_bound_expressions.size()); + return RC::INVALID_ARGUMENT; + } + + if (child_bound_expressions[0].get() != child_expr.get()) { + child_expr.reset(child_bound_expressions[0].release()); + } + } + + auto aggregate_expr = make_unique(aggregate_type, std::move(child_expr)); + aggregate_expr->set_name(unbound_aggregate_expr->name()); + rc = check_aggregate_expression(*aggregate_expr); + if (OB_FAIL(rc)) { + return rc; + } + + bound_expressions.emplace_back(std::move(aggregate_expr)); + return RC::SUCCESS; +} diff --git a/src/observer/sql/parser/expression_binder.h b/src/observer/sql/parser/expression_binder.h new file mode 100644 index 000000000..d52f049c4 --- /dev/null +++ b/src/observer/sql/parser/expression_binder.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by Wangyunlai on 2024/05/29. +// + +#pragma once + +#include + +#include "sql/expr/expression.h" + +class BinderContext +{ +public: + BinderContext() = default; + virtual ~BinderContext() = default; + + void add_table(Table *table) { query_tables_.push_back(table); } + + Table *find_table(const char *table_name) const; + + const std::vector
&query_tables() const { return query_tables_; } + +private: + std::vector
query_tables_; +}; + +/** + * @brief 绑定表达式 + * @details 绑定表达式,就是在SQL解析后,得到文本描述的表达式,将表达式解析为具体的数据库对象 + */ +class ExpressionBinder +{ +public: + ExpressionBinder(BinderContext &context) : context_(context) {} + virtual ~ExpressionBinder() = default; + + RC bind_expression(std::unique_ptr &expr, std::vector> &bound_expressions); + +private: + RC bind_star_expression( + std::unique_ptr &star_expr, std::vector> &bound_expressions); + RC bind_unbound_field_expression( + std::unique_ptr &unbound_field_expr, std::vector> &bound_expressions); + RC bind_field_expression( + std::unique_ptr &field_expr, std::vector> &bound_expressions); + RC bind_value_expression( + std::unique_ptr &value_expr, std::vector> &bound_expressions); + RC bind_cast_expression( + std::unique_ptr &cast_expr, std::vector> &bound_expressions); + RC bind_comparison_expression( + std::unique_ptr &comparison_expr, std::vector> &bound_expressions); + RC bind_conjunction_expression( + std::unique_ptr &conjunction_expr, std::vector> &bound_expressions); + RC bind_arithmetic_expression( + std::unique_ptr &arithmetic_expr, std::vector> &bound_expressions); + RC bind_aggregate_expression( + std::unique_ptr &aggregate_expr, std::vector> &bound_expressions); + +private: + BinderContext &context_; +}; \ No newline at end of file diff --git a/src/observer/sql/parser/lex_sql.cpp b/src/observer/sql/parser/lex_sql.cpp index e04cf336f..14ce1bc3b 100644 --- a/src/observer/sql/parser/lex_sql.cpp +++ b/src/observer/sql/parser/lex_sql.cpp @@ -1,4 +1,5 @@ #line 2 "lex_sql.cpp" +#line 2 "lex_sql.l" /* 这里的代码会被复制到lex_sql.cpp的最开始位置 定义yy_size_t的原因是因为flex生成的代码,会使用yy_size_t与其他类型的数字 @@ -22,7 +23,10 @@ do { \ } \ while (0); -#line 26 "lex_sql.cpp" + + + +#line 30 "lex_sql.cpp" #define YY_INT_ALIGNED short int @@ -30,36 +34,12 @@ while (0); #define FLEX_SCANNER #define YY_FLEX_MAJOR_VERSION 2 -#define YY_FLEX_MINOR_VERSION 6 -#define YY_FLEX_SUBMINOR_VERSION 4 +#define YY_FLEX_MINOR_VERSION 5 +#define YY_FLEX_SUBMINOR_VERSION 35 #if YY_FLEX_SUBMINOR_VERSION > 0 #define FLEX_BETA #endif -#ifdef yyget_lval -#define yyget_lval_ALREADY_DEFINED -#else -#define yyget_lval yyget_lval -#endif - -#ifdef yyset_lval -#define yyset_lval_ALREADY_DEFINED -#else -#define yyset_lval yyset_lval -#endif - -#ifdef yyget_lloc -#define yyget_lloc_ALREADY_DEFINED -#else -#define yyget_lloc yyget_lloc -#endif - -#ifdef yyset_lloc -#define yyset_lloc_ALREADY_DEFINED -#else -#define yyset_lloc yyset_lloc -#endif - /* First, we deal with platform-specific or compiler-specific issues. */ /* begin standard C headers. */ @@ -100,6 +80,7 @@ typedef int flex_int32_t; typedef unsigned char flex_uint8_t; typedef unsigned short int flex_uint16_t; typedef unsigned int flex_uint32_t; +#endif /* ! C99 */ /* Limits of integral types. */ #ifndef INT8_MIN @@ -130,32 +111,38 @@ typedef unsigned int flex_uint32_t; #define UINT32_MAX (4294967295U) #endif -#ifndef SIZE_MAX -#define SIZE_MAX (~(size_t)0) -#endif +#endif /* ! FLEXINT_H */ -#endif /* ! C99 */ +#ifdef __cplusplus -#endif /* ! FLEXINT_H */ +/* The "const" storage-class-modifier is valid. */ +#define YY_USE_CONST -/* begin standard C++ headers. */ +#else /* ! __cplusplus */ -/* TODO: this is always defined, so inline it */ -#define yyconst const +/* C99 requires __STDC__ to be defined as 1. */ +#if defined (__STDC__) + +#define YY_USE_CONST + +#endif /* defined (__STDC__) */ +#endif /* ! __cplusplus */ -#if defined(__GNUC__) && __GNUC__ >= 3 -#define yynoreturn __attribute__((__noreturn__)) +#ifdef YY_USE_CONST +#define yyconst const #else -#define yynoreturn +#define yyconst #endif /* Returned upon end-of-file. */ #define YY_NULL 0 -/* Promotes a possibly negative, possibly signed char to an - * integer in range [0..255] for use as an array index. +/* Promotes a possibly negative, possibly signed char to an unsigned + * integer for use as an array index. If the signed char is negative, + * we want to instead treat it as an 8-bit unsigned char, hence the + * double cast. */ -#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) +#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) /* An opaque pointer. */ #ifndef YY_TYPEDEF_YY_SCANNER_T @@ -179,29 +166,25 @@ typedef void* yyscan_t; * definition of BEGIN. */ #define BEGIN yyg->yy_start = 1 + 2 * + /* Translate the current start state into a value that can be later handed * to BEGIN to return to the state. The YYSTATE alias is for lex * compatibility. */ #define YY_START ((yyg->yy_start - 1) / 2) #define YYSTATE YY_START + /* Action number for EOF rule of a given start state. */ #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) + /* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE yyrestart( yyin , yyscanner ) +#define YY_NEW_FILE yyrestart(yyin ,yyscanner ) + #define YY_END_OF_BUFFER_CHAR 0 /* Size of default input buffer. */ #ifndef YY_BUF_SIZE -#ifdef __ia64__ -/* On IA-64, the buffer size is 16k, not 8k. - * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case. - * Ditto for the __ia64__ case accordingly. - */ -#define YY_BUF_SIZE 32768 -#else #define YY_BUF_SIZE 16384 -#endif /* __ia64__ */ #endif /* The state buf must be large enough to hold one state per character in the main buffer. @@ -213,17 +196,11 @@ typedef void* yyscan_t; typedef struct yy_buffer_state *YY_BUFFER_STATE; #endif -#ifndef YY_TYPEDEF_YY_SIZE_T -#define YY_TYPEDEF_YY_SIZE_T -typedef size_t yy_size_t; -#endif - #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 #define EOB_ACT_LAST_MATCH 2 - + #define YY_LESS_LINENO(n) - #define YY_LINENO_REWIND_TO(ptr) /* Return all but the first "n" matched characters back to the input stream. */ #define yyless(n) \ @@ -238,8 +215,14 @@ typedef size_t yy_size_t; YY_DO_BEFORE_ACTION; /* set up yytext again */ \ } \ while ( 0 ) + #define unput(c) yyunput( c, yyg->yytext_ptr , yyscanner ) +#ifndef YY_TYPEDEF_YY_SIZE_T +#define YY_TYPEDEF_YY_SIZE_T +typedef size_t yy_size_t; +#endif + #ifndef YY_STRUCT_YY_BUFFER_STATE #define YY_STRUCT_YY_BUFFER_STATE struct yy_buffer_state @@ -252,7 +235,7 @@ struct yy_buffer_state /* Size of input buffer in bytes, not including room for EOB * characters. */ - int yy_buf_size; + yy_size_t yy_buf_size; /* Number of characters read into yy_ch_buf, not including EOB * characters. @@ -280,7 +263,7 @@ struct yy_buffer_state int yy_bs_lineno; /**< The line count. */ int yy_bs_column; /**< The column count. */ - + /* Whether to try to fill the input buffer when we reach the * end of it. */ @@ -314,79 +297,86 @@ struct yy_buffer_state #define YY_CURRENT_BUFFER ( yyg->yy_buffer_stack \ ? yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] \ : NULL) + /* Same as previous macro, but useful when we know that the buffer stack is not * NULL or when we need an lvalue. For internal use only. */ #define YY_CURRENT_BUFFER_LVALUE yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] -void yyrestart ( FILE *input_file , yyscan_t yyscanner ); -void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size , yyscan_t yyscanner ); -void yy_delete_buffer ( YY_BUFFER_STATE b , yyscan_t yyscanner ); -void yy_flush_buffer ( YY_BUFFER_STATE b , yyscan_t yyscanner ); -void yypush_buffer_state ( YY_BUFFER_STATE new_buffer , yyscan_t yyscanner ); -void yypop_buffer_state ( yyscan_t yyscanner ); +void yyrestart (FILE *input_file ,yyscan_t yyscanner ); +void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +YY_BUFFER_STATE yy_create_buffer (FILE *file,int size ,yyscan_t yyscanner ); +void yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +void yy_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +void yypop_buffer_state (yyscan_t yyscanner ); + +static void yyensure_buffer_stack (yyscan_t yyscanner ); +static void yy_load_buffer_state (yyscan_t yyscanner ); +static void yy_init_buffer (YY_BUFFER_STATE b,FILE *file ,yyscan_t yyscanner ); -static void yyensure_buffer_stack ( yyscan_t yyscanner ); -static void yy_load_buffer_state ( yyscan_t yyscanner ); -static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file , yyscan_t yyscanner ); -#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER , yyscanner) +#define YY_FLUSH_BUFFER yy_flush_buffer(YY_CURRENT_BUFFER ,yyscanner) -YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_string ( const char *yy_str , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len , yyscan_t yyscanner ); +YY_BUFFER_STATE yy_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner ); +YY_BUFFER_STATE yy_scan_string (yyconst char *yy_str ,yyscan_t yyscanner ); +YY_BUFFER_STATE yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); -void *yyalloc ( yy_size_t , yyscan_t yyscanner ); -void *yyrealloc ( void *, yy_size_t , yyscan_t yyscanner ); -void yyfree ( void * , yyscan_t yyscanner ); +void *yyalloc (yy_size_t ,yyscan_t yyscanner ); +void *yyrealloc (void *,yy_size_t ,yyscan_t yyscanner ); +void yyfree (void * ,yyscan_t yyscanner ); #define yy_new_buffer yy_create_buffer + #define yy_set_interactive(is_interactive) \ { \ if ( ! YY_CURRENT_BUFFER ){ \ yyensure_buffer_stack (yyscanner); \ YY_CURRENT_BUFFER_LVALUE = \ - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); \ + yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \ } \ YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ } + #define yy_set_bol(at_bol) \ { \ if ( ! YY_CURRENT_BUFFER ){\ yyensure_buffer_stack (yyscanner); \ YY_CURRENT_BUFFER_LVALUE = \ - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); \ + yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \ } \ YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ } + #define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) /* Begin user sect3 */ -#define yywrap(yyscanner) (/*CONSTCOND*/1) +#define yywrap(n) 1 #define YY_SKIP_YYWRAP -typedef flex_uint8_t YY_CHAR; + +typedef unsigned char YY_CHAR; typedef int yy_state_type; #define yytext_ptr yytext_r -static yy_state_type yy_get_previous_state ( yyscan_t yyscanner ); -static yy_state_type yy_try_NUL_trans ( yy_state_type current_state , yyscan_t yyscanner); -static int yy_get_next_buffer ( yyscan_t yyscanner ); -static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner ); +static yy_state_type yy_get_previous_state (yyscan_t yyscanner ); +static yy_state_type yy_try_NUL_trans (yy_state_type current_state ,yyscan_t yyscanner); +static int yy_get_next_buffer (yyscan_t yyscanner ); +static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner ); /* Done after the current pattern has been matched and before the * corresponding action - sets up yytext. */ #define YY_DO_BEFORE_ACTION \ yyg->yytext_ptr = yy_bp; \ - yyleng = (int) (yy_cp - yy_bp); \ + yyleng = (size_t) (yy_cp - yy_bp); \ yyg->yy_hold_char = *yy_cp; \ *yy_cp = '\0'; \ yyg->yy_c_buf_p = yy_cp; -#define YY_NUM_RULES 57 -#define YY_END_OF_BUFFER 58 + +#define YY_NUM_RULES 61 +#define YY_END_OF_BUFFER 62 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -394,29 +384,31 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[166] = +static yyconst flex_int16_t yy_accept[183] = { 0, - 0, 0, 0, 0, 58, 56, 1, 2, 56, 56, - 56, 40, 41, 52, 50, 42, 51, 6, 53, 3, - 5, 47, 43, 49, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 57, 46, 0, 54, 0, 55, 3, 0, 44, - 45, 48, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 15, - 39, 39, 39, 39, 39, 39, 39, 39, 4, 22, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 32, 39, 39, - - 39, 28, 39, 39, 39, 39, 39, 39, 39, 19, - 33, 39, 39, 36, 39, 9, 11, 7, 39, 39, - 20, 8, 39, 39, 39, 24, 35, 39, 39, 16, - 17, 39, 39, 39, 39, 29, 39, 39, 39, 39, - 34, 14, 39, 39, 39, 39, 12, 39, 39, 21, - 30, 10, 26, 39, 37, 23, 39, 18, 13, 27, - 25, 38, 39, 31, 0 + 0, 0, 0, 0, 62, 60, 1, 2, 60, 60, + 60, 44, 45, 56, 54, 46, 55, 6, 57, 3, + 5, 51, 47, 53, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 61, 50, 0, 58, 0, 59, 3, 0, + 48, 49, 52, 43, 43, 43, 43, 40, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 15, 43, 43, 43, 43, 43, 43, + 43, 43, 43, 4, 22, 43, 43, 43, 43, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + + 43, 43, 43, 43, 32, 43, 43, 43, 28, 43, + 43, 43, 43, 43, 43, 43, 43, 19, 33, 43, + 43, 36, 43, 9, 11, 7, 43, 43, 43, 20, + 43, 8, 43, 43, 43, 24, 35, 43, 43, 16, + 43, 17, 43, 43, 43, 43, 29, 43, 43, 43, + 43, 34, 43, 39, 14, 43, 43, 43, 43, 43, + 12, 43, 43, 21, 30, 10, 26, 43, 42, 37, + 23, 43, 18, 43, 13, 27, 25, 38, 43, 41, + 31, 0 } ; -static const YY_CHAR yy_ec[256] = +static yyconst flex_int32_t yy_ec[256] = { 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, @@ -448,7 +440,7 @@ static const YY_CHAR yy_ec[256] = 1, 1, 1, 1, 1 } ; -static const YY_CHAR yy_meta[67] = +static yyconst flex_int32_t yy_meta[67] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, @@ -459,111 +451,120 @@ static const YY_CHAR yy_meta[67] = 2, 2, 2, 2, 2, 2 } ; -static const flex_int16_t yy_base[171] = +static yyconst flex_int16_t yy_base[188] = { 0, - 0, 0, 0, 0, 435, 436, 436, 436, 416, 428, - 426, 436, 436, 436, 436, 436, 416, 436, 436, 54, - 436, 52, 436, 412, 53, 57, 60, 59, 58, 61, - 406, 74, 69, 67, 73, 76, 111, 70, 78, 101, - 112, 436, 436, 414, 436, 412, 436, 115, 402, 436, - 436, 436, 0, 401, 117, 116, 119, 126, 128, 132, - 129, 134, 138, 133, 140, 142, 167, 164, 166, 400, - 169, 177, 178, 163, 188, 199, 189, 192, 399, 398, - 190, 212, 211, 213, 217, 221, 215, 229, 214, 233, - 235, 237, 241, 239, 238, 249, 245, 261, 244, 267, - - 263, 397, 264, 268, 269, 265, 273, 277, 284, 394, - 391, 287, 288, 390, 289, 389, 388, 386, 304, 291, - 383, 378, 292, 301, 295, 376, 375, 322, 323, 374, - 366, 318, 315, 332, 336, 331, 326, 344, 346, 343, - 329, 310, 348, 347, 358, 350, 360, 359, 362, 307, - 305, 293, 240, 367, 202, 191, 364, 154, 143, 136, - 97, 93, 122, 84, 436, 423, 425, 427, 76, 75 + 0, 0, 0, 0, 484, 485, 485, 485, 465, 477, + 475, 485, 485, 485, 485, 485, 465, 485, 485, 54, + 485, 52, 485, 454, 53, 57, 60, 59, 70, 91, + 61, 67, 75, 456, 87, 77, 98, 113, 109, 58, + 119, 111, 485, 485, 465, 485, 463, 485, 86, 453, + 485, 485, 485, 0, 451, 126, 121, 450, 115, 137, + 127, 143, 128, 139, 150, 157, 153, 160, 163, 168, + 173, 175, 186, 448, 180, 190, 199, 203, 193, 202, + 216, 201, 214, 447, 446, 225, 226, 228, 227, 230, + 237, 243, 239, 145, 231, 253, 250, 251, 256, 258, + + 260, 265, 271, 278, 264, 281, 285, 286, 445, 288, + 294, 290, 293, 299, 302, 310, 300, 444, 443, 307, + 312, 441, 316, 439, 438, 437, 317, 325, 340, 436, + 323, 433, 329, 334, 330, 432, 424, 327, 352, 423, + 355, 422, 361, 342, 363, 367, 419, 364, 368, 380, + 378, 412, 375, 404, 386, 381, 382, 397, 385, 392, + 395, 409, 407, 353, 347, 336, 275, 393, 263, 261, + 179, 399, 171, 416, 162, 99, 83, 74, 420, 73, + 69, 485, 473, 475, 477, 76, 75 } ; -static const flex_int16_t yy_def[171] = +static yyconst flex_int16_t yy_def[188] = { 0, - 165, 1, 166, 166, 165, 165, 165, 165, 165, 167, - 168, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 165, 165, 167, 165, 168, 165, 165, 165, 165, - 165, 165, 170, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 165, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, - 169, 169, 169, 169, 0, 165, 165, 165, 165, 165 + 182, 1, 183, 183, 182, 182, 182, 182, 182, 184, + 185, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 182, 182, 184, 182, 185, 182, 182, 182, + 182, 182, 182, 187, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 182, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + 186, 0, 182, 182, 182, 182, 182 } ; -static const flex_int16_t yy_nxt[503] = +static yyconst flex_int16_t yy_nxt[552] = { 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 33, 31, 31, - 34, 31, 31, 35, 31, 36, 37, 38, 39, 40, - 41, 31, 31, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 31, 34, 31, 31, 35, 31, 36, 37, - 38, 39, 40, 41, 31, 31, 49, 53, 48, 50, - 51, 53, 53, 53, 53, 53, 53, 54, 61, 57, - 56, 53, 62, 53, 53, 55, 58, 53, 53, 75, - 53, 65, 53, 59, 63, 60, 66, 67, 53, 64, - - 69, 68, 61, 57, 56, 70, 62, 53, 55, 71, - 58, 53, 76, 75, 65, 53, 59, 63, 60, 66, - 77, 67, 64, 69, 68, 53, 53, 49, 70, 48, - 53, 53, 71, 53, 72, 76, 53, 73, 78, 80, - 53, 81, 53, 53, 77, 83, 53, 53, 53, 82, - 53, 164, 53, 74, 53, 85, 53, 53, 72, 84, - 90, 73, 78, 80, 87, 81, 86, 91, 53, 83, - 88, 89, 82, 92, 164, 93, 74, 53, 53, 85, - 53, 53, 84, 53, 90, 99, 95, 87, 96, 86, - 91, 53, 53, 88, 89, 104, 92, 94, 93, 100, - - 97, 98, 53, 53, 53, 53, 53, 101, 105, 99, - 95, 103, 96, 53, 102, 108, 53, 109, 104, 107, - 94, 106, 100, 97, 98, 53, 53, 53, 53, 53, - 101, 53, 105, 110, 103, 53, 113, 102, 115, 108, - 114, 109, 107, 53, 112, 106, 111, 53, 117, 53, - 116, 53, 53, 53, 53, 53, 120, 110, 53, 53, - 113, 123, 115, 53, 114, 119, 127, 112, 125, 111, - 118, 117, 121, 122, 116, 53, 124, 53, 53, 53, - 120, 53, 53, 53, 133, 123, 129, 53, 119, 131, - 127, 53, 125, 118, 126, 121, 122, 128, 53, 132, - - 124, 53, 53, 53, 130, 53, 53, 53, 133, 53, - 129, 134, 135, 131, 137, 53, 136, 126, 53, 53, - 128, 53, 132, 140, 53, 138, 139, 130, 141, 53, - 144, 143, 53, 142, 134, 135, 53, 53, 137, 136, - 53, 147, 145, 53, 146, 53, 53, 140, 138, 139, - 53, 141, 148, 144, 143, 149, 142, 53, 53, 150, - 53, 53, 53, 151, 53, 147, 145, 152, 146, 153, - 154, 155, 53, 53, 53, 148, 53, 157, 53, 149, - 53, 53, 160, 150, 156, 163, 151, 158, 53, 53, - 53, 152, 53, 153, 154, 155, 159, 53, 161, 162, - - 53, 157, 53, 53, 53, 53, 160, 156, 53, 163, - 158, 53, 53, 79, 53, 53, 79, 47, 45, 159, - 53, 161, 162, 42, 42, 44, 44, 46, 46, 52, - 48, 47, 45, 43, 165, 5, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - - 165, 165 + 26, 27, 28, 29, 30, 31, 32, 33, 34, 34, + 35, 34, 34, 36, 34, 37, 38, 39, 40, 41, + 42, 34, 34, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 34, 34, 36, 34, 37, 38, + 39, 40, 41, 42, 34, 34, 50, 54, 49, 51, + 52, 54, 54, 54, 54, 54, 54, 55, 63, 59, + 57, 54, 64, 54, 54, 56, 60, 54, 54, 54, + 71, 54, 81, 61, 65, 62, 70, 54, 50, 58, + + 49, 54, 63, 59, 57, 54, 64, 72, 56, 74, + 60, 66, 54, 54, 71, 81, 61, 65, 62, 70, + 73, 67, 58, 54, 68, 54, 69, 54, 80, 54, + 72, 75, 74, 54, 66, 54, 76, 83, 82, 77, + 54, 54, 54, 73, 67, 87, 86, 68, 85, 69, + 78, 54, 80, 54, 75, 79, 88, 54, 89, 54, + 76, 83, 82, 77, 54, 91, 90, 54, 87, 92, + 86, 54, 85, 78, 54, 93, 54, 54, 79, 125, + 88, 89, 54, 94, 95, 54, 97, 54, 91, 54, + 90, 96, 92, 54, 54, 98, 99, 102, 93, 103, + + 54, 100, 125, 101, 54, 106, 94, 54, 95, 97, + 107, 104, 105, 54, 96, 54, 54, 54, 98, 99, + 108, 102, 113, 103, 100, 112, 101, 109, 54, 106, + 54, 115, 110, 107, 104, 105, 111, 116, 114, 54, + 54, 54, 54, 108, 54, 54, 113, 118, 112, 121, + 109, 54, 117, 54, 115, 110, 122, 54, 120, 111, + 124, 116, 114, 119, 54, 54, 123, 54, 126, 128, + 54, 118, 54, 121, 54, 54, 117, 54, 54, 54, + 122, 120, 129, 127, 124, 54, 119, 130, 133, 54, + 123, 126, 54, 128, 132, 54, 131, 136, 134, 54, + + 54, 135, 54, 137, 54, 129, 127, 54, 54, 139, + 130, 142, 133, 54, 54, 138, 54, 132, 144, 131, + 136, 54, 134, 143, 54, 135, 54, 137, 140, 141, + 54, 54, 147, 139, 148, 142, 151, 54, 138, 54, + 145, 54, 144, 54, 54, 146, 143, 158, 54, 149, + 54, 140, 141, 150, 54, 147, 54, 154, 148, 153, + 151, 54, 152, 145, 156, 157, 54, 54, 146, 54, + 155, 158, 149, 159, 160, 54, 150, 54, 54, 162, + 154, 54, 54, 153, 161, 152, 163, 156, 157, 54, + 164, 166, 54, 155, 54, 54, 54, 159, 160, 54, + + 54, 165, 162, 167, 170, 168, 54, 54, 161, 54, + 163, 54, 169, 54, 164, 166, 172, 174, 54, 171, + 179, 54, 173, 54, 165, 178, 54, 167, 170, 168, + 54, 175, 176, 54, 54, 169, 54, 54, 54, 180, + 172, 174, 171, 177, 179, 173, 54, 54, 178, 181, + 54, 54, 54, 54, 175, 54, 176, 54, 54, 54, + 54, 84, 54, 180, 54, 54, 177, 84, 48, 46, + 54, 53, 181, 43, 43, 45, 45, 47, 47, 49, + 48, 46, 44, 182, 5, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182 } ; -static const flex_int16_t yy_chk[503] = +static yyconst flex_int16_t yy_chk[552] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -572,55 +573,60 @@ static const flex_int16_t yy_chk[503] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 20, 25, 20, 22, - 22, 26, 29, 28, 27, 30, 170, 169, 28, 27, - 26, 34, 28, 33, 38, 25, 27, 35, 32, 38, - 36, 30, 39, 27, 28, 27, 30, 32, 164, 29, - - 34, 33, 28, 27, 26, 35, 28, 162, 25, 36, - 27, 161, 39, 38, 30, 40, 27, 28, 27, 30, - 40, 32, 29, 34, 33, 37, 41, 48, 35, 48, - 56, 55, 36, 57, 37, 39, 163, 37, 41, 55, - 58, 56, 59, 61, 40, 58, 60, 64, 62, 57, - 160, 163, 63, 37, 65, 60, 66, 159, 37, 59, - 64, 37, 41, 55, 62, 56, 61, 64, 158, 58, - 62, 63, 57, 65, 163, 66, 37, 74, 68, 60, - 69, 67, 59, 71, 64, 69, 68, 62, 68, 61, - 64, 72, 73, 62, 63, 74, 65, 67, 66, 71, - - 68, 68, 75, 77, 81, 156, 78, 72, 75, 69, - 68, 73, 68, 76, 72, 78, 155, 81, 74, 77, - 67, 76, 71, 68, 68, 83, 82, 84, 89, 87, - 72, 85, 75, 82, 73, 86, 85, 72, 87, 78, - 86, 81, 77, 88, 84, 76, 83, 90, 89, 91, - 88, 92, 95, 94, 153, 93, 92, 82, 99, 97, - 85, 95, 87, 96, 86, 91, 99, 84, 97, 83, - 90, 89, 93, 94, 88, 98, 96, 101, 103, 106, - 92, 100, 104, 105, 106, 95, 101, 107, 91, 104, - 99, 108, 97, 90, 98, 93, 94, 100, 109, 105, - - 96, 112, 113, 115, 103, 120, 123, 152, 106, 125, - 101, 107, 108, 104, 112, 124, 109, 98, 119, 151, - 100, 150, 105, 119, 142, 113, 115, 103, 120, 133, - 125, 124, 132, 123, 107, 108, 128, 129, 112, 109, - 137, 132, 128, 141, 129, 136, 134, 119, 113, 115, - 135, 120, 133, 125, 124, 134, 123, 140, 138, 135, - 139, 144, 143, 137, 146, 132, 128, 138, 129, 139, - 140, 143, 145, 148, 147, 133, 149, 145, 157, 134, - 131, 154, 148, 135, 144, 157, 137, 146, 130, 127, - 126, 138, 122, 139, 140, 143, 147, 121, 149, 154, - - 118, 145, 117, 116, 114, 111, 148, 144, 110, 157, - 146, 102, 80, 79, 70, 54, 49, 46, 44, 147, - 31, 149, 154, 166, 166, 167, 167, 168, 168, 24, - 17, 11, 10, 9, 5, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - - 165, 165 + 22, 26, 40, 28, 27, 31, 187, 186, 28, 27, + 26, 32, 28, 181, 29, 25, 27, 180, 178, 33, + 32, 36, 40, 27, 28, 27, 31, 177, 49, 26, + + 49, 35, 28, 27, 26, 30, 28, 33, 25, 36, + 27, 29, 37, 176, 32, 40, 27, 28, 27, 31, + 35, 30, 26, 39, 30, 42, 30, 38, 39, 59, + 33, 37, 36, 41, 29, 57, 38, 42, 41, 38, + 56, 61, 63, 35, 30, 59, 57, 30, 56, 30, + 38, 60, 39, 64, 37, 38, 60, 62, 61, 94, + 38, 42, 41, 38, 65, 63, 62, 67, 59, 64, + 57, 66, 56, 38, 68, 64, 175, 69, 38, 94, + 60, 61, 70, 65, 66, 173, 67, 71, 63, 72, + 62, 66, 64, 171, 75, 68, 69, 72, 64, 72, + + 73, 70, 94, 71, 76, 73, 65, 79, 66, 67, + 75, 72, 72, 77, 66, 82, 80, 78, 68, 69, + 76, 72, 80, 72, 70, 79, 71, 76, 83, 73, + 81, 82, 77, 75, 72, 72, 78, 83, 81, 86, + 87, 89, 88, 76, 90, 95, 80, 87, 79, 90, + 76, 91, 86, 93, 82, 77, 91, 92, 89, 78, + 93, 83, 81, 88, 97, 98, 92, 96, 95, 97, + 99, 87, 100, 90, 101, 170, 86, 169, 105, 102, + 91, 89, 98, 96, 93, 103, 88, 99, 102, 167, + 92, 95, 104, 97, 101, 106, 100, 105, 103, 107, + + 108, 104, 110, 106, 112, 98, 96, 113, 111, 108, + 99, 112, 102, 114, 117, 107, 115, 101, 114, 100, + 105, 120, 103, 113, 116, 104, 121, 106, 110, 111, + 123, 127, 117, 108, 120, 112, 127, 131, 107, 128, + 115, 138, 114, 133, 135, 116, 113, 138, 134, 121, + 166, 110, 111, 123, 129, 117, 144, 131, 120, 129, + 127, 165, 128, 115, 134, 135, 139, 164, 116, 141, + 133, 138, 121, 139, 141, 143, 123, 145, 148, 144, + 131, 146, 149, 129, 143, 128, 145, 134, 135, 153, + 146, 149, 151, 133, 150, 156, 157, 139, 141, 159, + + 155, 148, 144, 150, 156, 151, 160, 168, 143, 161, + 145, 158, 153, 172, 146, 149, 158, 160, 154, 157, + 172, 163, 159, 162, 148, 168, 152, 150, 156, 151, + 174, 161, 162, 147, 179, 153, 142, 140, 137, 174, + 158, 160, 157, 163, 172, 159, 136, 132, 168, 179, + 130, 126, 125, 124, 161, 122, 162, 119, 118, 109, + 85, 84, 74, 174, 58, 55, 163, 50, 47, 45, + 34, 24, 179, 183, 183, 184, 184, 185, 185, 17, + 11, 10, 9, 5, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, + 182 } ; /* The intent behind this definition is that it'll catch @@ -655,7 +661,6 @@ extern int atoi(); extern double atof(); #define RETURN_TOKEN(token) LOG_DEBUG("%s", #token);return token -#line 659 "lex_sql.cpp" /* Prevent the need for linking with -lfl */ #define YY_NO_INPUT 1 /* 不区分大小写 */ @@ -664,7 +669,7 @@ extern double atof(); /* 1. 匹配的规则长的优先 */ /* 2. 写在最前面的优先 */ /* yylval 就可以认为是 yacc 中 %union 定义的结构体(union 结构) */ -#line 668 "lex_sql.cpp" +#line 673 "lex_sql.cpp" #define INITIAL 0 #define STR 1 @@ -719,7 +724,7 @@ struct yyguts_t }; /* end struct yyguts_t */ -static int yy_init_globals ( yyscan_t yyscanner ); +static int yy_init_globals (yyscan_t yyscanner ); /* This must go here because YYSTYPE and YYLTYPE are included * from bison output in section 1.*/ @@ -729,48 +734,44 @@ static int yy_init_globals ( yyscan_t yyscanner ); int yylex_init (yyscan_t* scanner); -int yylex_init_extra ( YY_EXTRA_TYPE user_defined, yyscan_t* scanner); +int yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner); /* Accessor methods to globals. These are made visible to non-reentrant scanners for convenience. */ -int yylex_destroy ( yyscan_t yyscanner ); +int yylex_destroy (yyscan_t yyscanner ); -int yyget_debug ( yyscan_t yyscanner ); +int yyget_debug (yyscan_t yyscanner ); -void yyset_debug ( int debug_flag , yyscan_t yyscanner ); +void yyset_debug (int debug_flag ,yyscan_t yyscanner ); -YY_EXTRA_TYPE yyget_extra ( yyscan_t yyscanner ); +YY_EXTRA_TYPE yyget_extra (yyscan_t yyscanner ); -void yyset_extra ( YY_EXTRA_TYPE user_defined , yyscan_t yyscanner ); +void yyset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner ); -FILE *yyget_in ( yyscan_t yyscanner ); +FILE *yyget_in (yyscan_t yyscanner ); -void yyset_in ( FILE * _in_str , yyscan_t yyscanner ); +void yyset_in (FILE * in_str ,yyscan_t yyscanner ); -FILE *yyget_out ( yyscan_t yyscanner ); +FILE *yyget_out (yyscan_t yyscanner ); -void yyset_out ( FILE * _out_str , yyscan_t yyscanner ); +void yyset_out (FILE * out_str ,yyscan_t yyscanner ); - int yyget_leng ( yyscan_t yyscanner ); +int yyget_leng (yyscan_t yyscanner ); -char *yyget_text ( yyscan_t yyscanner ); +char *yyget_text (yyscan_t yyscanner ); -int yyget_lineno ( yyscan_t yyscanner ); +int yyget_lineno (yyscan_t yyscanner ); -void yyset_lineno ( int _line_number , yyscan_t yyscanner ); +void yyset_lineno (int line_number ,yyscan_t yyscanner ); -int yyget_column ( yyscan_t yyscanner ); +YYSTYPE * yyget_lval (yyscan_t yyscanner ); -void yyset_column ( int _column_no , yyscan_t yyscanner ); +void yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner ); -YYSTYPE * yyget_lval ( yyscan_t yyscanner ); - -void yyset_lval ( YYSTYPE * yylval_param , yyscan_t yyscanner ); - - YYLTYPE *yyget_lloc ( yyscan_t yyscanner ); + YYLTYPE *yyget_lloc (yyscan_t yyscanner ); - void yyset_lloc ( YYLTYPE * yylloc_param , yyscan_t yyscanner ); + void yyset_lloc (YYLTYPE * yylloc_param ,yyscan_t yyscanner ); /* Macros after this point can all be overridden by user definitions in * section 1. @@ -778,41 +779,33 @@ void yyset_lval ( YYSTYPE * yylval_param , yyscan_t yyscanner ); #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap ( yyscan_t yyscanner ); +extern "C" int yywrap (yyscan_t yyscanner ); #else -extern int yywrap ( yyscan_t yyscanner ); +extern int yywrap (yyscan_t yyscanner ); #endif #endif -#ifndef YY_NO_UNPUT - -#endif - #ifndef yytext_ptr -static void yy_flex_strncpy ( char *, const char *, int , yyscan_t yyscanner); +static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen ( const char * , yyscan_t yyscanner); +static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner); #endif #ifndef YY_NO_INPUT + #ifdef __cplusplus -static int yyinput ( yyscan_t yyscanner ); +static int yyinput (yyscan_t yyscanner ); #else -static int input ( yyscan_t yyscanner ); +static int input (yyscan_t yyscanner ); #endif #endif /* Amount of stuff to slurp up with each read. */ #ifndef YY_READ_BUF_SIZE -#ifdef __ia64__ -/* On IA-64, the buffer size is 16k, not 8k */ -#define YY_READ_BUF_SIZE 16384 -#else #define YY_READ_BUF_SIZE 8192 -#endif /* __ia64__ */ #endif /* Copy whatever the last rule matched to the standard output. */ @@ -820,7 +813,7 @@ static int input ( yyscan_t yyscanner ); /* This used to be an fputs(), but since the string might contain NUL's, * we now use fwrite(). */ -#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#define ECHO fwrite( yytext, yyleng, 1, yyout ) #endif /* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, @@ -844,7 +837,7 @@ static int input ( yyscan_t yyscanner ); else \ { \ errno=0; \ - while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + while ( (result = fread(buf, 1, max_size, yyin))==0 && ferror(yyin)) \ { \ if( errno != EINTR) \ { \ @@ -886,7 +879,7 @@ static int input ( yyscan_t yyscanner ); #define YY_DECL_IS_OURS 1 extern int yylex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner); + (YYSTYPE * yylval_param,YYLTYPE * yylloc_param ,yyscan_t yyscanner); #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner) @@ -901,7 +894,7 @@ extern int yylex \ /* Code executed at the end of each rule. */ #ifndef YY_BREAK -#define YY_BREAK /*LINTED*/break; +#define YY_BREAK break; #endif #define YY_RULE_SETUP \ @@ -911,11 +904,16 @@ extern int yylex \ */ YY_DECL { - yy_state_type yy_current_state; - char *yy_cp, *yy_bp; - int yy_act; + register yy_state_type yy_current_state; + register char *yy_cp, *yy_bp; + register int yy_act; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; +#line 75 "lex_sql.l" + + +#line 916 "lex_sql.cpp" + yylval = yylval_param; yylloc = yylloc_param; @@ -940,19 +938,13 @@ YY_DECL if ( ! YY_CURRENT_BUFFER ) { yyensure_buffer_stack (yyscanner); YY_CURRENT_BUFFER_LVALUE = - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); + yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); } - yy_load_buffer_state( yyscanner ); + yy_load_buffer_state(yyscanner ); } - { -#line 75 "lex_sql.l" - - -#line 954 "lex_sql.cpp" - - while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ + while ( 1 ) /* loops until end-of-file is reached */ { yy_cp = yyg->yy_c_buf_p; @@ -968,7 +960,7 @@ YY_DECL yy_match: do { - YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)] ; + register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -977,13 +969,13 @@ YY_DECL while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 166 ) - yy_c = yy_meta[yy_c]; + if ( yy_current_state >= 183 ) + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } - while ( yy_base[yy_current_state] != 436 ); + while ( yy_base[yy_current_state] != 485 ); yy_find_action: yy_act = yy_accept[yy_current_state]; @@ -1201,92 +1193,112 @@ RETURN_TOKEN(EXPLAIN); case 39: YY_RULE_SETUP #line 117 "lex_sql.l" -yylval->string=strdup(yytext); RETURN_TOKEN(ID); +RETURN_TOKEN(GROUP); YY_BREAK case 40: YY_RULE_SETUP #line 118 "lex_sql.l" -RETURN_TOKEN(LBRACE); +RETURN_TOKEN(BY); YY_BREAK case 41: YY_RULE_SETUP #line 119 "lex_sql.l" -RETURN_TOKEN(RBRACE); +RETURN_TOKEN(STORAGE); YY_BREAK case 42: YY_RULE_SETUP -#line 121 "lex_sql.l" -RETURN_TOKEN(COMMA); +#line 120 "lex_sql.l" +RETURN_TOKEN(FORMAT); YY_BREAK case 43: YY_RULE_SETUP -#line 122 "lex_sql.l" -RETURN_TOKEN(EQ); +#line 121 "lex_sql.l" +yylval->string=strdup(yytext); RETURN_TOKEN(ID); YY_BREAK case 44: YY_RULE_SETUP -#line 123 "lex_sql.l" -RETURN_TOKEN(LE); +#line 122 "lex_sql.l" +RETURN_TOKEN(LBRACE); YY_BREAK case 45: YY_RULE_SETUP -#line 124 "lex_sql.l" -RETURN_TOKEN(NE); +#line 123 "lex_sql.l" +RETURN_TOKEN(RBRACE); YY_BREAK case 46: YY_RULE_SETUP #line 125 "lex_sql.l" -RETURN_TOKEN(NE); +RETURN_TOKEN(COMMA); YY_BREAK case 47: YY_RULE_SETUP #line 126 "lex_sql.l" -RETURN_TOKEN(LT); +RETURN_TOKEN(EQ); YY_BREAK case 48: YY_RULE_SETUP #line 127 "lex_sql.l" -RETURN_TOKEN(GE); +RETURN_TOKEN(LE); YY_BREAK case 49: YY_RULE_SETUP #line 128 "lex_sql.l" -RETURN_TOKEN(GT); +RETURN_TOKEN(NE); YY_BREAK case 50: -#line 131 "lex_sql.l" +YY_RULE_SETUP +#line 129 "lex_sql.l" +RETURN_TOKEN(NE); + YY_BREAK case 51: -#line 132 "lex_sql.l" +YY_RULE_SETUP +#line 130 "lex_sql.l" +RETURN_TOKEN(LT); + YY_BREAK case 52: -#line 133 "lex_sql.l" +YY_RULE_SETUP +#line 131 "lex_sql.l" +RETURN_TOKEN(GE); + YY_BREAK case 53: YY_RULE_SETUP -#line 133 "lex_sql.l" -{ return yytext[0]; } +#line 132 "lex_sql.l" +RETURN_TOKEN(GT); YY_BREAK case 54: -/* rule 54 can match eol */ +#line 135 "lex_sql.l" +case 55: +#line 136 "lex_sql.l" +case 56: +#line 137 "lex_sql.l" +case 57: +YY_RULE_SETUP +#line 137 "lex_sql.l" +{ return yytext[0]; } + YY_BREAK +case 58: +/* rule 58 can match eol */ YY_RULE_SETUP -#line 134 "lex_sql.l" +#line 138 "lex_sql.l" yylval->string = strdup(yytext); RETURN_TOKEN(SSS); YY_BREAK -case 55: -/* rule 55 can match eol */ +case 59: +/* rule 59 can match eol */ YY_RULE_SETUP -#line 135 "lex_sql.l" +#line 139 "lex_sql.l" yylval->string = strdup(yytext); RETURN_TOKEN(SSS); YY_BREAK -case 56: +case 60: YY_RULE_SETUP -#line 137 "lex_sql.l" +#line 141 "lex_sql.l" LOG_DEBUG("Unknown character [%c]",yytext[0]); return yytext[0]; YY_BREAK -case 57: +case 61: YY_RULE_SETUP -#line 138 "lex_sql.l" +#line 142 "lex_sql.l" ECHO; YY_BREAK -#line 1290 "lex_sql.cpp" +#line 1302 "lex_sql.cpp" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(STR): yyterminate(); @@ -1365,7 +1377,7 @@ case YY_STATE_EOF(STR): { yyg->yy_did_buffer_switch_on_eof = 0; - if ( yywrap( yyscanner ) ) + if ( yywrap(yyscanner ) ) { /* Note: because we've taken care in * yy_get_next_buffer() to have set up @@ -1418,7 +1430,6 @@ case YY_STATE_EOF(STR): "fatal flex scanner internal error--no action found" ); } /* end of action switch */ } /* end of scanning one token */ - } /* end of user's declarations */ } /* end of yylex */ /* yy_get_next_buffer - try to read in a new buffer @@ -1431,9 +1442,9 @@ case YY_STATE_EOF(STR): static int yy_get_next_buffer (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; - char *source = yyg->yytext_ptr; - int number_to_move, i; + register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + register char *source = yyg->yytext_ptr; + register int number_to_move, i; int ret_val; if ( yyg->yy_c_buf_p > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] ) @@ -1462,7 +1473,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) /* Try to read more data. */ /* First move last chars to start of buffer. */ - number_to_move = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr - 1); + number_to_move = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr) - 1; for ( i = 0; i < number_to_move; ++i ) *(dest++) = *(source++); @@ -1482,7 +1493,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) { /* Not enough room in the buffer - grow it. */ /* just a shorter name for the current buffer */ - YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE; + YY_BUFFER_STATE b = YY_CURRENT_BUFFER; int yy_c_buf_p_offset = (int) (yyg->yy_c_buf_p - b->yy_ch_buf); @@ -1498,12 +1509,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) b->yy_ch_buf = (char *) /* Include room in for 2 EOB chars. */ - yyrealloc( (void *) b->yy_ch_buf, - (yy_size_t) (b->yy_buf_size + 2) , yyscanner ); + yyrealloc((void *) b->yy_ch_buf,b->yy_buf_size + 2 ,yyscanner ); } else /* Can't grow it, we don't own it. */ - b->yy_ch_buf = NULL; + b->yy_ch_buf = 0; if ( ! b->yy_ch_buf ) YY_FATAL_ERROR( @@ -1521,7 +1531,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) /* Read in more data. */ YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), - yyg->yy_n_chars, num_to_read ); + yyg->yy_n_chars, (size_t) num_to_read ); YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars; } @@ -1531,7 +1541,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) if ( number_to_move == YY_MORE_ADJ ) { ret_val = EOB_ACT_END_OF_FILE; - yyrestart( yyin , yyscanner); + yyrestart(yyin ,yyscanner); } else @@ -1545,15 +1555,12 @@ static int yy_get_next_buffer (yyscan_t yyscanner) else ret_val = EOB_ACT_CONTINUE_SCAN; - if ((yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + if ((yy_size_t) (yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { /* Extend the array by 50%, plus the number we really need. */ - int new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1); - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( - (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size , yyscanner ); + yy_size_t new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc((void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf,new_size ,yyscanner ); if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); - /* "- 2" to take care of EOB's */ - YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); } yyg->yy_n_chars += number_to_move; @@ -1569,15 +1576,15 @@ static int yy_get_next_buffer (yyscan_t yyscanner) static yy_state_type yy_get_previous_state (yyscan_t yyscanner) { - yy_state_type yy_current_state; - char *yy_cp; + register yy_state_type yy_current_state; + register char *yy_cp; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yy_current_state = yyg->yy_start; for ( yy_cp = yyg->yytext_ptr + YY_MORE_ADJ; yy_cp < yyg->yy_c_buf_p; ++yy_cp ) { - YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -1586,10 +1593,10 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 166 ) - yy_c = yy_meta[yy_c]; + if ( yy_current_state >= 183 ) + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; } return yy_current_state; @@ -1602,11 +1609,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) */ static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state , yyscan_t yyscanner) { - int yy_is_jam; + register int yy_is_jam; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* This var may be unused depending upon options. */ - char *yy_cp = yyg->yy_c_buf_p; + register char *yy_cp = yyg->yy_c_buf_p; - YY_CHAR yy_c = 1; + register YY_CHAR yy_c = 1; if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -1615,20 +1622,15 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 166 ) - yy_c = yy_meta[yy_c]; + if ( yy_current_state >= 183 ) + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - yy_is_jam = (yy_current_state == 165); + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + yy_is_jam = (yy_current_state == 182); - (void)yyg; return yy_is_jam ? 0 : yy_current_state; } -#ifndef YY_NO_UNPUT - -#endif - #ifndef YY_NO_INPUT #ifdef __cplusplus static int yyinput (yyscan_t yyscanner) @@ -1654,7 +1656,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) else { /* need more input */ - int offset = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr); + int offset = yyg->yy_c_buf_p - yyg->yytext_ptr; ++yyg->yy_c_buf_p; switch ( yy_get_next_buffer( yyscanner ) ) @@ -1671,14 +1673,14 @@ static int yy_get_next_buffer (yyscan_t yyscanner) */ /* Reset buffer status. */ - yyrestart( yyin , yyscanner); + yyrestart(yyin ,yyscanner); /*FALLTHROUGH*/ case EOB_ACT_END_OF_FILE: { - if ( yywrap( yyscanner ) ) - return 0; + if ( yywrap(yyscanner ) ) + return EOF; if ( ! yyg->yy_did_buffer_switch_on_eof ) YY_NEW_FILE; @@ -1716,11 +1718,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) if ( ! YY_CURRENT_BUFFER ){ yyensure_buffer_stack (yyscanner); YY_CURRENT_BUFFER_LVALUE = - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); + yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); } - yy_init_buffer( YY_CURRENT_BUFFER, input_file , yyscanner); - yy_load_buffer_state( yyscanner ); + yy_init_buffer(YY_CURRENT_BUFFER,input_file ,yyscanner); + yy_load_buffer_state(yyscanner ); } /** Switch to a different input buffer. @@ -1749,7 +1751,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) } YY_CURRENT_BUFFER_LVALUE = new_buffer; - yy_load_buffer_state( yyscanner ); + yy_load_buffer_state(yyscanner ); /* We don't actually know whether we did this switch during * EOF (yywrap()) processing, but the only time this flag @@ -1778,7 +1780,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) { YY_BUFFER_STATE b; - b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) , yyscanner ); + b = (YY_BUFFER_STATE) yyalloc(sizeof( struct yy_buffer_state ) ,yyscanner ); if ( ! b ) YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); @@ -1787,13 +1789,13 @@ static void yy_load_buffer_state (yyscan_t yyscanner) /* yy_ch_buf has to be 2 characters longer than the size given because * we need to put in 2 end-of-buffer characters. */ - b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) , yyscanner ); + b->yy_ch_buf = (char *) yyalloc(b->yy_buf_size + 2 ,yyscanner ); if ( ! b->yy_ch_buf ) YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); b->yy_is_our_buffer = 1; - yy_init_buffer( b, file , yyscanner); + yy_init_buffer(b,file ,yyscanner); return b; } @@ -1813,11 +1815,15 @@ static void yy_load_buffer_state (yyscan_t yyscanner) YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; if ( b->yy_is_our_buffer ) - yyfree( (void *) b->yy_ch_buf , yyscanner ); + yyfree((void *) b->yy_ch_buf ,yyscanner ); - yyfree( (void *) b , yyscanner ); + yyfree((void *) b ,yyscanner ); } +#ifndef __cplusplus +extern int isatty (int ); +#endif /* __cplusplus */ + /* Initializes or reinitializes a buffer. * This function is sometimes called more than once on the same buffer, * such as during a yyrestart() or at EOF. @@ -1828,7 +1834,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) int oerrno = errno; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yy_flush_buffer( b , yyscanner); + yy_flush_buffer(b ,yyscanner); b->yy_input_file = file; b->yy_fill_buffer = 1; @@ -1872,7 +1878,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) b->yy_buffer_status = YY_BUFFER_NEW; if ( b == YY_CURRENT_BUFFER ) - yy_load_buffer_state( yyscanner ); + yy_load_buffer_state(yyscanner ); } /** Pushes the new state onto the stack. The new state becomes @@ -1904,7 +1910,7 @@ void yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) YY_CURRENT_BUFFER_LVALUE = new_buffer; /* copied from yy_switch_to_buffer. */ - yy_load_buffer_state( yyscanner ); + yy_load_buffer_state(yyscanner ); yyg->yy_did_buffer_switch_on_eof = 1; } @@ -1918,13 +1924,13 @@ void yypop_buffer_state (yyscan_t yyscanner) if (!YY_CURRENT_BUFFER) return; - yy_delete_buffer(YY_CURRENT_BUFFER , yyscanner); + yy_delete_buffer(YY_CURRENT_BUFFER ,yyscanner); YY_CURRENT_BUFFER_LVALUE = NULL; if (yyg->yy_buffer_stack_top > 0) --yyg->yy_buffer_stack_top; if (YY_CURRENT_BUFFER) { - yy_load_buffer_state( yyscanner ); + yy_load_buffer_state(yyscanner ); yyg->yy_did_buffer_switch_on_eof = 1; } } @@ -1934,7 +1940,7 @@ void yypop_buffer_state (yyscan_t yyscanner) */ static void yyensure_buffer_stack (yyscan_t yyscanner) { - yy_size_t num_to_alloc; + int num_to_alloc; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if (!yyg->yy_buffer_stack) { @@ -1943,15 +1949,15 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) * scanner will even need a stack. We use 2 instead of 1 to avoid an * immediate realloc on the next call. */ - num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ + num_to_alloc = 1; yyg->yy_buffer_stack = (struct yy_buffer_state**)yyalloc (num_to_alloc * sizeof(struct yy_buffer_state*) , yyscanner); if ( ! yyg->yy_buffer_stack ) YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); - + memset(yyg->yy_buffer_stack, 0, num_to_alloc * sizeof(struct yy_buffer_state*)); - + yyg->yy_buffer_stack_max = num_to_alloc; yyg->yy_buffer_stack_top = 0; return; @@ -1960,7 +1966,7 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) if (yyg->yy_buffer_stack_top >= (yyg->yy_buffer_stack_max) - 1){ /* Increase the buffer to prepare for a possible push. */ - yy_size_t grow_size = 8 /* arbitrary grow size */; + int grow_size = 8 /* arbitrary grow size */; num_to_alloc = yyg->yy_buffer_stack_max + grow_size; yyg->yy_buffer_stack = (struct yy_buffer_state**)yyrealloc @@ -1980,7 +1986,7 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) * @param base the character buffer * @param size the size in bytes of the character buffer * @param yyscanner The scanner object. - * @return the newly allocated buffer state object. + * @return the newly allocated buffer state object. */ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscanner) { @@ -1990,23 +1996,23 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscann base[size-2] != YY_END_OF_BUFFER_CHAR || base[size-1] != YY_END_OF_BUFFER_CHAR ) /* They forgot to leave room for the EOB's. */ - return NULL; + return 0; - b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) , yyscanner ); + b = (YY_BUFFER_STATE) yyalloc(sizeof( struct yy_buffer_state ) ,yyscanner ); if ( ! b ) YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); - b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ b->yy_buf_pos = b->yy_ch_buf = base; b->yy_is_our_buffer = 0; - b->yy_input_file = NULL; + b->yy_input_file = 0; b->yy_n_chars = b->yy_buf_size; b->yy_is_interactive = 0; b->yy_at_bol = 1; b->yy_fill_buffer = 0; b->yy_buffer_status = YY_BUFFER_NEW; - yy_switch_to_buffer( b , yyscanner ); + yy_switch_to_buffer(b ,yyscanner ); return b; } @@ -2019,20 +2025,20 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscann * @note If you want to scan bytes that may contain NUL values, then use * yy_scan_bytes() instead. */ -YY_BUFFER_STATE yy_scan_string (const char * yystr , yyscan_t yyscanner) +YY_BUFFER_STATE yy_scan_string (yyconst char * yystr , yyscan_t yyscanner) { - return yy_scan_bytes( yystr, (int) strlen(yystr) , yyscanner); + return yy_scan_bytes(yystr,strlen(yystr) ,yyscanner); } /** Setup the input buffer state to scan the given bytes. The next call to yylex() will * scan from a @e copy of @a bytes. - * @param yybytes the byte buffer to scan - * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. + * @param bytes the byte buffer to scan + * @param len the number of bytes in the buffer pointed to by @a bytes. * @param yyscanner The scanner object. * @return the newly allocated buffer state object. */ -YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan_t yyscanner) +YY_BUFFER_STATE yy_scan_bytes (yyconst char * yybytes, int _yybytes_len , yyscan_t yyscanner) { YY_BUFFER_STATE b; char *buf; @@ -2040,8 +2046,8 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan int i; /* Get memory for full buffer, including space for trailing EOB's. */ - n = (yy_size_t) (_yybytes_len + 2); - buf = (char *) yyalloc( n , yyscanner ); + n = _yybytes_len + 2; + buf = (char *) yyalloc(n ,yyscanner ); if ( ! buf ) YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); @@ -2050,7 +2056,7 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; - b = yy_scan_buffer( buf, n , yyscanner); + b = yy_scan_buffer(buf,n ,yyscanner); if ( ! b ) YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); @@ -2066,11 +2072,9 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan #define YY_EXIT_FAILURE 2 #endif -static void yynoreturn yy_fatal_error (const char* msg , yyscan_t yyscanner) +static void yy_fatal_error (yyconst char* msg , yyscan_t yyscanner) { - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - (void)yyg; - fprintf( stderr, "%s\n", msg ); + (void) fprintf( stderr, "%s\n", msg ); exit( YY_EXIT_FAILURE ); } @@ -2108,7 +2112,7 @@ YY_EXTRA_TYPE yyget_extra (yyscan_t yyscanner) int yyget_lineno (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - + if (! YY_CURRENT_BUFFER) return 0; @@ -2121,7 +2125,7 @@ int yyget_lineno (yyscan_t yyscanner) int yyget_column (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - + if (! YY_CURRENT_BUFFER) return 0; @@ -2176,51 +2180,51 @@ void yyset_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner) } /** Set the current line number. - * @param _line_number line number + * @param line_number * @param yyscanner The scanner object. */ -void yyset_lineno (int _line_number , yyscan_t yyscanner) +void yyset_lineno (int line_number , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* lineno is only valid if an input buffer exists. */ if (! YY_CURRENT_BUFFER ) - YY_FATAL_ERROR( "yyset_lineno called with no buffer" ); + yy_fatal_error( "yyset_lineno called with no buffer" , yyscanner); - yylineno = _line_number; + yylineno = line_number; } /** Set the current column. - * @param _column_no column number + * @param line_number * @param yyscanner The scanner object. */ -void yyset_column (int _column_no , yyscan_t yyscanner) +void yyset_column (int column_no , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* column is only valid if an input buffer exists. */ if (! YY_CURRENT_BUFFER ) - YY_FATAL_ERROR( "yyset_column called with no buffer" ); + yy_fatal_error( "yyset_column called with no buffer" , yyscanner); - yycolumn = _column_no; + yycolumn = column_no; } /** Set the input stream. This does not discard the current * input buffer. - * @param _in_str A readable stream. + * @param in_str A readable stream. * @param yyscanner The scanner object. * @see yy_switch_to_buffer */ -void yyset_in (FILE * _in_str , yyscan_t yyscanner) +void yyset_in (FILE * in_str , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yyin = _in_str ; + yyin = in_str ; } -void yyset_out (FILE * _out_str , yyscan_t yyscanner) +void yyset_out (FILE * out_str , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yyout = _out_str ; + yyout = out_str ; } int yyget_debug (yyscan_t yyscanner) @@ -2229,10 +2233,10 @@ int yyget_debug (yyscan_t yyscanner) return yy_flex_debug; } -void yyset_debug (int _bdebug , yyscan_t yyscanner) +void yyset_debug (int bdebug , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yy_flex_debug = _bdebug ; + yy_flex_debug = bdebug ; } /* Accessor methods for yylval and yylloc */ @@ -2267,7 +2271,9 @@ void yyset_lloc (YYLTYPE * yylloc_param , yyscan_t yyscanner) * the ONLY reentrant function that doesn't take the scanner as the last argument. * That's why we explicitly handle the declaration, instead of using our macros. */ + int yylex_init(yyscan_t* ptr_yy_globals) + { if (ptr_yy_globals == NULL){ errno = EINVAL; @@ -2294,7 +2300,9 @@ int yylex_init(yyscan_t* ptr_yy_globals) * The user defined value in the first argument will be available to yyalloc in * the yyextra field. */ -int yylex_init_extra( YY_EXTRA_TYPE yy_user_defined, yyscan_t* ptr_yy_globals ) + +int yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ) + { struct yyguts_t dummy_yyguts; @@ -2304,20 +2312,20 @@ int yylex_init_extra( YY_EXTRA_TYPE yy_user_defined, yyscan_t* ptr_yy_globals ) errno = EINVAL; return 1; } - + *ptr_yy_globals = (yyscan_t) yyalloc ( sizeof( struct yyguts_t ), &dummy_yyguts ); - + if (*ptr_yy_globals == NULL){ errno = ENOMEM; return 1; } - + /* By setting to 0xAA, we expose bugs in yy_init_globals. Leave at 0x00 for releases. */ memset(*ptr_yy_globals,0x00,sizeof(struct yyguts_t)); - + yyset_extra (yy_user_defined, *ptr_yy_globals); - + return yy_init_globals ( *ptr_yy_globals ); } @@ -2328,10 +2336,10 @@ static int yy_init_globals (yyscan_t yyscanner) * This function is called from yylex_destroy(), so don't allocate here. */ - yyg->yy_buffer_stack = NULL; + yyg->yy_buffer_stack = 0; yyg->yy_buffer_stack_top = 0; yyg->yy_buffer_stack_max = 0; - yyg->yy_c_buf_p = NULL; + yyg->yy_c_buf_p = (char *) 0; yyg->yy_init = 0; yyg->yy_start = 0; @@ -2344,8 +2352,8 @@ static int yy_init_globals (yyscan_t yyscanner) yyin = stdin; yyout = stdout; #else - yyin = NULL; - yyout = NULL; + yyin = (FILE *) 0; + yyout = (FILE *) 0; #endif /* For future reference: Set errno on error, since we are called by @@ -2361,17 +2369,17 @@ int yylex_destroy (yyscan_t yyscanner) /* Pop the buffer stack, destroying each element. */ while(YY_CURRENT_BUFFER){ - yy_delete_buffer( YY_CURRENT_BUFFER , yyscanner ); + yy_delete_buffer(YY_CURRENT_BUFFER ,yyscanner ); YY_CURRENT_BUFFER_LVALUE = NULL; yypop_buffer_state(yyscanner); } /* Destroy the stack itself. */ - yyfree(yyg->yy_buffer_stack , yyscanner); + yyfree(yyg->yy_buffer_stack ,yyscanner); yyg->yy_buffer_stack = NULL; /* Destroy the start condition stack. */ - yyfree( yyg->yy_start_stack , yyscanner ); + yyfree(yyg->yy_start_stack ,yyscanner ); yyg->yy_start_stack = NULL; /* Reset the globals. This is important in a non-reentrant scanner so the next time @@ -2389,21 +2397,18 @@ int yylex_destroy (yyscan_t yyscanner) */ #ifndef yytext_ptr -static void yy_flex_strncpy (char* s1, const char * s2, int n , yyscan_t yyscanner) +static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yyscanner) { - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - (void)yyg; - - int i; + register int i; for ( i = 0; i < n; ++i ) s1[i] = s2[i]; } #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen (const char * s , yyscan_t yyscanner) +static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner) { - int n; + register int n; for ( n = 0; s[n]; ++n ) ; @@ -2413,16 +2418,11 @@ static int yy_flex_strlen (const char * s , yyscan_t yyscanner) void *yyalloc (yy_size_t size , yyscan_t yyscanner) { - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - (void)yyg; - return malloc(size); + return (void *) malloc( size ); } void *yyrealloc (void * ptr, yy_size_t size , yyscan_t yyscanner) { - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - (void)yyg; - /* The cast to (char *) in the following accommodates both * implementations that use char* generic pointers, and those * that use void* generic pointers. It works with the latter @@ -2430,22 +2430,21 @@ void *yyrealloc (void * ptr, yy_size_t size , yyscan_t yyscanner) * any pointer type to void*, and deal with argument conversions * as though doing an assignment. */ - return realloc(ptr, size); + return (void *) realloc( (char *) ptr, size ); } void yyfree (void * ptr , yyscan_t yyscanner) { - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - (void)yyg; free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ } #define YYTABLES_NAME "yytables" -#line 138 "lex_sql.l" +#line 142 "lex_sql.l" + void scan_string(const char *str, yyscan_t scanner) { - yy_switch_to_buffer(yy_scan_string(str, scanner), scanner); + yy_switch_to_buffer(yy_scan_string(str,scanner),scanner); } diff --git a/src/observer/sql/parser/lex_sql.h b/src/observer/sql/parser/lex_sql.h index e3761b2f9..2862800d5 100644 --- a/src/observer/sql/parser/lex_sql.h +++ b/src/observer/sql/parser/lex_sql.h @@ -3,6 +3,7 @@ #define yyIN_HEADER 1 #line 6 "lex_sql.h" +#line 2 "lex_sql.l" /* 这里的代码会被复制到lex_sql.cpp的最开始位置 定义yy_size_t的原因是因为flex生成的代码,会使用yy_size_t与其他类型的数字 @@ -26,7 +27,10 @@ do { \ } \ while (0); -#line 30 "lex_sql.h" + + + +#line 34 "lex_sql.h" #define YY_INT_ALIGNED short int @@ -34,36 +38,12 @@ while (0); #define FLEX_SCANNER #define YY_FLEX_MAJOR_VERSION 2 -#define YY_FLEX_MINOR_VERSION 6 -#define YY_FLEX_SUBMINOR_VERSION 4 +#define YY_FLEX_MINOR_VERSION 5 +#define YY_FLEX_SUBMINOR_VERSION 35 #if YY_FLEX_SUBMINOR_VERSION > 0 #define FLEX_BETA #endif -#ifdef yyget_lval -#define yyget_lval_ALREADY_DEFINED -#else -#define yyget_lval yyget_lval -#endif - -#ifdef yyset_lval -#define yyset_lval_ALREADY_DEFINED -#else -#define yyset_lval yyset_lval -#endif - -#ifdef yyget_lloc -#define yyget_lloc_ALREADY_DEFINED -#else -#define yyget_lloc yyget_lloc -#endif - -#ifdef yyset_lloc -#define yyset_lloc_ALREADY_DEFINED -#else -#define yyset_lloc yyset_lloc -#endif - /* First, we deal with platform-specific or compiler-specific issues. */ /* begin standard C headers. */ @@ -104,6 +84,7 @@ typedef int flex_int32_t; typedef unsigned char flex_uint8_t; typedef unsigned short int flex_uint16_t; typedef unsigned int flex_uint32_t; +#endif /* ! C99 */ /* Limits of integral types. */ #ifndef INT8_MIN @@ -134,23 +115,27 @@ typedef unsigned int flex_uint32_t; #define UINT32_MAX (4294967295U) #endif -#ifndef SIZE_MAX -#define SIZE_MAX (~(size_t)0) -#endif +#endif /* ! FLEXINT_H */ -#endif /* ! C99 */ +#ifdef __cplusplus -#endif /* ! FLEXINT_H */ +/* The "const" storage-class-modifier is valid. */ +#define YY_USE_CONST -/* begin standard C++ headers. */ +#else /* ! __cplusplus */ -/* TODO: this is always defined, so inline it */ -#define yyconst const +/* C99 requires __STDC__ to be defined as 1. */ +#if defined (__STDC__) + +#define YY_USE_CONST + +#endif /* defined (__STDC__) */ +#endif /* ! __cplusplus */ -#if defined(__GNUC__) && __GNUC__ >= 3 -#define yynoreturn __attribute__((__noreturn__)) +#ifdef YY_USE_CONST +#define yyconst const #else -#define yynoreturn +#define yyconst #endif /* An opaque pointer. */ @@ -172,15 +157,7 @@ typedef void* yyscan_t; /* Size of default input buffer. */ #ifndef YY_BUF_SIZE -#ifdef __ia64__ -/* On IA-64, the buffer size is 16k, not 8k. - * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case. - * Ditto for the __ia64__ case accordingly. - */ -#define YY_BUF_SIZE 32768 -#else #define YY_BUF_SIZE 16384 -#endif /* __ia64__ */ #endif #ifndef YY_TYPEDEF_YY_BUFFER_STATE @@ -205,7 +182,7 @@ struct yy_buffer_state /* Size of input buffer in bytes, not including room for EOB * characters. */ - int yy_buf_size; + yy_size_t yy_buf_size; /* Number of characters read into yy_ch_buf, not including EOB * characters. @@ -233,7 +210,7 @@ struct yy_buffer_state int yy_bs_lineno; /**< The line count. */ int yy_bs_column; /**< The column count. */ - + /* Whether to try to fill the input buffer when we reach the * end of it. */ @@ -244,25 +221,25 @@ struct yy_buffer_state }; #endif /* !YY_STRUCT_YY_BUFFER_STATE */ -void yyrestart ( FILE *input_file , yyscan_t yyscanner ); -void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size , yyscan_t yyscanner ); -void yy_delete_buffer ( YY_BUFFER_STATE b , yyscan_t yyscanner ); -void yy_flush_buffer ( YY_BUFFER_STATE b , yyscan_t yyscanner ); -void yypush_buffer_state ( YY_BUFFER_STATE new_buffer , yyscan_t yyscanner ); -void yypop_buffer_state ( yyscan_t yyscanner ); +void yyrestart (FILE *input_file ,yyscan_t yyscanner ); +void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +YY_BUFFER_STATE yy_create_buffer (FILE *file,int size ,yyscan_t yyscanner ); +void yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +void yy_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +void yypop_buffer_state (yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_string ( const char *yy_str , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len , yyscan_t yyscanner ); +YY_BUFFER_STATE yy_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner ); +YY_BUFFER_STATE yy_scan_string (yyconst char *yy_str ,yyscan_t yyscanner ); +YY_BUFFER_STATE yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); -void *yyalloc ( yy_size_t , yyscan_t yyscanner ); -void *yyrealloc ( void *, yy_size_t , yyscan_t yyscanner ); -void yyfree ( void * , yyscan_t yyscanner ); +void *yyalloc (yy_size_t ,yyscan_t yyscanner ); +void *yyrealloc (void *,yy_size_t ,yyscan_t yyscanner ); +void yyfree (void * ,yyscan_t yyscanner ); /* Begin user sect3 */ -#define yywrap(yyscanner) (/*CONSTCOND*/1) +#define yywrap(n) 1 #define YY_SKIP_YYWRAP #define yytext_ptr yytext_r @@ -287,48 +264,44 @@ void yyfree ( void * , yyscan_t yyscanner ); int yylex_init (yyscan_t* scanner); -int yylex_init_extra ( YY_EXTRA_TYPE user_defined, yyscan_t* scanner); +int yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner); /* Accessor methods to globals. These are made visible to non-reentrant scanners for convenience. */ -int yylex_destroy ( yyscan_t yyscanner ); - -int yyget_debug ( yyscan_t yyscanner ); - -void yyset_debug ( int debug_flag , yyscan_t yyscanner ); +int yylex_destroy (yyscan_t yyscanner ); -YY_EXTRA_TYPE yyget_extra ( yyscan_t yyscanner ); +int yyget_debug (yyscan_t yyscanner ); -void yyset_extra ( YY_EXTRA_TYPE user_defined , yyscan_t yyscanner ); +void yyset_debug (int debug_flag ,yyscan_t yyscanner ); -FILE *yyget_in ( yyscan_t yyscanner ); +YY_EXTRA_TYPE yyget_extra (yyscan_t yyscanner ); -void yyset_in ( FILE * _in_str , yyscan_t yyscanner ); +void yyset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner ); -FILE *yyget_out ( yyscan_t yyscanner ); +FILE *yyget_in (yyscan_t yyscanner ); -void yyset_out ( FILE * _out_str , yyscan_t yyscanner ); +void yyset_in (FILE * in_str ,yyscan_t yyscanner ); - int yyget_leng ( yyscan_t yyscanner ); +FILE *yyget_out (yyscan_t yyscanner ); -char *yyget_text ( yyscan_t yyscanner ); +void yyset_out (FILE * out_str ,yyscan_t yyscanner ); -int yyget_lineno ( yyscan_t yyscanner ); +int yyget_leng (yyscan_t yyscanner ); -void yyset_lineno ( int _line_number , yyscan_t yyscanner ); +char *yyget_text (yyscan_t yyscanner ); -int yyget_column ( yyscan_t yyscanner ); +int yyget_lineno (yyscan_t yyscanner ); -void yyset_column ( int _column_no , yyscan_t yyscanner ); +void yyset_lineno (int line_number ,yyscan_t yyscanner ); -YYSTYPE * yyget_lval ( yyscan_t yyscanner ); +YYSTYPE * yyget_lval (yyscan_t yyscanner ); -void yyset_lval ( YYSTYPE * yylval_param , yyscan_t yyscanner ); +void yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner ); - YYLTYPE *yyget_lloc ( yyscan_t yyscanner ); + YYLTYPE *yyget_lloc (yyscan_t yyscanner ); - void yyset_lloc ( YYLTYPE * yylloc_param , yyscan_t yyscanner ); + void yyset_lloc (YYLTYPE * yylloc_param ,yyscan_t yyscanner ); /* Macros after this point can all be overridden by user definitions in * section 1. @@ -336,18 +309,18 @@ void yyset_lval ( YYSTYPE * yylval_param , yyscan_t yyscanner ); #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap ( yyscan_t yyscanner ); +extern "C" int yywrap (yyscan_t yyscanner ); #else -extern int yywrap ( yyscan_t yyscanner ); +extern int yywrap (yyscan_t yyscanner ); #endif #endif #ifndef yytext_ptr -static void yy_flex_strncpy ( char *, const char *, int , yyscan_t yyscanner); +static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen ( const char * , yyscan_t yyscanner); +static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner); #endif #ifndef YY_NO_INPUT @@ -356,12 +329,7 @@ static int yy_flex_strlen ( const char * , yyscan_t yyscanner); /* Amount of stuff to slurp up with each read. */ #ifndef YY_READ_BUF_SIZE -#ifdef __ia64__ -/* On IA-64, the buffer size is 16k, not 8k */ -#define YY_READ_BUF_SIZE 16384 -#else #define YY_READ_BUF_SIZE 8192 -#endif /* __ia64__ */ #endif /* Number of entries by which start-condition stack grows. */ @@ -376,7 +344,7 @@ static int yy_flex_strlen ( const char * , yyscan_t yyscanner); #define YY_DECL_IS_OURS 1 extern int yylex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner); + (YYSTYPE * yylval_param,YYLTYPE * yylloc_param ,yyscan_t yyscanner); #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner) @@ -396,154 +364,9 @@ extern int yylex \ #undef YY_DECL #endif -#ifndef yy_create_buffer_ALREADY_DEFINED -#undef yy_create_buffer -#endif -#ifndef yy_delete_buffer_ALREADY_DEFINED -#undef yy_delete_buffer -#endif -#ifndef yy_scan_buffer_ALREADY_DEFINED -#undef yy_scan_buffer -#endif -#ifndef yy_scan_string_ALREADY_DEFINED -#undef yy_scan_string -#endif -#ifndef yy_scan_bytes_ALREADY_DEFINED -#undef yy_scan_bytes -#endif -#ifndef yy_init_buffer_ALREADY_DEFINED -#undef yy_init_buffer -#endif -#ifndef yy_flush_buffer_ALREADY_DEFINED -#undef yy_flush_buffer -#endif -#ifndef yy_load_buffer_state_ALREADY_DEFINED -#undef yy_load_buffer_state -#endif -#ifndef yy_switch_to_buffer_ALREADY_DEFINED -#undef yy_switch_to_buffer -#endif -#ifndef yypush_buffer_state_ALREADY_DEFINED -#undef yypush_buffer_state -#endif -#ifndef yypop_buffer_state_ALREADY_DEFINED -#undef yypop_buffer_state -#endif -#ifndef yyensure_buffer_stack_ALREADY_DEFINED -#undef yyensure_buffer_stack -#endif -#ifndef yylex_ALREADY_DEFINED -#undef yylex -#endif -#ifndef yyrestart_ALREADY_DEFINED -#undef yyrestart -#endif -#ifndef yylex_init_ALREADY_DEFINED -#undef yylex_init -#endif -#ifndef yylex_init_extra_ALREADY_DEFINED -#undef yylex_init_extra -#endif -#ifndef yylex_destroy_ALREADY_DEFINED -#undef yylex_destroy -#endif -#ifndef yyget_debug_ALREADY_DEFINED -#undef yyget_debug -#endif -#ifndef yyset_debug_ALREADY_DEFINED -#undef yyset_debug -#endif -#ifndef yyget_extra_ALREADY_DEFINED -#undef yyget_extra -#endif -#ifndef yyset_extra_ALREADY_DEFINED -#undef yyset_extra -#endif -#ifndef yyget_in_ALREADY_DEFINED -#undef yyget_in -#endif -#ifndef yyset_in_ALREADY_DEFINED -#undef yyset_in -#endif -#ifndef yyget_out_ALREADY_DEFINED -#undef yyget_out -#endif -#ifndef yyset_out_ALREADY_DEFINED -#undef yyset_out -#endif -#ifndef yyget_leng_ALREADY_DEFINED -#undef yyget_leng -#endif -#ifndef yyget_text_ALREADY_DEFINED -#undef yyget_text -#endif -#ifndef yyget_lineno_ALREADY_DEFINED -#undef yyget_lineno -#endif -#ifndef yyset_lineno_ALREADY_DEFINED -#undef yyset_lineno -#endif -#ifndef yyget_column_ALREADY_DEFINED -#undef yyget_column -#endif -#ifndef yyset_column_ALREADY_DEFINED -#undef yyset_column -#endif -#ifndef yywrap_ALREADY_DEFINED -#undef yywrap -#endif -#ifndef yyget_lval_ALREADY_DEFINED -#undef yyget_lval -#endif -#ifndef yyset_lval_ALREADY_DEFINED -#undef yyset_lval -#endif -#ifndef yyget_lloc_ALREADY_DEFINED -#undef yyget_lloc -#endif -#ifndef yyset_lloc_ALREADY_DEFINED -#undef yyset_lloc -#endif -#ifndef yyalloc_ALREADY_DEFINED -#undef yyalloc -#endif -#ifndef yyrealloc_ALREADY_DEFINED -#undef yyrealloc -#endif -#ifndef yyfree_ALREADY_DEFINED -#undef yyfree -#endif -#ifndef yytext_ALREADY_DEFINED -#undef yytext -#endif -#ifndef yyleng_ALREADY_DEFINED -#undef yyleng -#endif -#ifndef yyin_ALREADY_DEFINED -#undef yyin -#endif -#ifndef yyout_ALREADY_DEFINED -#undef yyout -#endif -#ifndef yy_flex_debug_ALREADY_DEFINED -#undef yy_flex_debug -#endif -#ifndef yylineno_ALREADY_DEFINED -#undef yylineno -#endif -#ifndef yytables_fload_ALREADY_DEFINED -#undef yytables_fload -#endif -#ifndef yytables_destroy_ALREADY_DEFINED -#undef yytables_destroy -#endif -#ifndef yyTABLES_NAME_ALREADY_DEFINED -#undef yyTABLES_NAME -#endif - -#line 138 "lex_sql.l" +#line 142 "lex_sql.l" -#line 548 "lex_sql.h" +#line 371 "lex_sql.h" #undef yyIN_HEADER #endif /* yyHEADER_H */ diff --git a/src/observer/sql/parser/lex_sql.l b/src/observer/sql/parser/lex_sql.l index 93d3a0a77..25fc9f05d 100644 --- a/src/observer/sql/parser/lex_sql.l +++ b/src/observer/sql/parser/lex_sql.l @@ -114,6 +114,10 @@ LOAD RETURN_TOKEN(LOAD); DATA RETURN_TOKEN(DATA); INFILE RETURN_TOKEN(INFILE); EXPLAIN RETURN_TOKEN(EXPLAIN); +GROUP RETURN_TOKEN(GROUP); +BY RETURN_TOKEN(BY); +STORAGE RETURN_TOKEN(STORAGE); +FORMAT RETURN_TOKEN(FORMAT); {ID} yylval->string=strdup(yytext); RETURN_TOKEN(ID); "(" RETURN_TOKEN(LBRACE); ")" RETURN_TOKEN(RBRACE); diff --git a/src/observer/sql/parser/parse.cpp b/src/observer/sql/parser/parse.cpp index def0ed144..06da1193d 100644 --- a/src/observer/sql/parser/parse.cpp +++ b/src/observer/sql/parser/parse.cpp @@ -18,14 +18,6 @@ See the Mulan PSL v2 for more details. */ RC parse(char *st, ParsedSqlNode *sqln); -CalcSqlNode::~CalcSqlNode() -{ - for (Expression *expr : expressions) { - delete expr; - } - expressions.clear(); -} - ParsedSqlNode::ParsedSqlNode() : flag(SCF_ERROR) {} ParsedSqlNode::ParsedSqlNode(SqlCommandFlag _flag) : flag(_flag) {} diff --git a/src/observer/sql/parser/parse_defs.h b/src/observer/sql/parser/parse_defs.h index b7ab46247..4672bc1e8 100644 --- a/src/observer/sql/parser/parse_defs.h +++ b/src/observer/sql/parser/parse_defs.h @@ -88,9 +88,10 @@ struct ConditionSqlNode struct SelectSqlNode { - std::vector attributes; ///< attributes in select clause - std::vector relations; ///< 查询的表 - std::vector conditions; ///< 查询条件,使用AND串联起来多个条件 + std::vector> expressions; ///< 查询的表达式 + std::vector relations; ///< 查询的表 + std::vector conditions; ///< 查询条件,使用AND串联起来多个条件 + std::vector> group_by; ///< group by clause }; /** @@ -99,9 +100,7 @@ struct SelectSqlNode */ struct CalcSqlNode { - std::vector expressions; ///< calc clause - - ~CalcSqlNode(); + std::vector> expressions; ///< calc clause }; /** @@ -156,8 +155,9 @@ struct AttrInfoSqlNode */ struct CreateTableSqlNode { - std::string relation_name; ///< Relation name - std::vector attr_infos; ///< attributes + std::string relation_name; ///< Relation name + std::vector attr_infos; ///< attributes + std::string storage_format; ///< storage format }; /** diff --git a/src/observer/sql/parser/value.cpp b/src/observer/sql/parser/value.cpp index e69dace9c..65c9ba4be 100644 --- a/src/observer/sql/parser/value.cpp +++ b/src/observer/sql/parser/value.cpp @@ -22,8 +22,8 @@ const char *ATTR_TYPE_NAME[] = {"undefined", "chars", "ints", "floats", "boolean const char *attr_type_to_string(AttrType type) { - if (type >= UNDEFINED && type <= FLOATS) { - return ATTR_TYPE_NAME[type]; + if (type >= AttrType::UNDEFINED && type <= AttrType::FLOATS) { + return ATTR_TYPE_NAME[static_cast(type)]; } return "unknown"; } @@ -34,7 +34,7 @@ AttrType attr_type_from_string(const char *s) return (AttrType)i; } } - return UNDEFINED; + return AttrType::UNDEFINED; } Value::Value(int val) { set_int(val); } @@ -48,18 +48,18 @@ Value::Value(const char *s, int len /*= 0*/) { set_string(s, len); } void Value::set_data(char *data, int length) { switch (attr_type_) { - case CHARS: { + case AttrType::CHARS: { set_string(data, length); } break; - case INTS: { + case AttrType::INTS: { num_value_.int_value_ = *(int *)data; length_ = length; } break; - case FLOATS: { + case AttrType::FLOATS: { num_value_.float_value_ = *(float *)data; length_ = length; } break; - case BOOLEANS: { + case AttrType::BOOLEANS: { num_value_.bool_value_ = *(int *)data != 0; length_ = length; } break; @@ -70,26 +70,26 @@ void Value::set_data(char *data, int length) } void Value::set_int(int val) { - attr_type_ = INTS; + attr_type_ = AttrType::INTS; num_value_.int_value_ = val; length_ = sizeof(val); } void Value::set_float(float val) { - attr_type_ = FLOATS; + attr_type_ = AttrType::FLOATS; num_value_.float_value_ = val; length_ = sizeof(val); } void Value::set_boolean(bool val) { - attr_type_ = BOOLEANS; + attr_type_ = AttrType::BOOLEANS; num_value_.bool_value_ = val; length_ = sizeof(val); } void Value::set_string(const char *s, int len /*= 0*/) { - attr_type_ = CHARS; + attr_type_ = AttrType::CHARS; if (len > 0) { len = strnlen(s, len); str_value_.assign(s, len); @@ -102,19 +102,19 @@ void Value::set_string(const char *s, int len /*= 0*/) void Value::set_value(const Value &value) { switch (value.attr_type_) { - case INTS: { + case AttrType::INTS: { set_int(value.get_int()); } break; - case FLOATS: { + case AttrType::FLOATS: { set_float(value.get_float()); } break; - case CHARS: { + case AttrType::CHARS: { set_string(value.get_string().c_str()); } break; - case BOOLEANS: { + case AttrType::BOOLEANS: { set_boolean(value.get_boolean()); } break; - case UNDEFINED: { + case AttrType::UNDEFINED: { ASSERT(false, "got an invalid value type"); } break; } @@ -123,7 +123,7 @@ void Value::set_value(const Value &value) const char *Value::data() const { switch (attr_type_) { - case CHARS: { + case AttrType::CHARS: { return str_value_.c_str(); } break; default: { @@ -136,16 +136,16 @@ std::string Value::to_string() const { std::stringstream os; switch (attr_type_) { - case INTS: { + case AttrType::INTS: { os << num_value_.int_value_; } break; - case FLOATS: { + case AttrType::FLOATS: { os << common::double_to_str(num_value_.float_value_); } break; - case BOOLEANS: { + case AttrType::BOOLEANS: { os << num_value_.bool_value_; } break; - case CHARS: { + case AttrType::CHARS: { os << str_value_; } break; default: { @@ -159,29 +159,29 @@ int Value::compare(const Value &other) const { if (this->attr_type_ == other.attr_type_) { switch (this->attr_type_) { - case INTS: { + case AttrType::INTS: { return common::compare_int((void *)&this->num_value_.int_value_, (void *)&other.num_value_.int_value_); } break; - case FLOATS: { + case AttrType::FLOATS: { return common::compare_float((void *)&this->num_value_.float_value_, (void *)&other.num_value_.float_value_); } break; - case CHARS: { + case AttrType::CHARS: { return common::compare_string((void *)this->str_value_.c_str(), this->str_value_.length(), (void *)other.str_value_.c_str(), other.str_value_.length()); } break; - case BOOLEANS: { + case AttrType::BOOLEANS: { return common::compare_int((void *)&this->num_value_.bool_value_, (void *)&other.num_value_.bool_value_); } default: { LOG_WARN("unsupported type: %d", this->attr_type_); } } - } else if (this->attr_type_ == INTS && other.attr_type_ == FLOATS) { + } else if (this->attr_type_ == AttrType::INTS && other.attr_type_ == AttrType::FLOATS) { float this_data = this->num_value_.int_value_; return common::compare_float((void *)&this_data, (void *)&other.num_value_.float_value_); - } else if (this->attr_type_ == FLOATS && other.attr_type_ == INTS) { + } else if (this->attr_type_ == AttrType::FLOATS && other.attr_type_ == AttrType::INTS) { float other_data = other.num_value_.int_value_; return common::compare_float((void *)&this->num_value_.float_value_, (void *)&other_data); } @@ -192,7 +192,7 @@ int Value::compare(const Value &other) const int Value::get_int() const { switch (attr_type_) { - case CHARS: { + case AttrType::CHARS: { try { return (int)(std::stol(str_value_)); } catch (std::exception const &ex) { @@ -200,13 +200,13 @@ int Value::get_int() const return 0; } } - case INTS: { + case AttrType::INTS: { return num_value_.int_value_; } - case FLOATS: { + case AttrType::FLOATS: { return (int)(num_value_.float_value_); } - case BOOLEANS: { + case AttrType::BOOLEANS: { return (int)(num_value_.bool_value_); } default: { @@ -220,7 +220,7 @@ int Value::get_int() const float Value::get_float() const { switch (attr_type_) { - case CHARS: { + case AttrType::CHARS: { try { return std::stof(str_value_); } catch (std::exception const &ex) { @@ -228,13 +228,13 @@ float Value::get_float() const return 0.0; } } break; - case INTS: { + case AttrType::INTS: { return float(num_value_.int_value_); } break; - case FLOATS: { + case AttrType::FLOATS: { return num_value_.float_value_; } break; - case BOOLEANS: { + case AttrType::BOOLEANS: { return float(num_value_.bool_value_); } break; default: { @@ -250,7 +250,7 @@ std::string Value::get_string() const { return this->to_string(); } bool Value::get_boolean() const { switch (attr_type_) { - case CHARS: { + case AttrType::CHARS: { try { float val = std::stof(str_value_); if (val >= EPSILON || val <= -EPSILON) { @@ -268,14 +268,14 @@ bool Value::get_boolean() const return !str_value_.empty(); } } break; - case INTS: { + case AttrType::INTS: { return num_value_.int_value_ != 0; } break; - case FLOATS: { + case AttrType::FLOATS: { float val = num_value_.float_value_; return val >= EPSILON || val <= -EPSILON; } break; - case BOOLEANS: { + case AttrType::BOOLEANS: { return num_value_.bool_value_; } break; default: { diff --git a/src/observer/sql/parser/value.h b/src/observer/sql/parser/value.h index e16e6685e..74d0eb590 100644 --- a/src/observer/sql/parser/value.h +++ b/src/observer/sql/parser/value.h @@ -20,7 +20,7 @@ See the Mulan PSL v2 for more details. */ * @brief 属性的类型 * */ -enum AttrType +enum class AttrType { UNDEFINED, CHARS, ///< 字符串类型 @@ -36,7 +36,7 @@ AttrType attr_type_from_string(const char *s); * @brief 属性的值 * */ -class Value +class Value final { public: Value() = default; @@ -80,7 +80,7 @@ class Value bool get_boolean() const; private: - AttrType attr_type_ = UNDEFINED; + AttrType attr_type_ = AttrType::UNDEFINED; int length_ = 0; union diff --git a/src/observer/sql/parser/yacc_sql.cpp b/src/observer/sql/parser/yacc_sql.cpp index 08104f338..87e43288f 100644 --- a/src/observer/sql/parser/yacc_sql.cpp +++ b/src/observer/sql/parser/yacc_sql.cpp @@ -110,8 +110,18 @@ ArithmeticExpr *create_arithmetic_expression(ArithmeticExpr::Type type, return expr; } +UnboundAggregateExpr *create_aggregate_expression(const char *aggregate_name, + Expression *child, + const char *sql_string, + YYLTYPE *llocp) +{ + UnboundAggregateExpr *expr = new UnboundAggregateExpr(aggregate_name, child); + expr->set_name(token_name(sql_string, llocp)); + return expr; +} + -#line 115 "yacc_sql.cpp" +#line 125 "yacc_sql.cpp" # ifndef YY_CAST # ifdef __cplusplus @@ -143,97 +153,102 @@ enum yysymbol_kind_t YYSYMBOL_YYerror = 1, /* error */ YYSYMBOL_YYUNDEF = 2, /* "invalid token" */ YYSYMBOL_SEMICOLON = 3, /* SEMICOLON */ - YYSYMBOL_CREATE = 4, /* CREATE */ - YYSYMBOL_DROP = 5, /* DROP */ - YYSYMBOL_TABLE = 6, /* TABLE */ - YYSYMBOL_TABLES = 7, /* TABLES */ - YYSYMBOL_INDEX = 8, /* INDEX */ - YYSYMBOL_CALC = 9, /* CALC */ - YYSYMBOL_SELECT = 10, /* SELECT */ - YYSYMBOL_DESC = 11, /* DESC */ - YYSYMBOL_SHOW = 12, /* SHOW */ - YYSYMBOL_SYNC = 13, /* SYNC */ - YYSYMBOL_INSERT = 14, /* INSERT */ - YYSYMBOL_DELETE = 15, /* DELETE */ - YYSYMBOL_UPDATE = 16, /* UPDATE */ - YYSYMBOL_LBRACE = 17, /* LBRACE */ - YYSYMBOL_RBRACE = 18, /* RBRACE */ - YYSYMBOL_COMMA = 19, /* COMMA */ - YYSYMBOL_TRX_BEGIN = 20, /* TRX_BEGIN */ - YYSYMBOL_TRX_COMMIT = 21, /* TRX_COMMIT */ - YYSYMBOL_TRX_ROLLBACK = 22, /* TRX_ROLLBACK */ - YYSYMBOL_INT_T = 23, /* INT_T */ - YYSYMBOL_STRING_T = 24, /* STRING_T */ - YYSYMBOL_FLOAT_T = 25, /* FLOAT_T */ - YYSYMBOL_HELP = 26, /* HELP */ - YYSYMBOL_EXIT = 27, /* EXIT */ - YYSYMBOL_DOT = 28, /* DOT */ - YYSYMBOL_INTO = 29, /* INTO */ - YYSYMBOL_VALUES = 30, /* VALUES */ - YYSYMBOL_FROM = 31, /* FROM */ - YYSYMBOL_WHERE = 32, /* WHERE */ - YYSYMBOL_AND = 33, /* AND */ - YYSYMBOL_SET = 34, /* SET */ - YYSYMBOL_ON = 35, /* ON */ - YYSYMBOL_LOAD = 36, /* LOAD */ - YYSYMBOL_DATA = 37, /* DATA */ - YYSYMBOL_INFILE = 38, /* INFILE */ - YYSYMBOL_EXPLAIN = 39, /* EXPLAIN */ - YYSYMBOL_EQ = 40, /* EQ */ - YYSYMBOL_LT = 41, /* LT */ - YYSYMBOL_GT = 42, /* GT */ - YYSYMBOL_LE = 43, /* LE */ - YYSYMBOL_GE = 44, /* GE */ - YYSYMBOL_NE = 45, /* NE */ - YYSYMBOL_NUMBER = 46, /* NUMBER */ - YYSYMBOL_FLOAT = 47, /* FLOAT */ - YYSYMBOL_ID = 48, /* ID */ - YYSYMBOL_SSS = 49, /* SSS */ - YYSYMBOL_50_ = 50, /* '+' */ - YYSYMBOL_51_ = 51, /* '-' */ - YYSYMBOL_52_ = 52, /* '*' */ - YYSYMBOL_53_ = 53, /* '/' */ - YYSYMBOL_UMINUS = 54, /* UMINUS */ - YYSYMBOL_YYACCEPT = 55, /* $accept */ - YYSYMBOL_commands = 56, /* commands */ - YYSYMBOL_command_wrapper = 57, /* command_wrapper */ - YYSYMBOL_exit_stmt = 58, /* exit_stmt */ - YYSYMBOL_help_stmt = 59, /* help_stmt */ - YYSYMBOL_sync_stmt = 60, /* sync_stmt */ - YYSYMBOL_begin_stmt = 61, /* begin_stmt */ - YYSYMBOL_commit_stmt = 62, /* commit_stmt */ - YYSYMBOL_rollback_stmt = 63, /* rollback_stmt */ - YYSYMBOL_drop_table_stmt = 64, /* drop_table_stmt */ - YYSYMBOL_show_tables_stmt = 65, /* show_tables_stmt */ - YYSYMBOL_desc_table_stmt = 66, /* desc_table_stmt */ - YYSYMBOL_create_index_stmt = 67, /* create_index_stmt */ - YYSYMBOL_drop_index_stmt = 68, /* drop_index_stmt */ - YYSYMBOL_create_table_stmt = 69, /* create_table_stmt */ - YYSYMBOL_attr_def_list = 70, /* attr_def_list */ - YYSYMBOL_attr_def = 71, /* attr_def */ - YYSYMBOL_number = 72, /* number */ - YYSYMBOL_type = 73, /* type */ - YYSYMBOL_insert_stmt = 74, /* insert_stmt */ - YYSYMBOL_value_list = 75, /* value_list */ - YYSYMBOL_value = 76, /* value */ - YYSYMBOL_delete_stmt = 77, /* delete_stmt */ - YYSYMBOL_update_stmt = 78, /* update_stmt */ - YYSYMBOL_select_stmt = 79, /* select_stmt */ - YYSYMBOL_calc_stmt = 80, /* calc_stmt */ - YYSYMBOL_expression_list = 81, /* expression_list */ - YYSYMBOL_expression = 82, /* expression */ - YYSYMBOL_select_attr = 83, /* select_attr */ - YYSYMBOL_rel_attr = 84, /* rel_attr */ - YYSYMBOL_attr_list = 85, /* attr_list */ - YYSYMBOL_rel_list = 86, /* rel_list */ - YYSYMBOL_where = 87, /* where */ - YYSYMBOL_condition_list = 88, /* condition_list */ - YYSYMBOL_condition = 89, /* condition */ - YYSYMBOL_comp_op = 90, /* comp_op */ - YYSYMBOL_load_data_stmt = 91, /* load_data_stmt */ - YYSYMBOL_explain_stmt = 92, /* explain_stmt */ - YYSYMBOL_set_variable_stmt = 93, /* set_variable_stmt */ - YYSYMBOL_opt_semicolon = 94 /* opt_semicolon */ + YYSYMBOL_BY = 4, /* BY */ + YYSYMBOL_CREATE = 5, /* CREATE */ + YYSYMBOL_DROP = 6, /* DROP */ + YYSYMBOL_GROUP = 7, /* GROUP */ + YYSYMBOL_TABLE = 8, /* TABLE */ + YYSYMBOL_TABLES = 9, /* TABLES */ + YYSYMBOL_INDEX = 10, /* INDEX */ + YYSYMBOL_CALC = 11, /* CALC */ + YYSYMBOL_SELECT = 12, /* SELECT */ + YYSYMBOL_DESC = 13, /* DESC */ + YYSYMBOL_SHOW = 14, /* SHOW */ + YYSYMBOL_SYNC = 15, /* SYNC */ + YYSYMBOL_INSERT = 16, /* INSERT */ + YYSYMBOL_DELETE = 17, /* DELETE */ + YYSYMBOL_UPDATE = 18, /* UPDATE */ + YYSYMBOL_LBRACE = 19, /* LBRACE */ + YYSYMBOL_RBRACE = 20, /* RBRACE */ + YYSYMBOL_COMMA = 21, /* COMMA */ + YYSYMBOL_TRX_BEGIN = 22, /* TRX_BEGIN */ + YYSYMBOL_TRX_COMMIT = 23, /* TRX_COMMIT */ + YYSYMBOL_TRX_ROLLBACK = 24, /* TRX_ROLLBACK */ + YYSYMBOL_INT_T = 25, /* INT_T */ + YYSYMBOL_STRING_T = 26, /* STRING_T */ + YYSYMBOL_FLOAT_T = 27, /* FLOAT_T */ + YYSYMBOL_HELP = 28, /* HELP */ + YYSYMBOL_EXIT = 29, /* EXIT */ + YYSYMBOL_DOT = 30, /* DOT */ + YYSYMBOL_INTO = 31, /* INTO */ + YYSYMBOL_VALUES = 32, /* VALUES */ + YYSYMBOL_FROM = 33, /* FROM */ + YYSYMBOL_WHERE = 34, /* WHERE */ + YYSYMBOL_AND = 35, /* AND */ + YYSYMBOL_SET = 36, /* SET */ + YYSYMBOL_ON = 37, /* ON */ + YYSYMBOL_LOAD = 38, /* LOAD */ + YYSYMBOL_DATA = 39, /* DATA */ + YYSYMBOL_INFILE = 40, /* INFILE */ + YYSYMBOL_EXPLAIN = 41, /* EXPLAIN */ + YYSYMBOL_STORAGE = 42, /* STORAGE */ + YYSYMBOL_FORMAT = 43, /* FORMAT */ + YYSYMBOL_EQ = 44, /* EQ */ + YYSYMBOL_LT = 45, /* LT */ + YYSYMBOL_GT = 46, /* GT */ + YYSYMBOL_LE = 47, /* LE */ + YYSYMBOL_GE = 48, /* GE */ + YYSYMBOL_NE = 49, /* NE */ + YYSYMBOL_NUMBER = 50, /* NUMBER */ + YYSYMBOL_FLOAT = 51, /* FLOAT */ + YYSYMBOL_ID = 52, /* ID */ + YYSYMBOL_SSS = 53, /* SSS */ + YYSYMBOL_54_ = 54, /* '+' */ + YYSYMBOL_55_ = 55, /* '-' */ + YYSYMBOL_56_ = 56, /* '*' */ + YYSYMBOL_57_ = 57, /* '/' */ + YYSYMBOL_UMINUS = 58, /* UMINUS */ + YYSYMBOL_YYACCEPT = 59, /* $accept */ + YYSYMBOL_commands = 60, /* commands */ + YYSYMBOL_command_wrapper = 61, /* command_wrapper */ + YYSYMBOL_exit_stmt = 62, /* exit_stmt */ + YYSYMBOL_help_stmt = 63, /* help_stmt */ + YYSYMBOL_sync_stmt = 64, /* sync_stmt */ + YYSYMBOL_begin_stmt = 65, /* begin_stmt */ + YYSYMBOL_commit_stmt = 66, /* commit_stmt */ + YYSYMBOL_rollback_stmt = 67, /* rollback_stmt */ + YYSYMBOL_drop_table_stmt = 68, /* drop_table_stmt */ + YYSYMBOL_show_tables_stmt = 69, /* show_tables_stmt */ + YYSYMBOL_desc_table_stmt = 70, /* desc_table_stmt */ + YYSYMBOL_create_index_stmt = 71, /* create_index_stmt */ + YYSYMBOL_drop_index_stmt = 72, /* drop_index_stmt */ + YYSYMBOL_create_table_stmt = 73, /* create_table_stmt */ + YYSYMBOL_attr_def_list = 74, /* attr_def_list */ + YYSYMBOL_attr_def = 75, /* attr_def */ + YYSYMBOL_number = 76, /* number */ + YYSYMBOL_type = 77, /* type */ + YYSYMBOL_insert_stmt = 78, /* insert_stmt */ + YYSYMBOL_value_list = 79, /* value_list */ + YYSYMBOL_value = 80, /* value */ + YYSYMBOL_storage_format = 81, /* storage_format */ + YYSYMBOL_delete_stmt = 82, /* delete_stmt */ + YYSYMBOL_update_stmt = 83, /* update_stmt */ + YYSYMBOL_select_stmt = 84, /* select_stmt */ + YYSYMBOL_calc_stmt = 85, /* calc_stmt */ + YYSYMBOL_expression_list = 86, /* expression_list */ + YYSYMBOL_expression = 87, /* expression */ + YYSYMBOL_rel_attr = 88, /* rel_attr */ + YYSYMBOL_relation = 89, /* relation */ + YYSYMBOL_rel_list = 90, /* rel_list */ + YYSYMBOL_where = 91, /* where */ + YYSYMBOL_condition_list = 92, /* condition_list */ + YYSYMBOL_condition = 93, /* condition */ + YYSYMBOL_comp_op = 94, /* comp_op */ + YYSYMBOL_group_by = 95, /* group_by */ + YYSYMBOL_load_data_stmt = 96, /* load_data_stmt */ + YYSYMBOL_explain_stmt = 97, /* explain_stmt */ + YYSYMBOL_set_variable_stmt = 98, /* set_variable_stmt */ + YYSYMBOL_opt_semicolon = 99 /* opt_semicolon */ }; typedef enum yysymbol_kind_t yysymbol_kind_t; @@ -564,19 +579,19 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 65 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 151 +#define YYLAST 140 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 55 +#define YYNTOKENS 59 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 40 +#define YYNNTS 41 /* YYNRULES -- Number of rules. */ -#define YYNRULES 89 +#define YYNRULES 91 /* YYNSTATES -- Number of states. */ -#define YYNSTATES 163 +#define YYNSTATES 165 /* YYMAXUTOK -- Last valid token kind. */ -#define YYMAXUTOK 305 +#define YYMAXUTOK 309 /* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM @@ -594,7 +609,7 @@ static const yytype_int8 yytranslate[] = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 52, 50, 2, 51, 2, 53, 2, 2, + 2, 2, 56, 54, 2, 55, 2, 57, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -620,22 +635,23 @@ static const yytype_int8 yytranslate[] = 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, - 45, 46, 47, 48, 49, 54 + 45, 46, 47, 48, 49, 50, 51, 52, 53, 58 }; #if YYDEBUG /* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ static const yytype_int16 yyrline[] = { - 0, 173, 173, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, - 198, 199, 200, 204, 210, 215, 221, 227, 233, 239, - 246, 252, 260, 274, 284, 304, 307, 320, 328, 338, - 341, 342, 343, 346, 363, 366, 377, 381, 385, 394, - 406, 421, 443, 453, 458, 469, 472, 475, 478, 481, - 485, 488, 496, 503, 515, 520, 531, 534, 548, 551, - 564, 567, 573, 576, 581, 588, 600, 612, 624, 639, - 640, 641, 642, 643, 644, 648, 661, 669, 679, 680 + 0, 188, 188, 196, 197, 198, 199, 200, 201, 202, + 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, + 213, 214, 215, 219, 225, 230, 236, 242, 248, 254, + 261, 267, 275, 289, 299, 323, 326, 339, 347, 357, + 360, 361, 362, 365, 382, 385, 396, 400, 404, 413, + 416, 423, 435, 450, 475, 484, 489, 500, 503, 506, + 509, 512, 516, 519, 524, 530, 537, 542, 552, 557, + 562, 576, 579, 585, 588, 593, 600, 612, 624, 636, + 651, 652, 653, 654, 655, 656, 662, 667, 680, 688, + 698, 699 }; #endif @@ -651,22 +667,22 @@ static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; First, the terminals, then, starting at YYNTOKENS, nonterminals. */ static const char *const yytname[] = { - "\"end of file\"", "error", "\"invalid token\"", "SEMICOLON", "CREATE", - "DROP", "TABLE", "TABLES", "INDEX", "CALC", "SELECT", "DESC", "SHOW", - "SYNC", "INSERT", "DELETE", "UPDATE", "LBRACE", "RBRACE", "COMMA", - "TRX_BEGIN", "TRX_COMMIT", "TRX_ROLLBACK", "INT_T", "STRING_T", + "\"end of file\"", "error", "\"invalid token\"", "SEMICOLON", "BY", + "CREATE", "DROP", "GROUP", "TABLE", "TABLES", "INDEX", "CALC", "SELECT", + "DESC", "SHOW", "SYNC", "INSERT", "DELETE", "UPDATE", "LBRACE", "RBRACE", + "COMMA", "TRX_BEGIN", "TRX_COMMIT", "TRX_ROLLBACK", "INT_T", "STRING_T", "FLOAT_T", "HELP", "EXIT", "DOT", "INTO", "VALUES", "FROM", "WHERE", - "AND", "SET", "ON", "LOAD", "DATA", "INFILE", "EXPLAIN", "EQ", "LT", - "GT", "LE", "GE", "NE", "NUMBER", "FLOAT", "ID", "SSS", "'+'", "'-'", - "'*'", "'/'", "UMINUS", "$accept", "commands", "command_wrapper", - "exit_stmt", "help_stmt", "sync_stmt", "begin_stmt", "commit_stmt", - "rollback_stmt", "drop_table_stmt", "show_tables_stmt", + "AND", "SET", "ON", "LOAD", "DATA", "INFILE", "EXPLAIN", "STORAGE", + "FORMAT", "EQ", "LT", "GT", "LE", "GE", "NE", "NUMBER", "FLOAT", "ID", + "SSS", "'+'", "'-'", "'*'", "'/'", "UMINUS", "$accept", "commands", + "command_wrapper", "exit_stmt", "help_stmt", "sync_stmt", "begin_stmt", + "commit_stmt", "rollback_stmt", "drop_table_stmt", "show_tables_stmt", "desc_table_stmt", "create_index_stmt", "drop_index_stmt", "create_table_stmt", "attr_def_list", "attr_def", "number", "type", - "insert_stmt", "value_list", "value", "delete_stmt", "update_stmt", - "select_stmt", "calc_stmt", "expression_list", "expression", - "select_attr", "rel_attr", "attr_list", "rel_list", "where", - "condition_list", "condition", "comp_op", "load_data_stmt", + "insert_stmt", "value_list", "value", "storage_format", "delete_stmt", + "update_stmt", "select_stmt", "calc_stmt", "expression_list", + "expression", "rel_attr", "relation", "rel_list", "where", + "condition_list", "condition", "comp_op", "group_by", "load_data_stmt", "explain_stmt", "set_variable_stmt", "opt_semicolon", YY_NULLPTR }; @@ -691,23 +707,23 @@ yysymbol_name (yysymbol_kind_t yysymbol) STATE-NUM. */ static const yytype_int8 yypact[] = { - -2, 51, 74, 9, -25, -26, 22, -97, 1, 4, - -6, -97, -97, -97, -97, -97, 6, -1, -2, 72, - 80, -97, -97, -97, -97, -97, -97, -97, -97, -97, + 50, 10, 11, -16, -16, -30, 2, -97, -5, -6, + -23, -97, -97, -97, -97, -97, -22, -7, 50, 31, + 35, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, -97, - -97, 36, 37, 38, 39, 9, -97, -97, -97, 9, - -97, -97, 12, 60, -97, 58, 71, -97, -97, 43, - 44, 59, 54, 57, -97, -97, -97, -97, 79, 62, - -97, 63, -12, -97, 9, 9, 9, 9, 9, 52, - 53, 55, -97, 69, 70, 56, 32, 61, 64, 65, - 66, -97, -97, -48, -48, -97, -97, -97, 86, 71, - 89, 27, -97, 67, -97, 82, 21, 90, 91, -97, - 68, 70, -97, 32, 26, 26, -97, 84, 32, 109, - -97, -97, -97, 101, 64, 102, 73, 86, -97, 100, - -97, -97, -97, -97, -97, -97, 27, 27, 27, 70, - 75, 76, 90, -97, 106, -97, 32, 107, -97, -97, - -97, -97, -97, -97, -97, -97, 108, -97, -97, 100, - -97, -97, -97 + -97, -2, 17, 24, 37, -16, -97, -97, 47, -97, + -16, -97, -97, -97, -8, -97, 21, -97, -97, 38, + 40, 51, 49, 54, -97, -97, -97, -97, 76, 59, + -97, 60, -12, 46, -97, -16, -16, -16, -16, -16, + 48, 67, 71, 55, -41, 53, 56, 57, 58, -97, + -97, -97, -32, -32, -97, -97, -97, 90, 71, 93, + -46, -97, 69, -97, 83, -11, 94, 97, -97, 48, + -97, -41, 36, 36, -97, 82, -41, 110, -97, -97, + -97, 100, 56, 101, 68, -97, -97, 102, -97, -97, + -97, -97, -97, -97, -46, -46, -46, 71, 70, 74, + 94, 84, 105, -41, 107, -97, -97, -97, -97, -97, + -97, -97, -97, 108, -97, 86, -97, -97, 102, -97, + -97, 87, -97, 78, -97 }; /* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. @@ -717,39 +733,41 @@ static const yytype_int8 yydefact[] = { 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 26, 27, 28, 24, 23, 0, 0, 0, 0, - 88, 22, 21, 14, 15, 16, 17, 9, 10, 11, + 90, 22, 21, 14, 15, 16, 17, 9, 10, 11, 12, 13, 8, 5, 7, 6, 4, 3, 18, 19, - 20, 0, 0, 0, 0, 0, 46, 47, 48, 0, - 61, 52, 53, 64, 62, 0, 66, 31, 30, 0, - 0, 0, 0, 0, 86, 1, 89, 2, 0, 0, - 29, 0, 0, 60, 0, 0, 0, 0, 0, 0, - 0, 0, 63, 0, 70, 0, 0, 0, 0, 0, - 0, 59, 54, 55, 56, 57, 58, 65, 68, 66, - 0, 72, 49, 0, 87, 0, 0, 35, 0, 33, - 0, 70, 67, 0, 0, 0, 71, 73, 0, 0, - 40, 41, 42, 38, 0, 0, 0, 68, 51, 44, - 79, 80, 81, 82, 83, 84, 0, 0, 72, 70, - 0, 0, 35, 34, 0, 69, 0, 0, 76, 78, - 75, 77, 74, 50, 85, 39, 0, 36, 32, 44, - 43, 37, 45 + 20, 0, 0, 0, 0, 0, 46, 47, 66, 48, + 0, 65, 63, 54, 55, 64, 0, 31, 30, 0, + 0, 0, 0, 0, 88, 1, 91, 2, 0, 0, + 29, 0, 0, 0, 62, 0, 0, 0, 0, 0, + 0, 0, 71, 0, 0, 0, 0, 0, 0, 61, + 67, 56, 57, 58, 59, 60, 68, 69, 71, 0, + 73, 51, 0, 89, 0, 0, 35, 0, 33, 0, + 86, 0, 0, 0, 72, 74, 0, 0, 40, 41, + 42, 38, 0, 0, 0, 70, 53, 44, 80, 81, + 82, 83, 84, 85, 0, 0, 73, 71, 0, 0, + 35, 49, 0, 0, 0, 77, 79, 76, 78, 75, + 52, 87, 39, 0, 36, 0, 34, 32, 44, 43, + 37, 0, 45, 0, 50 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int8 yypgoto[] = { - -97, -97, 110, -97, -97, -97, -97, -97, -97, -97, - -97, -97, -97, -97, -97, -15, 5, -97, -97, -97, - -29, -85, -97, -97, -97, -97, 77, -28, -97, -4, - 35, 8, -96, -7, -97, 23, -97, -97, -97, -97 + -97, -97, 114, -97, -97, -97, -97, -97, -97, -97, + -97, -97, -97, -97, -97, -3, 12, -97, -97, -97, + -25, -83, -97, -97, -97, -97, -97, -4, 25, -77, + -97, 26, -96, 0, -97, 27, -97, -97, -97, -97, + -97 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_uint8 yydefgoto[] = { 0, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 125, 107, 156, 123, 33, - 147, 50, 34, 35, 36, 37, 51, 52, 55, 115, - 82, 111, 102, 116, 117, 136, 38, 39, 40, 67 + 28, 29, 30, 31, 32, 123, 106, 153, 121, 33, + 144, 52, 156, 34, 35, 36, 37, 53, 54, 55, + 97, 98, 101, 114, 115, 134, 126, 38, 39, 40, + 67 }; /* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If @@ -757,79 +775,78 @@ static const yytype_uint8 yydefgoto[] = number is the opposite. If YYTABLE_NINF, syntax error. */ static const yytype_uint8 yytable[] = { - 56, 104, 1, 2, 77, 78, 91, 3, 4, 5, - 6, 7, 8, 9, 10, 128, 114, 72, 11, 12, - 13, 73, 57, 53, 14, 15, 45, 54, 129, 58, - 59, 74, 16, 139, 17, 60, 63, 18, 75, 76, - 77, 78, 61, 153, 120, 121, 122, 93, 94, 95, - 96, 148, 150, 114, 62, 46, 47, 41, 48, 42, - 49, 159, 75, 76, 77, 78, 130, 131, 132, 133, - 134, 135, 65, 46, 47, 53, 48, 99, 46, 47, - 43, 48, 44, 66, 68, 69, 70, 71, 79, 80, - 81, 83, 84, 85, 86, 87, 88, 89, 90, 100, - 97, 98, 101, 53, 103, 110, 113, 118, 126, 124, - 105, 119, 106, 108, 109, 140, 127, 138, 141, 146, - 143, 144, 155, 154, 158, 160, 161, 157, 64, 142, - 162, 152, 149, 151, 112, 145, 0, 0, 137, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 92 + 56, 103, 110, 45, 46, 47, 48, 49, 89, 46, + 47, 58, 49, 75, 118, 119, 120, 112, 41, 43, + 42, 44, 57, 113, 78, 79, 59, 60, 127, 61, + 62, 65, 63, 137, 46, 47, 48, 49, 66, 50, + 51, 150, 76, 77, 78, 79, 76, 77, 78, 79, + 68, 145, 147, 112, 80, 1, 2, 146, 148, 113, + 158, 3, 4, 5, 6, 7, 8, 9, 10, 69, + 72, 91, 11, 12, 13, 74, 70, 73, 14, 15, + 128, 129, 130, 131, 132, 133, 16, 83, 17, 71, + 81, 18, 82, 84, 85, 86, 87, 88, 90, 99, + 96, 92, 93, 94, 95, 100, 104, 102, 105, 107, + 108, 109, 111, 116, 117, 122, 124, 136, 138, 139, + 142, 141, 151, 143, 152, 157, 155, 159, 160, 161, + 164, 163, 64, 162, 140, 125, 149, 154, 0, 0, + 135 }; static const yytype_int16 yycheck[] = { - 4, 86, 4, 5, 52, 53, 18, 9, 10, 11, - 12, 13, 14, 15, 16, 111, 101, 45, 20, 21, - 22, 49, 48, 48, 26, 27, 17, 52, 113, 7, - 29, 19, 34, 118, 36, 31, 37, 39, 50, 51, - 52, 53, 48, 139, 23, 24, 25, 75, 76, 77, - 78, 136, 137, 138, 48, 46, 47, 6, 49, 8, - 51, 146, 50, 51, 52, 53, 40, 41, 42, 43, - 44, 45, 0, 46, 47, 48, 49, 81, 46, 47, - 6, 49, 8, 3, 48, 48, 48, 48, 28, 31, - 19, 48, 48, 34, 40, 38, 17, 35, 35, 30, - 48, 48, 32, 48, 48, 19, 17, 40, 17, 19, - 49, 29, 48, 48, 48, 6, 48, 33, 17, 19, - 18, 48, 46, 48, 18, 18, 18, 142, 18, 124, - 159, 138, 136, 137, 99, 127, -1, -1, 115, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 74 + 4, 84, 98, 19, 50, 51, 52, 53, 20, 50, + 51, 9, 53, 21, 25, 26, 27, 100, 8, 8, + 10, 10, 52, 100, 56, 57, 31, 33, 111, 52, + 52, 0, 39, 116, 50, 51, 52, 53, 3, 55, + 56, 137, 54, 55, 56, 57, 54, 55, 56, 57, + 52, 134, 135, 136, 33, 5, 6, 134, 135, 136, + 143, 11, 12, 13, 14, 15, 16, 17, 18, 52, + 45, 75, 22, 23, 24, 50, 52, 30, 28, 29, + 44, 45, 46, 47, 48, 49, 36, 36, 38, 52, + 52, 41, 52, 44, 40, 19, 37, 37, 52, 32, + 52, 76, 77, 78, 79, 34, 53, 52, 52, 52, + 52, 21, 19, 44, 31, 21, 19, 35, 8, 19, + 52, 20, 52, 21, 50, 20, 42, 20, 20, 43, + 52, 44, 18, 158, 122, 109, 136, 140, -1, -1, + 113 }; /* YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of state STATE-NUM. */ static const yytype_int8 yystos[] = { - 0, 4, 5, 9, 10, 11, 12, 13, 14, 15, - 16, 20, 21, 22, 26, 27, 34, 36, 39, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 74, 77, 78, 79, 80, 91, 92, - 93, 6, 8, 6, 8, 17, 46, 47, 49, 51, - 76, 81, 82, 48, 52, 83, 84, 48, 7, 29, - 31, 48, 48, 37, 57, 0, 3, 94, 48, 48, - 48, 48, 82, 82, 19, 50, 51, 52, 53, 28, - 31, 19, 85, 48, 48, 34, 40, 38, 17, 35, - 35, 18, 81, 82, 82, 82, 82, 48, 48, 84, - 30, 32, 87, 48, 76, 49, 48, 71, 48, 48, - 19, 86, 85, 17, 76, 84, 88, 89, 40, 29, - 23, 24, 25, 73, 19, 70, 17, 48, 87, 76, - 40, 41, 42, 43, 44, 45, 90, 90, 33, 76, - 6, 17, 71, 18, 48, 86, 19, 75, 76, 84, - 76, 84, 88, 87, 48, 46, 72, 70, 18, 76, - 18, 18, 75 + 0, 5, 6, 11, 12, 13, 14, 15, 16, 17, + 18, 22, 23, 24, 28, 29, 36, 38, 41, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + 71, 72, 73, 78, 82, 83, 84, 85, 96, 97, + 98, 8, 10, 8, 10, 19, 50, 51, 52, 53, + 55, 56, 80, 86, 87, 88, 86, 52, 9, 31, + 33, 52, 52, 39, 61, 0, 3, 99, 52, 52, + 52, 52, 87, 30, 87, 21, 54, 55, 56, 57, + 33, 52, 52, 36, 44, 40, 19, 37, 37, 20, + 52, 86, 87, 87, 87, 87, 52, 89, 90, 32, + 34, 91, 52, 80, 53, 52, 75, 52, 52, 21, + 91, 19, 80, 88, 92, 93, 44, 31, 25, 26, + 27, 77, 21, 74, 19, 90, 95, 80, 44, 45, + 46, 47, 48, 49, 94, 94, 35, 80, 8, 19, + 75, 20, 52, 21, 79, 80, 88, 80, 88, 92, + 91, 52, 50, 76, 74, 42, 81, 20, 80, 20, + 20, 43, 79, 44, 52 }; /* YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. */ static const yytype_int8 yyr1[] = { - 0, 55, 56, 57, 57, 57, 57, 57, 57, 57, - 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, - 57, 57, 57, 58, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 68, 69, 70, 70, 71, 71, 72, - 73, 73, 73, 74, 75, 75, 76, 76, 76, 77, - 78, 79, 80, 81, 81, 82, 82, 82, 82, 82, - 82, 82, 83, 83, 84, 84, 85, 85, 86, 86, - 87, 87, 88, 88, 88, 89, 89, 89, 89, 90, - 90, 90, 90, 90, 90, 91, 92, 93, 94, 94 + 0, 59, 60, 61, 61, 61, 61, 61, 61, 61, + 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, + 61, 61, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, 74, 74, 75, 75, 76, + 77, 77, 77, 78, 79, 79, 80, 80, 80, 81, + 81, 82, 83, 84, 85, 86, 86, 87, 87, 87, + 87, 87, 87, 87, 87, 87, 88, 88, 89, 90, + 90, 91, 91, 92, 92, 92, 93, 93, 93, 93, + 94, 94, 94, 94, 94, 94, 95, 96, 97, 98, + 99, 99 }; /* YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. */ @@ -838,12 +855,13 @@ static const yytype_int8 yyr2[] = 0, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, - 2, 2, 8, 5, 7, 0, 3, 5, 2, 1, - 1, 1, 1, 8, 0, 3, 1, 1, 1, 4, - 7, 6, 2, 1, 3, 3, 3, 3, 3, 3, - 2, 1, 1, 2, 1, 3, 0, 3, 0, 3, - 0, 2, 0, 1, 3, 3, 3, 3, 3, 1, - 1, 1, 1, 1, 1, 7, 2, 4, 0, 1 + 2, 2, 8, 5, 8, 0, 3, 5, 2, 1, + 1, 1, 1, 8, 0, 3, 1, 1, 1, 0, + 4, 4, 7, 6, 2, 1, 3, 3, 3, 3, + 3, 3, 2, 1, 1, 1, 1, 3, 1, 1, + 3, 0, 2, 0, 1, 3, 3, 3, 3, 3, + 1, 1, 1, 1, 1, 1, 0, 7, 2, 4, + 0, 1 }; @@ -1705,93 +1723,93 @@ YYLTYPE yylloc = yyloc_default; switch (yyn) { case 2: /* commands: command_wrapper opt_semicolon */ -#line 174 "yacc_sql.y" +#line 189 "yacc_sql.y" { std::unique_ptr sql_node = std::unique_ptr((yyvsp[-1].sql_node)); sql_result->add_sql_node(std::move(sql_node)); } -#line 1714 "yacc_sql.cpp" +#line 1732 "yacc_sql.cpp" break; case 23: /* exit_stmt: EXIT */ -#line 204 "yacc_sql.y" +#line 219 "yacc_sql.y" { (void)yynerrs; // 这么写为了消除yynerrs未使用的告警。如果你有更好的方法欢迎提PR (yyval.sql_node) = new ParsedSqlNode(SCF_EXIT); } -#line 1723 "yacc_sql.cpp" +#line 1741 "yacc_sql.cpp" break; case 24: /* help_stmt: HELP */ -#line 210 "yacc_sql.y" +#line 225 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_HELP); } -#line 1731 "yacc_sql.cpp" +#line 1749 "yacc_sql.cpp" break; case 25: /* sync_stmt: SYNC */ -#line 215 "yacc_sql.y" +#line 230 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_SYNC); } -#line 1739 "yacc_sql.cpp" +#line 1757 "yacc_sql.cpp" break; case 26: /* begin_stmt: TRX_BEGIN */ -#line 221 "yacc_sql.y" +#line 236 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_BEGIN); } -#line 1747 "yacc_sql.cpp" +#line 1765 "yacc_sql.cpp" break; case 27: /* commit_stmt: TRX_COMMIT */ -#line 227 "yacc_sql.y" +#line 242 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_COMMIT); } -#line 1755 "yacc_sql.cpp" +#line 1773 "yacc_sql.cpp" break; case 28: /* rollback_stmt: TRX_ROLLBACK */ -#line 233 "yacc_sql.y" +#line 248 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_ROLLBACK); } -#line 1763 "yacc_sql.cpp" +#line 1781 "yacc_sql.cpp" break; case 29: /* drop_table_stmt: DROP TABLE ID */ -#line 239 "yacc_sql.y" +#line 254 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_DROP_TABLE); (yyval.sql_node)->drop_table.relation_name = (yyvsp[0].string); free((yyvsp[0].string)); } -#line 1773 "yacc_sql.cpp" +#line 1791 "yacc_sql.cpp" break; case 30: /* show_tables_stmt: SHOW TABLES */ -#line 246 "yacc_sql.y" +#line 261 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_SHOW_TABLES); } -#line 1781 "yacc_sql.cpp" +#line 1799 "yacc_sql.cpp" break; case 31: /* desc_table_stmt: DESC ID */ -#line 252 "yacc_sql.y" +#line 267 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_DESC_TABLE); (yyval.sql_node)->desc_table.relation_name = (yyvsp[0].string); free((yyvsp[0].string)); } -#line 1791 "yacc_sql.cpp" +#line 1809 "yacc_sql.cpp" break; case 32: /* create_index_stmt: CREATE INDEX ID ON ID LBRACE ID RBRACE */ -#line 261 "yacc_sql.y" +#line 276 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_CREATE_INDEX); CreateIndexSqlNode &create_index = (yyval.sql_node)->create_index; @@ -1802,11 +1820,11 @@ YYLTYPE yylloc = yyloc_default; free((yyvsp[-3].string)); free((yyvsp[-1].string)); } -#line 1806 "yacc_sql.cpp" +#line 1824 "yacc_sql.cpp" break; case 33: /* drop_index_stmt: DROP INDEX ID ON ID */ -#line 275 "yacc_sql.y" +#line 290 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_DROP_INDEX); (yyval.sql_node)->drop_index.index_name = (yyvsp[-2].string); @@ -1814,40 +1832,44 @@ YYLTYPE yylloc = yyloc_default; free((yyvsp[-2].string)); free((yyvsp[0].string)); } -#line 1818 "yacc_sql.cpp" +#line 1836 "yacc_sql.cpp" break; - case 34: /* create_table_stmt: CREATE TABLE ID LBRACE attr_def attr_def_list RBRACE */ -#line 285 "yacc_sql.y" + case 34: /* create_table_stmt: CREATE TABLE ID LBRACE attr_def attr_def_list RBRACE storage_format */ +#line 300 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_CREATE_TABLE); CreateTableSqlNode &create_table = (yyval.sql_node)->create_table; - create_table.relation_name = (yyvsp[-4].string); - free((yyvsp[-4].string)); + create_table.relation_name = (yyvsp[-5].string); + free((yyvsp[-5].string)); - std::vector *src_attrs = (yyvsp[-1].attr_infos); + std::vector *src_attrs = (yyvsp[-2].attr_infos); if (src_attrs != nullptr) { create_table.attr_infos.swap(*src_attrs); delete src_attrs; } - create_table.attr_infos.emplace_back(*(yyvsp[-2].attr_info)); + create_table.attr_infos.emplace_back(*(yyvsp[-3].attr_info)); std::reverse(create_table.attr_infos.begin(), create_table.attr_infos.end()); - delete (yyvsp[-2].attr_info); + delete (yyvsp[-3].attr_info); + if ((yyvsp[0].string) != nullptr) { + create_table.storage_format = (yyvsp[0].string); + free((yyvsp[0].string)); + } } -#line 1839 "yacc_sql.cpp" +#line 1861 "yacc_sql.cpp" break; case 35: /* attr_def_list: %empty */ -#line 304 "yacc_sql.y" +#line 323 "yacc_sql.y" { (yyval.attr_infos) = nullptr; } -#line 1847 "yacc_sql.cpp" +#line 1869 "yacc_sql.cpp" break; case 36: /* attr_def_list: COMMA attr_def attr_def_list */ -#line 308 "yacc_sql.y" +#line 327 "yacc_sql.y" { if ((yyvsp[0].attr_infos) != nullptr) { (yyval.attr_infos) = (yyvsp[0].attr_infos); @@ -1857,11 +1879,11 @@ YYLTYPE yylloc = yyloc_default; (yyval.attr_infos)->emplace_back(*(yyvsp[-1].attr_info)); delete (yyvsp[-1].attr_info); } -#line 1861 "yacc_sql.cpp" +#line 1883 "yacc_sql.cpp" break; case 37: /* attr_def: ID type LBRACE number RBRACE */ -#line 321 "yacc_sql.y" +#line 340 "yacc_sql.y" { (yyval.attr_info) = new AttrInfoSqlNode; (yyval.attr_info)->type = (AttrType)(yyvsp[-3].number); @@ -1869,11 +1891,11 @@ YYLTYPE yylloc = yyloc_default; (yyval.attr_info)->length = (yyvsp[-1].number); free((yyvsp[-4].string)); } -#line 1873 "yacc_sql.cpp" +#line 1895 "yacc_sql.cpp" break; case 38: /* attr_def: ID type */ -#line 329 "yacc_sql.y" +#line 348 "yacc_sql.y" { (yyval.attr_info) = new AttrInfoSqlNode; (yyval.attr_info)->type = (AttrType)(yyvsp[0].number); @@ -1881,35 +1903,35 @@ YYLTYPE yylloc = yyloc_default; (yyval.attr_info)->length = 4; free((yyvsp[-1].string)); } -#line 1885 "yacc_sql.cpp" +#line 1907 "yacc_sql.cpp" break; case 39: /* number: NUMBER */ -#line 338 "yacc_sql.y" +#line 357 "yacc_sql.y" {(yyval.number) = (yyvsp[0].number);} -#line 1891 "yacc_sql.cpp" +#line 1913 "yacc_sql.cpp" break; case 40: /* type: INT_T */ -#line 341 "yacc_sql.y" - { (yyval.number)=INTS; } -#line 1897 "yacc_sql.cpp" +#line 360 "yacc_sql.y" + { (yyval.number) = static_cast(AttrType::INTS); } +#line 1919 "yacc_sql.cpp" break; case 41: /* type: STRING_T */ -#line 342 "yacc_sql.y" - { (yyval.number)=CHARS; } -#line 1903 "yacc_sql.cpp" +#line 361 "yacc_sql.y" + { (yyval.number) = static_cast(AttrType::CHARS); } +#line 1925 "yacc_sql.cpp" break; case 42: /* type: FLOAT_T */ -#line 343 "yacc_sql.y" - { (yyval.number)=FLOATS; } -#line 1909 "yacc_sql.cpp" +#line 362 "yacc_sql.y" + { (yyval.number) = static_cast(AttrType::FLOATS); } +#line 1931 "yacc_sql.cpp" break; case 43: /* insert_stmt: INSERT INTO ID VALUES LBRACE value value_list RBRACE */ -#line 347 "yacc_sql.y" +#line 366 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_INSERT); (yyval.sql_node)->insertion.relation_name = (yyvsp[-5].string); @@ -1922,19 +1944,19 @@ YYLTYPE yylloc = yyloc_default; delete (yyvsp[-2].value); free((yyvsp[-5].string)); } -#line 1926 "yacc_sql.cpp" +#line 1948 "yacc_sql.cpp" break; case 44: /* value_list: %empty */ -#line 363 "yacc_sql.y" +#line 382 "yacc_sql.y" { (yyval.value_list) = nullptr; } -#line 1934 "yacc_sql.cpp" +#line 1956 "yacc_sql.cpp" break; case 45: /* value_list: COMMA value value_list */ -#line 366 "yacc_sql.y" +#line 385 "yacc_sql.y" { if ((yyvsp[0].value_list) != nullptr) { (yyval.value_list) = (yyvsp[0].value_list); @@ -1944,40 +1966,56 @@ YYLTYPE yylloc = yyloc_default; (yyval.value_list)->emplace_back(*(yyvsp[-1].value)); delete (yyvsp[-1].value); } -#line 1948 "yacc_sql.cpp" +#line 1970 "yacc_sql.cpp" break; case 46: /* value: NUMBER */ -#line 377 "yacc_sql.y" +#line 396 "yacc_sql.y" { (yyval.value) = new Value((int)(yyvsp[0].number)); (yyloc) = (yylsp[0]); } -#line 1957 "yacc_sql.cpp" +#line 1979 "yacc_sql.cpp" break; case 47: /* value: FLOAT */ -#line 381 "yacc_sql.y" +#line 400 "yacc_sql.y" { (yyval.value) = new Value((float)(yyvsp[0].floats)); (yyloc) = (yylsp[0]); } -#line 1966 "yacc_sql.cpp" +#line 1988 "yacc_sql.cpp" break; case 48: /* value: SSS */ -#line 385 "yacc_sql.y" +#line 404 "yacc_sql.y" { char *tmp = common::substr((yyvsp[0].string),1,strlen((yyvsp[0].string))-2); (yyval.value) = new Value(tmp); free(tmp); free((yyvsp[0].string)); } -#line 1977 "yacc_sql.cpp" +#line 1999 "yacc_sql.cpp" + break; + + case 49: /* storage_format: %empty */ +#line 413 "yacc_sql.y" + { + (yyval.string) = nullptr; + } +#line 2007 "yacc_sql.cpp" break; - case 49: /* delete_stmt: DELETE FROM ID where */ -#line 395 "yacc_sql.y" + case 50: /* storage_format: STORAGE FORMAT EQ ID */ +#line 417 "yacc_sql.y" + { + (yyval.string) = (yyvsp[0].string); + } +#line 2015 "yacc_sql.cpp" + break; + + case 51: /* delete_stmt: DELETE FROM ID where */ +#line 424 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_DELETE); (yyval.sql_node)->deletion.relation_name = (yyvsp[-1].string); @@ -1987,11 +2025,11 @@ YYLTYPE yylloc = yyloc_default; } free((yyvsp[-1].string)); } -#line 1991 "yacc_sql.cpp" +#line 2029 "yacc_sql.cpp" break; - case 50: /* update_stmt: UPDATE ID SET ID EQ value where */ -#line 407 "yacc_sql.y" + case 52: /* update_stmt: UPDATE ID SET ID EQ value where */ +#line 436 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_UPDATE); (yyval.sql_node)->update.relation_name = (yyvsp[-5].string); @@ -2004,163 +2042,158 @@ YYLTYPE yylloc = yyloc_default; free((yyvsp[-5].string)); free((yyvsp[-3].string)); } -#line 2008 "yacc_sql.cpp" +#line 2046 "yacc_sql.cpp" break; - case 51: /* select_stmt: SELECT select_attr FROM ID rel_list where */ -#line 422 "yacc_sql.y" + case 53: /* select_stmt: SELECT expression_list FROM rel_list where group_by */ +#line 451 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_SELECT); - if ((yyvsp[-4].rel_attr_list) != nullptr) { - (yyval.sql_node)->selection.attributes.swap(*(yyvsp[-4].rel_attr_list)); - delete (yyvsp[-4].rel_attr_list); + if ((yyvsp[-4].expression_list) != nullptr) { + (yyval.sql_node)->selection.expressions.swap(*(yyvsp[-4].expression_list)); + delete (yyvsp[-4].expression_list); } - if ((yyvsp[-1].relation_list) != nullptr) { - (yyval.sql_node)->selection.relations.swap(*(yyvsp[-1].relation_list)); - delete (yyvsp[-1].relation_list); + + if ((yyvsp[-2].relation_list) != nullptr) { + (yyval.sql_node)->selection.relations.swap(*(yyvsp[-2].relation_list)); + delete (yyvsp[-2].relation_list); } - (yyval.sql_node)->selection.relations.push_back((yyvsp[-2].string)); - std::reverse((yyval.sql_node)->selection.relations.begin(), (yyval.sql_node)->selection.relations.end()); - if ((yyvsp[0].condition_list) != nullptr) { - (yyval.sql_node)->selection.conditions.swap(*(yyvsp[0].condition_list)); - delete (yyvsp[0].condition_list); + if ((yyvsp[-1].condition_list) != nullptr) { + (yyval.sql_node)->selection.conditions.swap(*(yyvsp[-1].condition_list)); + delete (yyvsp[-1].condition_list); + } + + if ((yyvsp[0].expression_list) != nullptr) { + (yyval.sql_node)->selection.group_by.swap(*(yyvsp[0].expression_list)); + delete (yyvsp[0].expression_list); } - free((yyvsp[-2].string)); } -#line 2032 "yacc_sql.cpp" +#line 2073 "yacc_sql.cpp" break; - case 52: /* calc_stmt: CALC expression_list */ -#line 444 "yacc_sql.y" + case 54: /* calc_stmt: CALC expression_list */ +#line 476 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_CALC); - std::reverse((yyvsp[0].expression_list)->begin(), (yyvsp[0].expression_list)->end()); (yyval.sql_node)->calc.expressions.swap(*(yyvsp[0].expression_list)); delete (yyvsp[0].expression_list); } -#line 2043 "yacc_sql.cpp" +#line 2083 "yacc_sql.cpp" break; - case 53: /* expression_list: expression */ -#line 454 "yacc_sql.y" + case 55: /* expression_list: expression */ +#line 485 "yacc_sql.y" { - (yyval.expression_list) = new std::vector; + (yyval.expression_list) = new std::vector>; (yyval.expression_list)->emplace_back((yyvsp[0].expression)); } -#line 2052 "yacc_sql.cpp" +#line 2092 "yacc_sql.cpp" break; - case 54: /* expression_list: expression COMMA expression_list */ -#line 459 "yacc_sql.y" + case 56: /* expression_list: expression COMMA expression_list */ +#line 490 "yacc_sql.y" { if ((yyvsp[0].expression_list) != nullptr) { (yyval.expression_list) = (yyvsp[0].expression_list); } else { - (yyval.expression_list) = new std::vector; + (yyval.expression_list) = new std::vector>; } - (yyval.expression_list)->emplace_back((yyvsp[-2].expression)); + (yyval.expression_list)->emplace((yyval.expression_list)->begin(), (yyvsp[-2].expression)); } -#line 2065 "yacc_sql.cpp" +#line 2105 "yacc_sql.cpp" break; - case 55: /* expression: expression '+' expression */ -#line 469 "yacc_sql.y" + case 57: /* expression: expression '+' expression */ +#line 500 "yacc_sql.y" { (yyval.expression) = create_arithmetic_expression(ArithmeticExpr::Type::ADD, (yyvsp[-2].expression), (yyvsp[0].expression), sql_string, &(yyloc)); } -#line 2073 "yacc_sql.cpp" +#line 2113 "yacc_sql.cpp" break; - case 56: /* expression: expression '-' expression */ -#line 472 "yacc_sql.y" + case 58: /* expression: expression '-' expression */ +#line 503 "yacc_sql.y" { (yyval.expression) = create_arithmetic_expression(ArithmeticExpr::Type::SUB, (yyvsp[-2].expression), (yyvsp[0].expression), sql_string, &(yyloc)); } -#line 2081 "yacc_sql.cpp" +#line 2121 "yacc_sql.cpp" break; - case 57: /* expression: expression '*' expression */ -#line 475 "yacc_sql.y" + case 59: /* expression: expression '*' expression */ +#line 506 "yacc_sql.y" { (yyval.expression) = create_arithmetic_expression(ArithmeticExpr::Type::MUL, (yyvsp[-2].expression), (yyvsp[0].expression), sql_string, &(yyloc)); } -#line 2089 "yacc_sql.cpp" +#line 2129 "yacc_sql.cpp" break; - case 58: /* expression: expression '/' expression */ -#line 478 "yacc_sql.y" + case 60: /* expression: expression '/' expression */ +#line 509 "yacc_sql.y" { (yyval.expression) = create_arithmetic_expression(ArithmeticExpr::Type::DIV, (yyvsp[-2].expression), (yyvsp[0].expression), sql_string, &(yyloc)); } -#line 2097 "yacc_sql.cpp" +#line 2137 "yacc_sql.cpp" break; - case 59: /* expression: LBRACE expression RBRACE */ -#line 481 "yacc_sql.y" + case 61: /* expression: LBRACE expression RBRACE */ +#line 512 "yacc_sql.y" { (yyval.expression) = (yyvsp[-1].expression); (yyval.expression)->set_name(token_name(sql_string, &(yyloc))); } -#line 2106 "yacc_sql.cpp" +#line 2146 "yacc_sql.cpp" break; - case 60: /* expression: '-' expression */ -#line 485 "yacc_sql.y" + case 62: /* expression: '-' expression */ +#line 516 "yacc_sql.y" { (yyval.expression) = create_arithmetic_expression(ArithmeticExpr::Type::NEGATIVE, (yyvsp[0].expression), nullptr, sql_string, &(yyloc)); } -#line 2114 "yacc_sql.cpp" +#line 2154 "yacc_sql.cpp" break; - case 61: /* expression: value */ -#line 488 "yacc_sql.y" + case 63: /* expression: value */ +#line 519 "yacc_sql.y" { (yyval.expression) = new ValueExpr(*(yyvsp[0].value)); (yyval.expression)->set_name(token_name(sql_string, &(yyloc))); delete (yyvsp[0].value); } -#line 2124 "yacc_sql.cpp" +#line 2164 "yacc_sql.cpp" break; - case 62: /* select_attr: '*' */ -#line 496 "yacc_sql.y" - { - (yyval.rel_attr_list) = new std::vector; - RelAttrSqlNode attr; - attr.relation_name = ""; - attr.attribute_name = "*"; - (yyval.rel_attr_list)->emplace_back(attr); + case 64: /* expression: rel_attr */ +#line 524 "yacc_sql.y" + { + RelAttrSqlNode *node = (yyvsp[0].rel_attr); + (yyval.expression) = new UnboundFieldExpr(node->relation_name, node->attribute_name); + (yyval.expression)->set_name(token_name(sql_string, &(yyloc))); + delete (yyvsp[0].rel_attr); } -#line 2136 "yacc_sql.cpp" +#line 2175 "yacc_sql.cpp" break; - case 63: /* select_attr: rel_attr attr_list */ -#line 503 "yacc_sql.y" - { - if ((yyvsp[0].rel_attr_list) != nullptr) { - (yyval.rel_attr_list) = (yyvsp[0].rel_attr_list); - } else { - (yyval.rel_attr_list) = new std::vector; - } - (yyval.rel_attr_list)->emplace_back(*(yyvsp[-1].rel_attr)); - delete (yyvsp[-1].rel_attr); + case 65: /* expression: '*' */ +#line 530 "yacc_sql.y" + { + (yyval.expression) = new StarExpr(); } -#line 2150 "yacc_sql.cpp" +#line 2183 "yacc_sql.cpp" break; - case 64: /* rel_attr: ID */ -#line 515 "yacc_sql.y" + case 66: /* rel_attr: ID */ +#line 537 "yacc_sql.y" { (yyval.rel_attr) = new RelAttrSqlNode; (yyval.rel_attr)->attribute_name = (yyvsp[0].string); free((yyvsp[0].string)); } -#line 2160 "yacc_sql.cpp" +#line 2193 "yacc_sql.cpp" break; - case 65: /* rel_attr: ID DOT ID */ -#line 520 "yacc_sql.y" + case 67: /* rel_attr: ID DOT ID */ +#line 542 "yacc_sql.y" { (yyval.rel_attr) = new RelAttrSqlNode; (yyval.rel_attr)->relation_name = (yyvsp[-2].string); @@ -2168,101 +2201,88 @@ YYLTYPE yylloc = yyloc_default; free((yyvsp[-2].string)); free((yyvsp[0].string)); } -#line 2172 "yacc_sql.cpp" +#line 2205 "yacc_sql.cpp" break; - case 66: /* attr_list: %empty */ -#line 531 "yacc_sql.y" - { - (yyval.rel_attr_list) = nullptr; - } -#line 2180 "yacc_sql.cpp" - break; - - case 67: /* attr_list: COMMA rel_attr attr_list */ -#line 534 "yacc_sql.y" - { - if ((yyvsp[0].rel_attr_list) != nullptr) { - (yyval.rel_attr_list) = (yyvsp[0].rel_attr_list); - } else { - (yyval.rel_attr_list) = new std::vector; - } - - (yyval.rel_attr_list)->emplace_back(*(yyvsp[-1].rel_attr)); - delete (yyvsp[-1].rel_attr); + case 68: /* relation: ID */ +#line 552 "yacc_sql.y" + { + (yyval.string) = (yyvsp[0].string); } -#line 2195 "yacc_sql.cpp" +#line 2213 "yacc_sql.cpp" break; - case 68: /* rel_list: %empty */ -#line 548 "yacc_sql.y" - { - (yyval.relation_list) = nullptr; + case 69: /* rel_list: relation */ +#line 557 "yacc_sql.y" + { + (yyval.relation_list) = new std::vector(); + (yyval.relation_list)->push_back((yyvsp[0].string)); + free((yyvsp[0].string)); } -#line 2203 "yacc_sql.cpp" +#line 2223 "yacc_sql.cpp" break; - case 69: /* rel_list: COMMA ID rel_list */ -#line 551 "yacc_sql.y" - { + case 70: /* rel_list: relation COMMA rel_list */ +#line 562 "yacc_sql.y" + { if ((yyvsp[0].relation_list) != nullptr) { (yyval.relation_list) = (yyvsp[0].relation_list); } else { (yyval.relation_list) = new std::vector; } - (yyval.relation_list)->push_back((yyvsp[-1].string)); - free((yyvsp[-1].string)); + (yyval.relation_list)->insert((yyval.relation_list)->begin(), (yyvsp[-2].string)); + free((yyvsp[-2].string)); } -#line 2218 "yacc_sql.cpp" +#line 2238 "yacc_sql.cpp" break; - case 70: /* where: %empty */ -#line 564 "yacc_sql.y" + case 71: /* where: %empty */ +#line 576 "yacc_sql.y" { (yyval.condition_list) = nullptr; } -#line 2226 "yacc_sql.cpp" +#line 2246 "yacc_sql.cpp" break; - case 71: /* where: WHERE condition_list */ -#line 567 "yacc_sql.y" + case 72: /* where: WHERE condition_list */ +#line 579 "yacc_sql.y" { (yyval.condition_list) = (yyvsp[0].condition_list); } -#line 2234 "yacc_sql.cpp" +#line 2254 "yacc_sql.cpp" break; - case 72: /* condition_list: %empty */ -#line 573 "yacc_sql.y" + case 73: /* condition_list: %empty */ +#line 585 "yacc_sql.y" { (yyval.condition_list) = nullptr; } -#line 2242 "yacc_sql.cpp" +#line 2262 "yacc_sql.cpp" break; - case 73: /* condition_list: condition */ -#line 576 "yacc_sql.y" + case 74: /* condition_list: condition */ +#line 588 "yacc_sql.y" { (yyval.condition_list) = new std::vector; (yyval.condition_list)->emplace_back(*(yyvsp[0].condition)); delete (yyvsp[0].condition); } -#line 2252 "yacc_sql.cpp" +#line 2272 "yacc_sql.cpp" break; - case 74: /* condition_list: condition AND condition_list */ -#line 581 "yacc_sql.y" + case 75: /* condition_list: condition AND condition_list */ +#line 593 "yacc_sql.y" { (yyval.condition_list) = (yyvsp[0].condition_list); (yyval.condition_list)->emplace_back(*(yyvsp[-2].condition)); delete (yyvsp[-2].condition); } -#line 2262 "yacc_sql.cpp" +#line 2282 "yacc_sql.cpp" break; - case 75: /* condition: rel_attr comp_op value */ -#line 589 "yacc_sql.y" + case 76: /* condition: rel_attr comp_op value */ +#line 601 "yacc_sql.y" { (yyval.condition) = new ConditionSqlNode; (yyval.condition)->left_is_attr = 1; @@ -2274,11 +2294,11 @@ YYLTYPE yylloc = yyloc_default; delete (yyvsp[-2].rel_attr); delete (yyvsp[0].value); } -#line 2278 "yacc_sql.cpp" +#line 2298 "yacc_sql.cpp" break; - case 76: /* condition: value comp_op value */ -#line 601 "yacc_sql.y" + case 77: /* condition: value comp_op value */ +#line 613 "yacc_sql.y" { (yyval.condition) = new ConditionSqlNode; (yyval.condition)->left_is_attr = 0; @@ -2290,11 +2310,11 @@ YYLTYPE yylloc = yyloc_default; delete (yyvsp[-2].value); delete (yyvsp[0].value); } -#line 2294 "yacc_sql.cpp" +#line 2314 "yacc_sql.cpp" break; - case 77: /* condition: rel_attr comp_op rel_attr */ -#line 613 "yacc_sql.y" + case 78: /* condition: rel_attr comp_op rel_attr */ +#line 625 "yacc_sql.y" { (yyval.condition) = new ConditionSqlNode; (yyval.condition)->left_is_attr = 1; @@ -2306,11 +2326,11 @@ YYLTYPE yylloc = yyloc_default; delete (yyvsp[-2].rel_attr); delete (yyvsp[0].rel_attr); } -#line 2310 "yacc_sql.cpp" +#line 2330 "yacc_sql.cpp" break; - case 78: /* condition: value comp_op rel_attr */ -#line 625 "yacc_sql.y" + case 79: /* condition: value comp_op rel_attr */ +#line 637 "yacc_sql.y" { (yyval.condition) = new ConditionSqlNode; (yyval.condition)->left_is_attr = 0; @@ -2322,47 +2342,55 @@ YYLTYPE yylloc = yyloc_default; delete (yyvsp[-2].value); delete (yyvsp[0].rel_attr); } -#line 2326 "yacc_sql.cpp" +#line 2346 "yacc_sql.cpp" break; - case 79: /* comp_op: EQ */ -#line 639 "yacc_sql.y" + case 80: /* comp_op: EQ */ +#line 651 "yacc_sql.y" { (yyval.comp) = EQUAL_TO; } -#line 2332 "yacc_sql.cpp" +#line 2352 "yacc_sql.cpp" break; - case 80: /* comp_op: LT */ -#line 640 "yacc_sql.y" + case 81: /* comp_op: LT */ +#line 652 "yacc_sql.y" { (yyval.comp) = LESS_THAN; } -#line 2338 "yacc_sql.cpp" +#line 2358 "yacc_sql.cpp" break; - case 81: /* comp_op: GT */ -#line 641 "yacc_sql.y" + case 82: /* comp_op: GT */ +#line 653 "yacc_sql.y" { (yyval.comp) = GREAT_THAN; } -#line 2344 "yacc_sql.cpp" +#line 2364 "yacc_sql.cpp" break; - case 82: /* comp_op: LE */ -#line 642 "yacc_sql.y" + case 83: /* comp_op: LE */ +#line 654 "yacc_sql.y" { (yyval.comp) = LESS_EQUAL; } -#line 2350 "yacc_sql.cpp" +#line 2370 "yacc_sql.cpp" break; - case 83: /* comp_op: GE */ -#line 643 "yacc_sql.y" + case 84: /* comp_op: GE */ +#line 655 "yacc_sql.y" { (yyval.comp) = GREAT_EQUAL; } -#line 2356 "yacc_sql.cpp" +#line 2376 "yacc_sql.cpp" break; - case 84: /* comp_op: NE */ -#line 644 "yacc_sql.y" + case 85: /* comp_op: NE */ +#line 656 "yacc_sql.y" { (yyval.comp) = NOT_EQUAL; } -#line 2362 "yacc_sql.cpp" +#line 2382 "yacc_sql.cpp" + break; + + case 86: /* group_by: %empty */ +#line 662 "yacc_sql.y" + { + (yyval.expression_list) = nullptr; + } +#line 2390 "yacc_sql.cpp" break; - case 85: /* load_data_stmt: LOAD DATA INFILE SSS INTO TABLE ID */ -#line 649 "yacc_sql.y" + case 87: /* load_data_stmt: LOAD DATA INFILE SSS INTO TABLE ID */ +#line 668 "yacc_sql.y" { char *tmp_file_name = common::substr((yyvsp[-3].string), 1, strlen((yyvsp[-3].string)) - 2); @@ -2372,20 +2400,20 @@ YYLTYPE yylloc = yyloc_default; free((yyvsp[0].string)); free(tmp_file_name); } -#line 2376 "yacc_sql.cpp" +#line 2404 "yacc_sql.cpp" break; - case 86: /* explain_stmt: EXPLAIN command_wrapper */ -#line 662 "yacc_sql.y" + case 88: /* explain_stmt: EXPLAIN command_wrapper */ +#line 681 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_EXPLAIN); (yyval.sql_node)->explain.sql_node = std::unique_ptr((yyvsp[0].sql_node)); } -#line 2385 "yacc_sql.cpp" +#line 2413 "yacc_sql.cpp" break; - case 87: /* set_variable_stmt: SET ID EQ value */ -#line 670 "yacc_sql.y" + case 89: /* set_variable_stmt: SET ID EQ value */ +#line 689 "yacc_sql.y" { (yyval.sql_node) = new ParsedSqlNode(SCF_SET_VARIABLE); (yyval.sql_node)->set_variable.name = (yyvsp[-2].string); @@ -2393,11 +2421,11 @@ YYLTYPE yylloc = yyloc_default; free((yyvsp[-2].string)); delete (yyvsp[0].value); } -#line 2397 "yacc_sql.cpp" +#line 2425 "yacc_sql.cpp" break; -#line 2401 "yacc_sql.cpp" +#line 2429 "yacc_sql.cpp" default: break; } @@ -2626,7 +2654,7 @@ YYLTYPE yylloc = yyloc_default; return yyresult; } -#line 682 "yacc_sql.y" +#line 701 "yacc_sql.y" //_____________________________________________________________________ extern void scan_string(const char *str, yyscan_t scanner); diff --git a/src/observer/sql/parser/yacc_sql.hpp b/src/observer/sql/parser/yacc_sql.hpp index af42fa7a9..63a2545b8 100644 --- a/src/observer/sql/parser/yacc_sql.hpp +++ b/src/observer/sql/parser/yacc_sql.hpp @@ -55,53 +55,57 @@ extern int yydebug; YYerror = 256, /* error */ YYUNDEF = 257, /* "invalid token" */ SEMICOLON = 258, /* SEMICOLON */ - CREATE = 259, /* CREATE */ - DROP = 260, /* DROP */ - TABLE = 261, /* TABLE */ - TABLES = 262, /* TABLES */ - INDEX = 263, /* INDEX */ - CALC = 264, /* CALC */ - SELECT = 265, /* SELECT */ - DESC = 266, /* DESC */ - SHOW = 267, /* SHOW */ - SYNC = 268, /* SYNC */ - INSERT = 269, /* INSERT */ - DELETE = 270, /* DELETE */ - UPDATE = 271, /* UPDATE */ - LBRACE = 272, /* LBRACE */ - RBRACE = 273, /* RBRACE */ - COMMA = 274, /* COMMA */ - TRX_BEGIN = 275, /* TRX_BEGIN */ - TRX_COMMIT = 276, /* TRX_COMMIT */ - TRX_ROLLBACK = 277, /* TRX_ROLLBACK */ - INT_T = 278, /* INT_T */ - STRING_T = 279, /* STRING_T */ - FLOAT_T = 280, /* FLOAT_T */ - HELP = 281, /* HELP */ - EXIT = 282, /* EXIT */ - DOT = 283, /* DOT */ - INTO = 284, /* INTO */ - VALUES = 285, /* VALUES */ - FROM = 286, /* FROM */ - WHERE = 287, /* WHERE */ - AND = 288, /* AND */ - SET = 289, /* SET */ - ON = 290, /* ON */ - LOAD = 291, /* LOAD */ - DATA = 292, /* DATA */ - INFILE = 293, /* INFILE */ - EXPLAIN = 294, /* EXPLAIN */ - EQ = 295, /* EQ */ - LT = 296, /* LT */ - GT = 297, /* GT */ - LE = 298, /* LE */ - GE = 299, /* GE */ - NE = 300, /* NE */ - NUMBER = 301, /* NUMBER */ - FLOAT = 302, /* FLOAT */ - ID = 303, /* ID */ - SSS = 304, /* SSS */ - UMINUS = 305 /* UMINUS */ + BY = 259, /* BY */ + CREATE = 260, /* CREATE */ + DROP = 261, /* DROP */ + GROUP = 262, /* GROUP */ + TABLE = 263, /* TABLE */ + TABLES = 264, /* TABLES */ + INDEX = 265, /* INDEX */ + CALC = 266, /* CALC */ + SELECT = 267, /* SELECT */ + DESC = 268, /* DESC */ + SHOW = 269, /* SHOW */ + SYNC = 270, /* SYNC */ + INSERT = 271, /* INSERT */ + DELETE = 272, /* DELETE */ + UPDATE = 273, /* UPDATE */ + LBRACE = 274, /* LBRACE */ + RBRACE = 275, /* RBRACE */ + COMMA = 276, /* COMMA */ + TRX_BEGIN = 277, /* TRX_BEGIN */ + TRX_COMMIT = 278, /* TRX_COMMIT */ + TRX_ROLLBACK = 279, /* TRX_ROLLBACK */ + INT_T = 280, /* INT_T */ + STRING_T = 281, /* STRING_T */ + FLOAT_T = 282, /* FLOAT_T */ + HELP = 283, /* HELP */ + EXIT = 284, /* EXIT */ + DOT = 285, /* DOT */ + INTO = 286, /* INTO */ + VALUES = 287, /* VALUES */ + FROM = 288, /* FROM */ + WHERE = 289, /* WHERE */ + AND = 290, /* AND */ + SET = 291, /* SET */ + ON = 292, /* ON */ + LOAD = 293, /* LOAD */ + DATA = 294, /* DATA */ + INFILE = 295, /* INFILE */ + EXPLAIN = 296, /* EXPLAIN */ + STORAGE = 297, /* STORAGE */ + FORMAT = 298, /* FORMAT */ + EQ = 299, /* EQ */ + LT = 300, /* LT */ + GT = 301, /* GT */ + LE = 302, /* LE */ + GE = 303, /* GE */ + NE = 304, /* NE */ + NUMBER = 305, /* NUMBER */ + FLOAT = 306, /* FLOAT */ + ID = 307, /* ID */ + SSS = 308, /* SSS */ + UMINUS = 309 /* UMINUS */ }; typedef enum yytokentype yytoken_kind_t; #endif @@ -110,26 +114,26 @@ extern int yydebug; #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED union YYSTYPE { -#line 102 "yacc_sql.y" - - ParsedSqlNode * sql_node; - ConditionSqlNode * condition; - Value * value; - enum CompOp comp; - RelAttrSqlNode * rel_attr; - std::vector * attr_infos; - AttrInfoSqlNode * attr_info; - Expression * expression; - std::vector * expression_list; - std::vector * value_list; - std::vector * condition_list; - std::vector * rel_attr_list; - std::vector * relation_list; - char * string; - int number; - float floats; - -#line 133 "yacc_sql.hpp" +#line 116 "yacc_sql.y" + + ParsedSqlNode * sql_node; + ConditionSqlNode * condition; + Value * value; + enum CompOp comp; + RelAttrSqlNode * rel_attr; + std::vector * attr_infos; + AttrInfoSqlNode * attr_info; + Expression * expression; + std::vector> * expression_list; + std::vector * value_list; + std::vector * condition_list; + std::vector * rel_attr_list; + std::vector * relation_list; + char * string; + int number; + float floats; + +#line 137 "yacc_sql.hpp" }; typedef union YYSTYPE YYSTYPE; diff --git a/src/observer/sql/parser/yacc_sql.y b/src/observer/sql/parser/yacc_sql.y index 548bee449..328f775ad 100644 --- a/src/observer/sql/parser/yacc_sql.y +++ b/src/observer/sql/parser/yacc_sql.y @@ -41,6 +41,16 @@ ArithmeticExpr *create_arithmetic_expression(ArithmeticExpr::Type type, return expr; } +UnboundAggregateExpr *create_aggregate_expression(const char *aggregate_name, + Expression *child, + const char *sql_string, + YYLTYPE *llocp) +{ + UnboundAggregateExpr *expr = new UnboundAggregateExpr(aggregate_name, child); + expr->set_name(token_name(sql_string, llocp)); + return expr; +} + %} %define api.pure full @@ -55,8 +65,10 @@ ArithmeticExpr *create_arithmetic_expression(ArithmeticExpr::Type type, //标识tokens %token SEMICOLON + BY CREATE DROP + GROUP TABLE TABLES INDEX @@ -91,6 +103,8 @@ ArithmeticExpr *create_arithmetic_expression(ArithmeticExpr::Type type, DATA INFILE EXPLAIN + STORAGE + FORMAT EQ LT GT @@ -100,22 +114,22 @@ ArithmeticExpr *create_arithmetic_expression(ArithmeticExpr::Type type, /** union 中定义各种数据类型,真实生成的代码也是union类型,所以不能有非POD类型的数据 **/ %union { - ParsedSqlNode * sql_node; - ConditionSqlNode * condition; - Value * value; - enum CompOp comp; - RelAttrSqlNode * rel_attr; - std::vector * attr_infos; - AttrInfoSqlNode * attr_info; - Expression * expression; - std::vector * expression_list; - std::vector * value_list; - std::vector * condition_list; - std::vector * rel_attr_list; - std::vector * relation_list; - char * string; - int number; - float floats; + ParsedSqlNode * sql_node; + ConditionSqlNode * condition; + Value * value; + enum CompOp comp; + RelAttrSqlNode * rel_attr; + std::vector * attr_infos; + AttrInfoSqlNode * attr_info; + Expression * expression; + std::vector> * expression_list; + std::vector * value_list; + std::vector * condition_list; + std::vector * rel_attr_list; + std::vector * relation_list; + char * string; + int number; + float floats; } %token NUMBER @@ -129,6 +143,7 @@ ArithmeticExpr *create_arithmetic_expression(ArithmeticExpr::Type type, %type condition %type value %type number +%type relation %type comp_op %type rel_attr %type attr_def_list @@ -136,11 +151,11 @@ ArithmeticExpr *create_arithmetic_expression(ArithmeticExpr::Type type, %type value_list %type where %type condition_list -%type select_attr +%type storage_format %type rel_list -%type attr_list %type expression %type expression_list +%type group_by %type calc_stmt %type select_stmt %type insert_stmt @@ -281,7 +296,7 @@ drop_index_stmt: /*drop index 语句的语法解析树*/ } ; create_table_stmt: /*create table 语句的语法解析树*/ - CREATE TABLE ID LBRACE attr_def attr_def_list RBRACE + CREATE TABLE ID LBRACE attr_def attr_def_list RBRACE storage_format { $$ = new ParsedSqlNode(SCF_CREATE_TABLE); CreateTableSqlNode &create_table = $$->create_table; @@ -297,6 +312,10 @@ create_table_stmt: /*create table 语句的语法解析树*/ create_table.attr_infos.emplace_back(*$5); std::reverse(create_table.attr_infos.begin(), create_table.attr_infos.end()); delete $5; + if ($8 != nullptr) { + create_table.storage_format = $8; + free($8); + } } ; attr_def_list: @@ -338,9 +357,9 @@ number: NUMBER {$$ = $1;} ; type: - INT_T { $$=INTS; } - | STRING_T { $$=CHARS; } - | FLOAT_T { $$=FLOATS; } + INT_T { $$ = static_cast(AttrType::INTS); } + | STRING_T { $$ = static_cast(AttrType::CHARS); } + | FLOAT_T { $$ = static_cast(AttrType::FLOATS); } ; insert_stmt: /*insert 语句的语法解析树*/ INSERT INTO ID VALUES LBRACE value value_list RBRACE @@ -389,6 +408,16 @@ value: free($1); } ; +storage_format: + /* empty */ + { + $$ = nullptr; + } + | STORAGE FORMAT EQ ID + { + $$ = $4; + } + ; delete_stmt: /* delete 语句的语法解析树*/ DELETE FROM ID where @@ -418,32 +447,34 @@ update_stmt: /* update 语句的语法解析树*/ } ; select_stmt: /* select 语句的语法解析树*/ - SELECT select_attr FROM ID rel_list where + SELECT expression_list FROM rel_list where group_by { $$ = new ParsedSqlNode(SCF_SELECT); if ($2 != nullptr) { - $$->selection.attributes.swap(*$2); + $$->selection.expressions.swap(*$2); delete $2; } + + if ($4 != nullptr) { + $$->selection.relations.swap(*$4); + delete $4; + } + if ($5 != nullptr) { - $$->selection.relations.swap(*$5); + $$->selection.conditions.swap(*$5); delete $5; } - $$->selection.relations.push_back($4); - std::reverse($$->selection.relations.begin(), $$->selection.relations.end()); if ($6 != nullptr) { - $$->selection.conditions.swap(*$6); + $$->selection.group_by.swap(*$6); delete $6; } - free($4); } ; calc_stmt: CALC expression_list { $$ = new ParsedSqlNode(SCF_CALC); - std::reverse($2->begin(), $2->end()); $$->calc.expressions.swap(*$2); delete $2; } @@ -452,7 +483,7 @@ calc_stmt: expression_list: expression { - $$ = new std::vector; + $$ = new std::vector>; $$->emplace_back($1); } | expression COMMA expression_list @@ -460,9 +491,9 @@ expression_list: if ($3 != nullptr) { $$ = $3; } else { - $$ = new std::vector; + $$ = new std::vector>; } - $$->emplace_back($1); + $$->emplace($$->begin(), $1); } ; expression: @@ -490,25 +521,16 @@ expression: $$->set_name(token_name(sql_string, &@$)); delete $1; } - ; - -select_attr: - '*' { - $$ = new std::vector; - RelAttrSqlNode attr; - attr.relation_name = ""; - attr.attribute_name = "*"; - $$->emplace_back(attr); - } - | rel_attr attr_list { - if ($2 != nullptr) { - $$ = $2; - } else { - $$ = new std::vector; - } - $$->emplace_back(*$1); + | rel_attr { + RelAttrSqlNode *node = $1; + $$ = new UnboundFieldExpr(node->relation_name, node->attribute_name); + $$->set_name(token_name(sql_string, &@$)); delete $1; } + | '*' { + $$ = new StarExpr(); + } + // your code here ; rel_attr: @@ -526,39 +548,29 @@ rel_attr: } ; -attr_list: - /* empty */ - { - $$ = nullptr; - } - | COMMA rel_attr attr_list { - if ($3 != nullptr) { - $$ = $3; - } else { - $$ = new std::vector; - } - - $$->emplace_back(*$2); - delete $2; +relation: + ID { + $$ = $1; } ; - rel_list: - /* empty */ - { - $$ = nullptr; + relation { + $$ = new std::vector(); + $$->push_back($1); + free($1); } - | COMMA ID rel_list { + | relation COMMA rel_list { if ($3 != nullptr) { $$ = $3; } else { $$ = new std::vector; } - $$->push_back($2); - free($2); + $$->insert($$->begin(), $1); + free($1); } ; + where: /* empty */ { @@ -644,6 +656,13 @@ comp_op: | NE { $$ = NOT_EQUAL; } ; +// your code here +group_by: + /* empty */ + { + $$ = nullptr; + } + ; load_data_stmt: LOAD DATA INFILE SSS INTO TABLE ID { diff --git a/src/observer/sql/stmt/calc_stmt.h b/src/observer/sql/stmt/calc_stmt.h index 079880fed..d7a1b388c 100644 --- a/src/observer/sql/stmt/calc_stmt.h +++ b/src/observer/sql/stmt/calc_stmt.h @@ -38,12 +38,9 @@ class CalcStmt : public Stmt public: static RC create(CalcSqlNode &calc_sql, Stmt *&stmt) { - CalcStmt *calc_stmt = new CalcStmt(); - for (Expression *const expr : calc_sql.expressions) { - calc_stmt->expressions_.emplace_back(expr); - } - calc_sql.expressions.clear(); - stmt = calc_stmt; + CalcStmt *calc_stmt = new CalcStmt(); + calc_stmt->expressions_ = std::move(calc_sql.expressions); + stmt = calc_stmt; return RC::SUCCESS; } diff --git a/src/observer/sql/stmt/create_table_stmt.cpp b/src/observer/sql/stmt/create_table_stmt.cpp index 576e3cdfe..30603af66 100644 --- a/src/observer/sql/stmt/create_table_stmt.cpp +++ b/src/observer/sql/stmt/create_table_stmt.cpp @@ -12,12 +12,35 @@ See the Mulan PSL v2 for more details. */ // Created by Wangyunlai on 2023/6/13. // +#include "common/log/log.h" +#include "common/types.h" #include "sql/stmt/create_table_stmt.h" #include "event/sql_debug.h" RC CreateTableStmt::create(Db *db, const CreateTableSqlNode &create_table, Stmt *&stmt) { - stmt = new CreateTableStmt(create_table.relation_name, create_table.attr_infos); + StorageFormat storage_format = StorageFormat::UNKNOWN_FORMAT; + if (create_table.storage_format.length() == 0) { + storage_format = StorageFormat::ROW_FORMAT; + } else { + storage_format = get_storage_format(create_table.storage_format.c_str()); + } + if (storage_format == StorageFormat::UNKNOWN_FORMAT) { + return RC::INVALID_ARGUMENT; + } + stmt = new CreateTableStmt(create_table.relation_name, create_table.attr_infos, storage_format); sql_debug("create table statement: table name %s", create_table.relation_name.c_str()); return RC::SUCCESS; } + +StorageFormat CreateTableStmt::get_storage_format(const char *format_str) { + StorageFormat format = StorageFormat::UNKNOWN_FORMAT; + if (0 == strcasecmp(format_str, "ROW")) { + format = StorageFormat::ROW_FORMAT; + } else if (0 == strcasecmp(format_str, "PAX")) { + format = StorageFormat::PAX_FORMAT; + } else { + format = StorageFormat::UNKNOWN_FORMAT; + } + return format; +} diff --git a/src/observer/sql/stmt/create_table_stmt.h b/src/observer/sql/stmt/create_table_stmt.h index 710c54733..a7c0707e6 100644 --- a/src/observer/sql/stmt/create_table_stmt.h +++ b/src/observer/sql/stmt/create_table_stmt.h @@ -29,8 +29,9 @@ class Db; class CreateTableStmt : public Stmt { public: - CreateTableStmt(const std::string &table_name, const std::vector &attr_infos) - : table_name_(table_name), attr_infos_(attr_infos) + CreateTableStmt( + const std::string &table_name, const std::vector &attr_infos, StorageFormat storage_format) + : table_name_(table_name), attr_infos_(attr_infos), storage_format_(storage_format) {} virtual ~CreateTableStmt() = default; @@ -38,10 +39,13 @@ class CreateTableStmt : public Stmt const std::string &table_name() const { return table_name_; } const std::vector &attr_infos() const { return attr_infos_; } + const StorageFormat storage_format() const { return storage_format_; } - static RC create(Db *db, const CreateTableSqlNode &create_table, Stmt *&stmt); + static RC create(Db *db, const CreateTableSqlNode &create_table, Stmt *&stmt); + static StorageFormat get_storage_format(const char *format_str); private: std::string table_name_; std::vector attr_infos_; + StorageFormat storage_format_; }; \ No newline at end of file diff --git a/src/observer/sql/stmt/select_stmt.cpp b/src/observer/sql/stmt/select_stmt.cpp index 97263ec4e..a51d9e794 100644 --- a/src/observer/sql/stmt/select_stmt.cpp +++ b/src/observer/sql/stmt/select_stmt.cpp @@ -18,6 +18,10 @@ See the Mulan PSL v2 for more details. */ #include "sql/stmt/filter_stmt.h" #include "storage/db/db.h" #include "storage/table/table.h" +#include "sql/parser/expression_binder.h" + +using namespace std; +using namespace common; SelectStmt::~SelectStmt() { @@ -27,25 +31,18 @@ SelectStmt::~SelectStmt() } } -static void wildcard_fields(Table *table, std::vector &field_metas) -{ - const TableMeta &table_meta = table->table_meta(); - const int field_num = table_meta.field_num(); - for (int i = table_meta.sys_field_num(); i < field_num; i++) { - field_metas.push_back(Field(table, table_meta.field(i))); - } -} - -RC SelectStmt::create(Db *db, const SelectSqlNode &select_sql, Stmt *&stmt) +RC SelectStmt::create(Db *db, SelectSqlNode &select_sql, Stmt *&stmt) { if (nullptr == db) { LOG_WARN("invalid argument. db is null"); return RC::INVALID_ARGUMENT; } + BinderContext binder_context; + // collect tables in `from` statement - std::vector
tables; - std::unordered_map table_map; + vector
tables; + unordered_map table_map; for (size_t i = 0; i < select_sql.relations.size(); i++) { const char *table_name = select_sql.relations[i].c_str(); if (nullptr == table_name) { @@ -59,71 +56,31 @@ RC SelectStmt::create(Db *db, const SelectSqlNode &select_sql, Stmt *&stmt) return RC::SCHEMA_TABLE_NOT_EXIST; } + binder_context.add_table(table); tables.push_back(table); - table_map.insert(std::pair(table_name, table)); + table_map.insert({table_name, table}); } // collect query fields in `select` statement - std::vector query_fields; - for (int i = static_cast(select_sql.attributes.size()) - 1; i >= 0; i--) { - const RelAttrSqlNode &relation_attr = select_sql.attributes[i]; - - if (common::is_blank(relation_attr.relation_name.c_str()) && - 0 == strcmp(relation_attr.attribute_name.c_str(), "*")) { - for (Table *table : tables) { - wildcard_fields(table, query_fields); - } - - } else if (!common::is_blank(relation_attr.relation_name.c_str())) { - const char *table_name = relation_attr.relation_name.c_str(); - const char *field_name = relation_attr.attribute_name.c_str(); - - if (0 == strcmp(table_name, "*")) { - if (0 != strcmp(field_name, "*")) { - LOG_WARN("invalid field name while table is *. attr=%s", field_name); - return RC::SCHEMA_FIELD_MISSING; - } - for (Table *table : tables) { - wildcard_fields(table, query_fields); - } - } else { - auto iter = table_map.find(table_name); - if (iter == table_map.end()) { - LOG_WARN("no such table in from list: %s", table_name); - return RC::SCHEMA_FIELD_MISSING; - } - - Table *table = iter->second; - if (0 == strcmp(field_name, "*")) { - wildcard_fields(table, query_fields); - } else { - const FieldMeta *field_meta = table->table_meta().field(field_name); - if (nullptr == field_meta) { - LOG_WARN("no such field. field=%s.%s.%s", db->name(), table->name(), field_name); - return RC::SCHEMA_FIELD_MISSING; - } - - query_fields.push_back(Field(table, field_meta)); - } - } - } else { - if (tables.size() != 1) { - LOG_WARN("invalid. I do not know the attr's table. attr=%s", relation_attr.attribute_name.c_str()); - return RC::SCHEMA_FIELD_MISSING; - } - - Table *table = tables[0]; - const FieldMeta *field_meta = table->table_meta().field(relation_attr.attribute_name.c_str()); - if (nullptr == field_meta) { - LOG_WARN("no such field. field=%s.%s.%s", db->name(), table->name(), relation_attr.attribute_name.c_str()); - return RC::SCHEMA_FIELD_MISSING; - } - - query_fields.push_back(Field(table, field_meta)); + vector> bound_expressions; + ExpressionBinder expression_binder(binder_context); + + for (unique_ptr &expression : select_sql.expressions) { + RC rc = expression_binder.bind_expression(expression, bound_expressions); + if (OB_FAIL(rc)) { + LOG_INFO("bind expression failed. rc=%s", strrc(rc)); + return rc; } } - LOG_INFO("got %d tables in from stmt and %d fields in query stmt", tables.size(), query_fields.size()); + vector> group_by_expressions; + for (unique_ptr &expression : select_sql.group_by) { + RC rc = expression_binder.bind_expression(expression, group_by_expressions); + if (OB_FAIL(rc)) { + LOG_INFO("bind expression failed. rc=%s", strrc(rc)); + return rc; + } + } Table *default_table = nullptr; if (tables.size() == 1) { @@ -145,10 +102,11 @@ RC SelectStmt::create(Db *db, const SelectSqlNode &select_sql, Stmt *&stmt) // everything alright SelectStmt *select_stmt = new SelectStmt(); - // TODO add expression copy + select_stmt->tables_.swap(tables); - select_stmt->query_fields_.swap(query_fields); + select_stmt->query_expressions_.swap(bound_expressions); select_stmt->filter_stmt_ = filter_stmt; + select_stmt->group_by_.swap(group_by_expressions); stmt = select_stmt; return RC::SUCCESS; } diff --git a/src/observer/sql/stmt/select_stmt.h b/src/observer/sql/stmt/select_stmt.h index 5e30ac6ba..cfb048807 100644 --- a/src/observer/sql/stmt/select_stmt.h +++ b/src/observer/sql/stmt/select_stmt.h @@ -39,15 +39,18 @@ class SelectStmt : public Stmt StmtType type() const override { return StmtType::SELECT; } public: - static RC create(Db *db, const SelectSqlNode &select_sql, Stmt *&stmt); + static RC create(Db *db, SelectSqlNode &select_sql, Stmt *&stmt); public: const std::vector
&tables() const { return tables_; } - const std::vector &query_fields() const { return query_fields_; } FilterStmt *filter_stmt() const { return filter_stmt_; } + std::vector> &query_expressions() { return query_expressions_; } + std::vector> &group_by() { return group_by_; } + private: - std::vector query_fields_; - std::vector
tables_; - FilterStmt *filter_stmt_ = nullptr; + std::vector> query_expressions_; + std::vector
tables_; + FilterStmt *filter_stmt_ = nullptr; + std::vector> group_by_; }; diff --git a/src/observer/storage/buffer/buffer_pool_log.cpp b/src/observer/storage/buffer/buffer_pool_log.cpp index 30e6b3898..63b334b83 100644 --- a/src/observer/storage/buffer/buffer_pool_log.cpp +++ b/src/observer/storage/buffer/buffer_pool_log.cpp @@ -17,8 +17,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/clog/log_handler.h" #include "storage/clog/log_entry.h" -using namespace std; - string BufferPoolLogEntry::to_string() const { return string("buffer_pool_id=") + std::to_string(buffer_pool_id) + diff --git a/src/observer/storage/buffer/buffer_pool_log.h b/src/observer/storage/buffer/buffer_pool_log.h index ce77547b6..9be20b22e 100644 --- a/src/observer/storage/buffer/buffer_pool_log.h +++ b/src/observer/storage/buffer/buffer_pool_log.h @@ -14,7 +14,7 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include +#include "common/lang/string.h" #include "common/types.h" #include "common/rc.h" #include "storage/clog/log_replayer.h" @@ -45,9 +45,9 @@ class BufferPoolOperation Type type() const { return type_; } int32_t type_id() const { return static_cast(type_); } - std::string to_string() const + string to_string() const { - std::string ret = std::to_string(type_id()) + ":"; + string ret = std::to_string(type_id()) + ":"; switch (type_) { case Type::ALLOCATE: return ret + "ALLOCATE"; case Type::DEALLOCATE: return ret + "DEALLOCATE"; @@ -69,7 +69,7 @@ struct BufferPoolLogEntry int32_t operation_type; /// operation type PageNum page_num; /// page number - std::string to_string() const; + string to_string() const; }; /** @@ -126,4 +126,4 @@ class BufferPoolLogReplayer final : public LogReplayer private: BufferPoolManager &bp_manager_; -}; \ No newline at end of file +}; diff --git a/src/observer/storage/buffer/disk_buffer_pool.cpp b/src/observer/storage/buffer/disk_buffer_pool.cpp index f3cf1f831..92215cc7f 100644 --- a/src/observer/storage/buffer/disk_buffer_pool.cpp +++ b/src/observer/storage/buffer/disk_buffer_pool.cpp @@ -16,6 +16,7 @@ See the Mulan PSL v2 for more details. */ #include "common/io/io.h" #include "common/lang/mutex.h" +#include "common/lang/algorithm.h" #include "common/log/log.h" #include "common/math/crc.h" #include "storage/buffer/disk_buffer_pool.h" @@ -23,7 +24,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/db/db.h" using namespace common; -using namespace std; static const int MEM_POOL_ITEM_NUM = 20; diff --git a/src/observer/storage/buffer/disk_buffer_pool.h b/src/observer/storage/buffer/disk_buffer_pool.h index a8ee1220b..9ff02922f 100644 --- a/src/observer/storage/buffer/disk_buffer_pool.h +++ b/src/observer/storage/buffer/disk_buffer_pool.h @@ -23,13 +23,13 @@ See the Mulan PSL v2 for more details. */ #include #include #include -#include #include -#include #include "common/lang/bitmap.h" #include "common/lang/lru_cache.h" #include "common/lang/mutex.h" +#include "common/lang/memory.h" +#include "common/lang/unordered_map.h" #include "common/mm/mem_pool.h" #include "common/rc.h" #include "common/types.h" @@ -73,7 +73,7 @@ struct BPFileHeader */ static const int MAX_PAGE_NUM = (BP_PAGE_DATA_SIZE - sizeof(page_count) - sizeof(allocated_pages)) * 8; - std::string to_string() const; + string to_string() const; }; /** @@ -105,9 +105,9 @@ class BPFrameManager * @brief 列出所有指定文件的页面 * * @param buffer_pool_id buffer Pool标识 - * @return std::list 页帧列表 + * @return list 页帧列表 */ - std::list find_list(int buffer_pool_id); + list find_list(int buffer_pool_id); /** * @brief 分配一个新的页面 @@ -131,7 +131,7 @@ class BPFrameManager * @param purger 需要在释放frame之前,对页面做些什么操作。当前是刷新脏数据到磁盘 * @return 返回本次清理了多少个页面 */ - int purge_frames(int count, std::function purger); + int purge_frames(int count, function purger); size_t frame_num() const { return frames_.count(); } @@ -154,7 +154,7 @@ class BPFrameManager using FrameLruCache = common::LruCache; using FrameAllocator = common::MemPoolSimple; - std::mutex lock_; + mutex lock_; FrameLruCache frames_; FrameAllocator allocator_; }; @@ -300,12 +300,12 @@ class DiskBufferPool final int file_desc_ = -1; /// 文件描述符 /// 由于在最开始打开文件时,没有正确的buffer pool id不能加载header frame,所以单独从文件中读取此标识 - int32_t buffer_pool_id_ = -1; - Frame *hdr_frame_ = nullptr; /// 文件头页面 - BPFileHeader *file_header_ = nullptr; /// 文件头 - std::set disposed_pages_; /// 已经释放的页面 + int32_t buffer_pool_id_ = -1; + Frame *hdr_frame_ = nullptr; /// 文件头页面 + BPFileHeader *file_header_ = nullptr; /// 文件头 + set disposed_pages_; /// 已经释放的页面 - std::string file_name_; /// 文件名 + string file_name_; /// 文件名 common::Mutex lock_; common::Mutex wr_lock_; @@ -324,7 +324,7 @@ class BufferPoolManager final BufferPoolManager(int memory_size = 0); ~BufferPoolManager(); - RC init(std::unique_ptr dblwr_buffer); + RC init(unique_ptr dblwr_buffer); RC create_file(const char *file_name); RC open_file(LogHandler &log_handler, const char *file_name, DiskBufferPool *&bp); @@ -346,10 +346,10 @@ class BufferPoolManager final private: BPFrameManager frame_manager_{"BufPool"}; - std::unique_ptr dblwr_buffer_; + unique_ptr dblwr_buffer_; - common::Mutex lock_; - std::unordered_map buffer_pools_; - std::unordered_map id_to_buffer_pools_; - std::atomic next_buffer_pool_id_{1}; // 系统启动时,会打开所有的表,这样就可以知道当前系统最大的ID是多少了 + common::Mutex lock_; + unordered_map buffer_pools_; + unordered_map id_to_buffer_pools_; + atomic next_buffer_pool_id_{1}; // 系统启动时,会打开所有的表,这样就可以知道当前系统最大的ID是多少了 }; diff --git a/src/observer/storage/buffer/double_write_buffer.cpp b/src/observer/storage/buffer/double_write_buffer.cpp index e9abcbd54..46cacb56c 100644 --- a/src/observer/storage/buffer/double_write_buffer.cpp +++ b/src/observer/storage/buffer/double_write_buffer.cpp @@ -22,7 +22,6 @@ See the Mulan PSL v2 for more details. */ #include "common/log/log.h" #include "common/math/crc.h" -using namespace std; using namespace common; struct DoubleWritePage diff --git a/src/observer/storage/buffer/double_write_buffer.h b/src/observer/storage/buffer/double_write_buffer.h index 0e9715618..5bc64f56a 100644 --- a/src/observer/storage/buffer/double_write_buffer.h +++ b/src/observer/storage/buffer/double_write_buffer.h @@ -14,9 +14,8 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include - #include "common/lang/mutex.h" +#include "common/lang/unordered_map.h" #include "common/types.h" #include "common/rc.h" #include "storage/buffer/page.h" @@ -148,7 +147,7 @@ class DiskDoubleWriteBuffer : public DoubleWriteBuffer BufferPoolManager &bp_manager_; DoubleWriteBufferHeader header_; - std::unordered_map dblwr_pages_; + unordered_map dblwr_pages_; }; class VacuousDoubleWriteBuffer : public DoubleWriteBuffer diff --git a/src/observer/storage/buffer/frame.cpp b/src/observer/storage/buffer/frame.cpp index 3a0c172c5..d1152abc5 100644 --- a/src/observer/storage/buffer/frame.cpp +++ b/src/observer/storage/buffer/frame.cpp @@ -16,8 +16,6 @@ See the Mulan PSL v2 for more details. */ #include "session/session.h" #include "session/thread_data.h" -using namespace std; - FrameId::FrameId(int buffer_pool_id, PageNum page_num) : buffer_pool_id_(buffer_pool_id), page_num_(page_num) {} bool FrameId::equal_to(const FrameId &other) const @@ -120,7 +118,7 @@ void Frame::read_latch() { read_latch(get_default_debug_xid()); } void Frame::read_latch(intptr_t xid) { { - std::scoped_lock debug_lock(debug_lock_); + scoped_lock debug_lock(debug_lock_); ASSERT(pin_count_ > 0, "frame lock. read lock failed while pin count is invalid." "this=%p, pin=%d, frameId=%s, xid=%lx, lbt=%s", @@ -149,7 +147,7 @@ bool Frame::try_read_latch() { intptr_t xid = get_default_debug_xid(); { - std::scoped_lock debug_lock(debug_lock_); + scoped_lock debug_lock(debug_lock_); ASSERT(pin_count_ > 0, "frame try lock. read lock failed while pin count is invalid." "this=%p, pin=%d, frameId=%s, xid=%lx, lbt=%s", @@ -181,7 +179,7 @@ void Frame::read_unlatch() { read_unlatch(get_default_debug_xid()); } void Frame::read_unlatch(intptr_t xid) { { - std::scoped_lock debug_lock(debug_lock_); + scoped_lock debug_lock(debug_lock_); ASSERT(pin_count_.load() > 0, "frame lock. read unlock failed while pin count is invalid." "this=%p, pin=%d, frameId=%s, xid=%lx, lbt=%s", @@ -210,7 +208,7 @@ void Frame::read_unlatch(intptr_t xid) void Frame::pin() { - std::scoped_lock debug_lock(debug_lock_); + scoped_lock debug_lock(debug_lock_); [[maybe_unused]] intptr_t xid = get_default_debug_xid(); [[maybe_unused]] int pin_count = ++pin_count_; @@ -230,7 +228,7 @@ int Frame::unpin() "this=%p, pin=%d, frameId=%s, xid=%lx, lbt=%s", this, pin_count_.load(), frame_id_.to_string().c_str(), xid, lbt()); - std::scoped_lock debug_lock(debug_lock_); + scoped_lock debug_lock(debug_lock_); int pin_count = --pin_count_; TRACE("after frame unpin. " diff --git a/src/observer/storage/buffer/frame.h b/src/observer/storage/buffer/frame.h index 21aef8bc1..2c2e7d182 100644 --- a/src/observer/storage/buffer/frame.h +++ b/src/observer/storage/buffer/frame.h @@ -14,14 +14,13 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include #include -#include #include -#include #include "common/lang/mutex.h" +#include "common/lang/string.h" +#include "common/lang/atomic.h" +#include "common/lang/unordered_map.h" #include "common/log/log.h" #include "common/types.h" #include "storage/buffer/page.h" @@ -44,7 +43,7 @@ class FrameId void set_buffer_pool_id(int buffer_pool_id) { buffer_pool_id_ = buffer_pool_id; } void set_page_num(PageNum page_num) { page_num_ = page_num; } - std::string to_string() const; + string to_string() const; private: int buffer_pool_id_ = -1; @@ -167,24 +166,24 @@ class Frame void read_unlatch(); void read_unlatch(intptr_t xid); - std::string to_string() const; + string to_string() const; private: friend class BufferPool; - bool dirty_ = false; - std::atomic pin_count_{0}; - unsigned long acc_time_ = 0; - FrameId frame_id_; - Page page_; + bool dirty_ = false; + atomic pin_count_{0}; + unsigned long acc_time_ = 0; + FrameId frame_id_; + Page page_; /// 在非并发编译时,加锁解锁动作将什么都不做 common::RecursiveSharedMutex lock_; /// 使用一些手段来做测试,提前检测出头疼的死锁问题 /// 如果编译时没有增加调试选项,这些代码什么都不做 - common::DebugMutex debug_lock_; - intptr_t write_locker_ = 0; - int write_recursive_count_ = 0; - std::unordered_map read_lockers_; + common::DebugMutex debug_lock_; + intptr_t write_locker_ = 0; + int write_recursive_count_ = 0; + unordered_map read_lockers_; }; diff --git a/src/observer/storage/clog/disk_log_handler.cpp b/src/observer/storage/clog/disk_log_handler.cpp index afb383e2c..fdce4672f 100644 --- a/src/observer/storage/clog/disk_log_handler.cpp +++ b/src/observer/storage/clog/disk_log_handler.cpp @@ -16,8 +16,8 @@ See the Mulan PSL v2 for more details. */ #include "storage/clog/disk_log_handler.h" #include "storage/clog/log_file.h" #include "storage/clog/log_replayer.h" +#include "common/lang/chrono.h" -using namespace std; using namespace common; //////////////////////////////////////////////////////////////////////////////// @@ -99,7 +99,7 @@ RC DiskLogHandler::replay(LogReplayer &replayer, LSN start_lsn) return rc; } -RC DiskLogHandler::iterate(std::function consumer, LSN start_lsn) +RC DiskLogHandler::iterate(function consumer, LSN start_lsn) { vector log_files; RC rc = file_manager_.list_files(log_files, start_lsn); @@ -205,4 +205,4 @@ void DiskLogHandler::thread_func() } LOG_INFO("log handler thread stopped"); -} \ No newline at end of file +} diff --git a/src/observer/storage/clog/disk_log_handler.h b/src/observer/storage/clog/disk_log_handler.h index 680f0bb6b..3cb7a996b 100644 --- a/src/observer/storage/clog/disk_log_handler.h +++ b/src/observer/storage/clog/disk_log_handler.h @@ -14,13 +14,12 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include -#include - #include "common/types.h" #include "common/rc.h" +#include "common/lang/vector.h" +#include "common/lang/deque.h" +#include "common/lang/memory.h" +#include "common/lang/thread.h" #include "storage/clog/log_module.h" #include "storage/clog/log_file.h" #include "storage/clog/log_buffer.h" @@ -89,7 +88,7 @@ class DiskLogHandler : public LogHandler * @param consumer 消费者 * @param start_lsn 从哪个位置开始迭代 */ - RC iterate(std::function consumer, LSN start_lsn) override; + RC iterate(function consumer, LSN start_lsn) override; /** * @brief 等待指定的日志刷盘 @@ -111,7 +110,7 @@ class DiskLogHandler : public LogHandler * @param[in] module 日志模块 * @param[in] data 日志数据。具体的数据由各个模块自己定义 */ - RC _append(LSN &lsn, LogModule module, std::vector &&data) override; + RC _append(LSN &lsn, LogModule module, vector &&data) override; private: /** @@ -120,11 +119,11 @@ class DiskLogHandler : public LogHandler void thread_func(); private: - std::unique_ptr thread_; /// 刷新日志的线程 - std::atomic_bool running_{false}; /// 是否还要继续运行 + unique_ptr thread_; /// 刷新日志的线程 + atomic_bool running_{false}; /// 是否还要继续运行 LogFileManager file_manager_; /// 管理所有的日志文件 LogEntryBuffer entry_buffer_; /// 缓存日志 - std::string path_; /// 日志文件存放的目录 + string path_; /// 日志文件存放的目录 }; diff --git a/src/observer/storage/clog/integrated_log_replayer.cpp b/src/observer/storage/clog/integrated_log_replayer.cpp index 3d6082d66..008fa14ff 100644 --- a/src/observer/storage/clog/integrated_log_replayer.cpp +++ b/src/observer/storage/clog/integrated_log_replayer.cpp @@ -15,8 +15,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/clog/integrated_log_replayer.h" #include "storage/clog/log_entry.h" -using namespace std; - IntegratedLogReplayer::IntegratedLogReplayer(BufferPoolManager &bpm) : buffer_pool_log_replayer_(bpm), record_log_replayer_(bpm), diff --git a/src/observer/storage/clog/integrated_log_replayer.h b/src/observer/storage/clog/integrated_log_replayer.h index ec0796601..a281839c1 100644 --- a/src/observer/storage/clog/integrated_log_replayer.h +++ b/src/observer/storage/clog/integrated_log_replayer.h @@ -43,7 +43,7 @@ class IntegratedLogReplayer : public LogReplayer * @details * 区别于另一个构造函数,这个构造函数可以指定不同的事务日志回放器。比如进程启动时可以指定选择使用VacuousTrx还是MvccTrx。 */ - IntegratedLogReplayer(BufferPoolManager &bpm, std::unique_ptr trx_log_replayer); + IntegratedLogReplayer(BufferPoolManager &bpm, unique_ptr trx_log_replayer); virtual ~IntegratedLogReplayer() = default; //! @copydoc LogReplayer::replay @@ -53,8 +53,8 @@ class IntegratedLogReplayer : public LogReplayer RC on_done() override; private: - BufferPoolLogReplayer buffer_pool_log_replayer_; ///< 缓冲池日志回放器 - RecordLogReplayer record_log_replayer_; ///< record manager 日志回放器 - BplusTreeLogReplayer bplus_tree_log_replayer_; ///< bplus tree 日志回放器 - std::unique_ptr trx_log_replayer_; ///< trx 日志回放器 + BufferPoolLogReplayer buffer_pool_log_replayer_; ///< 缓冲池日志回放器 + RecordLogReplayer record_log_replayer_; ///< record manager 日志回放器 + BplusTreeLogReplayer bplus_tree_log_replayer_; ///< bplus tree 日志回放器 + unique_ptr trx_log_replayer_; ///< trx 日志回放器 }; \ No newline at end of file diff --git a/src/observer/storage/clog/log_buffer.cpp b/src/observer/storage/clog/log_buffer.cpp index 7c56e9b7d..4cbb10604 100644 --- a/src/observer/storage/clog/log_buffer.cpp +++ b/src/observer/storage/clog/log_buffer.cpp @@ -12,11 +12,10 @@ See the Mulan PSL v2 for more details. */ // Created by wangyunlai on 2024/01/31 // -#include #include "storage/clog/log_buffer.h" #include "storage/clog/log_file.h" +#include "common/lang/chrono.h" -using namespace std; using namespace common; RC LogEntryBuffer::init(LSN lsn, int32_t max_bytes /*= 0*/) @@ -104,4 +103,4 @@ int64_t LogEntryBuffer::bytes() const int32_t LogEntryBuffer::entry_number() const { return entries_.size(); -} \ No newline at end of file +} diff --git a/src/observer/storage/clog/log_buffer.h b/src/observer/storage/clog/log_buffer.h index 9d977b035..68046f72e 100644 --- a/src/observer/storage/clog/log_buffer.h +++ b/src/observer/storage/clog/log_buffer.h @@ -14,13 +14,12 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include - #include "common/rc.h" #include "common/types.h" #include "common/lang/mutex.h" +#include "common/lang/vector.h" +#include "common/lang/deque.h" +#include "common/lang/atomic.h" #include "storage/clog/log_module.h" #include "storage/clog/log_entry.h" @@ -45,7 +44,7 @@ class LogEntryBuffer * @brief 在缓冲区中追加一条日志 */ RC append(LSN &lsn, LogModule::Id module_id, vector &&data); - RC append(LSN &lsn, LogModule module, std::vector &&data); + RC append(LSN &lsn, LogModule module, vector &&data); /** * @brief 刷新缓冲区中的日志到磁盘 @@ -69,12 +68,12 @@ class LogEntryBuffer LSN flushed_lsn() const { return flushed_lsn_.load(); } private: - std::mutex mutex_; /// 当前数据结构一定会在多线程中访问,所以强制使用有效的锁,而不是有条件生效的common::Mutex - std::deque entries_; /// 日志缓冲区 - std::atomic bytes_; /// 当前缓冲区中的日志数据大小 + mutex mutex_; /// 当前数据结构一定会在多线程中访问,所以强制使用有效的锁,而不是有条件生效的common::Mutex + deque entries_; /// 日志缓冲区 + atomic bytes_; /// 当前缓冲区中的日志数据大小 - std::atomic current_lsn_{0}; - std::atomic flushed_lsn_{0}; + atomic current_lsn_{0}; + atomic flushed_lsn_{0}; int32_t max_bytes_ = 4 * 1024 * 1024; /// 缓冲区最大字节数 }; diff --git a/src/observer/storage/clog/log_entry.cpp b/src/observer/storage/clog/log_entry.cpp index 0289de9e6..3d34dad15 100644 --- a/src/observer/storage/clog/log_entry.cpp +++ b/src/observer/storage/clog/log_entry.cpp @@ -16,8 +16,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/clog/log_entry.h" #include "common/log/log.h" -using namespace std; - //////////////////////////////////////////////////////////////////////////////// // struct LogHeader diff --git a/src/observer/storage/clog/log_entry.h b/src/observer/storage/clog/log_entry.h index 3943ee919..a6c9d2a52 100644 --- a/src/observer/storage/clog/log_entry.h +++ b/src/observer/storage/clog/log_entry.h @@ -14,13 +14,12 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include - #include "common/rc.h" #include "common/types.h" #include "storage/clog/log_module.h" +#include "common/lang/vector.h" +#include "common/lang/string.h" +#include "common/lang/memory.h" /** * @brief 描述一条日志头 @@ -34,7 +33,7 @@ struct LogHeader final static const int32_t SIZE; /// 日志头大小 - std::string to_string() const; + string to_string() const; }; /** @@ -68,8 +67,8 @@ class LogEntry static int32_t max_payload_size() { return max_size() - LogHeader::SIZE; } public: - RC init(LSN lsn, LogModule::Id module_id, std::vector &&data); - RC init(LSN lsn, LogModule module, std::vector &&data); + RC init(LSN lsn, LogModule::Id module_id, vector &&data); + RC init(LSN lsn, LogModule module, vector &&data); const LogHeader &header() const { return header_; } const char *data() const { return data_.data(); } @@ -82,9 +81,9 @@ class LogEntry LogModule module() const { return LogModule(header_.module_id); } public: - std::string to_string() const; + string to_string() const; private: - LogHeader header_; /// 日志头 - std::vector data_; /// 日志数据 + LogHeader header_; /// 日志头 + vector data_; /// 日志数据 }; diff --git a/src/observer/storage/clog/log_file.cpp b/src/observer/storage/clog/log_file.cpp index f0addb0e8..c32653450 100644 --- a/src/observer/storage/clog/log_file.cpp +++ b/src/observer/storage/clog/log_file.cpp @@ -13,13 +13,14 @@ See the Mulan PSL v2 for more details. */ // #include -#include + +#include "common/lang/string_view.h" +#include "common/lang/charconv.h" #include "common/log/log.h" #include "storage/clog/log_file.h" #include "storage/clog/log_entry.h" #include "common/io/io.h" -using namespace std; using namespace common; RC LogFileReader::open(const char *filename) @@ -47,7 +48,7 @@ RC LogFileReader::close() return RC::SUCCESS; } -RC LogFileReader::iterate(std::function callback, LSN start_lsn /*=0*/) +RC LogFileReader::iterate(function callback, LSN start_lsn /*=0*/) { RC rc = skip_to(start_lsn); if (OB_FAIL(rc)) { @@ -278,7 +279,7 @@ RC LogFileManager::init(const char *directory, int max_entry_number_per_file) return RC::SUCCESS; } -RC LogFileManager::get_lsn_from_filename(const std::string &filename, LSN &lsn) +RC LogFileManager::get_lsn_from_filename(const string &filename, LSN &lsn) { if (!filename.starts_with(file_prefix_) || !filename.ends_with(file_suffix_)) { return RC::INVALID_ARGUMENT; @@ -295,7 +296,7 @@ RC LogFileManager::get_lsn_from_filename(const std::string &filename, LSN &lsn) return RC::SUCCESS; } -RC LogFileManager::list_files(std::vector &files, LSN start_lsn) +RC LogFileManager::list_files(vector &files, LSN start_lsn) { files.clear(); @@ -332,7 +333,7 @@ RC LogFileManager::next_file(LogFileWriter &file_writer) lsn = log_files_.rbegin()->first + max_entry_number_per_file_; } - string filename = file_prefix_ + to_string(lsn) + file_suffix_; + string filename = file_prefix_ + std::to_string(lsn) + file_suffix_; filesystem::path file_path = directory_ / filename; log_files_.emplace(lsn, file_path); diff --git a/src/observer/storage/clog/log_file.h b/src/observer/storage/clog/log_file.h index 2ed781b3d..6d61a212b 100644 --- a/src/observer/storage/clog/log_file.h +++ b/src/observer/storage/clog/log_file.h @@ -14,14 +14,13 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include -#include -#include - #include "common/rc.h" #include "common/types.h" +#include "common/lang/map.h" +#include "common/lang/functional.h" +#include "common/lang/filesystem.h" +#include "common/lang/fstream.h" +#include "common/lang/string.h" class LogEntry; @@ -39,7 +38,7 @@ class LogFileReader RC open(const char *filename); RC close(); - RC iterate(std::function callback, LSN start_lsn = 0); + RC iterate(function callback, LSN start_lsn = 0); private: /** @@ -50,8 +49,8 @@ class LogFileReader RC skip_to(LSN start_lsn); private: - int fd_ = -1; - std::string filename_; + int fd_ = -1; + string filename_; }; /** @@ -87,15 +86,15 @@ class LogFileWriter */ bool full() const; - std::string to_string() const; + string to_string() const; const char *filename() const { return filename_.c_str(); } private: - std::string filename_; /// 日志文件名 - int fd_ = -1; /// 日志文件描述符 - int last_lsn_ = 0; /// 写入的最后一条日志LSN - int end_lsn_ = 0; /// 当前日志文件中允许写入的最大的LSN,包括这条日志 + string filename_; /// 日志文件名 + int fd_ = -1; /// 日志文件描述符 + int last_lsn_ = 0; /// 写入的最后一条日志LSN + int end_lsn_ = 0; /// 当前日志文件中允许写入的最大的LSN,包括这条日志 }; /** @@ -124,7 +123,7 @@ class LogFileManager * @param files 满足条件的所有日志文件名 * @param start_lsn 想要查找的日志的最小LSN */ - RC list_files(std::vector &files, LSN start_lsn); + RC list_files(vector &files, LSN start_lsn); /** * @brief 获取最新的一个日志文件名 @@ -143,14 +142,14 @@ class LogFileManager * @brief 从文件名称中获取LSN * @details 如果日志文件名不符合要求,就返回失败 */ - static RC get_lsn_from_filename(const std::string &filename, LSN &lsn); + static RC get_lsn_from_filename(const string &filename, LSN &lsn); private: static constexpr const char *file_prefix_ = "clog_"; static constexpr const char *file_suffix_ = ".log"; - std::filesystem::path directory_; /// 日志文件存放的目录 - int max_entry_number_per_file_; /// 一个文件最大允许存放多少条日志 + filesystem::path directory_; /// 日志文件存放的目录 + int max_entry_number_per_file_; /// 一个文件最大允许存放多少条日志 - std::map log_files_; /// 日志文件名和第一个LSN的映射 + map log_files_; /// 日志文件名和第一个LSN的映射 }; diff --git a/src/observer/storage/clog/log_handler.cpp b/src/observer/storage/clog/log_handler.cpp index 0818a0637..14c49a576 100644 --- a/src/observer/storage/clog/log_handler.cpp +++ b/src/observer/storage/clog/log_handler.cpp @@ -19,8 +19,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/clog/disk_log_handler.h" #include "storage/clog/vacuous_log_handler.h" -using namespace std; - RC LogHandler::append(LSN &lsn, LogModule::Id module, span data) { vector data_vec(data.begin(), data.end()); diff --git a/src/observer/storage/clog/log_handler.h b/src/observer/storage/clog/log_handler.h index 9e3395da0..f91a10999 100644 --- a/src/observer/storage/clog/log_handler.h +++ b/src/observer/storage/clog/log_handler.h @@ -14,12 +14,12 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include - #include "common/rc.h" #include "common/types.h" +#include "common/lang/functional.h" +#include "common/lang/memory.h" +#include "common/lang/span.h" +#include "common/lang/vector.h" #include "storage/clog/log_module.h" /** @@ -76,7 +76,7 @@ class LogHandler * @param consumer 消费者 * @param start_lsn 从哪个LSN开始迭代 */ - virtual RC iterate(std::function consumer, LSN start_lsn) = 0; + virtual RC iterate(function consumer, LSN start_lsn) = 0; /** * @brief 写入一条日志 @@ -85,8 +85,8 @@ class LogHandler * @param data 日志数据 * @note 子类不应该重新实现这个函数 */ - virtual RC append(LSN &lsn, LogModule::Id module, std::span data); - virtual RC append(LSN &lsn, LogModule::Id module, std::vector &&data); + virtual RC append(LSN &lsn, LogModule::Id module, span data); + virtual RC append(LSN &lsn, LogModule::Id module, vector &&data); /** * @brief 等待某个LSN的日志被刷新到磁盘 @@ -103,5 +103,5 @@ class LogHandler * @brief 写入一条日志 * @details 子类应该重现实现这个函数 */ - virtual RC _append(LSN &lsn, LogModule module, std::vector &&data) = 0; + virtual RC _append(LSN &lsn, LogModule module, vector &&data) = 0; }; diff --git a/src/observer/storage/clog/vacuous_log_handler.h b/src/observer/storage/clog/vacuous_log_handler.h index ce1f26fa1..dbc31c213 100644 --- a/src/observer/storage/clog/vacuous_log_handler.h +++ b/src/observer/storage/clog/vacuous_log_handler.h @@ -32,14 +32,14 @@ class VacuousLogHandler : public LogHandler RC stop() override { return RC::SUCCESS; } RC await_termination() override { return RC::SUCCESS; } RC replay(LogReplayer &replayer, LSN start_lsn) override { return RC::SUCCESS; } - RC iterate(std::function consumer, LSN start_lsn) override { return RC::SUCCESS; } + RC iterate(function consumer, LSN start_lsn) override { return RC::SUCCESS; } RC wait_lsn(LSN lsn) override { return RC::SUCCESS; } LSN current_lsn() const override { return 0; } private: - RC _append(LSN &lsn, LogModule module, std::vector &&) override + RC _append(LSN &lsn, LogModule module, vector &&) override { lsn = 0; return RC::SUCCESS; diff --git a/src/observer/storage/common/chunk.cpp b/src/observer/storage/common/chunk.cpp new file mode 100644 index 000000000..9c7d5bd9d --- /dev/null +++ b/src/observer/storage/common/chunk.cpp @@ -0,0 +1,60 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "storage/common/chunk.h" + +void Chunk::add_column(unique_ptr col, int col_id) +{ + columns_.push_back(std::move(col)); + column_ids_.push_back(col_id); +} + +RC Chunk::reference(Chunk &chunk) +{ + reset(); + this->columns_.resize(chunk.column_num()); + for (size_t i = 0; i < columns_.size(); ++i) { + if (nullptr == columns_[i]) { + columns_[i] = make_unique(); + } + columns_[i]->reference(chunk.column(i)); + column_ids_.push_back(chunk.column_ids(i)); + } + return RC::SUCCESS; +} + +int Chunk::rows() const +{ + if (!columns_.empty()) { + return columns_[0]->count(); + } + return 0; +} + +int Chunk::capacity() const +{ + if (!columns_.empty()) { + return columns_[0]->capacity(); + } + return 0; +} + +void Chunk::reset_data() +{ + for (auto &col : columns_) { + col->reset_data(); + } +} + +void Chunk::reset() +{ + columns_.clear(); + column_ids_.clear(); +} \ No newline at end of file diff --git a/src/observer/storage/common/chunk.h b/src/observer/storage/common/chunk.h new file mode 100644 index 000000000..8da0d73f9 --- /dev/null +++ b/src/observer/storage/common/chunk.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include "common/rc.h" +#include "common/log/log.h" +#include "common/lang/memory.h" +#include "common/lang/vector.h" +#include "storage/common/column.h" + +/** + * @brief A Chunk represents a set of columns. + */ +class Chunk +{ +public: + Chunk() = default; + Chunk(const Chunk &) = delete; + Chunk(Chunk &&) = delete; + + int column_num() const { return columns_.size(); } + + Column &column(size_t idx) + { + ASSERT(idx < columns_.size(), "invalid column index"); + return *columns_[idx]; + } + + Column *column_ptr(size_t idx) + { + ASSERT(idx < columns_.size(), "invalid column index"); + return &column(idx); + } + + int column_ids(size_t i) + { + ASSERT(i < column_ids_.size(), "invalid column index"); + return column_ids_[i]; + } + + void add_column(unique_ptr col, int col_id); + + RC reference(Chunk &chunk); + + /** + * @brief 获取 Chunk 中的行数 + */ + int rows() const; + + /** + * @brief 获取 Chunk 的容量 + */ + int capacity() const; + + /** + * @brief 从 Chunk 中获得指定行指定列的 Value + * @param col_idx 列索引 + * @param row_idx 行索引 + * @return Value + * @note 没有检查 col_idx 和 row_idx 是否越界 + * + */ + Value get_value(int col_idx, int row_idx) const { return columns_[col_idx]->get_value(row_idx); } + + /** + * @brief 重置 Chunk 中的数据,不会修改 Chunk 的列属性。 + */ + void reset_data(); + + void reset(); + +private: + vector> columns_; + // TODO: remove it and support multi-tables, + // `columnd_ids` store the ids of child operator that need to be output + vector column_ids_; +}; \ No newline at end of file diff --git a/src/observer/storage/common/column.cpp b/src/observer/storage/common/column.cpp new file mode 100644 index 000000000..2d4f61dd6 --- /dev/null +++ b/src/observer/storage/common/column.cpp @@ -0,0 +1,130 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "common/log/log.h" +#include "storage/common/column.h" + +Column::Column(const FieldMeta &meta, size_t size) + : data_(nullptr), + count_(0), + capacity_(0), + own_(true), + attr_type_(meta.type()), + attr_len_(meta.len()), + column_type_(Type::NORMAL_COLUMN) +{ + // TODO: optimized the memory usage if it doesn't need to allocate memory + data_ = new char[size * attr_len_]; + capacity_ = size; +} + +Column::Column(AttrType attr_type, int attr_len, size_t capacity) +{ + attr_type_ = attr_type; + attr_len_ = attr_len; + data_ = new char[capacity * attr_len_]; + count_ = 0; + capacity_ = capacity; + own_ = true; + column_type_ = Type::NORMAL_COLUMN; +} + +void Column::init(const FieldMeta &meta, size_t size) +{ + reset(); + data_ = new char[size * meta.len()]; + count_ = 0; + capacity_ = size; + attr_type_ = meta.type(); + attr_len_ = meta.len(); + own_ = true; + column_type_ = Type::NORMAL_COLUMN; +} + +void Column::init(AttrType attr_type, int attr_len, size_t capacity) +{ + reset(); + data_ = new char[capacity * attr_len]; + count_ = 0; + capacity_ = capacity; + own_ = true; + attr_type_ = attr_type; + attr_len_ = attr_len; + column_type_ = Type::NORMAL_COLUMN; +} + +void Column::init(const Value &value) +{ + reset(); + attr_type_ = value.attr_type(); + attr_len_ = value.length(); + data_ = new char[attr_len_]; + count_ = 1; + capacity_ = 1; + own_ = true; + memcpy(data_, value.data(), attr_len_); + column_type_ = Type::CONSTANT_COLUMN; +} + +void Column::reset() +{ + if (data_ != nullptr && own_) { + delete[] data_; + } + data_ = nullptr; + count_ = 0; + capacity_ = 0; + own_ = false; + attr_type_ = AttrType::UNDEFINED; + attr_len_ = -1; +} + +RC Column::append_one(char *data) { return append(data, 1); } + +RC Column::append(char *data, int count) +{ + if (!own_) { + LOG_WARN("append data to non-owned column"); + return RC::INTERNAL; + } + if (count_ + count > capacity_) { + LOG_WARN("append data to full column"); + return RC::INTERNAL; + } + + memcpy(data_ + count_ * attr_len_, data, count * attr_len_); + count_ += count; + return RC::SUCCESS; +} + +Value Column::get_value(int index) const +{ + if (index >= count_ || index < 0) { + return Value(); + } + return Value(attr_type_, &data_[index * attr_len_], attr_len_); +} + +void Column::reference(const Column &column) +{ + if (this == &column) { + return; + } + reset(); + + this->data_ = column.data(); + this->capacity_ = column.capacity(); + this->count_ = column.count(); + this->own_ = false; + + this->column_type_ = column.column_type(); + this->attr_type_ = column.attr_type(); + this->attr_len_ = column.attr_len(); +} \ No newline at end of file diff --git a/src/observer/storage/common/column.h b/src/observer/storage/common/column.h new file mode 100644 index 000000000..122fb854f --- /dev/null +++ b/src/observer/storage/common/column.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +#include "storage/field/field_meta.h" + +/** + * @brief A column contains multiple values in contiguous memory with a specified type. + */ +// TODO: `Column` currently only support fixed-length type. +class Column +{ +public: + enum class Type + { + NORMAL_COLUMN, /// Normal column represents a list of fixed-length values + CONSTANT_COLUMN /// Constant column represents a single value + }; + + Column() = default; + Column(const Column &) = delete; + Column(Column &&) = delete; + + Column(const FieldMeta &meta, size_t size = DEFAULT_CAPACITY); + Column(AttrType attr_type, int attr_len, size_t size = DEFAULT_CAPACITY); + + void init(const FieldMeta &meta, size_t size = DEFAULT_CAPACITY); + void init(AttrType attr_type, int attr_len, size_t size = DEFAULT_CAPACITY); + void init(const Value &value); + + virtual ~Column() { reset(); } + + void reset(); + + RC append_one(char *data); + + /** + * @brief 向 Column 追加写入数据 + * @param data 要被写入数据的起始地址 + * @param count 要写入数据的长度(这里指列值的个数,而不是字节) + */ + RC append(char *data, int count); + + /** + * @brief 获取 index 位置的列值 + */ + Value get_value(int index) const; + + /** + * @brief 获取列数据的实际大小(字节) + */ + int data_len() const { return count_ * attr_len_; } + + char *data() const { return data_; } + + /** + * @brief 重置列数据,但不修改元信息 + */ + void reset_data() { count_ = 0; } + + /** + * @brief 引用另一个 Column + */ + void reference(const Column &column); + + void set_column_type(Type column_type) { column_type_ = column_type; } + void set_count(int count) { count_ = count; } + + int count() const { return count_; } + int capacity() const { return capacity_; } + AttrType attr_type() const { return attr_type_; } + int attr_len() const { return attr_len_; } + Type column_type() const { return column_type_; } + +private: + static constexpr size_t DEFAULT_CAPACITY = 8192; + + char *data_ = nullptr; + /// 当前列值数量 + int count_ = 0; + /// 当前容量,count_ <= capacity_ + int capacity_ = 0; + /// 是否拥有内存 + bool own_ = true; + /// 列属性类型 + AttrType attr_type_ = AttrType::UNDEFINED; + /// 列属性类型长度(目前只支持定长) + int attr_len_ = -1; + /// 列类型 + Type column_type_ = Type::NORMAL_COLUMN; +}; \ No newline at end of file diff --git a/src/observer/storage/common/condition_filter.cpp b/src/observer/storage/common/condition_filter.cpp index eb0b3f26b..47caa6d1c 100644 --- a/src/observer/storage/common/condition_filter.cpp +++ b/src/observer/storage/common/condition_filter.cpp @@ -38,7 +38,7 @@ DefaultConditionFilter::~DefaultConditionFilter() {} RC DefaultConditionFilter::init(const ConDesc &left, const ConDesc &right, AttrType attr_type, CompOp comp_op) { - if (attr_type < CHARS || attr_type > FLOATS) { + if (attr_type < AttrType::CHARS || attr_type > AttrType::FLOATS) { LOG_ERROR("Invalid condition with unsupported attribute type: %d", attr_type); return RC::INVALID_ARGUMENT; } @@ -61,8 +61,8 @@ RC DefaultConditionFilter::init(Table &table, const ConditionSqlNode &condition) ConDesc left; ConDesc right; - AttrType type_left = UNDEFINED; - AttrType type_right = UNDEFINED; + AttrType type_left = AttrType::UNDEFINED; + AttrType type_right = AttrType::UNDEFINED; if (1 == condition.left_is_attr) { left.is_attr = true; diff --git a/src/observer/storage/common/condition_filter.h b/src/observer/storage/common/condition_filter.h index ce63ebc0f..8fb4e5d18 100644 --- a/src/observer/storage/common/condition_filter.h +++ b/src/observer/storage/common/condition_filter.h @@ -61,7 +61,7 @@ class DefaultConditionFilter : public ConditionFilter private: ConDesc left_; ConDesc right_; - AttrType attr_type_ = UNDEFINED; + AttrType attr_type_ = AttrType::UNDEFINED; CompOp comp_op_ = NO_OP; }; diff --git a/src/observer/storage/common/meta_util.h b/src/observer/storage/common/meta_util.h index d8a1effb7..2eaacd619 100644 --- a/src/observer/storage/common/meta_util.h +++ b/src/observer/storage/common/meta_util.h @@ -13,7 +13,7 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include +#include "common/lang/string.h" static constexpr const char *DB_META_SUFFIX = ".db"; static constexpr const char *TABLE_META_SUFFIX = ".table"; @@ -21,7 +21,7 @@ static constexpr const char *TABLE_META_FILE_PATTERN = ".*\\.table$"; static constexpr const char *TABLE_DATA_SUFFIX = ".data"; static constexpr const char *TABLE_INDEX_SUFFIX = ".index"; -std::string db_meta_file(const char *base_dir, const char *db_name); -std::string table_meta_file(const char *base_dir, const char *table_name); -std::string table_data_file(const char *base_dir, const char *table_name); -std::string table_index_file(const char *base_dir, const char *table_name, const char *index_name); +string db_meta_file(const char *base_dir, const char *db_name); +string table_meta_file(const char *base_dir, const char *table_name); +string table_data_file(const char *base_dir, const char *table_name); +string table_index_file(const char *base_dir, const char *table_name, const char *index_name); diff --git a/src/observer/storage/db/db.cpp b/src/observer/storage/db/db.cpp index a49173e3f..42849f57a 100644 --- a/src/observer/storage/db/db.cpp +++ b/src/observer/storage/db/db.cpp @@ -30,7 +30,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/clog/disk_log_handler.h" #include "storage/clog/integrated_log_replayer.h" -using namespace std; using namespace common; Db::~Db() @@ -71,11 +70,11 @@ RC Db::init(const char *name, const char *dbpath, const char *trx_kit_name, cons trx_kit_.reset(trx_kit); buffer_pool_manager_ = make_unique(); - auto dblwr_buffer = make_unique(*buffer_pool_manager_); + auto dblwr_buffer = make_unique(*buffer_pool_manager_); - const char *double_write_buffer_filename = "dblwr.db"; + const char *double_write_buffer_filename = "dblwr.db"; filesystem::path double_write_buffer_file_path = filesystem::path(dbpath) / double_write_buffer_filename; - rc = dblwr_buffer->open_file(double_write_buffer_file_path.c_str()); + rc = dblwr_buffer->open_file(double_write_buffer_file_path.c_str()); if (OB_FAIL(rc)) { LOG_ERROR("Failed to open double write buffer file. file=%s, rc=%s", double_write_buffer_file_path.c_str(), strrc(rc)); @@ -88,15 +87,15 @@ RC Db::init(const char *name, const char *dbpath, const char *trx_kit_name, cons return rc; } - filesystem::path clog_path = filesystem::path(dbpath) / "clog"; - LogHandler *tmp_log_handler = nullptr; - rc = LogHandler::create(log_handler_name, tmp_log_handler); + filesystem::path clog_path = filesystem::path(dbpath) / "clog"; + LogHandler *tmp_log_handler = nullptr; + rc = LogHandler::create(log_handler_name, tmp_log_handler); if (OB_FAIL(rc)) { LOG_ERROR("Failed to create log handler: %s", log_handler_name); return rc; } log_handler_.reset(tmp_log_handler); - + rc = log_handler_->init(clog_path.c_str()); if (OB_FAIL(rc)) { LOG_WARN("failed to init log handler. dbpath=%s, rc=%s", dbpath, strrc(rc)); @@ -137,7 +136,7 @@ RC Db::init(const char *name, const char *dbpath, const char *trx_kit_name, cons return rc; } -RC Db::create_table(const char *table_name, span attributes) +RC Db::create_table(const char *table_name, span attributes, const StorageFormat storage_format) { RC rc = RC::SUCCESS; // check table_name @@ -147,10 +146,10 @@ RC Db::create_table(const char *table_name, span attribut } // 文件路径可以移到Table模块 - std::string table_file_path = table_meta_file(path_.c_str(), table_name); - Table *table = new Table(); - int32_t table_id = next_table_id_++; - rc = table->create(this, table_id, table_file_path.c_str(), table_name, path_.c_str(), attributes); + string table_file_path = table_meta_file(path_.c_str(), table_name); + Table *table = new Table(); + int32_t table_id = next_table_id_++; + rc = table->create(this, table_id, table_file_path.c_str(), table_name, path_.c_str(), attributes, storage_format); if (rc != RC::SUCCESS) { LOG_ERROR("Failed to create table %s.", table_name); delete table; @@ -243,7 +242,7 @@ RC Db::sync() } auto dblwr_buffer = static_cast(buffer_pool_manager_->get_dblwr_buffer()); - rc = dblwr_buffer->flush_page(); + rc = dblwr_buffer->flush_page(); LOG_INFO("double write buffer flush pages ret=%s", strrc(rc)); /* @@ -358,7 +357,7 @@ RC Db::flush_meta() return RC::IOERR_WRITE; } - string buffer = to_string(check_point_lsn_); + string buffer = std::to_string(check_point_lsn_); int n = write(fd, buffer.c_str(), buffer.size()); if (n < 0) { LOG_ERROR("Failed to write db meta file. db=%s, file=%s, errno=%s", @@ -388,7 +387,7 @@ RC Db::flush_meta() RC Db::init_dblwr_buffer() { auto dblwr_buffer = static_cast(buffer_pool_manager_->get_dblwr_buffer()); - RC rc = dblwr_buffer->recover(); + RC rc = dblwr_buffer->recover(); if (OB_FAIL(rc)) { LOG_ERROR("fail to recover in dblwr buffer"); return rc; diff --git a/src/observer/storage/db/db.h b/src/observer/storage/db/db.h index 6e60a409a..8eee8943e 100644 --- a/src/observer/storage/db/db.h +++ b/src/observer/storage/db/db.h @@ -14,13 +14,12 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include -#include -#include - #include "common/rc.h" +#include "common/lang/vector.h" +#include "common/lang/string.h" +#include "common/lang/unordered_map.h" +#include "common/lang/memory.h" +#include "common/lang/span.h" #include "sql/parser/parse_defs.h" #include "storage/buffer/disk_buffer_pool.h" #include "storage/clog/disk_log_handler.h" @@ -62,8 +61,10 @@ class Db * @brief 创建一个表 * @param table_name 表名 * @param attributes 表的属性 + * @param storage_format 表的存储格式 */ - RC create_table(const char *table_name, std::span attributes); + RC create_table(const char *table_name, span attributes, + const StorageFormat storage_format = StorageFormat::ROW_FORMAT); /** * @brief 根据表名查找表 @@ -78,7 +79,7 @@ class Db const char *name() const; /// @brief 列出所有的表 - void all_tables(std::vector &table_names) const; + void all_tables(vector &table_names) const; /** * @brief 将所有内存中的数据,刷新到磁盘中。 @@ -110,12 +111,12 @@ class Db RC init_dblwr_buffer(); private: - std::string name_; ///< 数据库名称 - std::string path_; ///< 数据库文件存放的目录 - std::unordered_map opened_tables_; ///< 当前所有打开的表 - std::unique_ptr buffer_pool_manager_; ///< 当前数据库的buffer pool管理器 - std::unique_ptr log_handler_; ///< 当前数据库的日志处理器 - std::unique_ptr trx_kit_; ///< 当前数据库的事务管理器 + string name_; ///< 数据库名称 + string path_; ///< 数据库文件存放的目录 + unordered_map opened_tables_; ///< 当前所有打开的表 + unique_ptr buffer_pool_manager_; ///< 当前数据库的buffer pool管理器 + unique_ptr log_handler_; ///< 当前数据库的日志处理器 + unique_ptr trx_kit_; ///< 当前数据库的事务管理器 /// 给每个table都分配一个ID,用来记录日志。这里假设所有的DDL都不会并发操作,所以相关的数据都不上锁 int32_t next_table_id_ = 0; diff --git a/src/observer/storage/default/default_handler.h b/src/observer/storage/default/default_handler.h index 57b7b7d17..56d7ae063 100644 --- a/src/observer/storage/default/default_handler.h +++ b/src/observer/storage/default/default_handler.h @@ -13,12 +13,11 @@ See the Mulan PSL v2 for more details. */ // #pragma once -#include -#include -#include -#include - #include "storage/db/db.h" +#include "common/lang/span.h" +#include "common/lang/map.h" +#include "common/lang/string.h" +#include "common/lang/memory.h" class Trx; class TrxKit; @@ -75,7 +74,7 @@ class DefaultHandler * @param relation_name 表名 * @param attributes 属性信息 */ - RC create_table(const char *dbname, const char *relation_name, std::span attributes); + RC create_table(const char *dbname, const char *relation_name, span attributes); /** * @brief 删除指定数据库下的表 @@ -92,9 +91,9 @@ class DefaultHandler RC sync(); private: - std::filesystem::path base_dir_; ///< 存储引擎的根目录 - std::filesystem::path db_dir_; ///< 数据库文件的根目录 - std::string trx_kit_name_; ///< 事务模型的名称 - std::string log_handler_name_; ///< 日志处理器的名称 - std::map opened_dbs_; ///< 打开的数据库 + filesystem::path base_dir_; ///< 存储引擎的根目录 + filesystem::path db_dir_; ///< 数据库文件的根目录 + string trx_kit_name_; ///< 事务模型的名称 + string log_handler_name_; ///< 日志处理器的名称 + map opened_dbs_; ///< 打开的数据库 }; diff --git a/src/observer/storage/field/field_meta.cpp b/src/observer/storage/field/field_meta.cpp index 4bae4c1d0..9be3bf2e2 100644 --- a/src/observer/storage/field/field_meta.cpp +++ b/src/observer/storage/field/field_meta.cpp @@ -24,16 +24,17 @@ const static Json::StaticString FIELD_TYPE("type"); const static Json::StaticString FIELD_OFFSET("offset"); const static Json::StaticString FIELD_LEN("len"); const static Json::StaticString FIELD_VISIBLE("visible"); +const static Json::StaticString FIELD_FIELD_ID("FIELD_id"); -FieldMeta::FieldMeta() : attr_type_(AttrType::UNDEFINED), attr_offset_(-1), attr_len_(0), visible_(false) {} +FieldMeta::FieldMeta() : attr_type_(AttrType::UNDEFINED), attr_offset_(-1), attr_len_(0), visible_(false), field_id_(0) {} -FieldMeta::FieldMeta(const char *name, AttrType attr_type, int attr_offset, int attr_len, bool visible) +FieldMeta::FieldMeta(const char *name, AttrType attr_type, int attr_offset, int attr_len, bool visible, int field_id) { - [[maybe_unused]] RC rc = this->init(name, attr_type, attr_offset, attr_len, visible); + [[maybe_unused]] RC rc = this->init(name, attr_type, attr_offset, attr_len, visible, field_id); ASSERT(rc == RC::SUCCESS, "failed to init field meta. rc=%s", strrc(rc)); } -RC FieldMeta::init(const char *name, AttrType attr_type, int attr_offset, int attr_len, bool visible) +RC FieldMeta::init(const char *name, AttrType attr_type, int attr_offset, int attr_len, bool visible, int field_id) { if (common::is_blank(name)) { LOG_WARN("Name cannot be empty"); @@ -51,6 +52,7 @@ RC FieldMeta::init(const char *name, AttrType attr_type, int attr_offset, int at attr_len_ = attr_len; attr_offset_ = attr_offset; visible_ = visible; + field_id_ = field_id; LOG_INFO("Init a field with name=%s", name); return RC::SUCCESS; @@ -66,6 +68,8 @@ int FieldMeta::len() const { return attr_len_; } bool FieldMeta::visible() const { return visible_; } +int FieldMeta::field_id() const { return field_id_; } + void FieldMeta::desc(std::ostream &os) const { os << "field name=" << name_ << ", type=" << attr_type_to_string(attr_type_) << ", len=" << attr_len_ @@ -79,6 +83,7 @@ void FieldMeta::to_json(Json::Value &json_value) const json_value[FIELD_OFFSET] = attr_offset_; json_value[FIELD_LEN] = attr_len_; json_value[FIELD_VISIBLE] = visible_; + json_value[FIELD_FIELD_ID] = field_id_; } RC FieldMeta::from_json(const Json::Value &json_value, FieldMeta &field) @@ -93,6 +98,7 @@ RC FieldMeta::from_json(const Json::Value &json_value, FieldMeta &field) const Json::Value &offset_value = json_value[FIELD_OFFSET]; const Json::Value &len_value = json_value[FIELD_LEN]; const Json::Value &visible_value = json_value[FIELD_VISIBLE]; + const Json::Value &field_id_value = json_value[FIELD_FIELD_ID]; if (!name_value.isString()) { LOG_ERROR("Field name is not a string. json value=%s", name_value.toStyledString().c_str()); @@ -115,9 +121,13 @@ RC FieldMeta::from_json(const Json::Value &json_value, FieldMeta &field) LOG_ERROR("Visible field is not a bool value. json value=%s", visible_value.toStyledString().c_str()); return RC::INTERNAL; } + if (!field_id_value.isInt()) { + LOG_ERROR("Field id is not an integer. json value=%s", field_id_value.toStyledString().c_str()); + return RC::INTERNAL; + } AttrType type = attr_type_from_string(type_value.asCString()); - if (UNDEFINED == type) { + if (AttrType::UNDEFINED == type) { LOG_ERROR("Got invalid field type. type=%d", type); return RC::INTERNAL; } @@ -126,5 +136,6 @@ RC FieldMeta::from_json(const Json::Value &json_value, FieldMeta &field) int offset = offset_value.asInt(); int len = len_value.asInt(); bool visible = visible_value.asBool(); - return field.init(name, type, offset, len, visible); + int field_id = field_id_value.asInt(); + return field.init(name, type, offset, len, visible, field_id); } diff --git a/src/observer/storage/field/field_meta.h b/src/observer/storage/field/field_meta.h index 799729ea0..03e9c797c 100644 --- a/src/observer/storage/field/field_meta.h +++ b/src/observer/storage/field/field_meta.h @@ -14,9 +14,8 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include - #include "common/rc.h" +#include "common/lang/string.h" #include "sql/parser/parse_defs.h" namespace Json { @@ -31,10 +30,10 @@ class FieldMeta { public: FieldMeta(); - FieldMeta(const char *name, AttrType attr_type, int attr_offset, int attr_len, bool visible); + FieldMeta(const char *name, AttrType attr_type, int attr_offset, int attr_len, bool visible, int field_id); ~FieldMeta() = default; - RC init(const char *name, AttrType attr_type, int attr_offset, int attr_len, bool visible); + RC init(const char *name, AttrType attr_type, int attr_offset, int attr_len, bool visible, int field_id); public: const char *name() const; @@ -42,18 +41,20 @@ class FieldMeta int offset() const; int len() const; bool visible() const; + int field_id() const; public: - void desc(std::ostream &os) const; + void desc(ostream &os) const; public: void to_json(Json::Value &json_value) const; static RC from_json(const Json::Value &json_value, FieldMeta &field); protected: - std::string name_; - AttrType attr_type_; - int attr_offset_; - int attr_len_; - bool visible_; + string name_; + AttrType attr_type_; + int attr_offset_; + int attr_len_; + bool visible_; + int field_id_; }; diff --git a/src/observer/storage/index/bplus_tree.cpp b/src/observer/storage/index/bplus_tree.cpp index 5b4601c1a..7c7a1103b 100644 --- a/src/observer/storage/index/bplus_tree.cpp +++ b/src/observer/storage/index/bplus_tree.cpp @@ -22,7 +22,6 @@ See the Mulan PSL v2 for more details. */ #include "sql/parser/parse_defs.h" #include "storage/buffer/disk_buffer_pool.h" -using namespace std; using namespace common; /** @@ -132,9 +131,9 @@ bool IndexNodeHandler::is_safe(BplusTreeOperationType op, bool is_root_node) return false; } -std::string to_string(const IndexNodeHandler &handler) +string to_string(const IndexNodeHandler &handler) { - std::stringstream ss; + stringstream ss; ss << "PageNum:" << handler.page_num() << ",is_leaf:" << handler.is_leaf() << "," << "key_num:" << handler.size() << "," @@ -348,9 +347,9 @@ RC LeafIndexNodeHandler::preappend(const char *item) char *LeafIndexNodeHandler::__item_at(int index) const { return leaf_node_->array + (index * item_size()); } -std::string to_string(const LeafIndexNodeHandler &handler, const KeyPrinter &printer) +string to_string(const LeafIndexNodeHandler &handler, const KeyPrinter &printer) { - std::stringstream ss; + stringstream ss; ss << to_string((const IndexNodeHandler &)handler) << ",next page:" << handler.next_page(); ss << ",values=[" << printer(handler.__key_at(0)); for (int i = 1; i < handler.size(); i++) { @@ -427,9 +426,9 @@ InternalIndexNodeHandler::InternalIndexNodeHandler(BplusTreeMiniTransaction &mtr : IndexNodeHandler(mtr, header, frame), internal_node_((InternalIndexNode *)frame->data()) {} -std::string to_string(const InternalIndexNodeHandler &node, const KeyPrinter &printer) +string to_string(const InternalIndexNodeHandler &node, const KeyPrinter &printer) { - std::stringstream ss; + stringstream ss; ss << to_string((const IndexNodeHandler &)node); ss << ",children:[" << "{key:" << printer(node.__key_at(0)) << "," @@ -1134,7 +1133,7 @@ bool BplusTreeHandler::validate_leaf_link(BplusTreeMiniTransaction &mtr) LeafIndexNodeHandler leaf_node(mtr, file_header_, frame); PageNum next_page_num = leaf_node.next_page(); - MemPoolItem::unique_ptr prev_key = mem_pool_item_->alloc_unique_ptr(); + MemPoolItem::item_unique_ptr prev_key = mem_pool_item_->alloc_unique_ptr(); memcpy(prev_key.get(), leaf_node.key_at(leaf_node.size() - 1), file_header_.key_length); bool result = true; @@ -1202,7 +1201,7 @@ RC BplusTreeHandler::left_most_page(BplusTreeMiniTransaction &mtr, Frame *&frame } RC BplusTreeHandler::find_leaf_internal(BplusTreeMiniTransaction &mtr, BplusTreeOperationType op, - const std::function &child_page_getter, Frame *&frame) + const function &child_page_getter, Frame *&frame) { LatchMemo &latch_memo = mtr.latch_memo(); @@ -1486,9 +1485,9 @@ RC BplusTreeHandler::create_new_tree(BplusTreeMiniTransaction &mtr, const char * return rc; } -MemPoolItem::unique_ptr BplusTreeHandler::make_key(const char *user_key, const RID &rid) +MemPoolItem::item_unique_ptr BplusTreeHandler::make_key(const char *user_key, const RID &rid) { - MemPoolItem::unique_ptr key = mem_pool_item_->alloc_unique_ptr(); + MemPoolItem::item_unique_ptr key = mem_pool_item_->alloc_unique_ptr(); if (key == nullptr) { LOG_WARN("Failed to alloc memory for key."); return nullptr; @@ -1505,7 +1504,7 @@ RC BplusTreeHandler::insert_entry(const char *user_key, const RID *rid) return RC::INVALID_ARGUMENT; } - MemPoolItem::unique_ptr pkey = make_key(user_key, *rid); + MemPoolItem::item_unique_ptr pkey = make_key(user_key, *rid); if (pkey == nullptr) { LOG_WARN("Failed to alloc memory for key."); return RC::NOMEM; @@ -1545,7 +1544,7 @@ RC BplusTreeHandler::insert_entry(const char *user_key, const RID *rid) return RC::SUCCESS; } -RC BplusTreeHandler::get_entry(const char *user_key, int key_len, std::list &rids) +RC BplusTreeHandler::get_entry(const char *user_key, int key_len, list &rids) { BplusTreeScanner scanner(*this); RC rc = scanner.open(user_key, key_len, true /*left_inclusive*/, user_key, key_len, true /*right_inclusive*/); @@ -1787,7 +1786,7 @@ RC BplusTreeHandler::delete_entry_internal(BplusTreeMiniTransaction &mtr, Frame RC BplusTreeHandler::delete_entry(const char *user_key, const RID *rid) { - MemPoolItem::unique_ptr pkey = mem_pool_item_->alloc_unique_ptr(); + MemPoolItem::item_unique_ptr pkey = mem_pool_item_->alloc_unique_ptr(); if (nullptr == pkey) { LOG_WARN("Failed to alloc memory for key. size=%d", file_header_.key_length); return RC::NOMEM; @@ -1869,7 +1868,7 @@ RC BplusTreeScanner::open(const char *left_user_key, int left_len, bool left_inc } else { char *fixed_left_key = const_cast(left_user_key); - if (tree_handler_.file_header_.attr_type == CHARS) { + if (tree_handler_.file_header_.attr_type == AttrType::CHARS) { bool should_inclusive_after_fix = false; rc = fix_user_key(left_user_key, left_len, true /*greater*/, &fixed_left_key, &should_inclusive_after_fix); if (OB_FAIL(rc)) { @@ -1882,7 +1881,7 @@ RC BplusTreeScanner::open(const char *left_user_key, int left_len, bool left_inc } } - MemPoolItem::unique_ptr left_pkey; + MemPoolItem::item_unique_ptr left_pkey; if (left_inclusive) { left_pkey = tree_handler_.make_key(fixed_left_key, *RID::min()); } else { @@ -1936,7 +1935,7 @@ RC BplusTreeScanner::open(const char *left_user_key, int left_len, bool left_inc char *fixed_right_key = const_cast(right_user_key); bool should_include_after_fix = false; - if (tree_handler_.file_header_.attr_type == CHARS) { + if (tree_handler_.file_header_.attr_type == AttrType::CHARS) { rc = fix_user_key(right_user_key, right_len, false /*want_greater*/, &fixed_right_key, &should_include_after_fix); if (OB_FAIL(rc)) { LOG_WARN("failed to fix right user key. rc=%s", strrc(rc)); @@ -2054,13 +2053,13 @@ RC BplusTreeScanner::fix_user_key( } // 这里很粗暴,变长字段才需要做调整,其它默认都不需要做调整 - assert(tree_handler_.file_header_.attr_type == CHARS); + assert(tree_handler_.file_header_.attr_type == AttrType::CHARS); assert(strlen(user_key) >= static_cast(key_len)); *should_inclusive = false; int32_t attr_length = tree_handler_.file_header_.attr_length; - char *key_buf = new (std::nothrow) char[attr_length]; + char *key_buf = new char[attr_length]; if (nullptr == key_buf) { return RC::NOMEM; } diff --git a/src/observer/storage/index/bplus_tree.h b/src/observer/storage/index/bplus_tree.h index 0149796cd..caea013eb 100644 --- a/src/observer/storage/index/bplus_tree.h +++ b/src/observer/storage/index/bplus_tree.h @@ -17,12 +17,12 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include -#include #include #include "common/lang/comparator.h" +#include "common/lang/memory.h" +#include "common/lang/sstream.h" +#include "common/lang/functional.h" #include "common/log/log.h" #include "sql/parser/parse_defs.h" #include "storage/buffer/disk_buffer_pool.h" @@ -67,13 +67,13 @@ class AttrComparator int operator()(const char *v1, const char *v2) const { switch (attr_type_) { - case INTS: { + case AttrType::INTS: { return common::compare_int((void *)v1, (void *)v2); } break; - case FLOATS: { + case AttrType::FLOATS: { return common::compare_float((void *)v1, (void *)v2); } - case CHARS: { + case AttrType::CHARS: { return common::compare_string((void *)v1, attr_length_, (void *)v2, attr_length_); } default: { @@ -131,7 +131,7 @@ class AttrPrinter int attr_length() const { return attr_length_; } - std::string operator()(const char *v) const + string operator()(const char *v) const { Value value(attr_type_, const_cast(v), attr_length_); return value.to_string(); @@ -153,9 +153,9 @@ class KeyPrinter const AttrPrinter &attr_printer() const { return attr_printer_; } - std::string operator()(const char *v) const + string operator()(const char *v) const { - std::stringstream ss; + stringstream ss; ss << "{key:" << attr_printer_(v) << ","; const RID *rid = (const RID *)(v + attr_printer_.attr_length()); @@ -187,13 +187,13 @@ struct IndexFileHeader int32_t key_length; ///< attr length + sizeof(RID) AttrType attr_type; ///< 键值的类型 - const std::string to_string() const + const string to_string() const { - std::stringstream ss; + stringstream ss; ss << "attr_length:" << attr_length << "," << "key_length:" << key_length << "," - << "attr_type:" << attr_type << "," + << "attr_type:" << attr_type_to_string(attr_type) << "," << "root_page:" << root_page << "," << "internal_max_size:" << internal_max_size << "," << "leaf_max_size:" << leaf_max_size << ";"; @@ -313,7 +313,7 @@ class IndexNodeHandler Frame *frame() const { return frame_; } - friend std::string to_string(const IndexNodeHandler &handler); + friend string to_string(const IndexNodeHandler &handler); RC recover_insert_items(int index, const char *items, int num); RC recover_remove_items(int index, int num); @@ -371,7 +371,7 @@ class LeafIndexNodeHandler final : public IndexNodeHandler bool validate(const KeyComparator &comparator, DiskBufferPool *bp) const; - friend std::string to_string(const LeafIndexNodeHandler &handler, const KeyPrinter &printer); + friend string to_string(const LeafIndexNodeHandler &handler, const KeyPrinter &printer); protected: char *__item_at(int index) const override; @@ -431,7 +431,7 @@ class InternalIndexNodeHandler final : public IndexNodeHandler bool validate(const KeyComparator &comparator, DiskBufferPool *bp) const; - friend std::string to_string(const InternalIndexNodeHandler &handler, const KeyPrinter &printer); + friend string to_string(const InternalIndexNodeHandler &handler, const KeyPrinter &printer); private: RC insert_items(int index, const char *items, int num); @@ -507,7 +507,7 @@ class BplusTreeHandler * @param key_len user_key的长度 * @param rid 返回值,记录记录所在的页面号和slot */ - RC get_entry(const char *user_key, int key_len, std::list &rids); + RC get_entry(const char *user_key, int key_len, list &rids); RC sync(); @@ -573,7 +573,7 @@ class BplusTreeHandler * @param[out] frame 返回找到的叶子节点 */ RC find_leaf_internal(BplusTreeMiniTransaction &mtr, BplusTreeOperationType op, - const std::function &child_page_getter, Frame *&frame); + const function &child_page_getter, Frame *&frame); /** * @brief 使用crabing protocol 获取页面 @@ -640,7 +640,7 @@ class BplusTreeHandler RC adjust_root(BplusTreeMiniTransaction &mtr, Frame *root_frame); private: - common::MemPoolItem::unique_ptr make_key(const char *user_key, const RID &rid); + common::MemPoolItem::item_unique_ptr make_key(const char *user_key, const RID &rid); protected: LogHandler *log_handler_ = nullptr; /// 日志处理器 @@ -655,7 +655,7 @@ class BplusTreeHandler KeyComparator key_comparator_; KeyPrinter key_printer_; - std::unique_ptr mem_pool_item_; + unique_ptr mem_pool_item_; private: friend class BplusTreeScanner; @@ -724,7 +724,7 @@ class BplusTreeScanner /// 起始位置和终止位置都是有效的数据 Frame *current_frame_ = nullptr; - common::MemPoolItem::unique_ptr right_key_; - int iter_index_ = -1; - bool first_emitted_ = false; + common::MemPoolItem::item_unique_ptr right_key_; + int iter_index_ = -1; + bool first_emitted_ = false; }; diff --git a/src/observer/storage/index/bplus_tree_log.cpp b/src/observer/storage/index/bplus_tree_log.cpp index 2f57f3238..e2ee32ff3 100644 --- a/src/observer/storage/index/bplus_tree_log.cpp +++ b/src/observer/storage/index/bplus_tree_log.cpp @@ -12,11 +12,10 @@ See the Mulan PSL v2 for more details. */ // Created by wangyunlai.wyl on 2024/02/05. // -#include -#include - #include "common/log/log.h" #include "common/lang/defer.h" +#include "common/lang/algorithm.h" +#include "common/lang/sstream.h" #include "storage/index/bplus_tree_log.h" #include "storage/index/bplus_tree.h" #include "storage/clog/log_handler.h" @@ -25,7 +24,6 @@ See the Mulan PSL v2 for more details. */ #include "common/lang/serializer.h" #include "storage/clog/vacuous_log_handler.h" -using namespace std; using namespace common; using namespace bplus_tree; diff --git a/src/observer/storage/index/bplus_tree_log.h b/src/observer/storage/index/bplus_tree_log.h index e878fff00..35eb45b25 100644 --- a/src/observer/storage/index/bplus_tree_log.h +++ b/src/observer/storage/index/bplus_tree_log.h @@ -15,13 +15,13 @@ See the Mulan PSL v2 for more details. */ #pragma once #include -#include -#include -#include -#include #include "common/types.h" #include "common/rc.h" +#include "common/lang/span.h" +#include "common/lang/memory.h" +#include "common/lang/vector.h" +#include "common/lang/string.h" #include "storage/index/latch_memo.h" #include "storage/clog/log_replayer.h" // #include "storage/index/bplus_tree_log_entry.h" @@ -88,7 +88,7 @@ class BplusTreeLogger final * @param items 插入的元素 * @param item_num 元素个数 */ - RC node_insert_items(IndexNodeHandler &node_handler, int index, std::span items, int item_num); + RC node_insert_items(IndexNodeHandler &node_handler, int index, span items, int item_num); /** * @brief 在某个页面中删除一些元素 * @param node_handler 页面处理器。同时也包含了页面编号、页帧 @@ -97,7 +97,7 @@ class BplusTreeLogger final * @param item_num 元素个数 * @details 会在内存中记录一些数据帮助回滚操作 */ - RC node_remove_items(IndexNodeHandler &node_handler, int index, std::span items, int item_num); + RC node_remove_items(IndexNodeHandler &node_handler, int index, span items, int item_num); /** * @brief 初始化一个空的叶子节点 @@ -116,12 +116,11 @@ class BplusTreeLogger final * @brief 创建一个新的根节点 */ RC internal_create_new_root( - IndexNodeHandler &node_handler, PageNum first_page_num, std::span key, PageNum page_num); + IndexNodeHandler &node_handler, PageNum first_page_num, span key, PageNum page_num); /** * @brief 更新某个内部页面上,更新指定位置的键值 */ - RC internal_update_key( - IndexNodeHandler &node_handler, int index, std::span key, std::span old_key); + RC internal_update_key(IndexNodeHandler &node_handler, int index, span key, span old_key); /** * @brief 修改某个页面的父节点编号 @@ -144,19 +143,19 @@ class BplusTreeLogger final /** * @brief 日志记录转字符串 */ - static std::string log_entry_to_string(const LogEntry &entry); + static string log_entry_to_string(const LogEntry &entry); private: RC __redo(LSN lsn, BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler, common::Deserializer &redo_buffer); protected: - RC append_log_entry(std::unique_ptr entry); + RC append_log_entry(unique_ptr entry); private: LogHandler &log_handler_; int32_t buffer_pool_id_ = -1; /// 关联的缓冲池ID - std::vector> entries_; /// 当前记录了的日志 + vector> entries_; /// 当前记录了的日志 bool need_log_ = true; /// 是否需要记录日志。在回滚或重做过程中,不需要记录日志。 }; diff --git a/src/observer/storage/index/bplus_tree_log_entry.cpp b/src/observer/storage/index/bplus_tree_log_entry.cpp index 8c32970aa..615bb52b4 100644 --- a/src/observer/storage/index/bplus_tree_log_entry.cpp +++ b/src/observer/storage/index/bplus_tree_log_entry.cpp @@ -15,7 +15,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/index/bplus_tree_log_entry.h" #include "common/lang/serializer.h" -using namespace std; using namespace common; namespace bplus_tree { diff --git a/src/observer/storage/index/bplus_tree_log_entry.h b/src/observer/storage/index/bplus_tree_log_entry.h index 279603e28..7abdc8bdf 100644 --- a/src/observer/storage/index/bplus_tree_log_entry.h +++ b/src/observer/storage/index/bplus_tree_log_entry.h @@ -14,11 +14,10 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include - #include "common/types.h" #include "common/rc.h" +#include "common/lang/span.h" +#include "common/lang/string.h" #include "storage/index/bplus_tree.h" class IndexNodeHandler; @@ -62,7 +61,7 @@ class LogOperation Type type() const { return type_; } int index() const { return static_cast(type_); } - std::string to_string() const; + string to_string() const; private: Type type_; @@ -112,7 +111,7 @@ class LogEntryHandler */ virtual RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) = 0; - virtual std::string to_string() const; + virtual string to_string() const; /** * @brief 从buffer中反序列化出一个LogEntryHandler @@ -126,11 +125,11 @@ class LogEntryHandler * 但是这里实现的时候,只使用了handler,所以在反序列化时,有些场景下根本不需要Frame,但是为了适配,允许传入一个null * frame。 在LogEntryHandler类中也做了特殊处理。就是虽然有frame指针对象,但是也另外单独记录了page_num。 */ - static RC from_buffer(std::function frame_getter, common::Deserializer &buffer, - std::unique_ptr &handler); static RC from_buffer( - DiskBufferPool &buffer_pool, common::Deserializer &buffer, std::unique_ptr &handler); - static RC from_buffer(common::Deserializer &deserializer, std::unique_ptr &handler); + function frame_getter, common::Deserializer &buffer, unique_ptr &handler); + static RC from_buffer( + DiskBufferPool &buffer_pool, common::Deserializer &buffer, unique_ptr &handler); + static RC from_buffer(common::Deserializer &deserializer, unique_ptr &handler); protected: LogOperation operation_type_; @@ -167,9 +166,9 @@ class InitHeaderPageLogEntryHandler : public LogEntryHandler RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - std::string to_string() const override; + string to_string() const override; - static RC deserialize(Frame *frame, common::Deserializer &buffer, std::unique_ptr &handler); + static RC deserialize(Frame *frame, common::Deserializer &buffer, unique_ptr &handler); const IndexFileHeader &file_header() const { return file_header_; } @@ -191,9 +190,9 @@ class UpdateRootPageLogEntryHandler : public LogEntryHandler RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - std::string to_string() const override; + string to_string() const override; - static RC deserialize(Frame *frame, common::Deserializer &buffer, std::unique_ptr &handler); + static RC deserialize(Frame *frame, common::Deserializer &buffer, unique_ptr &handler); PageNum root_page_num() const { return root_page_num_; } @@ -216,9 +215,9 @@ class SetParentPageLogEntryHandler : public NodeLogEntryHandler RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - std::string to_string() const override; + string to_string() const override; - static RC deserialize(Frame *frame, common::Deserializer &buffer, std::unique_ptr &handler); + static RC deserialize(Frame *frame, common::Deserializer &buffer, unique_ptr &handler); PageNum parent_page_num() const { return parent_page_num_; } @@ -241,10 +240,10 @@ class NormalOperationLogEntryHandler : public NodeLogEntryHandler RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - std::string to_string() const override; + string to_string() const override; static RC deserialize( - Frame *frame, LogOperation operation, common::Deserializer &buffer, std::unique_ptr &handler); + Frame *frame, LogOperation operation, common::Deserializer &buffer, unique_ptr &handler); int index() const { return index_; } int item_num() const { return item_num_; } @@ -252,9 +251,9 @@ class NormalOperationLogEntryHandler : public NodeLogEntryHandler int32_t item_bytes() const { return static_cast(items_.size()); } private: - int index_ = -1; - int item_num_ = -1; - std::vector items_; + int index_ = -1; + int item_num_ = -1; + vector items_; }; /** @@ -271,9 +270,9 @@ class LeafInitEmptyLogEntryHandler : public NodeLogEntryHandler RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override { return RC::SUCCESS; } RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - // std::string to_string() const override; + // string to_string() const override; - static RC deserialize(Frame *frame, common::Deserializer &buffer, std::unique_ptr &handler); + static RC deserialize(Frame *frame, common::Deserializer &buffer, unique_ptr &handler); }; /** @@ -290,9 +289,9 @@ class LeafSetNextPageLogEntryHandler : public NodeLogEntryHandler RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - std::string to_string() const override; + string to_string() const override; - static RC deserialize(Frame *frame, common::Deserializer &buffer, std::unique_ptr &handler); + static RC deserialize(Frame *frame, common::Deserializer &buffer, unique_ptr &handler); PageNum new_page_num() const { return new_page_num_; } @@ -315,9 +314,9 @@ class InternalInitEmptyLogEntryHandler : public NodeLogEntryHandler RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override { return RC::SUCCESS; } RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - // std::string to_string() const override; + // string to_string() const override; - static RC deserialize(Frame *frame, common::Deserializer &buffer, std::unique_ptr &handler); + static RC deserialize(Frame *frame, common::Deserializer &buffer, unique_ptr &handler); }; /** @@ -327,17 +326,16 @@ class InternalInitEmptyLogEntryHandler : public NodeLogEntryHandler class InternalCreateNewRootLogEntryHandler : public NodeLogEntryHandler { public: - InternalCreateNewRootLogEntryHandler( - Frame *frame, PageNum first_page_num, std::span key, PageNum page_num); + InternalCreateNewRootLogEntryHandler(Frame *frame, PageNum first_page_num, span key, PageNum page_num); virtual ~InternalCreateNewRootLogEntryHandler() = default; RC serialize_body(common::Serializer &buffer) const override; RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override { return RC::SUCCESS; } RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - std::string to_string() const override; + string to_string() const override; - static RC deserialize(Frame *frame, common::Deserializer &buffer, std::unique_ptr &handler); + static RC deserialize(Frame *frame, common::Deserializer &buffer, unique_ptr &handler); PageNum first_page_num() const { return first_page_num_; } PageNum page_num() const { return page_num_; } @@ -345,9 +343,9 @@ class InternalCreateNewRootLogEntryHandler : public NodeLogEntryHandler int32_t key_bytes() const { return static_cast(key_.size()); } private: - PageNum first_page_num_ = -1; - PageNum page_num_ = -1; - std::vector key_; + PageNum first_page_num_ = -1; + PageNum page_num_ = -1; + vector key_; }; /** @@ -357,25 +355,25 @@ class InternalCreateNewRootLogEntryHandler : public NodeLogEntryHandler class InternalUpdateKeyLogEntryHandler : public NodeLogEntryHandler { public: - InternalUpdateKeyLogEntryHandler(Frame *frame, int index, std::span key, std::span old_key); + InternalUpdateKeyLogEntryHandler(Frame *frame, int index, span key, span old_key); virtual ~InternalUpdateKeyLogEntryHandler() = default; RC serialize_body(common::Serializer &buffer) const override; RC rollback(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; RC redo(BplusTreeMiniTransaction &mtr, BplusTreeHandler &tree_handler) override; - std::string to_string() const override; + string to_string() const override; - static RC deserialize(Frame *frame, common::Deserializer &buffer, std::unique_ptr &handler); + static RC deserialize(Frame *frame, common::Deserializer &buffer, unique_ptr &handler); int index() const { return index_; } const char *key() const { return key_.data(); } int32_t key_bytes() const { return static_cast(key_.size()); } private: - int index_ = -1; - std::vector key_; - std::vector old_key_; + int index_ = -1; + vector key_; + vector old_key_; }; } // namespace bplus_tree diff --git a/src/observer/storage/index/index_meta.cpp b/src/observer/storage/index/index_meta.cpp index 1ce3ef6d3..32604b9ff 100644 --- a/src/observer/storage/index/index_meta.cpp +++ b/src/observer/storage/index/index_meta.cpp @@ -68,4 +68,4 @@ const char *IndexMeta::name() const { return name_.c_str(); } const char *IndexMeta::field() const { return field_.c_str(); } -void IndexMeta::desc(std::ostream &os) const { os << "index name=" << name_ << ", field=" << field_; } \ No newline at end of file +void IndexMeta::desc(ostream &os) const { os << "index name=" << name_ << ", field=" << field_; } \ No newline at end of file diff --git a/src/observer/storage/index/index_meta.h b/src/observer/storage/index/index_meta.h index 4ab410710..04d10642c 100644 --- a/src/observer/storage/index/index_meta.h +++ b/src/observer/storage/index/index_meta.h @@ -15,7 +15,7 @@ See the Mulan PSL v2 for more details. */ #pragma once #include "common/rc.h" -#include +#include "common/lang/string.h" class TableMeta; class FieldMeta; @@ -41,13 +41,13 @@ class IndexMeta const char *name() const; const char *field() const; - void desc(std::ostream &os) const; + void desc(ostream &os) const; public: void to_json(Json::Value &json_value) const; static RC from_json(const TableMeta &table, const Json::Value &json_value, IndexMeta &index); protected: - std::string name_; // index's name - std::string field_; // field's name + string name_; // index's name + string field_; // field's name }; diff --git a/src/observer/storage/index/latch_memo.h b/src/observer/storage/index/latch_memo.h index fc06dfa28..f0e0da8b9 100644 --- a/src/observer/storage/index/latch_memo.h +++ b/src/observer/storage/index/latch_memo.h @@ -14,10 +14,9 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include - #include "common/rc.h" +#include "common/lang/deque.h" +#include "common/lang/vector.h" #include "storage/buffer/page.h" class Frame; @@ -85,7 +84,7 @@ class LatchMemo final void release_item(LatchMemoItem &item); private: - DiskBufferPool *buffer_pool_ = nullptr; - std::deque items_; - std::vector disposed_pages_; /// 等待释放的页面 + DiskBufferPool *buffer_pool_ = nullptr; + deque items_; + vector disposed_pages_; /// 等待释放的页面 }; \ No newline at end of file diff --git a/src/observer/storage/persist/persist.h b/src/observer/storage/persist/persist.h index 863c1f122..69befa786 100644 --- a/src/observer/storage/persist/persist.h +++ b/src/observer/storage/persist/persist.h @@ -18,11 +18,11 @@ See the Mulan PSL v2 for more details. */ #include #include #include -#include #include #include #include "common/rc.h" +#include "common/lang/string.h" class PersistHandler { @@ -61,6 +61,6 @@ class PersistHandler RC seek(uint64_t offset); private: - std::string file_name_; - int file_desc_ = -1; + string file_name_; + int file_desc_ = -1; }; diff --git a/src/observer/storage/record/record.h b/src/observer/storage/record/record.h index 942ec6427..6749804dd 100644 --- a/src/observer/storage/record/record.h +++ b/src/observer/storage/record/record.h @@ -14,14 +14,14 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include #include -#include #include "common/log/log.h" #include "common/rc.h" #include "common/types.h" +#include "common/lang/vector.h" +#include "common/lang/sstream.h" +#include "common/lang/limits.h" #include "storage/field/field_meta.h" #include "storage/index/index_meta.h" @@ -39,9 +39,9 @@ struct RID RID() = default; RID(const PageNum _page_num, const SlotNum _slot_num) : page_num(_page_num), slot_num(_slot_num) {} - const std::string to_string() const + const string to_string() const { - std::stringstream ss; + stringstream ss; ss << "PageNum:" << page_num << ", SlotNum:" << slot_num; return ss.str(); } @@ -77,7 +77,7 @@ struct RID */ static RID *max() { - static RID rid{std::numeric_limits::max(), std::numeric_limits::max()}; + static RID rid{numeric_limits::max(), numeric_limits::max()}; return &rid; } }; @@ -135,7 +135,7 @@ class Record new (this) Record(other); return *this; } - + this->rid_ = other.rid_; memcpy(data_, other.data_, other.len_); return *this; } @@ -199,6 +199,33 @@ class Record return RC::SUCCESS; } + RC new_record(int len) + { + ASSERT(len!= 0, "the len of data should not be 0"); + char *tmp = (char *)malloc(len); + if (nullptr == tmp) { + LOG_WARN("failed to allocate memory. size=%d", len); + return RC::NOMEM; + } + set_data_owner(tmp, len); + return RC::SUCCESS; + } + + RC set_field(int field_offset, int field_len, char *data) + { + if (!owner_) { + LOG_ERROR("cannot set field when record does not own the memory"); + return RC::INTERNAL; + } + if (field_offset + field_len > len_) { + LOG_ERROR("invalid offset or length. offset=%d, length=%d, total length=%d", field_offset, field_len, len_); + return RC::INVALID_ARGUMENT; + } + + memcpy(data_ + field_offset, data, field_len); + return RC::SUCCESS; + } + char *data() { return this->data_; } const char *data() const { return this->data_; } int len() const { return this->len_; } diff --git a/src/observer/storage/record/record_log.cpp b/src/observer/storage/record/record_log.cpp index f2587e43e..3e8a15640 100644 --- a/src/observer/storage/record/record_log.cpp +++ b/src/observer/storage/record/record_log.cpp @@ -12,10 +12,9 @@ See the Mulan PSL v2 for more details. */ // Created by Wangyunlai on 2024/02/02. // -#include - #include "storage/record/record_log.h" #include "common/log/log.h" +#include "common/lang/sstream.h" #include "common/lang/defer.h" #include "storage/clog/log_handler.h" #include "storage/record/record.h" @@ -26,14 +25,13 @@ See the Mulan PSL v2 for more details. */ #include "storage/buffer/frame.h" #include "storage/record/record_log.h" -using namespace std; using namespace common; // class RecordOperation string RecordOperation::to_string() const { - std::string ret = std::to_string(type_id()) + ":"; + string ret = std::to_string(type_id()) + ":"; switch (type_) { case Type::INIT_PAGE: return ret + "INIT_PAGE"; case Type::INSERT: return ret + "INSERT"; @@ -74,28 +72,37 @@ string RecordLogHeader::to_string() const //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // class RecordLogHandler -RC RecordLogHandler::init(LogHandler &log_handler, int32_t buffer_pool_id, int32_t record_size) +RC RecordLogHandler::init( + LogHandler &log_handler, int32_t buffer_pool_id, int32_t record_size, StorageFormat storage_format) { RC rc = RC::SUCCESS; log_handler_ = &log_handler; buffer_pool_id_ = buffer_pool_id; record_size_ = record_size; + storage_format_ = storage_format; return rc; } -RC RecordLogHandler::init_new_page(Frame *frame, PageNum page_num) +// data is the column index in page +RC RecordLogHandler::init_new_page(Frame *frame, PageNum page_num, span data) { - RecordLogHeader header; - header.buffer_pool_id = buffer_pool_id_; - header.operation_type = RecordOperation(RecordOperation::Type::INIT_PAGE).type_id(); - header.page_num = page_num; - header.record_size = record_size_; + const int log_payload_size = RecordLogHeader::SIZE + data.size(); + vector log_payload(log_payload_size); + RecordLogHeader *header = reinterpret_cast(log_payload.data()); + header->buffer_pool_id = buffer_pool_id_; + header->operation_type = RecordOperation(RecordOperation::Type::INIT_PAGE).type_id(); + header->page_num = page_num; + header->record_size = record_size_; + header->storage_format = static_cast(storage_format_); + header->column_num = data.size() / sizeof(int); + if (data.size() > 0) { + memcpy(log_payload.data() + RecordLogHeader::SIZE, data.data(), data.size()); + } LSN lsn = 0; - RC rc = log_handler_->append( - lsn, LogModule::Id::RECORD_MANAGER, span(reinterpret_cast(&header), RecordLogHeader::SIZE)); + RC rc = log_handler_->append(lsn, LogModule::Id::RECORD_MANAGER, std::move(log_payload)); if (OB_SUCC(rc) && lsn > 0) { frame->set_lsn(lsn); } @@ -104,17 +111,18 @@ RC RecordLogHandler::init_new_page(Frame *frame, PageNum page_num) RC RecordLogHandler::insert_record(Frame *frame, const RID &rid, const char *record) { - const int log_payload_size = RecordLogHeader::SIZE + record_size_; - vector log_payload(log_payload_size); - RecordLogHeader *header = reinterpret_cast(log_payload.data()); - header->buffer_pool_id = buffer_pool_id_; - header->operation_type = RecordOperation(RecordOperation::Type::INSERT).type_id(); - header->page_num = rid.page_num; - header->slot_num = rid.slot_num; + const int log_payload_size = RecordLogHeader::SIZE + record_size_; + vector log_payload(log_payload_size); + RecordLogHeader *header = reinterpret_cast(log_payload.data()); + header->buffer_pool_id = buffer_pool_id_; + header->operation_type = RecordOperation(RecordOperation::Type::INSERT).type_id(); + header->page_num = rid.page_num; + header->slot_num = rid.slot_num; + header->storage_format = static_cast(storage_format_); memcpy(log_payload.data() + RecordLogHeader::SIZE, record, record_size_); LSN lsn = 0; - RC rc = log_handler_->append(lsn, LogModule::Id::RECORD_MANAGER, std::move(log_payload)); + RC rc = log_handler_->append(lsn, LogModule::Id::RECORD_MANAGER, std::move(log_payload)); if (OB_SUCC(rc) && lsn > 0) { frame->set_lsn(lsn); } @@ -123,17 +131,18 @@ RC RecordLogHandler::insert_record(Frame *frame, const RID &rid, const char *rec RC RecordLogHandler::update_record(Frame *frame, const RID &rid, const char *record) { - const int log_payload_size = RecordLogHeader::SIZE + record_size_; - vector log_payload(log_payload_size); - RecordLogHeader *header = reinterpret_cast(log_payload.data()); - header->buffer_pool_id = buffer_pool_id_; - header->operation_type = RecordOperation(RecordOperation::Type::UPDATE).type_id(); - header->page_num = rid.page_num; - header->slot_num = rid.slot_num; + const int log_payload_size = RecordLogHeader::SIZE + record_size_; + vector log_payload(log_payload_size); + RecordLogHeader *header = reinterpret_cast(log_payload.data()); + header->buffer_pool_id = buffer_pool_id_; + header->operation_type = RecordOperation(RecordOperation::Type::UPDATE).type_id(); + header->page_num = rid.page_num; + header->slot_num = rid.slot_num; + header->storage_format = static_cast(storage_format_); memcpy(log_payload.data() + RecordLogHeader::SIZE, record, record_size_); LSN lsn = 0; - RC rc = log_handler_->append(lsn, LogModule::Id::RECORD_MANAGER, std::move(log_payload)); + RC rc = log_handler_->append(lsn, LogModule::Id::RECORD_MANAGER, std::move(log_payload)); if (OB_SUCC(rc) && lsn > 0) { frame->set_lsn(lsn); } @@ -147,10 +156,12 @@ RC RecordLogHandler::delete_record(Frame *frame, const RID &rid) header.operation_type = RecordOperation(RecordOperation::Type::DELETE).type_id(); header.page_num = rid.page_num; header.slot_num = rid.slot_num; + header.storage_format = static_cast(storage_format_); LSN lsn = 0; - RC rc = log_handler_->append( - lsn, LogModule::Id::RECORD_MANAGER, span(reinterpret_cast(&header), RecordLogHeader::SIZE)); + RC rc = log_handler_->append(lsn, + LogModule::Id::RECORD_MANAGER, + span(reinterpret_cast(&header), RecordLogHeader::SIZE)); if (OB_SUCC(rc) && lsn > 0) { frame->set_lsn(lsn); } @@ -179,8 +190,8 @@ RC RecordLogReplayer::replay(const LogEntry &entry) auto log_header = reinterpret_cast(entry.data()); DiskBufferPool *buffer_pool = nullptr; - Frame *frame = nullptr; - RC rc = bpm_.get_buffer_pool(log_header->buffer_pool_id, buffer_pool); + Frame *frame = nullptr; + RC rc = bpm_.get_buffer_pool(log_header->buffer_pool_id, buffer_pool); if (OB_FAIL(rc)) { LOG_WARN("fail to get buffer pool. buffer pool id=%d, rc=%s", log_header->buffer_pool_id, strrc(rc)); return rc; @@ -232,11 +243,16 @@ RC RecordLogReplayer::replay(const LogEntry &entry) RC RecordLogReplayer::replay_init_page(DiskBufferPool &buffer_pool, const RecordLogHeader &log_header) { - VacuousLogHandler vacuous_log_handler; - RecordPageHandler record_page_handler; - - RC rc = record_page_handler.init_empty_page( - buffer_pool, vacuous_log_handler, log_header.page_num, log_header.record_size); + VacuousLogHandler vacuous_log_handler; + unique_ptr record_page_handler( + RecordPageHandler::create(StorageFormat(log_header.storage_format))); + + RC rc = record_page_handler->init_empty_page(buffer_pool, + vacuous_log_handler, + log_header.page_num, + log_header.record_size, + log_header.column_num, + log_header.data); if (OB_FAIL(rc)) { LOG_WARN("fail to init record page handler. page num=%d, rc=%s", log_header.page_num, strrc(rc)); return rc; @@ -247,10 +263,11 @@ RC RecordLogReplayer::replay_init_page(DiskBufferPool &buffer_pool, const Record RC RecordLogReplayer::replay_insert(DiskBufferPool &buffer_pool, const RecordLogHeader &log_header) { - VacuousLogHandler vacuous_log_handler; - RecordPageHandler record_page_handler; + VacuousLogHandler vacuous_log_handler; + unique_ptr record_page_handler( + RecordPageHandler::create(StorageFormat(log_header.storage_format))); - RC rc = record_page_handler.init(buffer_pool, vacuous_log_handler, log_header.page_num, ReadWriteMode::READ_WRITE); + RC rc = record_page_handler->init(buffer_pool, vacuous_log_handler, log_header.page_num, ReadWriteMode::READ_WRITE); if (OB_FAIL(rc)) { LOG_WARN("fail to init record page handler. page num=%d, rc=%s", log_header.page_num, strrc(rc)); return rc; @@ -258,7 +275,7 @@ RC RecordLogReplayer::replay_insert(DiskBufferPool &buffer_pool, const RecordLog const char *record = log_header.data; RID rid(log_header.page_num, log_header.slot_num); - rc = record_page_handler.insert_record(record, &rid); + rc = record_page_handler->insert_record(record, &rid); if (OB_FAIL(rc)) { LOG_WARN("fail to recover insert record. page num=%d, slot num=%d, rc=%s", log_header.page_num, log_header.slot_num, strrc(rc)); @@ -270,17 +287,18 @@ RC RecordLogReplayer::replay_insert(DiskBufferPool &buffer_pool, const RecordLog RC RecordLogReplayer::replay_delete(DiskBufferPool &buffer_pool, const RecordLogHeader &log_header) { - VacuousLogHandler vacuous_log_handler; - RecordPageHandler record_page_handler; + VacuousLogHandler vacuous_log_handler; + unique_ptr record_page_handler( + RecordPageHandler::create(StorageFormat(log_header.storage_format))); - RC rc = record_page_handler.init(buffer_pool, vacuous_log_handler, log_header.page_num, ReadWriteMode::READ_WRITE); + RC rc = record_page_handler->init(buffer_pool, vacuous_log_handler, log_header.page_num, ReadWriteMode::READ_WRITE); if (OB_FAIL(rc)) { LOG_WARN("fail to init record page handler. page num=%d, rc=%s", log_header.page_num, strrc(rc)); return rc; } RID rid(log_header.page_num, log_header.slot_num); - rc = record_page_handler.delete_record(&rid); + rc = record_page_handler->delete_record(&rid); if (OB_FAIL(rc)) { LOG_WARN("fail to recover delete record. page num=%d, slot num=%d, rc=%s", log_header.page_num, log_header.slot_num, strrc(rc)); @@ -292,17 +310,17 @@ RC RecordLogReplayer::replay_delete(DiskBufferPool &buffer_pool, const RecordLog RC RecordLogReplayer::replay_update(DiskBufferPool &buffer_pool, const RecordLogHeader &header) { - VacuousLogHandler vacuous_log_handler; - RecordPageHandler record_page_handler; + VacuousLogHandler vacuous_log_handler; + unique_ptr record_page_handler(RecordPageHandler::create(StorageFormat(header.storage_format))); - RC rc = record_page_handler.init(buffer_pool, vacuous_log_handler, header.page_num, ReadWriteMode::READ_WRITE); + RC rc = record_page_handler->init(buffer_pool, vacuous_log_handler, header.page_num, ReadWriteMode::READ_WRITE); if (OB_FAIL(rc)) { LOG_WARN("fail to init record page handler. page num=%d, rc=%s", header.page_num, strrc(rc)); return rc; } RID rid(header.page_num, header.slot_num); - rc = record_page_handler.update_record(rid, header.data); + rc = record_page_handler->update_record(rid, header.data); if (OB_FAIL(rc)) { LOG_WARN("fail to recover update record. page num=%d, slot num=%d, rc=%s", header.page_num, header.slot_num, strrc(rc)); diff --git a/src/observer/storage/record/record_log.h b/src/observer/storage/record/record_log.h index 3f65de84a..516328a80 100644 --- a/src/observer/storage/record/record_log.h +++ b/src/observer/storage/record/record_log.h @@ -15,10 +15,13 @@ See the Mulan PSL v2 for more details. */ #pragma once #include -#include + #include "common/types.h" #include "common/rc.h" +#include "common/lang/span.h" +#include "common/lang/string.h" #include "storage/clog/log_replayer.h" +#include "sql/parser/parse_defs.h" struct RID; class LogHandler; @@ -49,7 +52,7 @@ class RecordOperation Type type() const { return type_; } int32_t type_id() const { return static_cast(type_); } - std::string to_string() const; + string to_string() const; private: Type type_; @@ -60,6 +63,8 @@ struct RecordLogHeader int32_t buffer_pool_id; int32_t operation_type; PageNum page_num; + int32_t storage_format; + int32_t column_num; union { SlotNum slot_num; @@ -68,7 +73,7 @@ struct RecordLogHeader char data[0]; - std::string to_string() const; + string to_string() const; static const int32_t SIZE; }; @@ -79,7 +84,7 @@ class RecordLogHandler final RecordLogHandler() = default; ~RecordLogHandler() = default; - RC init(LogHandler &log_handler, int32_t buffer_pool_id, int32_t record_size); + RC init(LogHandler &log_handler, int32_t buffer_pool_id, int32_t record_size, StorageFormat storage_format); /** * @brief 初始化一个新的页面 @@ -92,8 +97,9 @@ class RecordLogHandler final * 或者页面在访问时会出现异常。 * @param frame 页帧 * @param page_num 页面编号 + * @param data 页面数据目前主要是 `column index` */ - RC init_new_page(Frame *frame, PageNum page_num); + RC init_new_page(Frame *frame, PageNum page_num, span data); /** * @brief 插入一条记录 @@ -120,9 +126,10 @@ class RecordLogHandler final RC update_record(Frame *frame, const RID &rid, const char *record); private: - LogHandler *log_handler_ = nullptr; - int32_t buffer_pool_id_ = -1; - int32_t record_size_ = -1; + LogHandler *log_handler_ = nullptr; + int32_t buffer_pool_id_ = -1; + int32_t record_size_ = -1; + StorageFormat storage_format_ = StorageFormat::ROW_FORMAT; }; /** diff --git a/src/observer/storage/record/record_manager.cpp b/src/observer/storage/record/record_manager.cpp index 1faf6dd07..00f403d9e 100644 --- a/src/observer/storage/record/record_manager.cpp +++ b/src/observer/storage/record/record_manager.cpp @@ -17,11 +17,17 @@ See the Mulan PSL v2 for more details. */ #include "storage/trx/trx.h" #include "storage/clog/log_handler.h" -using namespace std; using namespace common; static constexpr int PAGE_HEADER_SIZE = (sizeof(PageHeader)); - +RecordPageHandler *RecordPageHandler::create(StorageFormat format) +{ + if (format == StorageFormat::ROW_FORMAT) { + return new RowRecordPageHandler(); + } else { + return new PaxRecordPageHandler(); + } +} /** * @brief 8字节对齐 * 注: ceiling(a / b) = floor((a + b - 1) / b) @@ -35,12 +41,14 @@ int align8(int size) { return (size + 7) & ~7; } * * @param page_size 页面的大小 * @param record_size 记录的大小 + * @param fixed_size 除 PAGE_HEADER 外,页面中其余固定长度占用,目前为PAX存储格式中的 + * 列偏移索引大小(column index)。 */ -int page_record_capacity(int page_size, int record_size) +int page_record_capacity(int page_size, int record_size, int fixed_size) { // (record_capacity * record_size) + record_capacity/8 + 1 <= (page_size - fix_size) // ==> record_capacity = ((page_size - fix_size) - 1) / (record_size + 0.125) - return (int)((page_size - PAGE_HEADER_SIZE - 1) / (record_size + 0.125)); + return (int)((page_size - PAGE_HEADER_SIZE - fixed_size - 1) / (record_size + 0.125)); } /** @@ -54,11 +62,8 @@ int page_bitmap_size(int record_capacity) { return (record_capacity + 7) / 8; } string PageHeader::to_string() const { stringstream ss; - ss << "record_num:" << record_num - << ",record_real_size:" << record_real_size - << ",record_size:" << record_size - << ",record_capacity:" << record_capacity - << ",first_record_offset:" << first_record_offset; + ss << "record_num:" << record_num << ",column_num:" << column_num << ",record_real_size:" << record_real_size + << ",record_size:" << record_size << ",record_capacity:" << record_capacity << ",data_offset:" << data_offset; return ss.str(); } @@ -66,11 +71,11 @@ string PageHeader::to_string() const RecordPageIterator::RecordPageIterator() {} RecordPageIterator::~RecordPageIterator() {} -void RecordPageIterator::init(RecordPageHandler &record_page_handler, SlotNum start_slot_num /*=0*/) +void RecordPageIterator::init(RecordPageHandler *record_page_handler, SlotNum start_slot_num /*=0*/) { - record_page_handler_ = &record_page_handler; - page_num_ = record_page_handler.get_page_num(); - bitmap_.init(record_page_handler.bitmap_, record_page_handler.page_header_->record_capacity); + record_page_handler_ = record_page_handler; + page_num_ = record_page_handler->get_page_num(); + bitmap_.init(record_page_handler->bitmap_, record_page_handler->page_header_->record_capacity); next_slot_num_ = bitmap_.next_setted_bit(start_slot_num); } @@ -78,8 +83,7 @@ bool RecordPageIterator::has_next() { return -1 != next_slot_num_; } RC RecordPageIterator::next(Record &record) { - record.set_rid(page_num_, next_slot_num_); - record.set_data(record_page_handler_->get_record_data(record.rid().slot_num)); + record_page_handler_->get_record(RID(page_num_, next_slot_num_), record); if (next_slot_num_ >= 0) { next_slot_num_ = bitmap_.next_setted_bit(next_slot_num_ + 1); @@ -116,12 +120,12 @@ RC RecordPageHandler::init(DiskBufferPool &buffer_pool, LogHandler &log_handler, frame_->write_latch(); } disk_buffer_pool_ = &buffer_pool; - - rw_mode_ = mode; - page_header_ = (PageHeader *)(data); - bitmap_ = data + PAGE_HEADER_SIZE; - (void)log_handler_.init(log_handler, buffer_pool.id(), page_header_->record_real_size); + rw_mode_ = mode; + page_header_ = (PageHeader *)(data); + bitmap_ = data + PAGE_HEADER_SIZE; + + (void)log_handler_.init(log_handler, buffer_pool.id(), page_header_->record_real_size, storage_format_); LOG_TRACE("Successfully init page_num %d.", page_num); return ret; @@ -144,7 +148,7 @@ RC RecordPageHandler::recover_init(DiskBufferPool &buffer_pool, PageNum page_num frame_->write_latch(); disk_buffer_pool_ = &buffer_pool; - rw_mode_ = ReadWriteMode::READ_WRITE; + rw_mode_ = ReadWriteMode::READ_WRITE; page_header_ = (PageHeader *)(data); bitmap_ = data + PAGE_HEADER_SIZE; @@ -155,7 +159,7 @@ RC RecordPageHandler::recover_init(DiskBufferPool &buffer_pool, PageNum page_num } RC RecordPageHandler::init_empty_page( - DiskBufferPool &buffer_pool, LogHandler &log_handler, PageNum page_num, int record_size) + DiskBufferPool &buffer_pool, LogHandler &log_handler, PageNum page_num, int record_size, TableMeta *table_meta) { RC rc = init(buffer_pool, log_handler, page_num, ReadWriteMode::READ_WRITE); if (OB_FAIL(rc)) { @@ -163,27 +167,86 @@ RC RecordPageHandler::init_empty_page( return rc; } - (void)log_handler_.init(log_handler, buffer_pool.id(), record_size); + (void)log_handler_.init(log_handler, buffer_pool.id(), record_size, storage_format_); + + int column_num = 0; + // only pax format need column index + if (table_meta != nullptr && storage_format_ == StorageFormat::PAX_FORMAT) { + column_num = table_meta->field_num(); + } + page_header_->record_num = 0; + page_header_->column_num = column_num; + page_header_->record_real_size = record_size; + page_header_->record_size = align8(record_size); + page_header_->record_capacity = page_record_capacity( + BP_PAGE_DATA_SIZE, page_header_->record_size, column_num * sizeof(int) /* other fixed size*/); + page_header_->col_idx_offset = align8(PAGE_HEADER_SIZE + page_bitmap_size(page_header_->record_capacity)); + page_header_->data_offset = align8(PAGE_HEADER_SIZE + page_bitmap_size(page_header_->record_capacity)) + + column_num * sizeof(int) /* column index*/; + this->fix_record_capacity(); + ASSERT(page_header_->data_offset + page_header_->record_capacity * page_header_->record_size + <= BP_PAGE_DATA_SIZE, + "Record overflow the page size"); + + bitmap_ = frame_->data() + PAGE_HEADER_SIZE; + memset(bitmap_, 0, page_bitmap_size(page_header_->record_capacity)); + // column_index[i] store the end offset of column `i` or the start offset of column `i+1` + int *column_index = reinterpret_cast(frame_->data() + page_header_->col_idx_offset); + for (int i = 0; i < column_num; ++i) { + ASSERT(i == table_meta->field(i)->field_id(), "i should be the col_id of fields[i]"); + if (i == 0) { + column_index[i] = table_meta->field(i)->len() * page_header_->record_capacity; + } else { + column_index[i] = table_meta->field(i)->len() * page_header_->record_capacity + column_index[i - 1]; + } + } - rc = log_handler_.init_new_page(frame_, page_num); + rc = log_handler_.init_new_page(frame_, page_num, span((const char *)column_index, column_num * sizeof(int))); if (OB_FAIL(rc)) { LOG_ERROR("Failed to init empty page: write log failed. page_num:record_size %d:%d. rc=%s", page_num, record_size, strrc(rc)); return rc; } - page_header_->record_num = 0; - page_header_->record_real_size = record_size; - page_header_->record_size = align8(record_size); - page_header_->record_capacity = page_record_capacity(BP_PAGE_DATA_SIZE, page_header_->record_size); - page_header_->first_record_offset = align8(PAGE_HEADER_SIZE + page_bitmap_size(page_header_->record_capacity)); + return RC::SUCCESS; +} + +RC RecordPageHandler::init_empty_page(DiskBufferPool &buffer_pool, LogHandler &log_handler, PageNum page_num, + int record_size, int column_num, const char *col_idx_data) +{ + RC rc = init(buffer_pool, log_handler, page_num, ReadWriteMode::READ_WRITE); + if (OB_FAIL(rc)) { + LOG_ERROR("Failed to init empty page page_num:record_size %d:%d. rc=%s", page_num, record_size, strrc(rc)); + return rc; + } + + (void)log_handler_.init(log_handler, buffer_pool.id(), record_size, storage_format_); + + page_header_->record_num = 0; + page_header_->column_num = column_num; + page_header_->record_real_size = record_size; + page_header_->record_size = align8(record_size); + page_header_->record_capacity = + page_record_capacity(BP_PAGE_DATA_SIZE, page_header_->record_size, page_header_->column_num * sizeof(int)); + page_header_->col_idx_offset = align8(PAGE_HEADER_SIZE + page_bitmap_size(page_header_->record_capacity)); + page_header_->data_offset = align8(PAGE_HEADER_SIZE + page_bitmap_size(page_header_->record_capacity)) + + column_num * sizeof(int) /* column index*/; this->fix_record_capacity(); - ASSERT(page_header_->first_record_offset + page_header_->record_capacity * page_header_->record_size + ASSERT(page_header_->data_offset + page_header_->record_capacity * page_header_->record_size <= BP_PAGE_DATA_SIZE, "Record overflow the page size"); bitmap_ = frame_->data() + PAGE_HEADER_SIZE; memset(bitmap_, 0, page_bitmap_size(page_header_->record_capacity)); + // column_index[i] store the end offset of column `i` the start offset of column `i+1` + int *column_index = reinterpret_cast(frame_->data() + page_header_->col_idx_offset); + memcpy(column_index, col_idx_data, column_num * sizeof(int)); + + if (OB_FAIL(rc)) { + LOG_ERROR("Failed to init empty page: write log failed. page_num:record_size %d:%d. rc=%s", + page_num, record_size, strrc(rc)); + return rc; + } return RC::SUCCESS; } @@ -203,7 +266,7 @@ RC RecordPageHandler::cleanup() return RC::SUCCESS; } -RC RecordPageHandler::insert_record(const char *data, RID *rid) +RC RowRecordPageHandler::insert_record(const char *data, RID *rid) { ASSERT(rw_mode_ != ReadWriteMode::READ_ONLY, "cannot insert record into page while the page is readonly"); @@ -240,7 +303,7 @@ RC RecordPageHandler::insert_record(const char *data, RID *rid) return RC::SUCCESS; } -RC RecordPageHandler::recover_insert_record(const char *data, const RID &rid) +RC RowRecordPageHandler::recover_insert_record(const char *data, const RID &rid) { if (rid.slot_num >= page_header_->record_capacity) { LOG_WARN("slot_num illegal, slot_num(%d) > record_capacity(%d).", rid.slot_num, page_header_->record_capacity); @@ -263,17 +326,11 @@ RC RecordPageHandler::recover_insert_record(const char *data, const RID &rid) return RC::SUCCESS; } -RC RecordPageHandler::delete_record(const RID *rid) +RC RowRecordPageHandler::delete_record(const RID *rid) { ASSERT(rw_mode_ != ReadWriteMode::READ_ONLY, "cannot delete record from page while the page is readonly"); - if (rid->slot_num >= page_header_->record_capacity) { - LOG_ERROR("Invalid slot_num %d, exceed page's record capacity, frame=%s, page_header=%s", - rid->slot_num, frame_->to_string().c_str(), page_header_->to_string().c_str()); - return RC::INVALID_ARGUMENT; - } - Bitmap bitmap(bitmap_, page_header_->record_capacity); if (bitmap.get_bit(rid->slot_num)) { bitmap.clear_bit(rid->slot_num); @@ -293,7 +350,7 @@ RC RecordPageHandler::delete_record(const RID *rid) } } -RC RecordPageHandler::update_record(const RID &rid, const char *data) +RC RowRecordPageHandler::update_record(const RID &rid, const char *data) { ASSERT(rw_mode_ != ReadWriteMode::READ_ONLY, "cannot delete record from page while the page is readonly"); @@ -328,7 +385,7 @@ RC RecordPageHandler::update_record(const RID &rid, const char *data) } } -RC RecordPageHandler::get_record(const RID &rid, Record &record) +RC RowRecordPageHandler::get_record(const RID &rid, Record &record) { if (rid.slot_num >= page_header_->record_capacity) { LOG_ERROR("Invalid slot_num %d, exceed page's record capacity, frame=%s, page_header=%s", @@ -357,11 +414,74 @@ PageNum RecordPageHandler::get_page_num() const bool RecordPageHandler::is_full() const { return page_header_->record_num >= page_header_->record_capacity; } +RC PaxRecordPageHandler::insert_record(const char *data, RID *rid) +{ + // your code here + exit(-1); +} + +RC PaxRecordPageHandler::delete_record(const RID *rid) +{ + ASSERT(rw_mode_ != ReadWriteMode::READ_ONLY, + "cannot delete record from page while the page is readonly"); + + Bitmap bitmap(bitmap_, page_header_->record_capacity); + if (bitmap.get_bit(rid->slot_num)) { + bitmap.clear_bit(rid->slot_num); + page_header_->record_num--; + frame_->mark_dirty(); + + RC rc = log_handler_.delete_record(frame_, *rid); + if (OB_FAIL(rc)) { + LOG_ERROR("Failed to delete record. page_num %d:%d. rc=%s", disk_buffer_pool_->file_desc(), frame_->page_num(), strrc(rc)); + // return rc; // ignore errors + } + + return RC::SUCCESS; + } else { + LOG_DEBUG("Invalid slot_num %d, slot is empty, page_num %d.", rid->slot_num, frame_->page_num()); + return RC::RECORD_NOT_EXIST; + } +} + +RC PaxRecordPageHandler::get_record(const RID &rid, Record &record) +{ + // your code here + exit(-1); +} + +// TODO: specify the column_ids that chunk needed. currenly we get all columns +RC PaxRecordPageHandler::get_chunk(Chunk &chunk) +{ + // your code here + exit(-1); +} + +char *PaxRecordPageHandler::get_field_data(SlotNum slot_num, int col_id) +{ + int *col_idx = reinterpret_cast(frame_->data() + page_header_->col_idx_offset); + if (col_id == 0) { + return frame_->data() + page_header_->data_offset + (get_field_len(col_id) * slot_num); + } else { + return frame_->data() + page_header_->data_offset + col_idx[col_id - 1] + (get_field_len(col_id) * slot_num); + } +} + +int PaxRecordPageHandler::get_field_len(int col_id) +{ + int *col_idx = reinterpret_cast(frame_->data() + page_header_->col_idx_offset); + if (col_id == 0) { + return col_idx[col_id] / page_header_->record_capacity; + } else { + return (col_idx[col_id] - col_idx[col_id - 1]) / page_header_->record_capacity; + } +} + //////////////////////////////////////////////////////////////////////////////// RecordFileHandler::~RecordFileHandler() { this->close(); } -RC RecordFileHandler::init(DiskBufferPool &buffer_pool, LogHandler &log_handler) +RC RecordFileHandler::init(DiskBufferPool &buffer_pool, LogHandler &log_handler, TableMeta *table_meta) { if (disk_buffer_pool_ != nullptr) { LOG_ERROR("record file handler has been openned."); @@ -369,7 +489,8 @@ RC RecordFileHandler::init(DiskBufferPool &buffer_pool, LogHandler &log_handler) } disk_buffer_pool_ = &buffer_pool; - log_handler_ = &log_handler; + log_handler_ = &log_handler; + table_meta_ = table_meta; RC rc = init_free_pages(); @@ -382,7 +503,8 @@ void RecordFileHandler::close() if (disk_buffer_pool_ != nullptr) { free_pages_.clear(); disk_buffer_pool_ = nullptr; - log_handler_ = nullptr; + log_handler_ = nullptr; + table_meta_ = nullptr; } } @@ -396,22 +518,22 @@ RC RecordFileHandler::init_free_pages() BufferPoolIterator bp_iterator; bp_iterator.init(*disk_buffer_pool_, 1); - RecordPageHandler record_page_handler; - PageNum current_page_num = 0; + unique_ptr record_page_handler(RecordPageHandler::create(storage_format_)); + PageNum current_page_num = 0; while (bp_iterator.has_next()) { current_page_num = bp_iterator.next(); - rc = record_page_handler.init(*disk_buffer_pool_, *log_handler_, current_page_num, ReadWriteMode::READ_ONLY); + rc = record_page_handler->init(*disk_buffer_pool_, *log_handler_, current_page_num, ReadWriteMode::READ_ONLY); if (rc != RC::SUCCESS) { LOG_WARN("failed to init record page handler. page num=%d, rc=%d:%s", current_page_num, rc, strrc(rc)); return rc; } - if (!record_page_handler.is_full()) { + if (!record_page_handler->is_full()) { free_pages_.insert(current_page_num); } - record_page_handler.cleanup(); + record_page_handler->cleanup(); } LOG_INFO("record file handler init free pages done. free page num=%d, rc=%s", free_pages_.size(), strrc(rc)); return rc; @@ -421,9 +543,9 @@ RC RecordFileHandler::insert_record(const char *data, int record_size, RID *rid) { RC ret = RC::SUCCESS; - RecordPageHandler record_page_handler; - bool page_found = false; - PageNum current_page_num = 0; + unique_ptr record_page_handler(RecordPageHandler::create(storage_format_)); + bool page_found = false; + PageNum current_page_num = 0; // 当前要访问free_pages对象,所以需要加锁。在非并发编译模式下,不需要考虑这个锁 lock_.lock(); @@ -432,18 +554,18 @@ RC RecordFileHandler::insert_record(const char *data, int record_size, RID *rid) while (!free_pages_.empty()) { current_page_num = *free_pages_.begin(); - ret = record_page_handler.init(*disk_buffer_pool_, *log_handler_, current_page_num, ReadWriteMode::READ_WRITE); + ret = record_page_handler->init(*disk_buffer_pool_, *log_handler_, current_page_num, ReadWriteMode::READ_WRITE); if (OB_FAIL(ret)) { lock_.unlock(); LOG_WARN("failed to init record page handler. page num=%d, rc=%d:%s", current_page_num, ret, strrc(ret)); return ret; } - if (!record_page_handler.is_full()) { + if (!record_page_handler->is_full()) { page_found = true; break; } - record_page_handler.cleanup(); + record_page_handler->cleanup(); free_pages_.erase(free_pages_.begin()); } lock_.unlock(); // 如果找到了一个有效的页面,那么此时已经拿到了页面的写锁 @@ -458,7 +580,8 @@ RC RecordFileHandler::insert_record(const char *data, int record_size, RID *rid) current_page_num = frame->page_num(); - ret = record_page_handler.init_empty_page(*disk_buffer_pool_, *log_handler_, current_page_num, record_size); + ret = record_page_handler->init_empty_page( + *disk_buffer_pool_, *log_handler_, current_page_num, record_size, table_meta_); if (OB_FAIL(ret)) { frame->unpin(); LOG_ERROR("Failed to init empty page. ret:%d", ret); @@ -479,41 +602,41 @@ RC RecordFileHandler::insert_record(const char *data, int record_size, RID *rid) } // 找到空闲位置 - return record_page_handler.insert_record(data, rid); + return record_page_handler->insert_record(data, rid); } RC RecordFileHandler::recover_insert_record(const char *data, int record_size, const RID &rid) { RC ret = RC::SUCCESS; - RecordPageHandler record_page_handler; + unique_ptr record_page_handler(RecordPageHandler::create(storage_format_)); - ret = record_page_handler.recover_init(*disk_buffer_pool_, rid.page_num); + ret = record_page_handler->recover_init(*disk_buffer_pool_, rid.page_num); if (OB_FAIL(ret)) { LOG_WARN("failed to init record page handler. page num=%d, rc=%s", rid.page_num, strrc(ret)); return ret; } - return record_page_handler.recover_insert_record(data, rid); + return record_page_handler->recover_insert_record(data, rid); } RC RecordFileHandler::delete_record(const RID *rid) { RC rc = RC::SUCCESS; - RecordPageHandler page_handler; + unique_ptr record_page_handler(RecordPageHandler::create(storage_format_)); - rc = page_handler.init(*disk_buffer_pool_, *log_handler_, rid->page_num, ReadWriteMode::READ_WRITE); + rc = record_page_handler->init(*disk_buffer_pool_, *log_handler_, rid->page_num, ReadWriteMode::READ_WRITE); if (OB_FAIL(rc)) { LOG_ERROR("Failed to init record page handler.page number=%d. rc=%s", rid->page_num, strrc(rc)); return rc; } - rc = page_handler.delete_record(rid); + rc = record_page_handler->delete_record(rid); // 📢 这里注意要清理掉资源,否则会与insert_record中的加锁顺序冲突而可能出现死锁 // delete record的加锁逻辑是拿到页面锁,删除指定记录,然后加上和释放record manager锁 // insert record是加上 record manager锁,然后拿到指定页面锁再释放record manager锁 - page_handler.cleanup(); + record_page_handler->cleanup(); if (OB_SUCC(rc)) { // 因为这里已经释放了页面锁,并发时,其它线程可能又把该页面填满了,那就不应该再放入 free_pages_ // 中。但是这里可以不关心,因为在查找空闲页面时,会自动过滤掉已经满的页面 @@ -527,16 +650,16 @@ RC RecordFileHandler::delete_record(const RID *rid) RC RecordFileHandler::get_record(const RID &rid, Record &record) { - RecordPageHandler page_handler; + unique_ptr page_handler(RecordPageHandler::create(storage_format_)); - RC rc = page_handler.init(*disk_buffer_pool_, *log_handler_, rid.page_num, ReadWriteMode::READ_WRITE); + RC rc = page_handler->init(*disk_buffer_pool_, *log_handler_, rid.page_num, ReadWriteMode::READ_WRITE); if (OB_FAIL(rc)) { LOG_ERROR("Failed to init record page handler.page number=%d", rid.page_num); return rc; } Record inplace_record; - rc = page_handler.get_record(rid, inplace_record); + rc = page_handler->get_record(rid, inplace_record); if (OB_FAIL(rc)) { LOG_WARN("failed to get record from record page handle. rid=%s, rc=%s", rid.to_string().c_str(), strrc(rc)); return rc; @@ -547,18 +670,18 @@ RC RecordFileHandler::get_record(const RID &rid, Record &record) return rc; } -RC RecordFileHandler::visit_record(const RID &rid, std::function updater) +RC RecordFileHandler::visit_record(const RID &rid, function updater) { - RecordPageHandler page_handler; + unique_ptr page_handler(RecordPageHandler::create(storage_format_)); - RC rc = page_handler.init(*disk_buffer_pool_, *log_handler_, rid.page_num, ReadWriteMode::READ_WRITE); + RC rc = page_handler->init(*disk_buffer_pool_, *log_handler_, rid.page_num, ReadWriteMode::READ_WRITE); if (OB_FAIL(rc)) { LOG_ERROR("Failed to init record page handler.page number=%d", rid.page_num); return rc; } Record inplace_record; - rc = page_handler.get_record(rid, inplace_record); + rc = page_handler->get_record(rid, inplace_record); if (OB_FAIL(rc)) { LOG_WARN("failed to get record from record page handle. rid=%s, rc=%s", rid.to_string().c_str(), strrc(rc)); return rc; @@ -572,7 +695,7 @@ RC RecordFileHandler::visit_record(const RID &rid, std::function bool updated = updater(record); if (updated) { - rc = page_handler.update_record(rid, record.data()); + rc = page_handler->update_record(rid, record.data()); } return rc; } @@ -581,8 +704,8 @@ RC RecordFileHandler::visit_record(const RID &rid, std::function RecordFileScanner::~RecordFileScanner() { close_scan(); } -RC RecordFileScanner::open_scan( - Table *table, DiskBufferPool &buffer_pool, Trx *trx, LogHandler &log_handler, ReadWriteMode mode, ConditionFilter *condition_filter) +RC RecordFileScanner::open_scan(Table *table, DiskBufferPool &buffer_pool, Trx *trx, LogHandler &log_handler, + ReadWriteMode mode, ConditionFilter *condition_filter) { close_scan(); @@ -598,6 +721,11 @@ RC RecordFileScanner::open_scan( return rc; } condition_filter_ = condition_filter; + if (table == nullptr || table->table_meta().storage_format() == StorageFormat::ROW_FORMAT) { + record_page_handler_ = new RowRecordPageHandler(); + } else { + record_page_handler_ = new PaxRecordPageHandler(); + } return rc; } @@ -625,8 +753,8 @@ RC RecordFileScanner::fetch_next_record() // 上个页面遍历完了,或者还没有开始遍历某个页面,那么就从一个新的页面开始遍历查找 while (bp_iterator_.has_next()) { PageNum page_num = bp_iterator_.next(); - record_page_handler_.cleanup(); - rc = record_page_handler_.init(*disk_buffer_pool_, *log_handler_, page_num, rw_mode_); + record_page_handler_->cleanup(); + rc = record_page_handler_->init(*disk_buffer_pool_, *log_handler_, page_num, rw_mode_); if (OB_FAIL(rc)) { LOG_WARN("failed to init record page handler. page_num=%d, rc=%s", page_num, strrc(rc)); return rc; @@ -644,7 +772,7 @@ RC RecordFileScanner::fetch_next_record() // 所有的页面都遍历完了,没有数据了 next_record_.rid().slot_num = -1; - record_page_handler_.cleanup(); + record_page_handler_->cleanup(); return RC::RECORD_EOF; } @@ -657,7 +785,7 @@ RC RecordFileScanner::fetch_next_record_in_page() while (record_page_iterator_.has_next()) { rc = record_page_iterator_.next(next_record_); if (rc != RC::SUCCESS) { - const auto page_num = record_page_handler_.get_page_num(); + const auto page_num = record_page_handler_->get_page_num(); LOG_TRACE("failed to get next record from page. page_num=%d, rc=%s", page_num, strrc(rc)); return rc; } @@ -696,8 +824,11 @@ RC RecordFileScanner::close_scan() if (condition_filter_ != nullptr) { condition_filter_ = nullptr; } - - record_page_handler_.cleanup(); + if (record_page_handler_ != nullptr) { + record_page_handler_->cleanup(); + delete record_page_handler_; + record_page_handler_ = nullptr; + } return RC::SUCCESS; } @@ -719,5 +850,73 @@ RC RecordFileScanner::update_current(const Record &record) return RC::INVALID_ARGUMENT; } - return record_page_handler_.update_record(record.rid(), record.data()); + return record_page_handler_->update_record(record.rid(), record.data()); +} + +ChunkFileScanner::~ChunkFileScanner() { close_scan(); } + +RC ChunkFileScanner::close_scan() +{ + if (disk_buffer_pool_ != nullptr) { + disk_buffer_pool_ = nullptr; + } + + if (record_page_handler_ != nullptr) { + record_page_handler_->cleanup(); + delete record_page_handler_; + record_page_handler_ = nullptr; + } + + return RC::SUCCESS; +} + +RC ChunkFileScanner::open_scan_chunk( + Table *table, DiskBufferPool &buffer_pool, LogHandler &log_handler, ReadWriteMode mode) +{ + close_scan(); + + table_ = table; + disk_buffer_pool_ = &buffer_pool; + log_handler_ = &log_handler; + rw_mode_ = mode; + + RC rc = bp_iterator_.init(buffer_pool, 1); + if (rc != RC::SUCCESS) { + LOG_WARN("failed to init bp iterator. rc=%d:%s", rc, strrc(rc)); + return rc; + } + if (table == nullptr || table->table_meta().storage_format() == StorageFormat::ROW_FORMAT) { + record_page_handler_ = new RowRecordPageHandler(); + } else { + record_page_handler_ = new PaxRecordPageHandler(); + } + + return rc; +} + +RC ChunkFileScanner::next_chunk(Chunk &chunk) +{ + RC rc = RC::SUCCESS; + + while (bp_iterator_.has_next()) { + PageNum page_num = bp_iterator_.next(); + record_page_handler_->cleanup(); + rc = record_page_handler_->init(*disk_buffer_pool_, *log_handler_, page_num, rw_mode_); + if (OB_FAIL(rc)) { + LOG_WARN("failed to init record page handler. page_num=%d, rc=%s", page_num, strrc(rc)); + return rc; + } + rc = record_page_handler_->get_chunk(chunk); + if (rc == RC::SUCCESS) { + return rc; + } else if (rc == RC::RECORD_EOF) { + break; + } else { + LOG_WARN("failed to get chunk from page. page_num=%d, rc=%s", page_num, strrc(rc)); + return rc; + } + } + + record_page_handler_->cleanup(); + return RC::RECORD_EOF; } diff --git a/src/observer/storage/record/record_manager.h b/src/observer/storage/record/record_manager.h index c82a4d054..421e6e76c 100644 --- a/src/observer/storage/record/record_manager.h +++ b/src/observer/storage/record/record_manager.h @@ -14,11 +14,10 @@ See the Mulan PSL v2 for more details. */ // #pragma once -#include -#include - #include "common/lang/bitmap.h" +#include "common/lang/sstream.h" #include "storage/buffer/disk_buffer_pool.h" +#include "storage/common/chunk.h" #include "storage/record/record.h" #include "storage/record/record_log.h" #include "common/types.h" @@ -66,13 +65,15 @@ class Table; */ struct PageHeader { - int32_t record_num; ///< 当前页面记录的个数 - int32_t record_real_size; ///< 每条记录的实际大小 - int32_t record_size; ///< 每条记录占用实际空间大小(可能对齐) - int32_t record_capacity; ///< 最大记录个数 - int32_t first_record_offset; ///< 第一条记录的偏移量 - - std::string to_string() const; + int32_t record_num; ///< 当前页面记录的个数 + int32_t column_num; ///< 当前页面记录所包含的列数 + int32_t record_real_size; ///< 每条记录的实际大小 + int32_t record_size; ///< 每条记录占用实际空间大小(可能对齐) + int32_t record_capacity; ///< 最大记录个数 + int32_t col_idx_offset; ///< 列索引偏移量 + int32_t data_offset; ///< 第一条记录的偏移量 + + string to_string() const; }; /** @@ -91,7 +92,7 @@ class RecordPageIterator * @param record_page_handler 负责某个页面上记录增删改查的对象 * @param start_slot_num 从哪个记录开始扫描,默认是0 */ - void init(RecordPageHandler &record_page_handler, SlotNum start_slot_num = 0); + void init(RecordPageHandler *record_page_handler, SlotNum start_slot_num = 0); /** * @brief 判断是否有下一个记录 @@ -120,18 +121,13 @@ class RecordPageIterator /** * @brief 负责处理一个页面中各种操作,比如插入记录、删除记录或者查找记录 * @ingroup RecordManager - * @details 当前定长记录模式下每个页面的组织大概是这样的: - * @code - * | PageHeader | record allocate bitmap | - * |------------|------------------------| - * | record1 | record2 | ..... | recordN | - * @endcode */ class RecordPageHandler { public: - RecordPageHandler() = default; - ~RecordPageHandler(); + RecordPageHandler(StorageFormat storage_format) : storage_format_(storage_format) {} + virtual ~RecordPageHandler(); + static RecordPageHandler *create(StorageFormat format); /** * @brief 初始化 @@ -156,8 +152,21 @@ class RecordPageHandler * @param buffer_pool 关联某个文件时,都通过buffer pool来做读写文件 * @param page_num 当前处理哪个页面 * @param record_size 每个记录的大小 + * @param table_meta 表的元数据 + */ + RC init_empty_page( + DiskBufferPool &buffer_pool, LogHandler &log_handler, PageNum page_num, int record_size, TableMeta *table_meta); + + /** + * @brief 对一个新的页面做初始化,初始化关于该页面记录信息的页头PageHeader,该函数用于日志回放时。 + * @param buffer_pool 关联某个文件时,都通过buffer pool来做读写文件 + * @param page_num 当前处理哪个页面 + * @param record_size 每个记录的大小 + * @param col_num 表中包含的列数 + * @param col_idx_data 列索引数据 */ - RC init_empty_page(DiskBufferPool &buffer_pool, LogHandler &log_handler, PageNum page_num, int record_size); + RC init_empty_page(DiskBufferPool &buffer_pool, LogHandler &log_handler, PageNum page_num, int record_size, + int col_num, const char *col_idx_data); /** * @brief 操作结束后做的清理工作,比如释放页面、解锁 @@ -170,7 +179,7 @@ class RecordPageHandler * @param data 要插入的记录 * @param rid 如果插入成功,通过这个参数返回插入的位置 */ - RC insert_record(const char *data, RID *rid); + virtual RC insert_record(const char *data, RID *rid) { return RC::UNIMPLENMENT; } /** * @brief 数据库恢复时,在指定位置插入数据 @@ -178,28 +187,36 @@ class RecordPageHandler * @param data 要插入的数据行 * @param rid 插入的位置 */ - RC recover_insert_record(const char *data, const RID &rid); + virtual RC recover_insert_record(const char *data, const RID &rid) { return RC::UNIMPLENMENT; } /** * @brief 删除指定的记录 * * @param rid 要删除的记录标识 */ - RC delete_record(const RID *rid); + virtual RC delete_record(const RID *rid) { return RC::UNIMPLENMENT; } /** * @brief * */ - RC update_record(const RID &rid, const char *data); + virtual RC update_record(const RID &rid, const char *data) { return RC::UNIMPLENMENT; } /** * @brief 获取指定位置的记录数据 * * @param rid 指定的位置 - * @param inplace_record 返回指定的数据。这里不会将数据复制出来,而是使用指针,所以调用者必须保证数据使用期间受到保护 + * @param record 获取到的记录结果 + */ + virtual RC get_record(const RID &rid, Record &record) { return RC::UNIMPLENMENT; } + + /** + * @brief 获取整个页面中指定列的所有记录。 + * + * @param chunk 由 chunk.column(i).col_id() 指定列。 + * 只需由 PaxRecordPageHandler 实现。 */ - RC get_record(const RID &rid, Record &inplace_record); + virtual RC get_chunk(Chunk &chunk) { return RC::UNIMPLENMENT; } /** * @brief 返回该记录页的页号 @@ -220,8 +237,7 @@ class RecordPageHandler */ void fix_record_capacity() { - int32_t last_record_offset = - page_header_->first_record_offset + page_header_->record_capacity * page_header_->record_size; + int32_t last_record_offset = page_header_->data_offset + page_header_->record_capacity * page_header_->record_size; while (last_record_offset > BP_PAGE_DATA_SIZE) { page_header_->record_capacity -= 1; last_record_offset -= page_header_->record_size; @@ -235,7 +251,7 @@ class RecordPageHandler */ char *get_record_data(SlotNum slot_num) { - return frame_->data() + page_header_->first_record_offset + (page_header_->record_size * slot_num); + return frame_->data() + page_header_->data_offset + (page_header_->record_size * slot_num); } protected: @@ -245,11 +261,94 @@ class RecordPageHandler ReadWriteMode rw_mode_ = ReadWriteMode::READ_WRITE; ///< 当前的操作是否都是只读的 PageHeader *page_header_ = nullptr; ///< 当前页面上页面头 char *bitmap_ = nullptr; ///< 当前页面上record分配状态信息bitmap内存起始位置 + StorageFormat storage_format_; -private: +protected: friend class RecordPageIterator; }; +/** + * @brief 负责处理行存页面中各种操作 + * @ingroup RecordManager + * @details 行存格式实现,当前定长记录模式下每个页面的组织大概是这样的: + * @code + * | PageHeader | record allocate bitmap | + * |------------|------------------------| + * | record1 | record2 | ..... | recordN | + * @endcode + */ +class RowRecordPageHandler : public RecordPageHandler +{ +public: + RowRecordPageHandler() : RecordPageHandler(StorageFormat::ROW_FORMAT) {} + + virtual RC insert_record(const char *data, RID *rid) override; + + virtual RC recover_insert_record(const char *data, const RID &rid) override; + + virtual RC delete_record(const RID *rid) override; + + virtual RC update_record(const RID &rid, const char *data) override; + + /** + * @brief 获取指定位置的记录数据 + * + * @param rid 指定的位置 + * @param record 返回指定的数据。这里不会将数据复制出来,而是使用指针,所以调用者必须保证数据使用期间受到保护 + */ + virtual RC get_record(const RID &rid, Record &record) override; +}; + +/** + * @brief 负责处理 PAX 存储格式的页面中各种操作 + * @ingroup RecordManager + * @details PAX 格式实现,当前定长记录模式下每个页面的组织大概是这样的: + * @code + * | PageHeader | record allocate bitmap | column index | + * |------------|------------------------| ------------- | + * | column1 | column2 | ..................... | columnN | + * @endcode + * 更多细节可参考:docs/design/miniob-pax-storage.md + */ +class PaxRecordPageHandler : public RecordPageHandler +{ +public: + PaxRecordPageHandler() : RecordPageHandler(StorageFormat::PAX_FORMAT) {} + + /** + * @brief 插入一条记录 + * + * @param data 要插入的记录 + * @param rid 如果插入成功,通过这个参数返回插入的位置 + * 注意:需要将record 按列拆分,在 Page 内按 PAX 格式存储。 + */ + virtual RC insert_record(const char *data, RID *rid) override; + + virtual RC delete_record(const RID *rid) override; + + /** + * @brief 获取指定位置的记录数据 + * + * @param rid 指定的位置 + * @param record 返回指定的数据。 + * 注意:需要将列数据组装成 Record 并返回。 + */ + virtual RC get_record(const RID &rid, Record &record) override; + + /** + * @brief 以 Chunk 格式获取整个页面中指定列的所有记录。 + * + * @param chunk 由 chunk.column(i).col_id() 指定列。 + */ + virtual RC get_chunk(Chunk &chunk) override; + +private: + // get the field data by `slot_num` and `column id` + char *get_field_data(SlotNum slot_num, int col_id); + + // get the field length by `column id`, all columns are fixed length. + int get_field_len(int col_id); +}; /** * @brief 管理整个文件中记录的增删改查 * @ingroup RecordManager @@ -258,7 +357,7 @@ class RecordPageHandler class RecordFileHandler { public: - RecordFileHandler() = default; + RecordFileHandler(StorageFormat storage_format) : storage_format_(storage_format){}; ~RecordFileHandler(); /** @@ -266,7 +365,7 @@ class RecordFileHandler * * @param buffer_pool 当前操作的是哪个文件 */ - RC init(DiskBufferPool &buffer_pool, LogHandler &log_handler); + RC init(DiskBufferPool &buffer_pool, LogHandler &log_handler, TableMeta *table_meta); /** * @brief 关闭,做一些资源清理的工作 @@ -300,7 +399,7 @@ class RecordFileHandler RC get_record(const RID &rid, Record &record); - RC visit_record(const RID &rid, std::function updater); + RC visit_record(const RID &rid, function updater); private: /** @@ -309,10 +408,12 @@ class RecordFileHandler RC init_free_pages(); private: - DiskBufferPool *disk_buffer_pool_ = nullptr; - LogHandler *log_handler_ = nullptr; ///< 记录日志的处理器 - std::unordered_set free_pages_; ///< 没有填充满的页面集合 - common::Mutex lock_; ///< 当编译时增加-DCONCURRENCY=ON 选项时,才会真正的支持并发 + DiskBufferPool *disk_buffer_pool_ = nullptr; + LogHandler *log_handler_ = nullptr; ///< 记录日志的处理器 + unordered_set free_pages_; ///< 没有填充满的页面集合 + common::Mutex lock_; ///< 当编译时增加-DCONCURRENCY=ON 选项时,才会真正的支持并发 + StorageFormat storage_format_; + TableMeta *table_meta_; }; /** @@ -372,9 +473,44 @@ class RecordFileScanner LogHandler *log_handler_ = nullptr; ReadWriteMode rw_mode_ = ReadWriteMode::READ_WRITE; ///< 遍历出来的数据,是否可能对它做修改 - BufferPoolIterator bp_iterator_; ///< 遍历buffer pool的所有页面 - ConditionFilter *condition_filter_ = nullptr; ///< 过滤record - RecordPageHandler record_page_handler_; ///< 处理文件某页面的记录 - RecordPageIterator record_page_iterator_; ///< 遍历某个页面上的所有record - Record next_record_; ///< 获取的记录放在这里缓存起来 + BufferPoolIterator bp_iterator_; ///< 遍历buffer pool的所有页面 + ConditionFilter *condition_filter_ = nullptr; ///< 过滤record + RecordPageHandler *record_page_handler_ = nullptr; ///< 处理文件某页面的记录 + RecordPageIterator record_page_iterator_; ///< 遍历某个页面上的所有record + Record next_record_; ///< 获取的记录放在这里缓存起来 +}; + +/** + * @brief 遍历某个文件中所有记录,每次返回一个 Chunk + * @ingroup RecordManager + * @details 遍历所有的页面,每次以 Chunk 格式返回一个页面内的所有数据 + */ +class ChunkFileScanner +{ +public: + ChunkFileScanner() = default; + ~ChunkFileScanner(); + + // TODO: not support filter and transaction + RC open_scan_chunk(Table *table, DiskBufferPool &buffer_pool, LogHandler &log_handler, ReadWriteMode mode); + + /** + * @brief 关闭一个文件扫描,释放相应的资源 + */ + RC close_scan(); + + /** + * @brief 每次调用获取一个页面中的所有记录。 + */ + RC next_chunk(Chunk &chunk); + +private: + Table *table_ = nullptr; ///< 当前遍历的是哪张表。 + + DiskBufferPool *disk_buffer_pool_ = nullptr; ///< 当前访问的文件 + LogHandler *log_handler_ = nullptr; + ReadWriteMode rw_mode_ = ReadWriteMode::READ_WRITE; ///< 遍历出来的数据,是否可能对它做修改 + + BufferPoolIterator bp_iterator_; ///< 遍历buffer pool的所有页面 + RecordPageHandler *record_page_handler_ = nullptr; ///< 处理文件某页面的记录 }; diff --git a/src/observer/storage/table/table.cpp b/src/observer/storage/table/table.cpp index 835934cb0..910dd259e 100644 --- a/src/observer/storage/table/table.cpp +++ b/src/observer/storage/table/table.cpp @@ -12,12 +12,13 @@ See the Mulan PSL v2 for more details. */ // Created by Meiyi & Wangyunlai on 2021/5/13. // -#include #include #include #include "common/defs.h" #include "common/lang/string.h" +#include "common/lang/span.h" +#include "common/lang/algorithm.h" #include "common/log/log.h" #include "common/global_context.h" #include "storage/db/db.h" @@ -28,7 +29,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/index/index.h" #include "storage/record/record_manager.h" #include "storage/table/table.h" -#include "storage/table/table_meta.h" #include "storage/trx/trx.h" Table::~Table() @@ -43,7 +43,7 @@ Table::~Table() data_buffer_pool_ = nullptr; } - for (std::vector::iterator it = indexes_.begin(); it != indexes_.end(); ++it) { + for (vector::iterator it = indexes_.begin(); it != indexes_.end(); ++it) { Index *index = *it; delete index; } @@ -53,7 +53,7 @@ Table::~Table() } RC Table::create(Db *db, int32_t table_id, const char *path, const char *name, const char *base_dir, - span attributes) + span attributes, StorageFormat storage_format) { if (table_id < 0) { LOG_WARN("invalid table id. table_id=%d, table_name=%s", table_id, name); @@ -88,14 +88,14 @@ RC Table::create(Db *db, int32_t table_id, const char *path, const char *name, c close(fd); // 创建文件 - const std::vector *trx_fields = db->trx_kit().trx_fields(); - if ((rc = table_meta_.init(table_id, name, trx_fields, attributes)) != RC::SUCCESS) { + const vector *trx_fields = db->trx_kit().trx_fields(); + if ((rc = table_meta_.init(table_id, name, trx_fields, attributes, storage_format)) != RC::SUCCESS) { LOG_ERROR("Failed to init table meta. name:%s, ret:%d", name, rc); return rc; // delete table file } - std::fstream fs; - fs.open(path, std::ios_base::out | std::ios_base::binary); + fstream fs; + fs.open(path, ios_base::out | ios_base::binary); if (!fs.is_open()) { LOG_ERROR("Failed to open file for write. file name=%s, errmsg=%s", path, strerror(errno)); return RC::IOERR_OPEN; @@ -105,10 +105,10 @@ RC Table::create(Db *db, int32_t table_id, const char *path, const char *name, c table_meta_.serialize(fs); fs.close(); - db_ = db; + db_ = db; base_dir_ = base_dir; - std::string data_file = table_data_file(base_dir, name); + string data_file = table_data_file(base_dir, name); BufferPoolManager &bpm = db->buffer_pool_manager(); rc = bpm.create_file(data_file.c_str()); if (rc != RC::SUCCESS) { @@ -130,9 +130,9 @@ RC Table::create(Db *db, int32_t table_id, const char *path, const char *name, c RC Table::open(Db *db, const char *meta_file, const char *base_dir) { // 加载元数据文件 - std::fstream fs; - std::string meta_file_path = std::string(base_dir) + common::FILE_PATH_SPLIT_STR + meta_file; - fs.open(meta_file_path, std::ios_base::in | std::ios_base::binary); + fstream fs; + string meta_file_path = string(base_dir) + common::FILE_PATH_SPLIT_STR + meta_file; + fs.open(meta_file_path, ios_base::in | ios_base::binary); if (!fs.is_open()) { LOG_ERROR("Failed to open meta file for read. file name=%s, errmsg=%s", meta_file_path.c_str(), strerror(errno)); return RC::IOERR_OPEN; @@ -144,7 +144,7 @@ RC Table::open(Db *db, const char *meta_file, const char *base_dir) } fs.close(); - db_ = db; + db_ = db; base_dir_ = base_dir; // 加载数据文件 @@ -168,7 +168,7 @@ RC Table::open(Db *db, const char *meta_file, const char *base_dir) } BplusTreeIndex *index = new BplusTreeIndex(); - std::string index_file = table_index_file(base_dir, name(), index_meta->name()); + string index_file = table_index_file(base_dir, name(), index_meta->name()); rc = index->open(this, index_file.c_str(), *index_meta, *field_meta); if (rc != RC::SUCCESS) { @@ -210,7 +210,7 @@ RC Table::insert_record(Record &record) return rc; } -RC Table::visit_record(const RID &rid, std::function visitor) +RC Table::visit_record(const RID &rid, function visitor) { return record_handler_->visit_record(rid, visitor); } @@ -282,7 +282,7 @@ RC Table::make_record(int value_num, const Value *values, Record &record) const FieldMeta *field = table_meta_.field(i + normal_field_start_index); const Value &value = values[i]; size_t copy_len = field->len(); - if (field->type() == CHARS) { + if (field->type() == AttrType::CHARS) { const size_t data_len = value.length(); if (copy_len > data_len) { copy_len = data_len + 1; @@ -297,18 +297,18 @@ RC Table::make_record(int value_num, const Value *values, Record &record) RC Table::init_record_handler(const char *base_dir) { - std::string data_file = table_data_file(base_dir, table_meta_.name()); + string data_file = table_data_file(base_dir, table_meta_.name()); BufferPoolManager &bpm = db_->buffer_pool_manager(); - RC rc = bpm.open_file(db_->log_handler(), data_file.c_str(), data_buffer_pool_); + RC rc = bpm.open_file(db_->log_handler(), data_file.c_str(), data_buffer_pool_); if (rc != RC::SUCCESS) { LOG_ERROR("Failed to open disk buffer pool for file:%s. rc=%d:%s", data_file.c_str(), rc, strrc(rc)); return rc; } - record_handler_ = new RecordFileHandler(); + record_handler_ = new RecordFileHandler(table_meta_.storage_format()); - rc = record_handler_->init(*data_buffer_pool_, db_->log_handler()); + rc = record_handler_->init(*data_buffer_pool_, db_->log_handler(), &table_meta_); if (rc != RC::SUCCESS) { LOG_ERROR("Failed to init record handler. rc=%s", strrc(rc)); data_buffer_pool_->close_file(); @@ -330,6 +330,15 @@ RC Table::get_record_scanner(RecordFileScanner &scanner, Trx *trx, ReadWriteMode return rc; } +RC Table::get_chunk_scanner(ChunkFileScanner &scanner, Trx *trx, ReadWriteMode mode) +{ + RC rc = scanner.open_scan_chunk(this, *data_buffer_pool_, db_->log_handler(), mode); + if (rc != RC::SUCCESS) { + LOG_ERROR("failed to open scanner. rc=%s", strrc(rc)); + } + return rc; +} + RC Table::create_index(Trx *trx, const FieldMeta *field_meta, const char *index_name) { if (common::is_blank(index_name) || nullptr == field_meta) { @@ -348,7 +357,7 @@ RC Table::create_index(Trx *trx, const FieldMeta *field_meta, const char *index_ // 创建索引相关数据 BplusTreeIndex *index = new BplusTreeIndex(); - std::string index_file = table_index_file(base_dir_.c_str(), name(), index_name); + string index_file = table_index_file(base_dir_.c_str(), name(), index_name); rc = index->create(this, index_file.c_str(), new_index_meta, *field_meta); if (rc != RC::SUCCESS) { @@ -398,9 +407,9 @@ RC Table::create_index(Trx *trx, const FieldMeta *field_meta, const char *index_ /// 内存中有一份元数据,磁盘文件也有一份元数据。修改磁盘文件时,先创建一个临时文件,写入完成后再rename为正式文件 /// 这样可以防止文件内容不完整 // 创建元数据临时文件 - std::string tmp_file = table_meta_file(base_dir_.c_str(), name()) + ".tmp"; - std::fstream fs; - fs.open(tmp_file, std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + string tmp_file = table_meta_file(base_dir_.c_str(), name()) + ".tmp"; + fstream fs; + fs.open(tmp_file, ios_base::out | ios_base::binary | ios_base::trunc); if (!fs.is_open()) { LOG_ERROR("Failed to open file for write. file name=%s, errmsg=%s", tmp_file.c_str(), strerror(errno)); return RC::IOERR_OPEN; // 创建索引中途出错,要做还原操作 @@ -412,7 +421,7 @@ RC Table::create_index(Trx *trx, const FieldMeta *field_meta, const char *index_ fs.close(); // 覆盖原始元数据文件 - std::string meta_file = table_meta_file(base_dir_.c_str(), name()); + string meta_file = table_meta_file(base_dir_.c_str(), name()); int ret = rename(tmp_file.c_str(), meta_file.c_str()); if (ret != 0) { @@ -430,7 +439,7 @@ RC Table::create_index(Trx *trx, const FieldMeta *field_meta, const char *index_ RC Table::delete_record(const RID &rid) { - RC rc = RC::SUCCESS; + RC rc = RC::SUCCESS; Record record; rc = get_record(rid, record); if (OB_FAIL(rc)) { diff --git a/src/observer/storage/table/table.h b/src/observer/storage/table/table.h index 66b64d29e..5fc14021c 100644 --- a/src/observer/storage/table/table.h +++ b/src/observer/storage/table/table.h @@ -14,17 +14,17 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include - #include "storage/table/table_meta.h" #include "common/types.h" +#include "common/lang/span.h" +#include "common/lang/functional.h" struct RID; class Record; class DiskBufferPool; class RecordFileHandler; class RecordFileScanner; +class ChunkFileScanner; class ConditionFilter; class DefaultConditionFilter; class Index; @@ -52,7 +52,7 @@ class Table * @param attributes 字段 */ RC create(Db *db, int32_t table_id, const char *path, const char *name, const char *base_dir, - std::span attributes); + span attributes, StorageFormat storage_format); /** * 打开一个表 @@ -87,6 +87,8 @@ class Table RC get_record_scanner(RecordFileScanner &scanner, Trx *trx, ReadWriteMode mode); + RC get_chunk_scanner(ChunkFileScanner &scanner, Trx *trx, ReadWriteMode mode); + RecordFileHandler *record_handler() const { return record_handler_; } /** @@ -96,7 +98,7 @@ class Table * @param visitor * @return RC */ - RC visit_record(const RID &rid, std::function visitor); + RC visit_record(const RID &rid, function visitor); public: int32_t table_id() const { return table_meta_.table_id(); } @@ -120,10 +122,10 @@ class Table Index *find_index_by_field(const char *field_name) const; private: - Db *db_ = nullptr; - std::string base_dir_; - TableMeta table_meta_; - DiskBufferPool *data_buffer_pool_ = nullptr; /// 数据文件关联的buffer pool - RecordFileHandler *record_handler_ = nullptr; /// 记录操作 - std::vector indexes_; + Db *db_ = nullptr; + string base_dir_; + TableMeta table_meta_; + DiskBufferPool *data_buffer_pool_ = nullptr; /// 数据文件关联的buffer pool + RecordFileHandler *record_handler_ = nullptr; /// 记录操作 + vector indexes_; }; diff --git a/src/observer/storage/table/table_meta.cpp b/src/observer/storage/table/table_meta.cpp index e7f00eee1..f87cb894b 100644 --- a/src/observer/storage/table/table_meta.cpp +++ b/src/observer/storage/table/table_meta.cpp @@ -12,19 +12,17 @@ See the Mulan PSL v2 for more details. */ // Created by Meiyi & Wangyunlai on 2021/5/12. // -#include - #include "common/lang/string.h" +#include "common/lang/algorithm.h" #include "common/log/log.h" #include "common/global_context.h" #include "storage/table/table_meta.h" #include "storage/trx/trx.h" #include "json/json.h" -using namespace std; - static const Json::StaticString FIELD_TABLE_ID("table_id"); static const Json::StaticString FIELD_TABLE_NAME("table_name"); +static const Json::StaticString FIELD_STORAGE_FORMAT("storage_format"); static const Json::StaticString FIELD_FIELDS("fields"); static const Json::StaticString FIELD_INDEXES("indexes"); @@ -33,6 +31,7 @@ TableMeta::TableMeta(const TableMeta &other) name_(other.name_), fields_(other.fields_), indexes_(other.indexes_), + storage_format_(other.storage_format_), record_size_(other.record_size_) {} @@ -45,7 +44,7 @@ void TableMeta::swap(TableMeta &other) noexcept } RC TableMeta::init(int32_t table_id, const char *name, const std::vector *trx_fields, - span attributes) + span attributes, StorageFormat storage_format) { if (common::is_blank(name)) { LOG_ERROR("Name cannot be empty"); @@ -66,10 +65,9 @@ RC TableMeta::init(int32_t table_id, const char *name, const std::vectorsize()); - for (size_t i = 0; i < trx_fields->size(); i++) { const FieldMeta &field_meta = (*trx_fields)[i]; - fields_[i] = FieldMeta(field_meta.name(), field_meta.type(), field_offset, field_meta.len(), false /*visible*/); + fields_[i] = FieldMeta(field_meta.name(), field_meta.type(), field_offset, field_meta.len(), false /*visible*/, field_meta.field_id()); field_offset += field_meta.len(); } @@ -80,9 +78,9 @@ RC TableMeta::init(int32_t table_id, const char *name, const std::vector(storage_format_); Json::Value fields_value; for (const FieldMeta &field : fields_) { @@ -236,6 +235,14 @@ int TableMeta::deserialize(std::istream &is) return -1; } + const Json::Value &storage_format_value = table_value[FIELD_STORAGE_FORMAT]; + if (!table_id_value.isInt()) { + LOG_ERROR("Invalid storage format. json value=%s", storage_format_value.toStyledString().c_str()); + return -1; + } + + int32_t storage_format = storage_format_value.asInt(); + RC rc = RC::SUCCESS; int field_num = fields_value.size(); @@ -255,6 +262,7 @@ int TableMeta::deserialize(std::istream &is) std::sort(fields.begin(), fields.end(), comparator); table_id_ = table_id; + storage_format_ = static_cast(storage_format); name_.swap(table_name); fields_.swap(fields); record_size_ = fields_.back().offset() + fields_.back().len() - fields_.begin()->offset(); diff --git a/src/observer/storage/table/table_meta.h b/src/observer/storage/table/table_meta.h index 7a9e1d93e..6f438018a 100644 --- a/src/observer/storage/table/table_meta.h +++ b/src/observer/storage/table/table_meta.h @@ -20,6 +20,7 @@ See the Mulan PSL v2 for more details. */ #include "common/lang/serializable.h" #include "common/rc.h" +#include "common/types.h" #include "storage/field/field_meta.h" #include "storage/index/index_meta.h" @@ -30,27 +31,28 @@ See the Mulan PSL v2 for more details. */ class TableMeta : public common::Serializable { public: - TableMeta() = default; - ~TableMeta() = default; + TableMeta() = default; + virtual ~TableMeta() = default; TableMeta(const TableMeta &other); void swap(TableMeta &other) noexcept; RC init(int32_t table_id, const char *name, const std::vector *trx_fields, - std::span attributes); + std::span attributes, StorageFormat storage_format); RC add_index(const IndexMeta &index); public: - int32_t table_id() const { return table_id_; } - const char *name() const; - const FieldMeta *trx_field() const; - const FieldMeta *field(int index) const; - const FieldMeta *field(const char *name) const; - const FieldMeta *find_field_by_offset(int offset) const; - auto field_metas() const -> const std::vector *{ return &fields_; } - auto trx_fields() const -> std::span; + int32_t table_id() const { return table_id_; } + const char *name() const; + const FieldMeta *trx_field() const; + const FieldMeta *field(int index) const; + const FieldMeta *field(const char *name) const; + const FieldMeta *find_field_by_offset(int offset) const; + auto field_metas() const -> const std::vector *{ return &fields_; } + auto trx_fields() const -> std::span; + const StorageFormat storage_format() const { return storage_format_; } int field_num() const; // sys field included int sys_field_num() const; @@ -75,6 +77,7 @@ class TableMeta : public common::Serializable std::vector trx_fields_; std::vector fields_; // 包含sys_fields std::vector indexes_; + StorageFormat storage_format_; int record_size_ = 0; }; diff --git a/src/observer/storage/trx/mvcc_trx.cpp b/src/observer/storage/trx/mvcc_trx.cpp index dd03ec2d4..eefaef43c 100644 --- a/src/observer/storage/trx/mvcc_trx.cpp +++ b/src/observer/storage/trx/mvcc_trx.cpp @@ -12,15 +12,11 @@ See the Mulan PSL v2 for more details. */ // Created by Wangyunlai on 2023/04/24. // -#include -#include - #include "storage/trx/mvcc_trx.h" #include "storage/db/db.h" #include "storage/field/field.h" #include "storage/trx/mvcc_trx_log.h" - -using namespace std; +#include "common/lang/algorithm.h" MvccTrxKit::~MvccTrxKit() { @@ -36,8 +32,9 @@ RC MvccTrxKit::init() { // 事务使用一些特殊的字段,放到每行记录中,表示行记录的可见性。 fields_ = vector{ - FieldMeta("__trx_xid_begin", AttrType::INTS, 0 /*attr_offset*/, 4 /*attr_len*/, false /*visible*/), - FieldMeta("__trx_xid_end", AttrType::INTS, 0 /*attr_offset*/, 4 /*attr_len*/, false /*visible*/)}; + // field_id in trx fields is invisible. + FieldMeta("__trx_xid_begin", AttrType::INTS, 0 /*attr_offset*/, 4 /*attr_len*/, false /*visible*/, -1/*field_id*/), + FieldMeta("__trx_xid_end", AttrType::INTS, 0 /*attr_offset*/, 4 /*attr_len*/, false /*visible*/, -2/*field_id*/)}; LOG_INFO("init mvcc trx kit done."); return RC::SUCCESS; @@ -102,7 +99,7 @@ Trx *MvccTrxKit::find_trx(int32_t trx_id) return nullptr; } -void MvccTrxKit::all_trxes(std::vector &trxes) +void MvccTrxKit::all_trxes(vector &trxes) { lock_.lock(); trxes = trxes_; diff --git a/src/observer/storage/trx/mvcc_trx.h b/src/observer/storage/trx/mvcc_trx.h index b6e7430d9..4a16ba0ae 100644 --- a/src/observer/storage/trx/mvcc_trx.h +++ b/src/observer/storage/trx/mvcc_trx.h @@ -14,8 +14,7 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include - +#include "common/lang/vector.h" #include "storage/trx/trx.h" #include "storage/trx/mvcc_trx_log.h" @@ -29,8 +28,8 @@ class MvccTrxKit : public TrxKit MvccTrxKit() = default; virtual ~MvccTrxKit(); - RC init() override; - const std::vector *trx_fields() const override; + RC init() override; + const vector *trx_fields() const override; Trx *create_trx(LogHandler &log_handler) override; Trx *create_trx(LogHandler &log_handler, int32_t trx_id) override; @@ -41,7 +40,7 @@ class MvccTrxKit : public TrxKit * @details 当前仅在recover场景下使用 */ Trx *find_trx(int32_t trx_id) override; - void all_trxes(std::vector &trxes) override; + void all_trxes(vector &trxes) override; LogReplayer *create_log_replayer(Db &db, LogHandler &log_handler) override; @@ -52,12 +51,12 @@ class MvccTrxKit : public TrxKit int32_t max_trx_id() const; private: - std::vector fields_; // 存储事务数据需要用到的字段元数据,所有表结构都需要带的 + vector fields_; // 存储事务数据需要用到的字段元数据,所有表结构都需要带的 - std::atomic current_trx_id_{0}; + atomic current_trx_id_{0}; - common::Mutex lock_; - std::vector trxes_; + common::Mutex lock_; + vector trxes_; }; /** @@ -105,11 +104,11 @@ class MvccTrx : public Trx void trx_fields(Table *table, Field &begin_xid_field, Field &end_xid_field) const; private: - static const int32_t MAX_TRX_ID = std::numeric_limits::max(); + static const int32_t MAX_TRX_ID = numeric_limits::max(); private: - // using OperationSet = std::unordered_set; - using OperationSet = std::vector; + // using OperationSet = unordered_set; + using OperationSet = vector; MvccTrxKit &trx_kit_; MvccTrxLogHandler log_handler_; diff --git a/src/observer/storage/trx/mvcc_trx_log.cpp b/src/observer/storage/trx/mvcc_trx_log.cpp index bf578d8c3..abb4e7ba1 100644 --- a/src/observer/storage/trx/mvcc_trx_log.cpp +++ b/src/observer/storage/trx/mvcc_trx_log.cpp @@ -21,7 +21,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/db/db.h" #include "storage/table/table.h" -using namespace std; using namespace common; string MvccTrxLogOperation::to_string() const diff --git a/src/observer/storage/trx/mvcc_trx_log.h b/src/observer/storage/trx/mvcc_trx_log.h index 4885e9062..4efc298df 100644 --- a/src/observer/storage/trx/mvcc_trx_log.h +++ b/src/observer/storage/trx/mvcc_trx_log.h @@ -14,11 +14,10 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include -#include - #include "common/rc.h" #include "common/types.h" +#include "common/lang/string.h" +#include "common/lang/unordered_map.h" #include "storage/record/record.h" #include "storage/clog/log_replayer.h" @@ -52,7 +51,7 @@ class MvccTrxLogOperation final Type type() const { return type_; } int32_t index() const { return static_cast(type_); } - std::string to_string() const; + string to_string() const; private: Type type_; @@ -69,7 +68,7 @@ struct MvccTrxLogHeader static const int32_t SIZE; ///< 头部大小 - std::string to_string() const; + string to_string() const; }; /** @@ -85,7 +84,7 @@ struct MvccTrxRecordLogEntry static const int32_t SIZE; ///< 日志大小 - std::string to_string() const; + string to_string() const; }; /** @@ -100,7 +99,7 @@ struct MvccTrxCommitLogEntry static const int32_t SIZE; - std::string to_string() const; + string to_string() const; }; /** @@ -161,5 +160,5 @@ class MvccTrxLogReplayer final : public LogReplayer LogHandler &log_handler_; ///< 日志处理器 ///< 事务ID到事务的映射。在重做结束后,如果还有未提交的事务,需要回滚。 - std::unordered_map trx_map_; + unordered_map trx_map_; }; \ No newline at end of file diff --git a/src/observer/storage/trx/trx.h b/src/observer/storage/trx/trx.h index 4bca38f0d..5ff9af011 100644 --- a/src/observer/storage/trx/trx.h +++ b/src/observer/storage/trx/trx.h @@ -14,12 +14,11 @@ See the Mulan PSL v2 for more details. */ #pragma once -#include #include -#include #include #include "common/rc.h" +#include "common/lang/mutex.h" #include "sql/parser/parse.h" #include "storage/field/field_meta.h" #include "storage/record/record_manager.h" @@ -114,8 +113,8 @@ class TrxKit TrxKit() = default; virtual ~TrxKit() = default; - virtual RC init() = 0; - virtual const std::vector *trx_fields() const = 0; + virtual RC init() = 0; + virtual const vector *trx_fields() const = 0; virtual Trx *create_trx(LogHandler &log_handler) = 0; @@ -124,7 +123,7 @@ class TrxKit */ virtual Trx *create_trx(LogHandler &log_handler, int32_t trx_id) = 0; virtual Trx *find_trx(int32_t trx_id) = 0; - virtual void all_trxes(std::vector &trxes) = 0; + virtual void all_trxes(vector &trxes) = 0; virtual void destroy_trx(Trx *trx) = 0; diff --git a/src/observer/storage/trx/vacuous_trx.cpp b/src/observer/storage/trx/vacuous_trx.cpp index c12a5ce85..1b67a46a0 100644 --- a/src/observer/storage/trx/vacuous_trx.cpp +++ b/src/observer/storage/trx/vacuous_trx.cpp @@ -14,8 +14,6 @@ See the Mulan PSL v2 for more details. */ #include "storage/trx/vacuous_trx.h" -using namespace std; - RC VacuousTrxKit::init() { return RC::SUCCESS; } const vector *VacuousTrxKit::trx_fields() const { return nullptr; } @@ -28,7 +26,7 @@ void VacuousTrxKit::destroy_trx(Trx *trx) { delete trx; } Trx *VacuousTrxKit::find_trx(int32_t /* trx_id */) { return nullptr; } -void VacuousTrxKit::all_trxes(std::vector &trxes) { return; } +void VacuousTrxKit::all_trxes(vector &trxes) { return; } LogReplayer *VacuousTrxKit::create_log_replayer(Db &, LogHandler &) { return new VacuousTrxLogReplayer; } diff --git a/src/observer/storage/trx/vacuous_trx.h b/src/observer/storage/trx/vacuous_trx.h index d6515f72b..f558c19c3 100644 --- a/src/observer/storage/trx/vacuous_trx.h +++ b/src/observer/storage/trx/vacuous_trx.h @@ -25,13 +25,13 @@ class VacuousTrxKit : public TrxKit VacuousTrxKit() = default; virtual ~VacuousTrxKit() = default; - RC init() override; - const std::vector *trx_fields() const override; + RC init() override; + const vector *trx_fields() const override; Trx *create_trx(LogHandler &log_handler) override; Trx *create_trx(LogHandler &log_handler, int32_t trx_id) override; Trx *find_trx(int32_t trx_id) override; - void all_trxes(std::vector &trxes) override; + void all_trxes(vector &trxes) override; void destroy_trx(Trx *trx) override; diff --git a/test/case/result/vectorized-aggregation-and-group-by.result b/test/case/result/vectorized-aggregation-and-group-by.result new file mode 100644 index 000000000..a0cf33726 --- /dev/null +++ b/test/case/result/vectorized-aggregation-and-group-by.result @@ -0,0 +1,173 @@ +INITIALIZATION +SET EXECUTION_MODE='CHUNK_ITERATOR'; +SUCCESS +CREATE TABLE AGGREGATION_FUNC(ID INT, NUM INT, PRICE FLOAT, ADDR CHAR) STORAGE FORMAT=PAX; +SUCCESS + +INSERT INTO AGGREGATION_FUNC VALUES (1, 18, 10.0, 'ABC'); +SUCCESS +INSERT INTO AGGREGATION_FUNC VALUES (2, 15, 20.0, 'ABC'); +SUCCESS +INSERT INTO AGGREGATION_FUNC VALUES (3, 12, 30.0, 'DEF'); +SUCCESS +INSERT INTO AGGREGATION_FUNC VALUES (4, 15, 30.0, 'DEI'); +SUCCESS +INSERT INTO AGGREGATION_FUNC VALUES (4, 15, 31.11, 'CEI'); +SUCCESS +INSERT INTO AGGREGATION_FUNC VALUES (4, 13, 2.22, 'WEI'); +SUCCESS + + + +1. UNGROUPED AGGREGATION + +SELECT SUM(NUM) FROM AGGREGATION_FUNC; +88 +SUM(NUM) + +SELECT SUM(NUM), SUM(PRICE) FROM AGGREGATION_FUNC; +88 | 123.33 +SUM(NUM) | SUM(PRICE) + +SELECT SUM(NUM) FROM AGGREGATION_FUNC WHERE ID>1; +70 +SUM(NUM) + +SELECT SUM(NUM) FROM AGGREGATION_FUNC WHERE ID<=3; +45 +SUM(NUM) + +SELECT SUM(NUM) FROM AGGREGATION_FUNC WHERE ID>=2 AND ID<=3; +27 +SUM(NUM) + +2. GROUP BY +SELECT NUM, SUM(PRICE) FROM AGGREGATION_FUNC GROUP BY NUM; +12 | 30 +13 | 2.22 +15 | 81.11 +18 | 10 +NUM | SUM(PRICE) + +SELECT NUM, SUM(PRICE) FROM AGGREGATION_FUNC WHERE ID>=2 GROUP BY NUM; +12 | 30 +13 | 2.22 +15 | 81.11 +NUM | SUM(PRICE) + +SELECT ID, SUM(PRICE) FROM AGGREGATION_FUNC GROUP BY ID; +1 | 10 +2 | 20 +3 | 30 +4 | 63.33 +ID | SUM(PRICE) + +SELECT SUM(PRICE), ID FROM AGGREGATION_FUNC GROUP BY ID; +10 | 1 +20 | 2 +30 | 3 +63.33 | 4 +SUM(PRICE) | ID + +SELECT ID, NUM, SUM(PRICE) FROM AGGREGATION_FUNC GROUP BY ID, NUM; +1 | 18 | 10 +2 | 15 | 20 +3 | 12 | 30 +4 | 13 | 2.22 +4 | 15 | 61.11 +ID | NUM | SUM(PRICE) + +SELECT ID, SUM(PRICE), NUM FROM AGGREGATION_FUNC GROUP BY ID, NUM; +1 | 10 | 18 +2 | 20 | 15 +3 | 30 | 12 +4 | 2.22 | 13 +4 | 61.11 | 15 +ID | SUM(PRICE) | NUM + +SELECT ID, SUM(NUM + 1), PRICE FROM AGGREGATION_FUNC GROUP BY ID, PRICE; +1 | 19 | 10 +2 | 16 | 20 +3 | 13 | 30 +4 | 14 | 2.22 +4 | 16 | 30 +4 | 16 | 31.11 +ID | SUM(NUM + 1) | PRICE + +SELECT ID, SUM(PRICE+PRICE), NUM FROM AGGREGATION_FUNC GROUP BY ID, NUM; +1 | 20 | 18 +2 | 40 | 15 +3 | 60 | 12 +4 | 122.22 | 15 +4 | 4.44 | 13 +ID | SUM(PRICE+PRICE) | NUM + +SELECT ID+ID-ID, SUM(NUM)+ID+1 FROM AGGREGATION_FUNC GROUP BY ID; +1 | 20 +2 | 18 +3 | 16 +4 | 48 +ID+ID-ID | SUM(NUM)+ID+1 + +SELECT ID+1, SUM(NUM)+SUM(NUM) FROM AGGREGATION_FUNC GROUP BY ID; +2 | 36 +3 | 30 +4 | 24 +5 | 86 +ID+1 | SUM(NUM)+SUM(NUM) + +SELECT ID+ID+1, SUM(NUM)+SUM(NUM) FROM AGGREGATION_FUNC GROUP BY ID+ID; +3 | 36 +5 | 30 +7 | 24 +9 | 86 +ID+ID+1 | SUM(NUM)+SUM(NUM) + +SELECT (ID+ID)+(ID+ID), SUM(NUM)+SUM(NUM) FROM AGGREGATION_FUNC GROUP BY ID+ID; +(ID+ID)+(ID+ID) | SUM(NUM)+SUM(NUM) +12 | 24 +16 | 86 +4 | 36 +8 | 30 + +SELECT ID+ID, SUM(NUM) + 1 + SUM(NUM), PRICE FROM AGGREGATION_FUNC GROUP BY ID+ID, PRICE; +2 | 37 | 10 +4 | 31 | 20 +6 | 25 | 30 +8 | 27 | 2.22 +8 | 31 | 30 +8 | 31 | 31.11 +ID+ID | SUM(NUM) + 1 + SUM(NUM) | PRICE + +SELECT ID+NUM+ID+NUM, SUM(PRICE+PRICE) FROM AGGREGATION_FUNC GROUP BY ID+NUM; +FAILURE + +SELECT (ID+NUM)+(ID+NUM), SUM(PRICE+PRICE) FROM AGGREGATION_FUNC GROUP BY ID+NUM; +(ID+NUM)+(ID+NUM) | SUM(PRICE+PRICE) +30 | 60 +34 | 44.44 +38 | 142.22 + +SELECT ADDR, SUM(ID+ID) FROM AGGREGATION_FUNC GROUP BY ADDR; +ABC | 6 +ADDR | SUM(ID+ID) +CEI | 8 +DEF | 6 +DEI | 8 +WEI | 8 + +SELECT ADDR, SUM(PRICE+PRICE) FROM AGGREGATION_FUNC GROUP BY ADDR; +ABC | 60 +ADDR | SUM(PRICE+PRICE) +CEI | 62.22 +DEF | 60 +DEI | 60 +WEI | 4.44 + +EXPLAIN SELECT ID, SUM(PRICE+PRICE), NUM FROM AGGREGATION_FUNC GROUP BY ID, NUM; +QUERY PLAN +OPERATOR(NAME) +PROJECT_VEC +└─EXPR_VEC + └─GROUP_BY_VEC + └─TABLE_SCAN_VEC(AGGREGATION_FUNC) diff --git a/test/case/result/vectorized-basic.result b/test/case/result/vectorized-basic.result new file mode 100644 index 000000000..a5af886bb --- /dev/null +++ b/test/case/result/vectorized-basic.result @@ -0,0 +1,92 @@ +BASIC INSERT + +SET EXECUTION_MODE='CHUNK_ITERATOR'; +SUCCESS +CREATE TABLE T_BASIC(ID INT, AGE INT, NAME CHAR, SCORE FLOAT) STORAGE FORMAT=PAX; +SUCCESS +INSERT INTO T_BASIC VALUES(1,1, 'A', 1.0); +SUCCESS +INSERT INTO T_BASIC VALUES(2,2, 'B', 2.0); +SUCCESS +INSERT INTO T_BASIC VALUES(4,4, 'C', 3.0); +SUCCESS +INSERT INTO T_BASIC VALUES(3,3, 'D', 4.0); +SUCCESS +INSERT INTO T_BASIC VALUES(5,5, 'E', 5.5); +SUCCESS +INSERT INTO T_BASIC VALUES(6,6, 'F', 6.6); +SUCCESS +INSERT INTO T_BASIC VALUES(7,7, 'G', 7.7); +SUCCESS + +SELECT * FROM T_BASIC; +1 | 1 | A | 1 +2 | 2 | B | 2 +3 | 3 | D | 4 +4 | 4 | C | 3 +5 | 5 | E | 5.5 +6 | 6 | F | 6.6 +7 | 7 | G | 7.7 +ID | AGE | NAME | SCORE + +BASIC DELETE +DELETE FROM T_BASIC WHERE ID=3; +SUCCESS +SELECT * FROM T_BASIC; +1 | 1 | A | 1 +2 | 2 | B | 2 +4 | 4 | C | 3 +5 | 5 | E | 5.5 +6 | 6 | F | 6.6 +7 | 7 | G | 7.7 +ID | AGE | NAME | SCORE + +BASIC SELECT +SELECT * FROM T_BASIC WHERE ID=1; +ID | AGE | NAME | SCORE +1 | 1 | A | 1 + +SELECT * FROM T_BASIC WHERE ID>=5; +5 | 5 | E | 5.5 +6 | 6 | F | 6.6 +7 | 7 | G | 7.7 +ID | AGE | NAME | SCORE + +SELECT * FROM T_BASIC WHERE AGE>1 AND AGE<3; +ID | AGE | NAME | SCORE +2 | 2 | B | 2 + +SELECT * FROM T_BASIC WHERE T_BASIC.ID=1 AND T_BASIC.AGE=1; +ID | AGE | NAME | SCORE +1 | 1 | A | 1 + +SELECT * FROM T_BASIC WHERE ID=1 AND AGE=1; +ID | AGE | NAME | SCORE +1 | 1 | A | 1 + +SELECT ID, AGE, NAME, SCORE FROM T_BASIC; +1 | 1 | A | 1 +2 | 2 | B | 2 +4 | 4 | C | 3 +5 | 5 | E | 5.5 +6 | 6 | F | 6.6 +7 | 7 | G | 7.7 +ID | AGE | NAME | SCORE + +SELECT T_BASIC.ID, T_BASIC.AGE, T_BASIC.NAME, T_BASIC.SCORE FROM T_BASIC; +1 | 1 | A | 1 +2 | 2 | B | 2 +4 | 4 | C | 3 +5 | 5 | E | 5.5 +6 | 6 | F | 6.6 +7 | 7 | G | 7.7 +ID | AGE | NAME | SCORE + +SELECT T_BASIC.ID, T_BASIC.AGE, NAME FROM T_BASIC; +1 | 1 | A +2 | 2 | B +4 | 4 | C +5 | 5 | E +6 | 6 | F +7 | 7 | G +ID | AGE | NAME diff --git a/test/case/test/vectorized-aggregation-and-group-by.test b/test/case/test/vectorized-aggregation-and-group-by.test new file mode 100644 index 000000000..21a90e0e4 --- /dev/null +++ b/test/case/test/vectorized-aggregation-and-group-by.test @@ -0,0 +1,61 @@ +-- echo initialization +set execution_mode='chunk_iterator'; +CREATE TABLE aggregation_func(id int, num int, price float, addr char) storage format=pax; + +INSERT INTO aggregation_func VALUES (1, 18, 10.0, 'abc'); +INSERT INTO aggregation_func VALUES (2, 15, 20.0, 'abc'); +INSERT INTO aggregation_func VALUES (3, 12, 30.0, 'def'); +INSERT INTO aggregation_func VALUES (4, 15, 30.0, 'dei'); +INSERT INTO aggregation_func VALUES (4, 15, 31.11, 'cei'); +INSERT INTO aggregation_func VALUES (4, 13, 2.22, 'wei'); + + + +-- echo 1. ungrouped aggregation + +-- sort SELECT sum(num) FROM aggregation_func; + +-- sort SELECT sum(num), sum(price) FROM aggregation_func; + +-- sort SELECT sum(num) FROM aggregation_func where id>1; + +-- sort SELECT sum(num) FROM aggregation_func where id<=3; + +-- sort SELECT sum(num) FROM aggregation_func where id>=2 and id<=3; + +-- echo 2. group by +-- sort SELECT num, sum(price) FROM aggregation_func group by num; + +-- sort SELECT num, sum(price) FROM aggregation_func where id>=2 group by num; + +-- sort SELECT id, sum(price) FROM aggregation_func group by id; + +-- sort SELECT sum(price), id FROM aggregation_func group by id; + +-- sort SELECT id, num, sum(price) FROM aggregation_func group by id, num; + +-- sort SELECT id, sum(price), num FROM aggregation_func group by id, num; + +-- sort SELECT id, SUM(num + 1), price FROM aggregation_func GROUP BY id, price; + +-- sort SELECT id, sum(price+price), num FROM aggregation_func group by id, num; + +-- sort SELECT id+id-id, sum(num)+id+1 FROM aggregation_func group by id; + +-- sort SELECT id+1, sum(num)+sum(num) FROM aggregation_func group by id; + +-- sort SELECT id+id+1, sum(num)+sum(num) FROM aggregation_func group by id+id; + +-- sort SELECT (id+id)+(id+id), sum(num)+sum(num) FROM aggregation_func group by id+id; + +-- sort SELECT id+id, sum(num) + 1 + sum(num), price FROM aggregation_func group by id+id, price; + +-- sort SELECT id+num+id+num, sum(price+price) FROM aggregation_func group by id+num; + +-- sort SELECT (id+num)+(id+num), sum(price+price) FROM aggregation_func group by id+num; + +-- sort SELECT addr, sum(id+id) FROM aggregation_func group by addr; + +-- sort SELECT addr, sum(price+price) FROM aggregation_func group by addr; + +explain SELECT id, sum(price+price), num FROM aggregation_func group by id, num; diff --git a/test/case/test/vectorized-basic.test b/test/case/test/vectorized-basic.test new file mode 100644 index 000000000..26b9d1f57 --- /dev/null +++ b/test/case/test/vectorized-basic.test @@ -0,0 +1,34 @@ +-- echo basic insert + +set execution_mode='chunk_iterator'; +create table t_basic(id int, age int, name char, score float) storage format=pax; +insert into t_basic values(1,1, 'a', 1.0); +insert into t_basic values(2,2, 'b', 2.0); +insert into t_basic values(4,4, 'c', 3.0); +insert into t_basic values(3,3, 'd', 4.0); +insert into t_basic values(5,5, 'e', 5.5); +insert into t_basic values(6,6, 'f', 6.6); +insert into t_basic values(7,7, 'g', 7.7); + +--sort select * from t_basic; + +-- echo basic delete +delete from t_basic where id=3; +-- sort select * from t_basic; + +-- echo basic select +select * from t_basic where id=1; + +-- sort select * from t_basic where id>=5; + +select * from t_basic where age>1 and age<3; + +select * from t_basic where t_basic.id=1 and t_basic.age=1; + +select * from t_basic where id=1 and age=1; + +-- sort select id, age, name, score from t_basic; + +-- sort select t_basic.id, t_basic.age, t_basic.name, t_basic.score from t_basic; + +-- sort select t_basic.id, t_basic.age, name from t_basic; \ No newline at end of file diff --git a/unittest/common/integer_generator_test.cpp b/unittest/common/integer_generator_test.cpp index e88993431..1e14814e1 100644 --- a/unittest/common/integer_generator_test.cpp +++ b/unittest/common/integer_generator_test.cpp @@ -15,7 +15,6 @@ See the Mulan PSL v2 for more details. */ #include "gtest/gtest.h" #include "common/math/integer_generator.h" -using namespace std; using namespace common; TEST(IntegerGenerator, test) diff --git a/unittest/common/log_test.cpp b/unittest/common/log_test.cpp index bb7537086..108bca119 100644 --- a/unittest/common/log_test.cpp +++ b/unittest/common/log_test.cpp @@ -30,7 +30,7 @@ LogTest::~LogTest() // Auto-generated destructor stub } -int LogTest::init(const std::string &logFile) +int LogTest::init(const string &logFile) { LoggerFactory::init_default(logFile); diff --git a/unittest/common/log_test.h b/unittest/common/log_test.h index e1928d544..e44c39970 100644 --- a/unittest/common/log_test.h +++ b/unittest/common/log_test.h @@ -15,7 +15,7 @@ See the Mulan PSL v2 for more details. */ #ifndef CTESTLOG_H_ #define CTESTLOG_H_ -#include +#include "common/lang/string.h" /* * @@ -26,7 +26,7 @@ class LogTest LogTest(); virtual ~LogTest(); - int init(const std::string &logFile = "test.log"); + int init(const string &logFile = "test.log"); void *log_loop(void *param); }; diff --git a/unittest/common/lower_bound_test.cpp b/unittest/common/lower_bound_test.cpp index 02c6656a7..299e596d9 100644 --- a/unittest/common/lower_bound_test.cpp +++ b/unittest/common/lower_bound_test.cpp @@ -14,29 +14,29 @@ See the Mulan PSL v2 for more details. */ #include "common/lang/lower_bound.h" #include "gtest/gtest.h" -#include +#include "common/lang/vector.h" using namespace common; TEST(lower_bound, test_lower_bound) { - int items[] = {1, 3, 5, 7, 9}; - const int count = sizeof(items) / sizeof(items[0]); - std::vector v(items, items + count); + int items[] = {1, 3, 5, 7, 9}; + const int count = sizeof(items) / sizeof(items[0]); + vector v(items, items + count); for (int i = 0; i <= 10; i++) { - bool found = false; - std::vector::iterator retval = lower_bound(v.begin(), v.end(), i, &found); + bool found = false; + vector::iterator retval = lower_bound(v.begin(), v.end(), i, &found); ASSERT_EQ(retval - v.begin(), i / 2); ASSERT_EQ(found, (i % 2 != 0)); } - int empty_items = {}; - const int empty_count = 0; - std::vector empty_v(empty_items, empty_items + empty_count); + int empty_items = {}; + const int empty_count = 0; + vector empty_v(empty_items, empty_items + empty_count); for (int i = 0; i <= 10; i++) { - bool found = false; - std::vector::iterator retval = lower_bound(empty_v.begin(), empty_v.end(), i, &found); + bool found = false; + vector::iterator retval = lower_bound(empty_v.begin(), empty_v.end(), i, &found); ASSERT_EQ(retval - empty_v.begin(), 0); ASSERT_EQ(found, false); } diff --git a/unittest/common/mem_pool_test.cpp b/unittest/common/mem_pool_test.cpp index fc8c56f7f..0c9dcb3a1 100644 --- a/unittest/common/mem_pool_test.cpp +++ b/unittest/common/mem_pool_test.cpp @@ -14,8 +14,8 @@ See the Mulan PSL v2 for more details. */ #include "common/mm/mem_pool.h" #include "gtest/gtest.h" -#include -#include +#include "common/lang/list.h" +#include "common/lang/iostream.h" using namespace common; @@ -25,7 +25,7 @@ TEST(test_mem_pool_item, test_mem_pool_item_basic) const int item_num_per_pool = 128; mem_pool_item.init(32, true, 1, item_num_per_pool); - std::list used_list; + list used_list; int alloc_num = 1000; @@ -34,7 +34,7 @@ TEST(test_mem_pool_item, test_mem_pool_item_basic) used_list.push_back(item); } - std::cout << mem_pool_item.to_string() << std::endl; + cout << mem_pool_item.to_string() << endl; int pool_size = ((alloc_num + item_num_per_pool - 1) / item_num_per_pool) * item_num_per_pool; ASSERT_EQ(alloc_num, mem_pool_item.get_used_num()); @@ -52,7 +52,7 @@ TEST(test_mem_pool_item, test_mem_pool_item_basic) mem_pool_item.free(check); } - std::cout << mem_pool_item.to_string() << std::endl; + cout << mem_pool_item.to_string() << endl; ASSERT_EQ(used_list.size(), mem_pool_item.get_used_num()); ASSERT_EQ(pool_size, mem_pool_item.get_size()); } diff --git a/unittest/common/pidfile_test.cpp b/unittest/common/pidfile_test.cpp index b56918c96..c7fdbd47d 100644 --- a/unittest/common/pidfile_test.cpp +++ b/unittest/common/pidfile_test.cpp @@ -28,14 +28,14 @@ int main() const char *programName = "test"; writePidFile(programName); - std::string pidFile = getPidPath(); + string pidFile = getPidPath(); char *p = NULL; size_t size = 0; readFromFile(pidFile, p, size); - std::string temp(p); - long long target = 0; + string temp(p); + long long target = 0; str_to_val(temp, target); free(p); diff --git a/unittest/common/serializer_test.cpp b/unittest/common/serializer_test.cpp index f967bf678..4e48a2cef 100644 --- a/unittest/common/serializer_test.cpp +++ b/unittest/common/serializer_test.cpp @@ -12,13 +12,11 @@ See the Mulan PSL v2 for more details. */ // Created by wangyunlai.wyl on 2024/02/21 // -#include - #include "gtest/gtest.h" #include "common/lang/serializer.h" +#include "common/lang/filesystem.h" #include "common/log/log.h" -using namespace std; using namespace common; TEST(Serializer, serializer) @@ -34,7 +32,7 @@ TEST(Serializer, serializer) serializer.write_int64(b0); serializer.write_int32(c0); serializer.write_int64(d0); - serializer.write(std::span(str, sizeof(str))); + serializer.write(span(str, sizeof(str))); Deserializer deserializer(serializer.data()); int32_t a; @@ -47,7 +45,7 @@ TEST(Serializer, serializer) deserializer.read_int64(d); char str2[sizeof(str)]; - deserializer.read(std::span(str2, sizeof(str2))); + deserializer.read(span(str2, sizeof(str2))); ASSERT_EQ(a, a0); ASSERT_EQ(b, b0); @@ -56,7 +54,7 @@ TEST(Serializer, serializer) ASSERT_EQ(0, memcmp(str, str2, sizeof(str))); char str3[sizeof(str)]; - int ret = deserializer.read(std::span(str3, sizeof(str3))); + int ret = deserializer.read(span(str3, sizeof(str3))); ASSERT_NE(ret, 0); } diff --git a/unittest/common/simple_queue_test.cpp b/unittest/common/simple_queue_test.cpp index 9bccaf095..9e66648ce 100644 --- a/unittest/common/simple_queue_test.cpp +++ b/unittest/common/simple_queue_test.cpp @@ -12,11 +12,10 @@ See the Mulan PSL v2 for more details. */ // Created by Wangyunlai on 2024/01/11. // -#include #include "gtest/gtest.h" +#include "common/lang/memory.h" #include "common/queue/simple_queue.h" -using namespace std; using namespace common; TEST(SimpleQueue, test) diff --git a/unittest/common/thread_pool_executor_test.cpp b/unittest/common/thread_pool_executor_test.cpp index 777ac968d..e122c3f93 100644 --- a/unittest/common/thread_pool_executor_test.cpp +++ b/unittest/common/thread_pool_executor_test.cpp @@ -12,16 +12,14 @@ See the Mulan PSL v2 for more details. */ // Created by Wangyunlai on 2024/01/11. // -#include -#include - #include "gtest/gtest.h" +#include "common/lang/memory.h" +#include "common/lang/atomic.h" #include "common/queue/queue.h" #include "common/queue/simple_queue.h" #include "common/thread/runnable.h" #include "common/thread/thread_pool_executor.h" -using namespace std; using namespace common; class TestRunnable : public Runnable diff --git a/unittest/observer/aggregate_hash_table_test.cpp b/unittest/observer/aggregate_hash_table_test.cpp new file mode 100644 index 000000000..6eb626249 --- /dev/null +++ b/unittest/observer/aggregate_hash_table_test.cpp @@ -0,0 +1,189 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include + +#include "gtest/gtest.h" +#include "sql/expr/aggregate_hash_table.h" + +using namespace std; + +TEST(AggregateHashTableTest, DISABLED_standard_hash_table) +{ + // single group by column, single aggregate column + { + Chunk group_chunk; + Chunk aggr_chunk; + std::unique_ptr column1 = std::make_unique(AttrType::INTS, 4); + std::unique_ptr column2 = std::make_unique(AttrType::INTS, 4); + for (int i = 0; i < 1023; i++) { + int key = i % 8; + column1->append_one((char *)&key); + column2->append_one((char *)&i); + } + group_chunk.add_column(std::move(column1), 0); + aggr_chunk.add_column(std::move(column2), 1); + + AggregateExpr aggregate_expr(AggregateExpr::Type::SUM, nullptr); + std::vector aggregate_exprs; + aggregate_exprs.push_back(&aggregate_expr); + auto standard_hash_table = std::make_unique(aggregate_exprs); + RC rc = standard_hash_table->add_chunk(group_chunk, aggr_chunk); + ASSERT_EQ(rc, RC::SUCCESS); + Chunk output_chunk; + output_chunk.add_column( + make_unique(group_chunk.column(0).attr_type(), group_chunk.column(0).attr_len()), 0); + output_chunk.add_column(make_unique(aggr_chunk.column(0).attr_type(), aggr_chunk.column(0).attr_len()), 1); + StandardAggregateHashTable::Scanner scanner(standard_hash_table.get()); + scanner.open_scan(); + rc = scanner.next(output_chunk); + ASSERT_EQ(rc, RC::SUCCESS); + ASSERT_EQ(output_chunk.rows(), 8); + for (int i = 0; i < 8; i++) { + std::cout << "column1: " << output_chunk.get_value(1, i).get_string() << " " + << "sum(column2)" << output_chunk.get_value(0, i).get_string() << std::endl; + } + } + // mutiple group by columns, mutiple aggregate columns + { + Chunk group_chunk; + Chunk aggr_chunk; + std::unique_ptr group1 = std::make_unique(AttrType::CHARS, 4); + std::unique_ptr group2 = std::make_unique(AttrType::INTS, 4); + + std::unique_ptr aggr1 = std::make_unique(AttrType::FLOATS, 4); + std::unique_ptr aggr2 = std::make_unique(AttrType::INTS, 4); + for (int i = 0; i < 1023; i++) { + float i_float = i + 0.5; + int i_group2 = i % 8; + + group1->append_one((char *)std::to_string(i % 8).c_str()); + group2->append_one((char *)&i_group2); + aggr1->append_one((char *)&i_float); + aggr2->append_one((char *)&i); + } + group_chunk.add_column(std::move(group1), 0); + group_chunk.add_column(std::move(group2), 1); + aggr_chunk.add_column(std::move(aggr1), 0); + aggr_chunk.add_column(std::move(aggr2), 1); + + AggregateExpr aggregate_expr(AggregateExpr::Type::SUM, nullptr); + std::vector aggregate_exprs; + aggregate_exprs.push_back(&aggregate_expr); + aggregate_exprs.push_back(&aggregate_expr); + auto standard_hash_table = std::make_unique(aggregate_exprs); + RC rc = standard_hash_table->add_chunk(group_chunk, aggr_chunk); + ASSERT_EQ(rc, RC::SUCCESS); + Chunk output_chunk; + output_chunk.add_column( + make_unique(group_chunk.column(0).attr_type(), group_chunk.column(0).attr_len()), 0); + output_chunk.add_column( + make_unique(group_chunk.column(1).attr_type(), group_chunk.column(1).attr_len()), 1); + output_chunk.add_column(make_unique(aggr_chunk.column(0).attr_type(), aggr_chunk.column(0).attr_len()), 1); + output_chunk.add_column(make_unique(aggr_chunk.column(1).attr_type(), aggr_chunk.column(1).attr_len()), 2); + StandardAggregateHashTable::Scanner scanner(standard_hash_table.get()); + scanner.open_scan(); + rc = scanner.next(output_chunk); + ASSERT_EQ(rc, RC::SUCCESS); + ASSERT_EQ(output_chunk.rows(), 8); + for (int i = 0; i < 8; i++) { + std::cout << "group1: " << output_chunk.get_value(0, i).get_string() << " " + << "group2: " << output_chunk.get_value(1, i).get_string() << " " + << "sum(aggr1): " << output_chunk.get_value(2, i).get_string() << " " + << "sum(aggr2): " << output_chunk.get_value(3, i).get_string() << std::endl; + } + } +} + +#ifdef USE_SIMD +TEST(AggregateHashTableTest, DISABLED_linear_probing_hash_table) +{ + // simple case + { + Chunk group_chunk; + Chunk aggr_chunk; + std::unique_ptr column1 = std::make_unique(AttrType::INTS, 4); + std::unique_ptr column2 = std::make_unique(AttrType::INTS, 4); + for (int i = 0; i < 1023; i++) { + int key = i % 8; + column1->append_one((char *)&key); + column2->append_one((char *)&i); + } + group_chunk.add_column(std::move(column1), 0); + aggr_chunk.add_column(std::move(column2), 1); + + auto linear_probing_hash_table = std::make_unique>(AggregateExpr::Type::SUM); + RC rc = linear_probing_hash_table->add_chunk(group_chunk, aggr_chunk); + ASSERT_EQ(rc, RC::SUCCESS); + Chunk output_chunk; + output_chunk.add_column( + make_unique(group_chunk.column(0).attr_type(), group_chunk.column(0).attr_len()), 0); + output_chunk.add_column(make_unique(aggr_chunk.column(0).attr_type(), aggr_chunk.column(0).attr_len()), 1); + LinearProbingAggregateHashTable::Scanner scanner(linear_probing_hash_table.get()); + scanner.open_scan(); + rc = scanner.next(output_chunk); + ASSERT_EQ(rc, RC::SUCCESS); + ASSERT_EQ(output_chunk.rows(), 8); + for (int i = 0; i < 8; i++) { + std::cout << "column1: " << output_chunk.get_value(1, i).get_string() << " " + << "sum(column2)" << output_chunk.get_value(0, i).get_string() << std::endl; + } + } + + // hash conflict + { + Chunk group_chunk; + Chunk aggr_chunk; + std::unique_ptr column1 = std::make_unique(AttrType::INTS, 4); + std::unique_ptr column2 = std::make_unique(AttrType::INTS, 4); + for (int i = 0; i < 1002; i++) { + int key, value = 1; + if (i % 2 == 0) { + key = 1; + } else { + key = 257; + } + + column1->append_one((char *)&key); + column2->append_one((char *)&value); + } + group_chunk.add_column(std::move(column1), 0); + aggr_chunk.add_column(std::move(column2), 1); + + auto linear_probing_hash_table = + std::make_unique>(AggregateExpr::Type::SUM, 256); + RC rc = linear_probing_hash_table->add_chunk(group_chunk, aggr_chunk); + ASSERT_EQ(rc, RC::SUCCESS); + Chunk output_chunk; + output_chunk.add_column( + make_unique(group_chunk.column(0).attr_type(), group_chunk.column(0).attr_len()), 0); + output_chunk.add_column(make_unique(aggr_chunk.column(0).attr_type(), aggr_chunk.column(0).attr_len()), 1); + LinearProbingAggregateHashTable::Scanner scanner(linear_probing_hash_table.get()); + scanner.open_scan(); + rc = scanner.next(output_chunk); + ASSERT_EQ(rc, RC::SUCCESS); + ASSERT_EQ(output_chunk.rows(), 2); + ASSERT_STREQ(output_chunk.get_value(1, 0).get_string().c_str(), "501"); + ASSERT_STREQ(output_chunk.get_value(1, 1).get_string().c_str(), "501"); + } +} +#endif + +int main(int argc, char **argv) +{ + + // 分析gtest程序的命令行参数 + testing::InitGoogleTest(&argc, argv); + + // 调用RUN_ALL_TESTS()运行所有测试用例 + // main函数返回RUN_ALL_TESTS()的运行结果 + return RUN_ALL_TESTS(); +} diff --git a/unittest/observer/arithmetic_expression_test.cpp b/unittest/observer/arithmetic_expression_test.cpp deleted file mode 100644 index e11a464cc..000000000 --- a/unittest/observer/arithmetic_expression_test.cpp +++ /dev/null @@ -1,149 +0,0 @@ -/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. -miniob is licensed under Mulan PSL v2. -You can use this software according to the terms and conditions of the Mulan PSL v2. -You may obtain a copy of Mulan PSL v2 at: - http://license.coscl.org.cn/MulanPSL2 -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -See the Mulan PSL v2 for more details. */ - -// -// Created by wangyunlai.wyl on 2023/08/14 -// - -#include - -#include "sql/expr/expression.h" -#include "gtest/gtest.h" - -using namespace std; -using namespace common; - -TEST(ArithmeticExpr, test_value_type) -{ - Value int_value1(1); - Value int_value2(2); - Value float_value1((float)1.1); - Value float_value2((float)2.2); - - unique_ptr left_expr(new ValueExpr(int_value1)); - unique_ptr right_expr(new ValueExpr(int_value2)); - ArithmeticExpr expr_int(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(AttrType::INTS, expr_int.value_type()); - - left_expr.reset(new ValueExpr(float_value1)); - right_expr.reset(new ValueExpr(float_value2)); - ArithmeticExpr expr_float(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(AttrType::FLOATS, expr_float.value_type()); - - left_expr.reset(new ValueExpr(int_value1)); - right_expr.reset(new ValueExpr(float_value2)); - ArithmeticExpr expr_int_float(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(AttrType::FLOATS, expr_int_float.value_type()); - - left_expr.reset(new ValueExpr(int_value1)); - right_expr.reset(new ValueExpr(int_value2)); - ArithmeticExpr expr_int_int(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(AttrType::FLOATS, expr_int_int.value_type()); -} - -TEST(ArithmeticExpr, test_try_get_value) -{ - Value int_value1(1); - Value int_value2(2); - Value float_value1((float)1.1); - Value float_value2((float)2.2); - - Value int_result; - Value float_result; - - unique_ptr left_expr(new ValueExpr(int_value1)); - unique_ptr right_expr(new ValueExpr(int_value2)); - ArithmeticExpr int_expr(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); - - ASSERT_EQ(int_expr.try_get_value(int_result), RC::SUCCESS); - ASSERT_EQ(int_result.get_int(), 3); - - int_expr.~ArithmeticExpr(); - - left_expr.reset(new ValueExpr(int_value1)); - right_expr.reset(new ValueExpr(int_value2)); - new (&int_expr)(ArithmeticExpr)(ArithmeticExpr::Type::SUB, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(int_expr.try_get_value(int_result), RC::SUCCESS); - ASSERT_EQ(int_result.get_int(), -1); - - left_expr.reset(new ValueExpr(int_value1)); - right_expr.reset(new ValueExpr(int_value2)); - int_expr.~ArithmeticExpr(); - new (&int_expr)(ArithmeticExpr)(ArithmeticExpr::Type::MUL, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(int_expr.try_get_value(int_result), RC::SUCCESS); - ASSERT_EQ(int_result.get_int(), 2); - - left_expr.reset(new ValueExpr(int_value1)); - right_expr.reset(new ValueExpr(int_value2)); - int_expr.~ArithmeticExpr(); - new (&int_expr)(ArithmeticExpr)(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(int_expr.try_get_value(float_result), RC::SUCCESS); - EXPECT_FLOAT_EQ(float_result.get_float(), 0.5); - - left_expr.reset(new ValueExpr(float_value1)); - right_expr.reset(new ValueExpr(float_value2)); - ArithmeticExpr float_expr(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(float_expr.try_get_value(float_result), RC::SUCCESS); - EXPECT_FLOAT_EQ(float_result.get_float(), 3.3); - - left_expr.reset(new ValueExpr(float_value1)); - right_expr.reset(new ValueExpr(float_value2)); - float_expr.~ArithmeticExpr(); - new (&float_expr)(ArithmeticExpr)(ArithmeticExpr::Type::SUB, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(float_expr.try_get_value(float_result), RC::SUCCESS); - EXPECT_FLOAT_EQ(float_result.get_float(), -1.1); - - left_expr.reset(new ValueExpr(float_value1)); - right_expr.reset(new ValueExpr(float_value2)); - float_expr.~ArithmeticExpr(); - new (&float_expr)(ArithmeticExpr)(ArithmeticExpr::Type::MUL, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(float_expr.try_get_value(float_result), RC::SUCCESS); - EXPECT_FLOAT_EQ(float_result.get_float(), 1.1 * 2.2); - - left_expr.reset(new ValueExpr(float_value1)); - right_expr.reset(new ValueExpr(float_value2)); - float_expr.~ArithmeticExpr(); - new (&float_expr)(ArithmeticExpr)(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(float_expr.try_get_value(float_result), RC::SUCCESS); - EXPECT_FLOAT_EQ(float_result.get_float(), 1.1 / 2.2); - - Value zero_int_value(0); - Value zero_float_value((float)0); - left_expr.reset(new ValueExpr(int_value1)); - right_expr.reset(new ValueExpr(zero_int_value)); - ArithmeticExpr zero_int_expr(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(zero_int_expr.try_get_value(float_result), RC::SUCCESS); - - left_expr.reset(new ValueExpr(float_value1)); - right_expr.reset(new ValueExpr(zero_float_value)); - ArithmeticExpr zero_float_expr(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); - ASSERT_EQ(zero_float_expr.try_get_value(float_result), RC::SUCCESS); - - left_expr.reset(new ValueExpr(int_value1)); - ArithmeticExpr negative_expr(ArithmeticExpr::Type::NEGATIVE, std::move(left_expr), nullptr); - ASSERT_EQ(negative_expr.try_get_value(int_result), RC::SUCCESS); - ASSERT_EQ(int_result.get_int(), -1); - - left_expr.reset(new ValueExpr(float_value1)); - ArithmeticExpr negative_float_expr(ArithmeticExpr::Type::NEGATIVE, std::move(left_expr), nullptr); - ASSERT_EQ(negative_float_expr.try_get_value(float_result), RC::SUCCESS); - EXPECT_FLOAT_EQ(float_result.get_float(), -1.1); -} - -int main(int argc, char **argv) -{ - - // 分析gtest程序的命令行参数 - testing::InitGoogleTest(&argc, argv); - - // 调用RUN_ALL_TESTS()运行所有测试用例 - // main函数返回RUN_ALL_TESTS()的运行结果 - return RUN_ALL_TESTS(); -} diff --git a/unittest/observer/arithmetic_operator_test.cpp b/unittest/observer/arithmetic_operator_test.cpp new file mode 100644 index 000000000..d01147e9d --- /dev/null +++ b/unittest/observer/arithmetic_operator_test.cpp @@ -0,0 +1,118 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include + +#include "sql/expr/arithmetic_operator.hpp" +#include "gtest/gtest.h" + +using namespace std; + +TEST(ArithmeticTest, arithmetic_test) +{ + // compare column + { + int size = 100; + std::vector a(size, 1); + std::vector b(size, 2); + std::vector result(size, 1); + compare_result(a.data(), b.data(), size, result, CompOp::EQUAL_TO); + for (int i = 0; i < size; ++i) { + ASSERT_EQ(result[i], 0); + } + } + // addition + { + int size = 100; + std::vector a(size, 1); + std::vector b(size, 2); + std::vector result(size, 0); + binary_operator(a.data(), b.data(), result.data(), size); + for (int i = 0; i < size; ++i) { + ASSERT_EQ(result[i], 3); + } + } + // subtraction + { + int size = 100; + std::vector a(size, 1); + std::vector b(size, 2); + std::vector result(size, 0); + binary_operator(a.data(), b.data(), result.data(), size); + for (int i = 0; i < size; ++i) { + ASSERT_EQ(result[i], -1); + } + } + // multiplication + { + int size = 100; + std::vector a(size, 1); + std::vector b(size, 2); + std::vector result(size, 0); + binary_operator(a.data(), b.data(), result.data(), size); + for (int i = 0; i < size; ++i) { + ASSERT_EQ(result[i], 2); + } + } + // division + { + int size = 100; + std::vector a(size, 2); + std::vector b(size, 2); + std::vector result(size, 0); + binary_operator(a.data(), b.data(), result.data(), size); + for (int i = 0; i < size; ++i) { + ASSERT_EQ(result[i], 1); + } + } + // negate + { + int size = 100; + std::vector a(size, 1); + std::vector b(size, 2); + std::vector result(size, 0); + unary_operator(a.data(), result.data(), size); + for (int i = 0; i < size; ++i) { + ASSERT_EQ(result[i], -1); + } + } +// sum +#ifdef USE_SIMD + { + int size = 100; + std::vector a(size, 0); + for (int i = 0; i < size; i++) { + a[i] = i; + } + int res = mm256_sum_epi32(a.data(), size); + ASSERT_EQ(res, 4950); + } + { + int size = 100; + std::vector a(size, 0); + for (int i = 0; i < size; i++) { + a[i] = i; + } + int res = mm256_sum_ps(a.data(), size); + ASSERT_FLOAT_EQ(res, 4950.0); + } +#endif +} + +int main(int argc, char **argv) +{ + + // 分析gtest程序的命令行参数 + testing::InitGoogleTest(&argc, argv); + + // 调用RUN_ALL_TESTS()运行所有测试用例 + // main函数返回RUN_ALL_TESTS()的运行结果 + return RUN_ALL_TESTS(); +} diff --git a/unittest/observer/bplus_tree_log_entry_test.cpp b/unittest/observer/bplus_tree_log_entry_test.cpp index 04d6031e2..6fa615713 100644 --- a/unittest/observer/bplus_tree_log_entry_test.cpp +++ b/unittest/observer/bplus_tree_log_entry_test.cpp @@ -30,7 +30,7 @@ TEST(BplusTreeLogEntry, init_header_page_log_entry) file_header.leaf_max_size = 200; file_header.attr_length = 20; file_header.key_length = 30; - file_header.attr_type = INTS; + file_header.attr_type = AttrType::INTS; Frame frame; frame.set_page_num(100); diff --git a/unittest/observer/bplus_tree_log_test.cpp b/unittest/observer/bplus_tree_log_test.cpp index f2cb28fc0..a300e2845 100644 --- a/unittest/observer/bplus_tree_log_test.cpp +++ b/unittest/observer/bplus_tree_log_test.cpp @@ -74,7 +74,7 @@ TEST(BplusTreeLog, base) ASSERT_EQ(RC::SUCCESS, log_handler->start()); auto bplus_tree = make_unique(); - ASSERT_EQ(RC::SUCCESS, bplus_tree->create(*log_handler, *buffer_pool, INTS, 4)); + ASSERT_EQ(RC::SUCCESS, bplus_tree->create(*log_handler, *buffer_pool, AttrType::INTS, 4)); // 2. insert some key-value pairs into the bplus tree const int insert_num = 10000; @@ -192,7 +192,7 @@ TEST(BplusTreeLog, concurrency) vector> bplus_trees; for (DiskBufferPool *buffer_pool : buffer_pools) { auto bplus_tree = make_unique(); - ASSERT_EQ(RC::SUCCESS, bplus_tree->create(*log_handler, *buffer_pool, INTS, 4)); + ASSERT_EQ(RC::SUCCESS, bplus_tree->create(*log_handler, *buffer_pool, AttrType::INTS, 4)); bplus_trees.push_back(std::move(bplus_tree)); } diff --git a/unittest/observer/bplus_tree_test.cpp b/unittest/observer/bplus_tree_test.cpp index 7d670d361..6f33db2e5 100644 --- a/unittest/observer/bplus_tree_test.cpp +++ b/unittest/observer/bplus_tree_test.cpp @@ -17,6 +17,8 @@ See the Mulan PSL v2 for more details. */ #include #include "common/log/log.h" +#include "common/lang/memory.h" +#include "common/lang/filesystem.h" #include "sql/parser/parse_defs.h" #include "storage/buffer/disk_buffer_pool.h" #include "storage/index/bplus_tree.h" @@ -328,7 +330,7 @@ TEST(test_bplus_tree, test_leaf_index_node_handle) index_file_header.leaf_max_size = 5; index_file_header.attr_length = 4; index_file_header.key_length = 4 + sizeof(RID); - index_file_header.attr_type = INTS; + index_file_header.attr_type = AttrType::INTS; VacuousLogHandler log_handler; BufferPoolManager bpm; @@ -351,7 +353,7 @@ TEST(test_bplus_tree, test_leaf_index_node_handle) Frame frame; KeyComparator key_comparator; - key_comparator.init(INTS, 4); + key_comparator.init(AttrType::INTS, 4); LeafIndexNodeHandler leaf_node(mtr, index_file_header, &frame); leaf_node.init_empty(); @@ -409,7 +411,7 @@ TEST(test_bplus_tree, test_internal_index_node_handle) index_file_header.leaf_max_size = 5; index_file_header.attr_length = 4; index_file_header.key_length = 4 + sizeof(RID); - index_file_header.attr_type = INTS; + index_file_header.attr_type = AttrType::INTS; VacuousLogHandler log_handler; BufferPoolManager bpm; @@ -433,7 +435,7 @@ TEST(test_bplus_tree, test_internal_index_node_handle) Frame frame; KeyComparator key_comparator; - key_comparator.init(INTS, 4); + key_comparator.init(AttrType::INTS, 4); InternalIndexNodeHandler internal_node(mtr, index_file_header, &frame); internal_node.init_empty(); @@ -536,7 +538,7 @@ TEST(test_bplus_tree, test_chars) ASSERT_NE(nullptr, buffer_pool); BplusTreeHandler handler; - ASSERT_EQ(RC::SUCCESS, handler.create(log_handler, *buffer_pool, CHARS, 8, ORDER, ORDER)); + ASSERT_EQ(RC::SUCCESS, handler.create(log_handler, *buffer_pool, AttrType::CHARS, 8, ORDER, ORDER)); char keys[][9] = {"abcdefg", "12345678", "12345678", "abcdefg", "abcdefga"}; @@ -586,7 +588,7 @@ TEST(test_bplus_tree, test_scanner) ASSERT_NE(nullptr, buffer_pool); BplusTreeHandler handler; - ASSERT_EQ(RC::SUCCESS, handler.create(log_handler, *buffer_pool, INTS, sizeof(int), ORDER, ORDER)); + ASSERT_EQ(RC::SUCCESS, handler.create(log_handler, *buffer_pool, AttrType::INTS, sizeof(int), ORDER, ORDER)); int count = 0; RC rc = RC::SUCCESS; @@ -811,7 +813,7 @@ TEST(test_bplus_tree, test_bplus_tree_insert) ASSERT_NE(nullptr, buffer_pool); BplusTreeHandler *handler = new BplusTreeHandler(); - ASSERT_EQ(RC::SUCCESS, handler->create(log_handler, *buffer_pool, INTS, sizeof(int), ORDER, ORDER)); + ASSERT_EQ(RC::SUCCESS, handler->create(log_handler, *buffer_pool, AttrType::INTS, sizeof(int), ORDER, ORDER)); test_insert(handler); diff --git a/unittest/observer/chunk_test.cpp b/unittest/observer/chunk_test.cpp new file mode 100644 index 000000000..c510fd8a5 --- /dev/null +++ b/unittest/observer/chunk_test.cpp @@ -0,0 +1,72 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include + +#include "storage/common/chunk.h" +#include "gtest/gtest.h" + +using namespace std; + +TEST(ChunkTest, chunk_test) +{ + // empty + { + Chunk chunk; + ASSERT_EQ(chunk.capacity(), 0); + ASSERT_EQ(chunk.column_num(), 0); + ASSERT_EQ(chunk.rows(), 0); + } + // normal + { + int row_num = 8; + Chunk chunk; + chunk.add_column(std::make_unique(AttrType::INTS, sizeof(int), row_num), 0); + chunk.add_column(std::make_unique(AttrType::FLOATS, sizeof(float), row_num), 1); + ASSERT_EQ(chunk.capacity(), row_num); + ASSERT_EQ(chunk.column_num(), 2); + ASSERT_EQ(chunk.rows(), 0); + for (int i = 0; i < row_num; i++) { + int value1 = i; + chunk.column(0).append_one((char *)&value1); + } + for (int i = 0; i < row_num; i++) { + float value2 = i + 0.5f; + chunk.column(1).append_one((char *)&value2); + } + ASSERT_EQ(chunk.rows(), row_num); + + Chunk chunk2; + chunk2.reference(chunk); + ASSERT_EQ(chunk2.capacity(), row_num); + ASSERT_EQ(chunk2.column_num(), 2); + ASSERT_EQ(chunk2.rows(), row_num); + for (int i = 0; i < row_num; i++) { + int value1 = i; + ASSERT_EQ(chunk2.get_value(0, i).get_int(), value1); + } + + for (int i = 0; i < row_num; i++) { + float value2 = i + 0.5f; + ASSERT_EQ(chunk2.get_value(1, i).get_float(), value2); + } + } +} + +int main(int argc, char **argv) +{ + + // 分析gtest程序的命令行参数 + testing::InitGoogleTest(&argc, argv); + + // 调用RUN_ALL_TESTS()运行所有测试用例 + // main函数返回RUN_ALL_TESTS()的运行结果 + return RUN_ALL_TESTS(); +} diff --git a/unittest/observer/composite_tuple_test.cpp b/unittest/observer/composite_tuple_test.cpp new file mode 100644 index 000000000..3caade56c --- /dev/null +++ b/unittest/observer/composite_tuple_test.cpp @@ -0,0 +1,162 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by wangyunlai.wyl on 2024/05/31 +// + +#include + +#include "sql/expr/composite_tuple.h" +#include "gtest/gtest.h" + +using namespace std; +using namespace common; + +TEST(CompositeTuple, test_empty) +{ + CompositeTuple tuple1; + ASSERT_EQ(0, tuple1.cell_num()); + + Value value; + TupleCellSpec spec; + ASSERT_NE(RC::SUCCESS, tuple1.cell_at(0, value)); + ASSERT_NE(RC::SUCCESS, tuple1.spec_at(0, spec)); + + ValueListTuple value_list_tuple; + + tuple1.add_tuple(make_unique(value_list_tuple)); + ASSERT_EQ(0, tuple1.cell_num()); + + ValueListTuple value_list_tuple2; + tuple1.add_tuple(make_unique(value_list_tuple2)); + ASSERT_EQ(0, tuple1.cell_num()); + +} + +TEST(CompositeTuple, test_1tuple) +{ + // 测试包含一个tuple + CompositeTuple composite_tuple; + + ValueListTuple value_list_tuple; + vector values; + vector specs; + for (int i = 0; i < 10; i++) + { + values.emplace_back(i); + specs.emplace_back(TupleCellSpec(to_string(i))); + } + + value_list_tuple.set_cells(values); + value_list_tuple.set_names(specs); + + composite_tuple.add_tuple(make_unique(value_list_tuple)); + ASSERT_EQ(10, composite_tuple.cell_num()); + + for (int i = 0; i < 10; i++) { + Value value; + TupleCellSpec spec; + ASSERT_EQ(RC::SUCCESS, composite_tuple.cell_at(i, value)); + ASSERT_EQ(RC::SUCCESS, composite_tuple.spec_at(i, spec)); + ASSERT_EQ(i, value.get_int()); + ASSERT_EQ(to_string(i), spec.alias()); + } + + Value value; + TupleCellSpec spec; + ASSERT_NE(RC::SUCCESS, composite_tuple.cell_at(10, value)); + ASSERT_NE(RC::SUCCESS, composite_tuple.spec_at(10, spec)); + ASSERT_NE(RC::SUCCESS, composite_tuple.cell_at(11, value)); + ASSERT_NE(RC::SUCCESS, composite_tuple.spec_at(11, spec)); +} + +TEST(CompositeTuple, test_2tuple) +{ + // 测试包含两个tuple + CompositeTuple composite_tuple; + vector values; + vector specs; + for (int i = 0; i < 10; i++) + { + values.emplace_back(i); + specs.emplace_back(TupleCellSpec(to_string(i))); + } + + ValueListTuple value_list_tuple; + value_list_tuple.set_cells(values); + value_list_tuple.set_names(specs); + + composite_tuple.add_tuple(make_unique(value_list_tuple)); + ASSERT_EQ(10, composite_tuple.cell_num()); + + composite_tuple.add_tuple(make_unique(value_list_tuple)); + ASSERT_EQ(20, composite_tuple.cell_num()); + + for (int i = 0; i < 10; i++) { + Value value; + TupleCellSpec spec; + ASSERT_EQ(RC::SUCCESS, composite_tuple.cell_at(i, value)); + ASSERT_EQ(RC::SUCCESS, composite_tuple.spec_at(i, spec)); + ASSERT_EQ(i, value.get_int()); + ASSERT_EQ(to_string(i), spec.alias()); + } + + for (int i = 0; i < 10; i++) { + Value value; + TupleCellSpec spec; + ASSERT_EQ(RC::SUCCESS, composite_tuple.cell_at(i + 10, value)); + ASSERT_EQ(RC::SUCCESS, composite_tuple.spec_at(i + 10, spec)); + ASSERT_EQ(i, value.get_int()); + ASSERT_EQ(to_string(i), spec.alias()); + } + + Value value; + TupleCellSpec spec; + ASSERT_NE(RC::SUCCESS, composite_tuple.cell_at(20, value)); + ASSERT_NE(RC::SUCCESS, composite_tuple.spec_at(20, spec)); + ASSERT_NE(RC::SUCCESS, composite_tuple.cell_at(21, value)); + ASSERT_NE(RC::SUCCESS, composite_tuple.spec_at(21, spec)); +} + +TEST(CompositeTuple, test_including_composite_tuple) +{ + // 测试包含一个 composite tuple + CompositeTuple composite_tuple; + vector values; + vector specs; + for (int i = 0; i < 10; i++) + { + values.emplace_back(i); + specs.emplace_back(TupleCellSpec(to_string(i))); + } + + ValueListTuple value_list_tuple; + value_list_tuple.set_cells(values); + value_list_tuple.set_names(specs); + + composite_tuple.add_tuple(make_unique(value_list_tuple)); + ASSERT_EQ(10, composite_tuple.cell_num()); + + CompositeTuple composite_tuple2; + composite_tuple2.add_tuple(make_unique(std::move(composite_tuple))); + ASSERT_EQ(10, composite_tuple2.cell_num()); +} + +int main(int argc, char **argv) +{ + + // 分析gtest程序的命令行参数 + testing::InitGoogleTest(&argc, argv); + + // 调用RUN_ALL_TESTS()运行所有测试用例 + // main函数返回RUN_ALL_TESTS()的运行结果 + return RUN_ALL_TESTS(); +} diff --git a/unittest/observer/expression_test.cpp b/unittest/observer/expression_test.cpp new file mode 100644 index 000000000..94d85a3b1 --- /dev/null +++ b/unittest/observer/expression_test.cpp @@ -0,0 +1,346 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +// +// Created by wangyunlai.wyl on 2023/08/14 +// + +#include + +#include "sql/expr/expression.h" +#include "sql/expr/tuple.h" +#include "gtest/gtest.h" + +using namespace std; +using namespace common; + +TEST(ArithmeticExpr, test_value_type) +{ + Value int_value1(1); + Value int_value2(2); + Value float_value1((float)1.1); + Value float_value2((float)2.2); + + unique_ptr left_expr(new ValueExpr(int_value1)); + unique_ptr right_expr(new ValueExpr(int_value2)); + ArithmeticExpr expr_int(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(AttrType::INTS, expr_int.value_type()); + + left_expr.reset(new ValueExpr(float_value1)); + right_expr.reset(new ValueExpr(float_value2)); + ArithmeticExpr expr_float(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(AttrType::FLOATS, expr_float.value_type()); + + left_expr.reset(new ValueExpr(int_value1)); + right_expr.reset(new ValueExpr(float_value2)); + ArithmeticExpr expr_int_float(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(AttrType::FLOATS, expr_int_float.value_type()); + + left_expr.reset(new ValueExpr(int_value1)); + right_expr.reset(new ValueExpr(int_value2)); + ArithmeticExpr expr_int_int(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(AttrType::FLOATS, expr_int_int.value_type()); +} + +TEST(ArithmeticExpr, test_try_get_value) +{ + Value int_value1(1); + Value int_value2(2); + Value float_value1((float)1.1); + Value float_value2((float)2.2); + + Value int_result; + Value float_result; + + unique_ptr left_expr(new ValueExpr(int_value1)); + unique_ptr right_expr(new ValueExpr(int_value2)); + ArithmeticExpr int_expr(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + + ASSERT_EQ(int_expr.try_get_value(int_result), RC::SUCCESS); + ASSERT_EQ(int_result.get_int(), 3); + + int_expr.~ArithmeticExpr(); + + left_expr.reset(new ValueExpr(int_value1)); + right_expr.reset(new ValueExpr(int_value2)); + new (&int_expr)(ArithmeticExpr)(ArithmeticExpr::Type::SUB, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(int_expr.try_get_value(int_result), RC::SUCCESS); + ASSERT_EQ(int_result.get_int(), -1); + + left_expr.reset(new ValueExpr(int_value1)); + right_expr.reset(new ValueExpr(int_value2)); + int_expr.~ArithmeticExpr(); + new (&int_expr)(ArithmeticExpr)(ArithmeticExpr::Type::MUL, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(int_expr.try_get_value(int_result), RC::SUCCESS); + ASSERT_EQ(int_result.get_int(), 2); + + left_expr.reset(new ValueExpr(int_value1)); + right_expr.reset(new ValueExpr(int_value2)); + int_expr.~ArithmeticExpr(); + new (&int_expr)(ArithmeticExpr)(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(int_expr.try_get_value(float_result), RC::SUCCESS); + EXPECT_FLOAT_EQ(float_result.get_float(), 0.5); + + left_expr.reset(new ValueExpr(float_value1)); + right_expr.reset(new ValueExpr(float_value2)); + ArithmeticExpr float_expr(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(float_expr.try_get_value(float_result), RC::SUCCESS); + EXPECT_FLOAT_EQ(float_result.get_float(), 3.3); + + left_expr.reset(new ValueExpr(float_value1)); + right_expr.reset(new ValueExpr(float_value2)); + float_expr.~ArithmeticExpr(); + new (&float_expr)(ArithmeticExpr)(ArithmeticExpr::Type::SUB, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(float_expr.try_get_value(float_result), RC::SUCCESS); + EXPECT_FLOAT_EQ(float_result.get_float(), -1.1); + + left_expr.reset(new ValueExpr(float_value1)); + right_expr.reset(new ValueExpr(float_value2)); + float_expr.~ArithmeticExpr(); + new (&float_expr)(ArithmeticExpr)(ArithmeticExpr::Type::MUL, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(float_expr.try_get_value(float_result), RC::SUCCESS); + EXPECT_FLOAT_EQ(float_result.get_float(), 1.1 * 2.2); + + left_expr.reset(new ValueExpr(float_value1)); + right_expr.reset(new ValueExpr(float_value2)); + float_expr.~ArithmeticExpr(); + new (&float_expr)(ArithmeticExpr)(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(float_expr.try_get_value(float_result), RC::SUCCESS); + EXPECT_FLOAT_EQ(float_result.get_float(), 1.1 / 2.2); + + Value zero_int_value(0); + Value zero_float_value((float)0); + left_expr.reset(new ValueExpr(int_value1)); + right_expr.reset(new ValueExpr(zero_int_value)); + ArithmeticExpr zero_int_expr(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(zero_int_expr.try_get_value(float_result), RC::SUCCESS); + + left_expr.reset(new ValueExpr(float_value1)); + right_expr.reset(new ValueExpr(zero_float_value)); + ArithmeticExpr zero_float_expr(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(zero_float_expr.try_get_value(float_result), RC::SUCCESS); + + left_expr.reset(new ValueExpr(int_value1)); + ArithmeticExpr negative_expr(ArithmeticExpr::Type::NEGATIVE, std::move(left_expr), nullptr); + ASSERT_EQ(negative_expr.try_get_value(int_result), RC::SUCCESS); + ASSERT_EQ(int_result.get_int(), -1); + + left_expr.reset(new ValueExpr(float_value1)); + ArithmeticExpr negative_float_expr(ArithmeticExpr::Type::NEGATIVE, std::move(left_expr), nullptr); + ASSERT_EQ(negative_float_expr.try_get_value(float_result), RC::SUCCESS); + EXPECT_FLOAT_EQ(float_result.get_float(), -1.1); +} + +TEST(ArithmeticExpr, get_column) +{ + // constant value + { + Value int_value1(1); + Value int_value2(2); + Value float_value1((float)1.1); + Value float_value2((float)2.2); + Chunk chunk; + + Value int_result; + Value float_result; + + unique_ptr left_expr(new ValueExpr(int_value1)); + unique_ptr right_expr(new ValueExpr(int_value2)); + ArithmeticExpr int_expr(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + Column column; + ASSERT_EQ(int_expr.get_column(chunk, column), RC::SUCCESS); + ASSERT_EQ(column.count(), 1); + ASSERT_EQ(column.get_value(0).get_int(), 3); + + left_expr.reset(new ValueExpr(int_value1)); + right_expr.reset(new ValueExpr(int_value2)); + int_expr.~ArithmeticExpr(); + column.reset_data(); + new (&int_expr)(ArithmeticExpr)(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(int_expr.get_column(chunk, column), RC::SUCCESS); + ASSERT_EQ(column.count(), 1); + ASSERT_EQ(column.get_value(0).get_int(), 3); + + left_expr.reset(new ValueExpr(float_value1)); + right_expr.reset(new ValueExpr(float_value2)); + ArithmeticExpr float_expr(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(float_expr.get_column(chunk, column), RC::SUCCESS); + ASSERT_EQ(column.count(), 1); + EXPECT_FLOAT_EQ(column.get_value(0).get_float(), 3.3); + } + + // column op constant value + { + int count = 8; + const int int_len = sizeof(int); + std::unique_ptr column_left = std::make_unique(AttrType::FLOATS, int_len, count); + Value float_value(2.0f); + unique_ptr right_expr(new ValueExpr(float_value)); + for (int i = 0; i < count; ++i) { + float left_value = i; + column_left->append_one((char *)&left_value); + } + Chunk chunk; + chunk.add_column(std::move(column_left), 0); + Column column_result; + FieldMeta field_meta1("col1", AttrType::INTS, 0, int_len, true, 0); + Field field1(nullptr, &field_meta1); + auto left_expr = std::make_unique(field1); + ArithmeticExpr expr(ArithmeticExpr::Type::DIV, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(expr.get_column(chunk, column_result), RC::SUCCESS); + for (int i = 0; i < count; ++i) { + float expect_value = static_cast(i) / 2.0f; + ASSERT_EQ(column_result.get_value(i).get_float(), expect_value); + } + } + + // column op column + { + int count = 8; + const int int_len = sizeof(int); + std::unique_ptr column_left = std::make_unique(AttrType::INTS, int_len, count); + std::unique_ptr column_right = std::make_unique(AttrType::INTS, int_len, count); + for (int i = 0; i < count; ++i) { + int left_value = i; + int right_value = i; + column_left->append_one((char *)&left_value); + column_right->append_one((char *)&right_value); + } + Chunk chunk; + chunk.add_column(std::move(column_left), 0); + chunk.add_column(std::move(column_right), 1); + Column column_result; + FieldMeta field_meta1("col1", AttrType::INTS, 0, int_len, true, 0); + FieldMeta field_meta2("col2", AttrType::INTS, 0, int_len, true, 1); + Field field1(nullptr, &field_meta1); + Field field2(nullptr, &field_meta2); + auto left_expr = std::make_unique(field1); + auto right_expr = std::make_unique(field2); + ArithmeticExpr expr(ArithmeticExpr::Type::ADD, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(expr.get_column(chunk, column_result), RC::SUCCESS); + for (int i = 0; i < count; ++i) { + int expect_value = i * 2; + ASSERT_EQ(column_result.get_value(i).get_int(), expect_value); + } + } +} + +TEST(ValueExpr, value_expr_test) +{ + ValueExpr value_expr1(Value(1)); + ValueExpr value_expr2(Value(2)); + Value result; + value_expr1.get_value(result); + ASSERT_EQ(result.get_int(), 1); + ASSERT_EQ(value_expr1.equal(value_expr2), false); + ASSERT_EQ(value_expr1.equal(value_expr1), true); +} + +TEST(CastExpr, cast_expr_test) +{ + Value int_value1(1); + unique_ptr value_expr1 = make_unique(int_value1); + CastExpr cast_expr(std::move(value_expr1), AttrType::INTS); + RowTuple tuple; + Value result; + cast_expr.get_value(tuple, result); + ASSERT_EQ(result.get_int(), 1); + result.set_int(0); + cast_expr.try_get_value(result); + ASSERT_EQ(result.get_int(), 1); +} + +TEST(ComparisonExpr, comparison_expr_test) +{ + { + Value int_value1(1); + Value int_value2(2); + bool bool_res = false; + Value bool_value; + + unique_ptr left_expr(new ValueExpr(int_value1)); + unique_ptr right_expr(new ValueExpr(int_value2)); + ComparisonExpr expr_eq(CompOp::EQUAL_TO, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(AttrType::BOOLEANS, expr_eq.value_type()); + ASSERT_EQ(expr_eq.compare_value(int_value1, int_value2, bool_res), RC::SUCCESS); + ASSERT_EQ(bool_res, false); + ASSERT_EQ(expr_eq.compare_value(int_value1, int_value1, bool_res), RC::SUCCESS); + ASSERT_EQ(bool_res, true); + ASSERT_EQ(expr_eq.try_get_value(bool_value), RC::SUCCESS); + ASSERT_EQ(bool_value.get_boolean(), false); + } + { + const int int_len = sizeof(int); + Value int_value(1); + FieldMeta field_meta("col1", AttrType::INTS, 0, int_len, true, 0); + Field field(nullptr, &field_meta); + unique_ptr right_expr = std::make_unique(field); + int count = 1024; + std::unique_ptr column_right = std::make_unique(AttrType::INTS, int_len, count); + for (int i = 0; i < count; ++i) { + int right_value = i; + column_right->append_one((char *)&right_value); + } + Chunk chunk; + std::vector select(count, 1); + chunk.add_column(std::move(column_right), 0); + + unique_ptr left_expr(new ValueExpr(int_value)); + ComparisonExpr expr_eq(CompOp::EQUAL_TO, std::move(left_expr), std::move(right_expr)); + ASSERT_EQ(AttrType::BOOLEANS, expr_eq.value_type()); + RC rc = expr_eq.eval(chunk, select); + ASSERT_EQ(rc, RC::SUCCESS); + for (int i = 0; i < count; ++i) { + if (i == 1) { + ASSERT_EQ(select[i], 1); + } else { + ASSERT_EQ(select[i], 0); + } + } + } +} + +TEST(AggregateExpr, aggregate_expr_test) +{ + Value int_value(1); + unique_ptr value_expr(new ValueExpr(int_value)); + AggregateExpr aggregate_expr(AggregateExpr::Type::SUM, std::move(value_expr)); + aggregate_expr.equal(aggregate_expr); + auto aggregator = aggregate_expr.create_aggregator(); + for (int i = 0; i < 100; i++) { + aggregator->accumulate(Value(i)); + } + Value result; + aggregator->evaluate(result); + ASSERT_EQ(result.get_int(), 4950); + AggregateExpr::Type aggr_type; + ASSERT_EQ(RC::SUCCESS, AggregateExpr::type_from_string("sum", aggr_type)); + ASSERT_EQ(aggr_type, AggregateExpr::Type::SUM); + ASSERT_EQ(RC::SUCCESS, AggregateExpr::type_from_string("count", aggr_type)); + ASSERT_EQ(aggr_type, AggregateExpr::Type::COUNT); + ASSERT_EQ(RC::SUCCESS, AggregateExpr::type_from_string("avg", aggr_type)); + ASSERT_EQ(aggr_type, AggregateExpr::Type::AVG); + ASSERT_EQ(RC::SUCCESS, AggregateExpr::type_from_string("max", aggr_type)); + ASSERT_EQ(aggr_type, AggregateExpr::Type::MAX); + ASSERT_EQ(RC::SUCCESS, AggregateExpr::type_from_string("min", aggr_type)); + ASSERT_EQ(aggr_type, AggregateExpr::Type::MIN); + ASSERT_EQ(RC::INVALID_ARGUMENT, AggregateExpr::type_from_string("invalid type", aggr_type)); +} + +int main(int argc, char **argv) +{ + + // 分析gtest程序的命令行参数 + testing::InitGoogleTest(&argc, argv); + + // 调用RUN_ALL_TESTS()运行所有测试用例 + // main函数返回RUN_ALL_TESTS()的运行结果 + return RUN_ALL_TESTS(); +} diff --git a/unittest/observer/parser_test.cpp b/unittest/observer/parser_test.cpp new file mode 100644 index 000000000..af753958f --- /dev/null +++ b/unittest/observer/parser_test.cpp @@ -0,0 +1,76 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include +#include + +#include "sql/expr/expression.h" +#include "sql/parser/parse.h" +#include "gtest/gtest.h" + +using namespace std; + +TEST(ParserTest, DISABLED_aggregation_test) +{ + ParsedSqlResult result; + { + const char *sql = "select sum(a), sum(b) from tab"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } + { + const char *sql = "select sum(a+b*c+d/e-f*(g-h)) from tab"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } +} + +TEST(ParserTest, DISABLED_group_by_test) +{ + ParsedSqlResult result; + { + const char *sql = "select a, sum(b) from tab group by a"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } + { + const char *sql = "select sum(a+b*c+d/e-f*(g+1)) from tab group by c+d-e"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } + { + const char *sql = "select a,b,sum(a+b*c+d/e-f*(g+1)),e,f,d from tab group by c+d-e"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } + { + const char *sql = "select a,b,sum(a+b*c+d/e-f*(g+1)),e,f,d from tab where a>1 and b< 10 and c='abc' group by c+d-e"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } + { + const char *sql = "select t.a,s.c,sum(u.rr) from t,s,u where t.a<10 and t.b!='vfdv' and u.c='abc' group by s.c+u.q"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } + { + const char *sql = "select a+a,sum(b+b)+sum(b) from tab group by a"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } + { + const char *sql = "select a+a,sum(b+b)+sum(b) from tab group by a+a"; + ASSERT_EQ(parse(sql, &result), RC::SUCCESS); + } +} + +int main(int argc, char **argv) +{ + + // 分析gtest程序的命令行参数 + testing::InitGoogleTest(&argc, argv); + + // 调用RUN_ALL_TESTS()运行所有测试用例 + // main函数返回RUN_ALL_TESTS()的运行结果 + return RUN_ALL_TESTS(); +} diff --git a/unittest/observer/pax_storage_test.cpp b/unittest/observer/pax_storage_test.cpp new file mode 100644 index 000000000..ab09fef22 --- /dev/null +++ b/unittest/observer/pax_storage_test.cpp @@ -0,0 +1,353 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include +#include +#include + +#define protected public +#define private public +#include "storage/table/table.h" +#undef protected +#undef private + +#include "storage/buffer/disk_buffer_pool.h" +#include "storage/record/record_manager.h" +#include "storage/trx/vacuous_trx.h" +#include "storage/clog/vacuous_log_handler.h" +#include "storage/clog/disk_log_handler.h" +#include "storage/buffer/double_write_buffer.h" +#include "common/math/integer_generator.h" +#include "common/thread/thread_pool_executor.h" +#include "storage/clog/integrated_log_replayer.h" +#include "gtest/gtest.h" + +using namespace std; +using namespace common; + +class PaxRecordFileScannerWithParam : public testing::TestWithParam +{}; + +TEST_P(PaxRecordFileScannerWithParam, DISABLED_test_file_iterator) +{ + int record_insert_num = GetParam(); + VacuousLogHandler log_handler; + + const char *record_manager_file = "record_manager.bp"; + filesystem::remove(record_manager_file); + + BufferPoolManager *bpm = new BufferPoolManager(); + ASSERT_EQ(RC::SUCCESS, bpm->init(make_unique())); + DiskBufferPool *bp = nullptr; + RC rc = bpm->create_file(record_manager_file); + ASSERT_EQ(rc, RC::SUCCESS); + + rc = bpm->open_file(log_handler, record_manager_file, bp); + ASSERT_EQ(rc, RC::SUCCESS); + + TableMeta table_meta; + table_meta.fields_.resize(2); + table_meta.fields_[0].attr_type_ = AttrType::INTS; + table_meta.fields_[0].attr_len_ = 4; + table_meta.fields_[0].field_id_ = 0; + table_meta.fields_[1].attr_type_ = AttrType::INTS; + table_meta.fields_[1].attr_len_ = 4; + table_meta.fields_[1].field_id_ = 1; + + RecordFileHandler file_handler(StorageFormat::PAX_FORMAT); + rc = file_handler.init(*bp, log_handler, &table_meta); + ASSERT_EQ(rc, RC::SUCCESS); + + VacuousTrx trx; + ChunkFileScanner chunk_scanner; + RecordFileScanner record_scanner; + Table table; + table.table_meta_.storage_format_ = StorageFormat::PAX_FORMAT; + // no record + // record iterator + rc = record_scanner.open_scan(&table, *bp, &trx, log_handler, ReadWriteMode::READ_ONLY, nullptr /*condition_filter*/); + ASSERT_EQ(rc, RC::SUCCESS); + + int count = 0; + Record record; + while (OB_SUCC(rc = record_scanner.next(record))) { + count++; + } + if (rc == RC::RECORD_EOF) { + rc = RC::SUCCESS; + } + record_scanner.close_scan(); + ASSERT_EQ(count, 0); + + // chunk iterator + rc = chunk_scanner.open_scan_chunk(&table, *bp, log_handler, ReadWriteMode::READ_ONLY); + ASSERT_EQ(rc, RC::SUCCESS); + Chunk chunk; + FieldMeta fm; + fm.init("col1", AttrType::INTS, 0, 4, true, 0); + auto col1 = std::make_unique(fm, 2048); + chunk.add_column(std::move(col1), 0); + count = 0; + while (OB_SUCC(rc = chunk_scanner.next_chunk(chunk))) { + count += chunk.rows(); + chunk.reset_data(); + } + ASSERT_EQ(rc, RC::RECORD_EOF); + ASSERT_EQ(count, 0); + + // insert some records + char record_data[8]; + std::vector rids; + for (int i = 0; i < record_insert_num; i++) { + RID rid; + rc = file_handler.insert_record(record_data, sizeof(record_data), &rid); + ASSERT_EQ(rc, RC::SUCCESS); + rids.push_back(rid); + } + + // record iterator + rc = record_scanner.open_scan(&table, *bp, &trx, log_handler, ReadWriteMode::READ_ONLY, nullptr /*condition_filter*/); + ASSERT_EQ(rc, RC::SUCCESS); + + count = 0; + while (OB_SUCC(rc = record_scanner.next(record))) { + count++; + } + ASSERT_EQ(RC::RECORD_EOF, rc); + + record_scanner.close_scan(); + ASSERT_EQ(count, rids.size()); + + // chunk iterator + rc = chunk_scanner.open_scan_chunk(&table, *bp, log_handler, ReadWriteMode::READ_ONLY); + ASSERT_EQ(rc, RC::SUCCESS); + chunk.reset_data(); + count = 0; + while (OB_SUCC(rc = chunk_scanner.next_chunk(chunk))) { + count += chunk.rows(); + chunk.reset_data(); + } + ASSERT_EQ(rc, RC::RECORD_EOF); + ASSERT_EQ(count, record_insert_num); + + // delete some records + for (int i = 0; i < record_insert_num; i += 2) { + rc = file_handler.delete_record(&rids[i]); + ASSERT_EQ(rc, RC::SUCCESS); + } + + // record iterator + rc = record_scanner.open_scan(&table, *bp, &trx, log_handler, ReadWriteMode::READ_ONLY, nullptr /*condition_filter*/); + ASSERT_EQ(rc, RC::SUCCESS); + + count = 0; + while (OB_SUCC(rc = record_scanner.next(record))) { + count++; + } + ASSERT_EQ(RC::RECORD_EOF, rc); + + record_scanner.close_scan(); + ASSERT_EQ(count, rids.size() / 2); + + // chunk iterator + rc = chunk_scanner.open_scan_chunk(&table, *bp, log_handler, ReadWriteMode::READ_ONLY); + ASSERT_EQ(rc, RC::SUCCESS); + chunk.reset_data(); + count = 0; + while (OB_SUCC(rc = chunk_scanner.next_chunk(chunk))) { + count += chunk.rows(); + chunk.reset_data(); + } + ASSERT_EQ(rc, RC::RECORD_EOF); + ASSERT_EQ(count, rids.size() / 2); + + bpm->close_file(record_manager_file); + delete bpm; +} + +class PaxPageHandlerTestWithParam : public testing::TestWithParam +{}; + +TEST_P(PaxPageHandlerTestWithParam, DISABLED_PaxPageHandler) +{ + int record_num = GetParam(); + VacuousLogHandler log_handler; + + const char *record_manager_file = "record_manager.bp"; + ::remove(record_manager_file); + + BufferPoolManager *bpm = new BufferPoolManager(); + ASSERT_EQ(RC::SUCCESS, bpm->init(make_unique())); + DiskBufferPool *bp = nullptr; + RC rc = bpm->create_file(record_manager_file); + ASSERT_EQ(rc, RC::SUCCESS); + + rc = bpm->open_file(log_handler, record_manager_file, bp); + ASSERT_EQ(rc, RC::SUCCESS); + + Frame *frame = nullptr; + rc = bp->allocate_page(&frame); + ASSERT_EQ(rc, RC::SUCCESS); + + const int record_size = 19; // 4 + 4 + 4 + 7 + RecordPageHandler *record_page_handle = new PaxRecordPageHandler(); + TableMeta table_meta; + table_meta.fields_.resize(4); + table_meta.fields_[0].attr_type_ = AttrType::INTS; + table_meta.fields_[0].attr_len_ = 4; + table_meta.fields_[0].field_id_ = 0; + table_meta.fields_[1].attr_type_ = AttrType::FLOATS; + table_meta.fields_[1].attr_len_ = 4; + table_meta.fields_[1].field_id_ = 1; + table_meta.fields_[2].attr_type_ = AttrType::CHARS; + table_meta.fields_[2].attr_len_ = 4; + table_meta.fields_[2].field_id_ = 2; + table_meta.fields_[3].attr_type_ = AttrType::CHARS; + table_meta.fields_[3].attr_len_ = 7; + table_meta.fields_[3].field_id_ = 3; + + rc = record_page_handle->init_empty_page(*bp, log_handler, frame->page_num(), record_size, &table_meta); + ASSERT_EQ(rc, RC::SUCCESS); + + RecordPageIterator iterator; + iterator.init(record_page_handle); + + int count = 0; + Record record; + while (iterator.has_next()) { + count++; + rc = iterator.next(record); + ASSERT_EQ(rc, RC::SUCCESS); + } + ASSERT_EQ(count, 0); + + char buf[record_size]; + int int_base = 122; + float float_base = 1.45; + memcpy(buf + 8, "12345678910", 4 + 7); + RID rid; + for (int i = 0; i < record_num; i++) { + int int_val = i + int_base; + float float_val = i + float_base; + memcpy(buf, &int_val, sizeof(int)); + memcpy(buf + 4, &float_val, sizeof(float)); + rc = record_page_handle->insert_record(buf, &rid); + ASSERT_EQ(rc, RC::SUCCESS); + } + + count = 0; + iterator.init(record_page_handle); + while (iterator.has_next()) { + int int_val = count + int_base; + float float_val = count + float_base; + rc = iterator.next(record); + ASSERT_EQ(memcmp(record.data(), &int_val, sizeof(int)), 0); + ASSERT_EQ(memcmp(record.data() + 4, &float_val, sizeof(float)), 0); + ASSERT_EQ(memcmp(record.data() + 8, "12345678910", 4 + 7), 0); + ASSERT_EQ(rc, RC::SUCCESS); + count++; + } + ASSERT_EQ(count, record_num); + + Chunk chunk1; + FieldMeta fm1, fm2, fm3, fm4; + fm1.init("col1", AttrType::INTS, 0, 4, true, 0); + fm2.init("col2", AttrType::FLOATS, 4, 4, true, 1); + fm3.init("col3", AttrType::CHARS, 8, 4, true, 2); + fm4.init("col4", AttrType::CHARS, 12, 7, true, 3); + auto col_1 = std::make_unique(fm1, 2048); + chunk1.add_column(std::move(col_1), 0); + auto col_2 = std::make_unique(fm2, 2048); + chunk1.add_column(std::move(col_2), 1); + auto col_3 = std::make_unique(fm3, 2048); + chunk1.add_column(std::move(col_3), 2); + auto col_4 = std::make_unique(fm4, 2048); + chunk1.add_column(std::move(col_4), 3); + rc = record_page_handle->get_chunk(chunk1); + ASSERT_EQ(rc, RC::SUCCESS); + ASSERT_EQ(chunk1.rows(), record_num); + for (int i = 0; i < record_num; i++) { + int int_val = i + int_base; + float float_val = i + float_base; + ASSERT_EQ(chunk1.get_value(0, i).get_int(), int_val); + ASSERT_EQ(chunk1.get_value(1, i).get_float(), float_val); + ASSERT_STREQ(chunk1.get_value(2, i).get_string().c_str(), "1234"); + ASSERT_STREQ(chunk1.get_value(3, i).get_string().c_str(), "5678910"); + } + + Chunk chunk2; + FieldMeta fm2_1; + fm2_1.init("col2", AttrType::FLOATS, 4, 4, true, 1); + auto col_2_1 = std::make_unique(fm2_1, 2048); + chunk2.add_column(std::move(col_2_1), 1); + rc = record_page_handle->get_chunk(chunk2); + ASSERT_EQ(rc, RC::SUCCESS); + ASSERT_EQ(chunk2.rows(), record_num); + for (int i = 0; i < record_num; i++) { + float float_val = i + float_base; + ASSERT_FLOAT_EQ(chunk2.get_value(0, i).get_float(), float_val); + } + + // delete record + IntegerGenerator generator(0, record_num - 1); + int delete_num = generator.next(); + std::set delete_slots; + for (int i = 0; i < delete_num; i++) { + + int slot_num = 0; + while (true) { + slot_num = generator.next(); + if (delete_slots.find(slot_num) == delete_slots.end()) break; + } + RID del_rid(1, slot_num); + rc = record_page_handle->delete_record(&del_rid); + delete_slots.insert(slot_num); + ASSERT_EQ(rc, RC::SUCCESS); + } + + // get chunk + chunk1.reset_data(); + record_page_handle->get_chunk(chunk1); + ASSERT_EQ(chunk1.rows(), record_num - delete_num); + + int col1_expected = (int_base + 0 + int_base + record_num - 1) * record_num /2; + int col1_actual = 0; + for (int i = 0; i < chunk1.rows(); i++) { + col1_actual += chunk1.get_value(0, i).get_int(); + } + for (auto it = delete_slots.begin(); it != delete_slots.end(); ++it) { + col1_actual += *it + int_base; + } + // check sum(col1) + ASSERT_EQ(col1_actual, col1_expected); + for (auto it = delete_slots.begin(); it != delete_slots.end(); ++it) { + RID get_rid(1, *it); + ASSERT_EQ(record_page_handle->get_record(get_rid, record), RC::RECORD_NOT_EXIST); + } + + rc = record_page_handle->cleanup(); + ASSERT_EQ(rc, RC::SUCCESS); + delete record_page_handle; + bpm->close_file(record_manager_file); + delete bpm; +} + +INSTANTIATE_TEST_SUITE_P(PaxFileScannerTests, PaxRecordFileScannerWithParam, testing::Values(1, 10, 100, 1000, 2000, 10000)); + +INSTANTIATE_TEST_SUITE_P(PaxPageTests, PaxPageHandlerTestWithParam, testing::Values(1, 10, 100, 337)); + +int main(int argc, char **argv) +{ + testing::InitGoogleTest(&argc, argv); + filesystem::path log_filename = filesystem::path(argv[0]).filename(); + LoggerFactory::init_default(log_filename.string() + ".log", LOG_LEVEL_TRACE); + return RUN_ALL_TESTS(); +} diff --git a/unittest/observer/record_manager_test.cpp b/unittest/observer/record_manager_test.cpp index 2aac4e43e..07a8b9c35 100644 --- a/unittest/observer/record_manager_test.cpp +++ b/unittest/observer/record_manager_test.cpp @@ -51,9 +51,9 @@ TEST(RecordPageHandler, test_record_page_handler) rc = bp->allocate_page(&frame); ASSERT_EQ(rc, RC::SUCCESS); - const int record_size = 8; - RecordPageHandler record_page_handle; - rc = record_page_handle.init_empty_page(*bp, log_handler, frame->page_num(), record_size); + const int record_size = 8; + RecordPageHandler *record_page_handle = new RowRecordPageHandler(); + rc = record_page_handle->init_empty_page(*bp, log_handler, frame->page_num(), record_size, nullptr); ASSERT_EQ(rc, RC::SUCCESS); RecordPageIterator iterator; @@ -72,7 +72,7 @@ TEST(RecordPageHandler, test_record_page_handler) RID rid; rid.page_num = 100; rid.slot_num = 100; - rc = record_page_handle.insert_record(buf, &rid); + rc = record_page_handle->insert_record(buf, &rid); ASSERT_EQ(rc, RC::SUCCESS); count = 0; @@ -87,7 +87,7 @@ TEST(RecordPageHandler, test_record_page_handler) for (int i = 0; i < 10; i++) { rid.page_num = i; rid.slot_num = i; - rc = record_page_handle.insert_record(buf, &rid); + rc = record_page_handle->insert_record(buf, &rid); ASSERT_EQ(rc, RC::SUCCESS); } @@ -103,7 +103,7 @@ TEST(RecordPageHandler, test_record_page_handler) for (int i = 0; i < 5; i++) { rid.page_num = i * 2; rid.slot_num = i * 2; - rc = record_page_handle.delete_record(&rid); + rc = record_page_handle->delete_record(&rid); ASSERT_EQ(rc, RC::SUCCESS); } @@ -116,9 +116,10 @@ TEST(RecordPageHandler, test_record_page_handler) } ASSERT_EQ(count, 6); - record_page_handle.cleanup(); + record_page_handle->cleanup(); bpm->close_file(record_manager_file); delete bpm; + delete record_page_handle; } TEST(RecordFileScanner, test_record_file_iterator) @@ -137,8 +138,8 @@ TEST(RecordFileScanner, test_record_file_iterator) rc = bpm->open_file(log_handler, record_manager_file, bp); ASSERT_EQ(rc, RC::SUCCESS); - RecordFileHandler file_handler; - rc = file_handler.init(*bp, log_handler); + RecordFileHandler file_handler(StorageFormat::ROW_FORMAT); + rc = file_handler.init(*bp, log_handler, nullptr); ASSERT_EQ(rc, RC::SUCCESS); VacuousTrx trx; @@ -150,7 +151,6 @@ TEST(RecordFileScanner, test_record_file_iterator) int count = 0; Record record; while (OB_SUCC(rc = file_scanner.next(record))) { - ASSERT_EQ(rc, RC::SUCCESS); count++; } if (rc == RC::RECORD_EOF) { @@ -175,7 +175,6 @@ TEST(RecordFileScanner, test_record_file_iterator) count = 0; while (OB_SUCC(rc = file_scanner.next(record))) { - ASSERT_EQ(rc, RC::SUCCESS); count++; } ASSERT_EQ(RC::RECORD_EOF, rc); @@ -194,7 +193,6 @@ TEST(RecordFileScanner, test_record_file_iterator) count = 0; while (OB_SUCC(rc = file_scanner.next(record))) { - ASSERT_EQ(rc, RC::SUCCESS); count++; } ASSERT_EQ(RC::RECORD_EOF, rc); @@ -235,8 +233,8 @@ TEST(RecordManager, durability) ASSERT_EQ(bpm.open_file(log_handler, record_manager_file.c_str(), buffer_pool), RC::SUCCESS); ASSERT_NE(buffer_pool, nullptr); - RecordFileHandler record_file_handler; - ASSERT_EQ(record_file_handler.init(*buffer_pool, log_handler), RC::SUCCESS); + RecordFileHandler record_file_handler(StorageFormat::ROW_FORMAT); + ASSERT_EQ(record_file_handler.init(*buffer_pool, log_handler, nullptr), RC::SUCCESS); const int record_size = 100; const char record_data[record_size] = "hello, world!"; @@ -344,8 +342,8 @@ TEST(RecordManager, durability) ASSERT_EQ(log_handler2.replay(log_replayer2, 0), RC::SUCCESS); ASSERT_EQ(log_handler2.start(), RC::SUCCESS); - RecordFileHandler record_file_handler2; - ASSERT_EQ(record_file_handler2.init(*buffer_pool2, log_handler2), RC::SUCCESS); + RecordFileHandler record_file_handler2(StorageFormat::ROW_FORMAT); + ASSERT_EQ(record_file_handler2.init(*buffer_pool2, log_handler2, nullptr), RC::SUCCESS); for (const auto &[rid, record] : record_map) { Record record_data; ASSERT_EQ(record_file_handler2.get_record(rid, record_data), RC::SUCCESS);