Skip to content

Commit

Permalink
Update vendored DuckDB sources to 4adc14b
Browse files Browse the repository at this point in the history
  • Loading branch information
duckdblabs-bot committed Sep 12, 2024
1 parent 4adc14b commit 015c757
Show file tree
Hide file tree
Showing 31 changed files with 260 additions and 242 deletions.
39 changes: 15 additions & 24 deletions src/duckdb/src/common/cgroups.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@ optional_idx CGroups::GetMemoryLimit(FileSystem &fs) {
}

optional_idx CGroups::GetCGroupV2MemoryLimit(FileSystem &fs) {
#ifdef DUCKDB_WASM
return optional_idx();
#else
#if defined(__linux__) && !defined(DUCKDB_WASM)
const char *cgroup_self = "/proc/self/cgroup";
const char *memory_max = "/sys/fs/cgroup/%s/memory.max";

Expand All @@ -45,13 +43,13 @@ optional_idx CGroups::GetCGroupV2MemoryLimit(FileSystem &fs) {
}

return ReadCGroupValue(fs, memory_max_path);
#else
return optional_idx();
#endif
}

optional_idx CGroups::GetCGroupV1MemoryLimit(FileSystem &fs) {
#ifdef DUCKDB_WASM
return optional_idx();
#else
#if defined(__linux__) && !defined(DUCKDB_WASM)
const char *cgroup_self = "/proc/self/cgroup";
const char *memory_limit = "/sys/fs/cgroup/memory/%s/memory.limit_in_bytes";

Expand All @@ -72,13 +70,13 @@ optional_idx CGroups::GetCGroupV1MemoryLimit(FileSystem &fs) {
}

return ReadCGroupValue(fs, memory_limit_path);
#else
return optional_idx();
#endif
}

string CGroups::ReadCGroupPath(FileSystem &fs, const char *cgroup_file) {
#ifdef DUCKDB_WASM
return "";
#else
#if defined(__linux__) && !defined(DUCKDB_WASM)
auto handle = fs.OpenFile(cgroup_file, FileFlags::FILE_FLAGS_READ);
char buffer[1024];
auto bytes_read = fs.Read(*handle, buffer, sizeof(buffer) - 1);
Expand All @@ -90,15 +88,12 @@ string CGroups::ReadCGroupPath(FileSystem &fs, const char *cgroup_file) {
if (pos != string::npos) {
return content.substr(pos + 2);
}

return "";
#endif
return "";
}

string CGroups::ReadMemoryCGroupPath(FileSystem &fs, const char *cgroup_file) {
#ifdef DUCKDB_WASM
return "";
#else
#if defined(__linux__) && !defined(DUCKDB_WASM)
auto handle = fs.OpenFile(cgroup_file, FileFlags::FILE_FLAGS_READ);
char buffer[1024];
auto bytes_read = fs.Read(*handle, buffer, sizeof(buffer) - 1);
Expand All @@ -115,15 +110,12 @@ string CGroups::ReadMemoryCGroupPath(FileSystem &fs, const char *cgroup_file) {
}
content.erase(0, pos + 1);
}

return "";
#endif
return "";
}

optional_idx CGroups::ReadCGroupValue(FileSystem &fs, const char *file_path) {
#ifdef DUCKDB_WASM
return optional_idx();
#else
#if defined(__linux__) && !defined(DUCKDB_WASM)
auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ);
char buffer[100];
auto bytes_read = fs.Read(*handle, buffer, 99);
Expand All @@ -133,15 +125,12 @@ optional_idx CGroups::ReadCGroupValue(FileSystem &fs, const char *file_path) {
if (TryCast::Operation<string_t, idx_t>(string_t(buffer), value)) {
return optional_idx(value);
}
return optional_idx();
#endif
return optional_idx();
}

idx_t CGroups::GetCPULimit(FileSystem &fs, idx_t physical_cores) {
#ifdef DUCKDB_WASM
return physical_cores;
#else

#if defined(__linux__) && !defined(DUCKDB_WASM)
static constexpr const char *cpu_max = "/sys/fs/cgroup/cpu.max";
static constexpr const char *cfs_quota = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
static constexpr const char *cfs_period = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
Expand Down Expand Up @@ -183,6 +172,8 @@ idx_t CGroups::GetCPULimit(FileSystem &fs, idx_t physical_cores) {
} else {
return physical_cores;
}
#else
return physical_cores;
#endif
}

Expand Down
62 changes: 3 additions & 59 deletions src/duckdb/src/common/operator/cast_operators.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -920,68 +920,12 @@ bool TryCast::Operation(double input, double &result, bool strict) {
//===--------------------------------------------------------------------===//
// Cast String -> Numeric
//===--------------------------------------------------------------------===//

template <>
bool TryCast::Operation(string_t input, bool &result, bool strict) {
auto input_data = reinterpret_cast<const uint8_t *>(input.GetData());
auto input_data = reinterpret_cast<const char *>(input.GetData());
auto input_size = input.GetSize();

switch (input_size) {
case 1: {
unsigned char c = UnsafeNumericCast<uint8_t>(std::tolower(*input_data));
if (c == 't' || (!strict && c == 'y') || (!strict && c == '1')) {
result = true;
return true;
} else if (c == 'f' || (!strict && c == 'n') || (!strict && c == '0')) {
result = false;
return true;
}
return false;
}
case 2: {
unsigned char n = UnsafeNumericCast<uint8_t>(std::tolower(input_data[0]));
unsigned char o = UnsafeNumericCast<uint8_t>(std::tolower(input_data[1]));
if (n == 'n' && o == 'o') {
result = false;
return true;
}
return false;
}
case 3: {
unsigned char y = UnsafeNumericCast<uint8_t>(std::tolower(input_data[0]));
unsigned char e = UnsafeNumericCast<uint8_t>(std::tolower(input_data[1]));
unsigned char s = UnsafeNumericCast<uint8_t>(std::tolower(input_data[2]));
if (y == 'y' && e == 'e' && s == 's') {
result = true;
return true;
}
return false;
}
case 4: {
unsigned char t = UnsafeNumericCast<uint8_t>(std::tolower(input_data[0]));
unsigned char r = UnsafeNumericCast<uint8_t>(std::tolower(input_data[1]));
unsigned char u = UnsafeNumericCast<uint8_t>(std::tolower(input_data[2]));
unsigned char e = UnsafeNumericCast<uint8_t>(std::tolower(input_data[3]));
if (t == 't' && r == 'r' && u == 'u' && e == 'e') {
result = true;
return true;
}
return false;
}
case 5: {
unsigned char f = UnsafeNumericCast<uint8_t>(std::tolower(input_data[0]));
unsigned char a = UnsafeNumericCast<uint8_t>(std::tolower(input_data[1]));
unsigned char l = UnsafeNumericCast<uint8_t>(std::tolower(input_data[2]));
unsigned char s = UnsafeNumericCast<uint8_t>(std::tolower(input_data[3]));
unsigned char e = UnsafeNumericCast<uint8_t>(std::tolower(input_data[4]));
if (f == 'f' && a == 'a' && l == 'l' && s == 's' && e == 'e') {
result = false;
return true;
}
return false;
}
default:
return false;
}
return TryCastStringBool(input_data, input_size, result, strict);
}
template <>
bool TryCast::Operation(string_t input, int8_t &result, bool strict) {
Expand Down
3 changes: 2 additions & 1 deletion src/duckdb/src/common/types/column/column_data_allocator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ BufferHandle ColumnDataAllocator::AllocateBlock(idx_t size) {
BlockMetaData data;
data.size = 0;
data.capacity = NumericCast<uint32_t>(max_size);
auto pin = alloc.buffer_manager->Allocate(MemoryTag::COLUMN_DATA, max_size, false, &data.handle);
auto pin = alloc.buffer_manager->Allocate(MemoryTag::COLUMN_DATA, max_size, false);
data.handle = pin.GetBlockHandle();
blocks.push_back(std::move(data));
allocated_size += max_size;
return pin;
Expand Down
3 changes: 2 additions & 1 deletion src/duckdb/src/common/types/row/tuple_data_allocator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ namespace duckdb {
using ValidityBytes = TupleDataLayout::ValidityBytes;

TupleDataBlock::TupleDataBlock(BufferManager &buffer_manager, idx_t capacity_p) : capacity(capacity_p), size(0) {
buffer_manager.Allocate(MemoryTag::HASH_TABLE, capacity, false, &handle);
auto buffer_handle = buffer_manager.Allocate(MemoryTag::HASH_TABLE, capacity, false);
handle = buffer_handle.GetBlockHandle();
}

TupleDataBlock::TupleDataBlock(TupleDataBlock &&other) noexcept : capacity(0), size(0) {
Expand Down
32 changes: 17 additions & 15 deletions src/duckdb/src/execution/index/art/iterator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -251,11 +251,7 @@ bool Iterator::Next() {
}

void Iterator::PopNode() {
// We are popping a gate node.
if (nodes.top().node.GetGateStatus() == GateStatus::GATE_SET) {
D_ASSERT(status == GateStatus::GATE_SET);
status = GateStatus::GATE_NOT_SET;
}
auto gate_status = nodes.top().node.GetGateStatus();

// Pop the byte and the node.
if (nodes.top().node.GetType() != NType::PREFIX) {
Expand All @@ -264,19 +260,25 @@ void Iterator::PopNode() {
nested_depth--;
D_ASSERT(nested_depth < Prefix::ROW_ID_SIZE);
}
nodes.pop();
return;
}

// Pop all prefix bytes and the node.
Prefix prefix(art, nodes.top().node);
auto prefix_byte_count = prefix.data[Prefix::Count(art)];
current_key.Pop(prefix_byte_count);
if (status == GateStatus::GATE_SET) {
nested_depth -= prefix_byte_count;
D_ASSERT(nested_depth < Prefix::ROW_ID_SIZE);
} else {
// Pop all prefix bytes and the node.
Prefix prefix(art, nodes.top().node);
auto prefix_byte_count = prefix.data[Prefix::Count(art)];
current_key.Pop(prefix_byte_count);

if (status == GateStatus::GATE_SET) {
nested_depth -= prefix_byte_count;
D_ASSERT(nested_depth < Prefix::ROW_ID_SIZE);
}
}
nodes.pop();

// We are popping a gate node.
if (gate_status == GateStatus::GATE_SET) {
D_ASSERT(status == GateStatus::GATE_SET);
status = GateStatus::GATE_NOT_SET;
}
}

} // namespace duckdb
43 changes: 9 additions & 34 deletions src/duckdb/src/execution/index/art/prefix.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -400,42 +400,15 @@ void Prefix::TransformToDeprecated(ART &art, Node &node, unsafe_unique_ptr<Fixed
return Node::TransformToDeprecated(art, ref, allocator);
}

// Fast path.
if (art.prefix_count <= DEPRECATED_COUNT) {
reference<Node> ref(node);
while (ref.get().GetType() == PREFIX && ref.get().GetGateStatus() == GateStatus::GATE_NOT_SET) {
Prefix prefix(art, ref, true, true);
if (!prefix.in_memory) {
return;
}

Node new_node;
new_node = allocator->New();
new_node.SetMetadata(static_cast<uint8_t>(PREFIX));

Prefix new_prefix(allocator, new_node, DEPRECATED_COUNT);
new_prefix.data[DEPRECATED_COUNT] = prefix.data[Count(art)];
memcpy(new_prefix.data, prefix.data, new_prefix.data[DEPRECATED_COUNT]);
*new_prefix.ptr = *prefix.ptr;

prefix.ptr->Clear();
Node::Free(art, ref);
ref.get() = new_node;
ref = *new_prefix.ptr;
}

return Node::TransformToDeprecated(art, ref, allocator);
}

// Else, we need to create a new prefix chain.
// We need to create a new prefix (chain).
Node new_node;
new_node = allocator->New();
new_node.SetMetadata(static_cast<uint8_t>(PREFIX));
Prefix new_prefix(allocator, new_node, DEPRECATED_COUNT);

reference<Node> ref(node);
while (ref.get().GetType() == PREFIX && ref.get().GetGateStatus() == GateStatus::GATE_NOT_SET) {
Prefix prefix(art, ref, true, true);
Node current_node = node;
while (current_node.GetType() == PREFIX && current_node.GetGateStatus() == GateStatus::GATE_NOT_SET) {
Prefix prefix(art, current_node, true, true);
if (!prefix.in_memory) {
return;
}
Expand All @@ -445,11 +418,13 @@ void Prefix::TransformToDeprecated(ART &art, Node &node, unsafe_unique_ptr<Fixed
}

*new_prefix.ptr = *prefix.ptr;
Node::GetAllocator(art, PREFIX).Free(ref);
ref = *new_prefix.ptr;
prefix.ptr->Clear();
Node::Free(art, current_node);
current_node = *new_prefix.ptr;
}

return Node::TransformToDeprecated(art, ref, allocator);
node = new_node;
return Node::TransformToDeprecated(art, *new_prefix.ptr, allocator);
}

Prefix Prefix::Append(ART &art, const uint8_t byte) {
Expand Down
7 changes: 4 additions & 3 deletions src/duckdb/src/execution/index/fixed_size_buffer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager)
block_handle(nullptr) {

auto &buffer_manager = block_manager.buffer_manager;
buffer_handle = buffer_manager.Allocate(MemoryTag::ART_INDEX, block_manager.GetBlockSize(), false, &block_handle);
buffer_handle = buffer_manager.Allocate(MemoryTag::ART_INDEX, block_manager.GetBlockSize(), false);
block_handle = buffer_handle.GetBlockHandle();
}

FixedSizeBuffer::FixedSizeBuffer(BlockManager &block_manager, const idx_t segment_count, const idx_t allocation_size,
Expand Down Expand Up @@ -137,8 +138,8 @@ void FixedSizeBuffer::Pin() {

// Copy the (partial) data into a new (not yet disk-backed) buffer handle.
shared_ptr<BlockHandle> new_block_handle;
auto new_buffer_handle =
buffer_manager.Allocate(MemoryTag::ART_INDEX, block_manager.GetBlockSize(), false, &new_block_handle);
auto new_buffer_handle = buffer_manager.Allocate(MemoryTag::ART_INDEX, block_manager.GetBlockSize(), false);
new_block_handle = new_buffer_handle.GetBlockHandle();
memcpy(new_buffer_handle.Ptr(), buffer_handle.Ptr() + block_pointer.offset, allocation_size);

buffer_handle = std::move(new_buffer_handle);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ void CSVBuffer::AllocateBuffer(idx_t buffer_size) {
auto &buffer_manager = BufferManager::GetBufferManager(context);
bool can_destroy = !is_pipe;
handle = buffer_manager.Allocate(MemoryTag::CSV_READER, MaxValue<idx_t>(buffer_manager.GetBlockSize(), buffer_size),
can_destroy, &block);
can_destroy);
block = handle.GetBlockHandle();
}

idx_t CSVBuffer::GetBufferSize() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,10 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size
}
bool success = true;
switch (parse_types[chunk_col_id].type_id) {
case LogicalTypeId::BOOLEAN:
success =
TryCastStringBool(value_ptr, size, static_cast<bool *>(vector_ptr[chunk_col_id])[number_of_rows], false);
break;
case LogicalTypeId::TINYINT:
success = TrySimpleIntegerCast(value_ptr, size, static_cast<int8_t *>(vector_ptr[chunk_col_id])[number_of_rows],
false);
Expand Down Expand Up @@ -644,9 +648,15 @@ bool LineError::HandleErrors(StringValueResult &result) {
result.error_handler.Error(csv_error);
}
if (is_error_in_line) {
result.borked_rows.insert(result.number_of_rows);
result.cur_col_id = 0;
result.chunk_col_id = 0;
if (result.sniffing) {
// If we are sniffing we just remove the line
result.RemoveLastLine();
} else {
// Otherwise, we add it to the borked rows to remove it later and just cleanup the column variables.
result.borked_rows.insert(result.number_of_rows);
result.cur_col_id = 0;
result.chunk_col_id = 0;
}
Reset();
return true;
}
Expand Down Expand Up @@ -1437,6 +1447,7 @@ bool StringValueScanner::CanDirectlyCast(const LogicalType &type, bool icu_loade
case LogicalTypeId::TIME:
case LogicalTypeId::DECIMAL:
case LogicalType::VARCHAR:
case LogicalType::BOOLEAN:
return true;
case LogicalType::TIMESTAMP_TZ:
// We only try to do direct cast of timestamp tz if the ICU extension is not loaded, otherwise, it needs to go
Expand Down Expand Up @@ -1493,7 +1504,7 @@ void StringValueScanner::SetStart() {
}
if (iterator.pos.buffer_pos == cur_buffer_handle->actual_size ||
scan_finder->iterator.GetBufferIdx() > iterator.GetBufferIdx()) {
// If things go terribly wrong, we never loop indefinetly.
// If things go terribly wrong, we never loop indefinitely.
iterator.pos.buffer_idx = scan_finder->iterator.pos.buffer_idx;
iterator.pos.buffer_pos = scan_finder->iterator.pos.buffer_pos;
result.last_position = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, result.buffer_size};
Expand Down
Loading

0 comments on commit 015c757

Please sign in to comment.