Enable verification - add ORDER BY *

.gitmodules

@@ -5,4 +5,4 @@
 [submodule "extension-ci-tools"]
 	path = extension-ci-tools
 	url = https://github.com/duckdb/extension-ci-tools
-	branch = main
+	branch = main

src/fuzzyduck.cpp

@@ -26,7 +26,10 @@ void FuzzyDuck::BeginFuzzing() {
 		auto &fs = FileSystem::GetFileSystem(context);
 		TryRemoveFile(complete_log);
 		complete_log_handle =
-		    fs.OpenFile(complete_log, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
+			fs.OpenFile(complete_log, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
+	}
+	if (enable_verification) {
+		RunQuery("PRAGMA enable_verification");
 	}
 }
 
@@ -64,15 +67,16 @@ void FuzzyDuck::FuzzAllFunctions() {
 }
 
 string FuzzyDuck::GenerateQuery() {
-	// generate the statement
+	// generate statement
 	StatementGenerator generator(context);
+	generator.verification_enabled = enable_verification;
 	// accumulate statement(s)
 	auto statement = string("");
 	if (generator.RandomPercentage(10)) {
 		// multi statement
 		idx_t number_of_statements = generator.RandomValue(1000);
 		LogTask("Generating Multi-Statement query of " + to_string(number_of_statements) + " statements with seed " +
-		        to_string(seed));
+			to_string(seed));
 		for (idx_t i = 0; i < number_of_statements; i++) {
 			statement += generator.GenerateStatement()->ToString() + "; ";
 		}
@@ -157,6 +161,7 @@ void FuzzyDuck::LogToCurrent(const string &message) {
 	file->Sync();
 	file->Close();
 }
+
 void FuzzyDuck::LogToComplete(const string &message) {
 	if (!complete_log_handle) {
 		return;

src/include/fuzzyduck.hpp

@@ -25,6 +25,7 @@ class FuzzyDuck {
 	string complete_log;
 	string log;
 	bool verbose_output = false;
+	bool enable_verification = false;
 	idx_t timeout = 30;
 
 public:

src/include/statement_generator.hpp

@@ -44,6 +44,7 @@ class StatementGenerator {
 public:
 	StatementGenerator(ClientContext &context);
 	StatementGenerator(StatementGenerator &parent);
+	StatementGenerator(StatementGenerator &parent, bool verification);
 	~StatementGenerator();
 
 public:
@@ -53,6 +54,7 @@ class StatementGenerator {
 
 	//! Returns true with a percentage change (0-100)
 	bool RandomPercentage(idx_t percentage);
+	bool verification_enabled = false;
 	idx_t RandomValue(idx_t max);
 	string GetRandomAttachedDataBase();
 	unique_ptr<SQLStatement> GenerateStatement(StatementType type); // came from private
@@ -63,9 +65,9 @@ class StatementGenerator {
 	unique_ptr<DeleteStatement> GenerateDelete();
 	unique_ptr<DetachStatement> GenerateDetach();
 	unique_ptr<MultiStatement> GenerateAttachUse();
-	unique_ptr<SelectStatement> GenerateSelect();
 	unique_ptr<SetStatement> GenerateSet();
 
+	unique_ptr<SelectStatement> GenerateSelect();
 	unique_ptr<QueryNode> GenerateQueryNode();
 
 	unique_ptr<AttachInfo> GenerateAttachInfo();
@@ -99,6 +101,8 @@ class StatementGenerator {
 	unique_ptr<ParsedExpression> GenerateCase();
 
 	unique_ptr<OrderModifier> GenerateOrderBy();
+	unique_ptr<OrderModifier> GenerateOrderByAll();
+
 
 	LogicalType GenerateLogicalType();
 

src/sqlsmith_extension.cpp

@@ -22,6 +22,7 @@ struct SQLSmithFunctionData : public TableFunctionData {
 	bool dump_all_queries = false;
 	bool dump_all_graphs = false;
 	bool verbose_output = false;
+	bool enable_verification = false;
 	string complete_log;
 	string log;
 	bool finished = false;
@@ -67,6 +68,7 @@ static void SQLSmithFunction(ClientContext &context, TableFunctionInput &data_p,
 	options.dump_all_queries = data.dump_all_queries;
 	options.dump_all_graphs = data.dump_all_graphs;
 	options.verbose_output = data.verbose_output;
+	options.enable_verification = data.enable_verification;
 	options.complete_log = data.complete_log;
 	options.log = data.log;
 	duckdb_sqlsmith::run_sqlsmith(DatabaseInstance::GetDatabase(context), options);
@@ -139,6 +141,8 @@ static duckdb::unique_ptr<FunctionData> FuzzyDuckBind(ClientContext &context, Ta
 			result->fuzzer.log = StringValue::Get(kv.second);
 		} else if (kv.first == "verbose_output") {
 			result->fuzzer.verbose_output = BooleanValue::Get(kv.second);
+		} else if (kv.first == "enable_verification") {
+			result->fuzzer.enable_verification = BooleanValue::Get(kv.second);
 		}
 	}
 	return_types.emplace_back(LogicalType::BOOLEAN);
@@ -186,6 +190,7 @@ void SqlsmithExtension::Load(DuckDB &db) {
 	fuzzy_duck_fun.named_parameters["log"] = LogicalType::VARCHAR;
 	fuzzy_duck_fun.named_parameters["complete_log"] = LogicalType::VARCHAR;
 	fuzzy_duck_fun.named_parameters["verbose_output"] = LogicalType::BOOLEAN;
+	fuzzy_duck_fun.named_parameters["enable_verification"] = LogicalType::BOOLEAN;
 	ExtensionUtil::RegisterFunction(db_instance, fuzzy_duck_fun);
 
 	TableFunction fuzz_all_functions("fuzz_all_functions", {}, FuzzAllFunctions, FuzzyDuckBind);

src/statement_generator.cpp

@@ -49,6 +49,14 @@
 	}
 }
 
+StatementGenerator::StatementGenerator(StatementGenerator &parent_p, bool verify)
+    : verification_enabled(verify), context(parent_p.context), parent(&parent_p), 
+	  generator_context(parent_p.generator_context), depth(parent_p.depth + 1) {
+	if (depth > MAX_DEPTH) {
+		throw InternalException("depth too high");
+	}
+}
+
 StatementGenerator::~StatementGenerator() {
 }
 
@@ -393,10 +401,12 @@
 	if (is_distinct) {
 		result->modifiers.push_back(make_uniq<DistinctModifier>());
 	}
-	if (RandomPercentage(20)) {
-		result->modifiers.push_back(GenerateOrderBy());
-	}
-	if (RandomPercentage(20)) {
+	if (verification_enabled) {
+		result->modifiers.push_back(GenerateOrderByAll());
+	} else if (!verification_enabled) {
+		if (RandomPercentage(5)) {
+			result->modifiers.push_back(GenerateOrderBy());
+		}
 		if (RandomPercentage(50)) {
 			auto limit_percent_modifier = make_uniq<LimitPercentModifier>();
 			if (RandomPercentage(30)) {
@@ -531,7 +541,7 @@
 	}
 	unique_ptr<SelectStatement> subquery;
 	{
-		StatementGenerator child_generator(*this);
+		StatementGenerator child_generator(*this, verification_enabled);
 		subquery = unique_ptr_cast<SQLStatement, SelectStatement>(child_generator.GenerateSelect());
 		for (auto &col : child_generator.current_column_names) {
 			current_column_names.push_back(std::move(col));
@@ -753,8 +763,8 @@
 	switch (function.type) {
 	case CatalogType::SCALAR_FUNCTION_ENTRY: {
 		auto &scalar_entry = function.Cast<ScalarFunctionCatalogEntry>();
-		auto actual_function = scalar_entry.functions.GetFunctionByOffset(RandomValue(scalar_entry.functions.Size()));
-
+		auto offset = RandomValue(scalar_entry.functions.Size());
+		auto actual_function = scalar_entry.functions.GetFunctionByOffset(offset);
 		name = scalar_entry.name;
 		arguments = actual_function.arguments;
 		min_parameters = actual_function.arguments.size();
@@ -767,7 +777,7 @@
 	case CatalogType::AGGREGATE_FUNCTION_ENTRY: {
 		auto &aggregate_entry = function.Cast<AggregateFunctionCatalogEntry>();
 		auto actual_function =
-		    aggregate_entry.functions.GetFunctionByOffset(RandomValue(aggregate_entry.functions.Size()));
+			aggregate_entry.functions.GetFunctionByOffset(RandomValue(aggregate_entry.functions.Size()));
 
 		name = aggregate_entry.name;
 		min_parameters = actual_function.arguments.size();
@@ -796,7 +806,7 @@
 	case CatalogType::MACRO_ENTRY: {
 		auto &macro_entry = function.Cast<MacroCatalogEntry>();
 		name = macro_entry.name;
 		min_parameters = macro_entry.function->parameters.size();
 		max_parameters = min_parameters;
 		break;
 	}
@@ -815,19 +825,27 @@
 	                                     distinct);
 }
 
+unique_ptr<OrderModifier> StatementGenerator::GenerateOrderByAll() {
+    auto result = make_uniq<OrderModifier>();
+	auto order_type = Choose<OrderType>({OrderType::ASCENDING, OrderType::DESCENDING, OrderType::ORDER_DEFAULT});
+	auto null_type = Choose<OrderByNullType>(
+		{OrderByNullType::NULLS_FIRST, OrderByNullType::NULLS_LAST, OrderByNullType::ORDER_DEFAULT});
+	result->orders.emplace_back(order_type, null_type, GenerateStar());
+    return result;
+}
+
 unique_ptr<OrderModifier> StatementGenerator::GenerateOrderBy() {
-	auto result = make_uniq<OrderModifier>();
+    auto result = make_uniq<OrderModifier>();
 	while (true) {
 		auto order_type = Choose<OrderType>({OrderType::ASCENDING, OrderType::DESCENDING, OrderType::ORDER_DEFAULT});
 		auto null_type = Choose<OrderByNullType>(
-		    {OrderByNullType::NULLS_FIRST, OrderByNullType::NULLS_LAST, OrderByNullType::ORDER_DEFAULT});
+			{OrderByNullType::NULLS_FIRST, OrderByNullType::NULLS_LAST, OrderByNullType::ORDER_DEFAULT});
 		result->orders.emplace_back(order_type, null_type, GenerateExpression());
-		// continue with a random chance
 		if (RandomPercentage(50)) {
 			break;
 		}
 	}
-	return result;
+    return result;
 }
 
 unique_ptr<ParsedExpression> StatementGenerator::GenerateOperator() {
@@ -1043,25 +1061,26 @@
 			result->relation_name = GenerateRelationName();
 		}
 	}
-
-	while (RandomPercentage(20)) {
-		auto column_name = GenerateColumnName();
-		if (column_name.empty()) {
-			break;
+	if (!verification_enabled) {
+		while (RandomPercentage(20)) {
+			auto column_name = GenerateColumnName();
+			if (column_name.empty()) {
+				break;
+			}
+			result->exclude_list.insert(column_name);
 		}
-		result->exclude_list.insert(column_name);
-	}
-	while (RandomPercentage(20)) {
-		auto column_name = GenerateColumnName();
-		if (column_name.empty()) {
-			break;
+		while (RandomPercentage(20)) {
+			auto column_name = GenerateColumnName();
+			if (column_name.empty()) {
+				break;
+			}
+			result->replace_list.insert(make_pair(column_name, GenerateExpression()));
 		}
-		result->replace_list.insert(make_pair(column_name, GenerateExpression()));
-	}
-	if (RandomPercentage(50) || expression_depth > 0) {
-		result->columns = true;
-		if (RandomPercentage(50)) {
-			result->expr = GenerateLambda();
+		if (RandomPercentage(50) || expression_depth > 0) {
+			result->columns = true;
+			if (RandomPercentage(50)) {
+				result->expr = GenerateLambda();
+			}
 		}
 	}
 	return std::move(result);
@@ -1120,9 +1139,8 @@
 		return GenerateConstant();
 	}
 	auto subquery = make_uniq<SubqueryExpression>();
-
 	{
-		StatementGenerator child_generator(*this);
+		StatementGenerator child_generator(*this, verification_enabled);
 		subquery->subquery = unique_ptr_cast<SQLStatement, SelectStatement>(child_generator.GenerateSelect());
 	}
 	subquery->subquery_type =

src/third_party/sqlsmith/include/sqlsmith.hh

@@ -16,6 +16,7 @@ struct SQLSmithOptions {
 	bool dump_all_queries = false;
 	bool dump_all_graphs = false;
 	bool verbose_output = false;
+	bool enable_verification = false;
 	std::string complete_log;
 	std::string log;
 };