diff --git a/.clang-tidy b/.clang-tidy index 07fc4979d..3d9b505a1 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -4,7 +4,6 @@ Checks: '*,misc-const-correctness,-bugprone-reserved-identifier,-hicpp-signed-bitwise,-llvmlibc-restrict-system-libc-headers,-altera-unroll-loops,-hicpp-named-parameter,-cert-dcl37-c,-cert-dcl51-cpp,-read,-cppcoreguidelines-init-variables,-cppcoreguidelines-avoid-non-const-global-variables,-altera-id-dependent-backward-branch,-performance-no-int-to-ptr,-altera-struct-pack-align,-google-readability-casting,-modernize-use-trailing-return-type,-llvmlibc-implementation-in-namespace,-llvmlibc-callee-namespace,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-fuchsia-default-arguments-declarations,-fuchsia-overloaded-operator,-cppcoreguidelines-pro-type-union-access,-fuchsia-default-arguments-calls,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-google-readability-todo,-llvm-header-guard,-readability-function-cognitive-complexity,-readability-identifier-length,-cppcoreguidelines-owning-memory,-cert-err58-cpp,-fuchsia-statically-constructed-objects,-google-build-using-namespace,-hicpp-avoid-goto,-cppcoreguidelines-avoid-goto,-hicpp-no-array-decay,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-bounds-constant-array-index,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers,-abseil-string-find-str-contains,-bugprone-unchecked-optional-access,-readability-use-anyofallof' WarningsAsErrors: '*' HeaderFilterRegex: '' -AnalyzeTemporaryDtors: false CheckOptions: - key: readability-function-cognitive-complexity.Threshold value: 35 diff --git a/cmake/objects.cmake b/cmake/objects.cmake index 931df3eaf..b89f46146 100644 --- a/cmake/objects.cmake +++ b/cmake/objects.cmake @@ -12,7 +12,6 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/expression.cpp ${libddwaf_SOURCE_DIR}/src/ruleset_info.cpp ${libddwaf_SOURCE_DIR}/src/ip_utils.cpp - ${libddwaf_SOURCE_DIR}/src/processor.cpp ${libddwaf_SOURCE_DIR}/src/iterator.cpp ${libddwaf_SOURCE_DIR}/src/log.cpp ${libddwaf_SOURCE_DIR}/src/obfuscator.cpp @@ -22,6 +21,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/platform.cpp ${libddwaf_SOURCE_DIR}/src/uuid.cpp ${libddwaf_SOURCE_DIR}/src/action_mapper.cpp + ${libddwaf_SOURCE_DIR}/src/builder/processor_builder.cpp ${libddwaf_SOURCE_DIR}/src/tokenizer/sql_base.cpp ${libddwaf_SOURCE_DIR}/src/tokenizer/pgsql.cpp ${libddwaf_SOURCE_DIR}/src/tokenizer/mysql.cpp @@ -30,13 +30,20 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/exclusion/input_filter.cpp ${libddwaf_SOURCE_DIR}/src/exclusion/object_filter.cpp ${libddwaf_SOURCE_DIR}/src/exclusion/rule_filter.cpp - ${libddwaf_SOURCE_DIR}/src/generator/extract_schema.cpp ${libddwaf_SOURCE_DIR}/src/parser/actions_parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/common.cpp ${libddwaf_SOURCE_DIR}/src/parser/parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/parser_v1.cpp - ${libddwaf_SOURCE_DIR}/src/parser/parser_v2.cpp ${libddwaf_SOURCE_DIR}/src/parser/rule_data_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/processor_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/expression_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/matcher_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/transformer_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/rule_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/rule_override_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/scanner_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp + ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp ${libddwaf_SOURCE_DIR}/src/condition/lfi_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/sqli_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/ssrf_detector.cpp diff --git a/fuzzer/lfi_detector/src/main.cpp b/fuzzer/lfi_detector/src/main.cpp index d3bd42f89..47d96aab9 100644 --- a/fuzzer/lfi_detector/src/main.cpp +++ b/fuzzer/lfi_detector/src/main.cpp @@ -20,7 +20,7 @@ extern "C" int LLVMFuzzerInitialize(const int * /*argc*/, char *** /*argv*/) return 0; } -template std::vector gen_param_def(Args... addresses) +template std::vector gen_param_def(Args... addresses) { return {{{{std::string{addresses}, get_target_index(addresses)}}}...}; } diff --git a/fuzzer/sqli_detector/src/main.cpp b/fuzzer/sqli_detector/src/main.cpp index ed2dabe7e..e73bbbed2 100644 --- a/fuzzer/sqli_detector/src/main.cpp +++ b/fuzzer/sqli_detector/src/main.cpp @@ -31,7 +31,7 @@ extern "C" int LLVMFuzzerInitialize(const int *argc, char ***argv) return 0; } -template std::vector gen_param_def(Args... addresses) +template std::vector gen_param_def(Args... addresses) { return {{{{std::string{addresses}, get_target_index(addresses)}}}...}; } diff --git a/fuzzer/ssrf_detector/src/main.cpp b/fuzzer/ssrf_detector/src/main.cpp index 77b143929..b0964c211 100644 --- a/fuzzer/ssrf_detector/src/main.cpp +++ b/fuzzer/ssrf_detector/src/main.cpp @@ -20,7 +20,7 @@ extern "C" int LLVMFuzzerInitialize(const int * /*argc*/, char *** /*argv*/) return 0; } -template std::vector gen_param_def(Args... addresses) +template std::vector gen_param_def(Args... addresses) { return {{{{std::string{addresses}, get_target_index(addresses)}}}...}; } diff --git a/src/argument_retriever.hpp b/src/argument_retriever.hpp new file mode 100644 index 000000000..5f8b9de39 --- /dev/null +++ b/src/argument_retriever.hpp @@ -0,0 +1,141 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "condition/base.hpp" +#include "traits.hpp" +#include "utils.hpp" + +namespace ddwaf { + +// A type of argument with a single address (target) mapping +template struct unary_argument { + // The memory associated with the address and the key path is owned + // by either the condition (condition_target) or the processor (processor_target). + std::string_view address{}; + std::span key_path; + bool ephemeral{false}; + T value; +}; + +template struct is_unary_argument : std::false_type {}; +template struct is_unary_argument> : std::true_type {}; + +// A type of argument which is considered to be optional +template using optional_argument = std::optional>; + +template struct is_optional_argument : std::false_type {}; +template struct is_optional_argument> : std::true_type {}; + +// A type of argument with multiple address(target) mappings +template using variadic_argument = std::vector>; + +template struct is_variadic_argument : std::false_type {}; +template struct is_variadic_argument> : std::true_type {}; + +template std::optional convert(const ddwaf_object *obj) +{ + if constexpr (std::is_same_v) { + return obj; + } + + if constexpr (std::is_same_v || std::is_same_v) { + if (obj->type == DDWAF_OBJ_STRING) { + return T{obj->stringValue, static_cast(obj->nbEntries)}; + } + } + + if constexpr (std::is_same_v || std::is_same_v) { + using limits = std::numeric_limits; + if (obj->type == DDWAF_OBJ_UNSIGNED && obj->uintValue <= limits::max()) { + return static_cast(obj->uintValue); + } + } + + if constexpr (std::is_same_v || std::is_same_v) { + using limits = std::numeric_limits; + if (obj->type == DDWAF_OBJ_SIGNED && obj->intValue >= limits::min() && + obj->intValue <= limits::max()) { + return static_cast(obj->intValue); + } + } + + if constexpr (std::is_same_v) { + if (obj->type == DDWAF_OBJ_BOOL) { + return static_cast(obj->boolean); + } + } + + return {}; +} + +struct default_argument_retriever { + static constexpr bool is_variadic = false; + static constexpr bool is_optional = false; +}; + +template struct argument_retriever : default_argument_retriever {}; + +template struct argument_retriever> : default_argument_retriever { + template + static std::optional> retrieve(const object_store &store, + const exclusion::object_set_ref &objects_excluded, const TargetType &target) + { + auto [object, attr] = store.get_target(target.index); + if (object == nullptr || objects_excluded.contains(object)) { + return std::nullopt; + } + + auto converted = convert(object); + if (!converted.has_value()) { + return std::nullopt; + } + + return unary_argument{target.name, target.key_path, + attr == object_store::attribute::ephemeral, std::move(converted.value())}; + } +}; + +template struct argument_retriever> : default_argument_retriever { + static constexpr bool is_optional = true; + + template + static optional_argument retrieve(const object_store &store, + const exclusion::object_set_ref &objects_excluded, const TargetType &target) + { + return argument_retriever>::retrieve(store, objects_excluded, target); + } +}; + +template struct argument_retriever> : default_argument_retriever { + static constexpr bool is_variadic = true; + + template + static variadic_argument retrieve(const object_store &store, + const exclusion::object_set_ref &objects_excluded, const std::vector &targets) + { + variadic_argument args; + for (const auto &target : targets) { + auto arg = + argument_retriever>::retrieve(store, objects_excluded, target); + if (!arg.has_value()) { + continue; + } + args.emplace_back(std::move(arg.value())); + } + return args; + } +}; + +} // namespace ddwaf diff --git a/src/builder/processor_builder.cpp b/src/builder/processor_builder.cpp new file mode 100644 index 000000000..b81de6337 --- /dev/null +++ b/src/builder/processor_builder.cpp @@ -0,0 +1,78 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include + +#include "builder/processor_builder.hpp" +#include "processor/extract_schema.hpp" + +namespace ddwaf { + +namespace { +std::set references_to_scanners( + const std::vector &references, const indexer &scanners) +{ + std::set scanner_refs; + for (const auto &ref : references) { + if (ref.type == parser::reference_type::id) { + const auto *scanner = scanners.find_by_id(ref.ref_id); + if (scanner == nullptr) { + continue; + } + scanner_refs.emplace(scanner); + } else if (ref.type == parser::reference_type::tags) { + auto current_refs = scanners.find_by_tags(ref.tags); + scanner_refs.merge(current_refs); + } + } + return scanner_refs; +} + +template struct typed_processor_builder; + +template <> struct typed_processor_builder { + std::shared_ptr build(const auto &spec, const auto &scanners) + { + auto ref_scanners = references_to_scanners(spec.scanners, scanners); + return std::make_shared( + spec.id, spec.expr, spec.mappings, std::move(ref_scanners), spec.evaluate, spec.output); + } +}; + +template +concept has_build_with_scanners = + requires(typed_processor_builder b, Spec spec, Scanners scanners) { + { + b.build(spec, scanners) + } -> std::same_as>; + }; + +template +[[nodiscard]] std::shared_ptr build_with_type( + const auto &spec, const auto &scanners) + requires std::is_base_of_v +{ + typed_processor_builder typed_builder; + if constexpr (has_build_with_scanners) { + return typed_builder.build(spec, scanners); + } else { + return typed_builder.build(spec); + } +} +} // namespace + +[[nodiscard]] std::shared_ptr processor_builder::build( + const indexer &scanners) const +{ + switch (type) { + case processor_type::extract_schema: + return build_with_type(*this, scanners); + default: + break; + } + return {}; +} +} // namespace ddwaf diff --git a/src/builder/processor_builder.hpp b/src/builder/processor_builder.hpp new file mode 100644 index 000000000..f2d220e1b --- /dev/null +++ b/src/builder/processor_builder.hpp @@ -0,0 +1,54 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include + +#include "indexer.hpp" +#include "parser/specification.hpp" +#include "processor/base.hpp" + +namespace ddwaf { + +enum class processor_type : unsigned { + extract_schema, + // Reserved + http_fingerprint, + session_fingerprint, + network_fingerprint, + header_fingerprint, +}; + +struct processor_builder { + [[nodiscard]] std::shared_ptr build( + const indexer &scanners) const; + + processor_type type; + std::string id; + std::shared_ptr expr; + std::vector mappings; + std::vector scanners; + bool evaluate{false}; + bool output{true}; +}; + +struct processor_container { + [[nodiscard]] bool empty() const { return pre.empty() && post.empty(); } + [[nodiscard]] std::size_t size() const { return pre.size() + post.size(); } + void clear() + { + pre.clear(); + post.clear(); + } + + std::vector pre; + std::vector post; +}; + +} // namespace ddwaf diff --git a/src/condition/base.hpp b/src/condition/base.hpp index d91ea9ad0..99f606400 100644 --- a/src/condition/base.hpp +++ b/src/condition/base.hpp @@ -56,9 +56,9 @@ struct parameter_specification { // Provides the definition of an individual address(target) to parameter mapping. // Each target must satisfy the associated parameter specification. -struct target_definition { +struct condition_target { std::string name; - target_index root{}; + target_index index{}; std::vector key_path{}; std::vector transformers{}; data_source source{data_source::values}; @@ -66,8 +66,8 @@ struct target_definition { // Provides the list of targets mapped to the given condition parameter. If the // parameter is non-variadic, only one mapping should be present. -struct parameter_definition { - std::vector targets; +struct condition_parameter { + std::vector targets; }; class base_condition { diff --git a/src/condition/lfi_detector.hpp b/src/condition/lfi_detector.hpp index f0df640e5..2b7262ae5 100644 --- a/src/condition/lfi_detector.hpp +++ b/src/condition/lfi_detector.hpp @@ -14,7 +14,7 @@ class lfi_detector : public base_impl { public: static constexpr std::array param_names{"resource", "params"}; - explicit lfi_detector(std::vector args, const object_limits &limits = {}) + explicit lfi_detector(std::vector args, const object_limits &limits = {}) : base_impl(std::move(args), limits) {} diff --git a/src/condition/scalar_condition.cpp b/src/condition/scalar_condition.cpp index f6f7d17d7..89684e25c 100644 --- a/src/condition/scalar_condition.cpp +++ b/src/condition/scalar_condition.cpp @@ -122,7 +122,7 @@ eval_result scalar_condition::eval(condition_cache &cache, const object_store &s } const auto &target = targets_[i]; - auto [object, attr] = store.get_target(target.root); + auto [object, attr] = store.get_target(target.index); if (object == nullptr || object == cache.targets[i]) { continue; } diff --git a/src/condition/scalar_condition.hpp b/src/condition/scalar_condition.hpp index a77b47868..6d7c9843f 100644 --- a/src/condition/scalar_condition.hpp +++ b/src/condition/scalar_condition.hpp @@ -13,7 +13,7 @@ namespace ddwaf { class scalar_condition : public base_condition { public: scalar_condition(std::unique_ptr &&matcher, std::string data_id, - std::vector args, const object_limits &limits = {}) + std::vector args, const object_limits &limits = {}) : matcher_(std::move(matcher)), data_id_(std::move(data_id)), limits_(limits) { if (args.size() > 1) { @@ -34,7 +34,7 @@ class scalar_condition : public base_condition { void get_addresses(std::unordered_map &addresses) const override { - for (const auto &target : targets_) { addresses.emplace(target.root, target.name); } + for (const auto &target : targets_) { addresses.emplace(target.index, target.name); } } static constexpr auto arguments() @@ -49,7 +49,7 @@ class scalar_condition : public base_condition { std::unique_ptr matcher_; std::string data_id_; - std::vector targets_; + std::vector targets_; const object_limits limits_; }; diff --git a/src/condition/sqli_detector.hpp b/src/condition/sqli_detector.hpp index 85f714726..a1a125419 100644 --- a/src/condition/sqli_detector.hpp +++ b/src/condition/sqli_detector.hpp @@ -15,7 +15,7 @@ class sqli_detector : public base_impl { public: static constexpr std::array param_names{"resource", "params", "db_type"}; - explicit sqli_detector(std::vector args, const object_limits &limits = {}) + explicit sqli_detector(std::vector args, const object_limits &limits = {}) : base_impl(std::move(args), limits) {} diff --git a/src/condition/ssrf_detector.cpp b/src/condition/ssrf_detector.cpp index c3282b1c7..e41ea0f78 100644 --- a/src/condition/ssrf_detector.cpp +++ b/src/condition/ssrf_detector.cpp @@ -241,7 +241,7 @@ ssrf_result ssrf_impl(const uri_decomposed &uri, const ddwaf_object ¶ms, } // namespace -ssrf_detector::ssrf_detector(std::vector args, const object_limits &limits) +ssrf_detector::ssrf_detector(std::vector args, const object_limits &limits) : base_impl(std::move(args), limits), dangerous_ip_matcher_(std::make_unique(dangerous_ips)), authorised_schemes_(authorised_schemes.begin(), authorised_schemes.end()) diff --git a/src/condition/ssrf_detector.hpp b/src/condition/ssrf_detector.hpp index 12e9cd88f..17d46640e 100644 --- a/src/condition/ssrf_detector.hpp +++ b/src/condition/ssrf_detector.hpp @@ -15,8 +15,7 @@ class ssrf_detector : public base_impl { public: static constexpr std::array param_names{"resource", "params"}; - explicit ssrf_detector( - std::vector args, const object_limits &limits = {}); + explicit ssrf_detector(std::vector args, const object_limits &limits = {}); protected: [[nodiscard]] eval_result eval_impl(const unary_argument &uri, diff --git a/src/condition/structured_condition.hpp b/src/condition/structured_condition.hpp index 0522b8239..b875b0725 100644 --- a/src/condition/structured_condition.hpp +++ b/src/condition/structured_condition.hpp @@ -13,132 +13,20 @@ #include #include +#include "argument_retriever.hpp" #include "condition/base.hpp" #include "traits.hpp" #include "utils.hpp" namespace ddwaf { -// A type of argument with a single address (target) mapping -template struct unary_argument { - std::string_view address{}; - std::span key_path; - bool ephemeral{false}; - T value; -}; - -template struct is_unary_argument : std::false_type {}; -template struct is_unary_argument> : std::true_type {}; - -// A type of argument which is considered to be optional -template using optional_argument = std::optional>; - -template struct is_optional_argument : std::false_type {}; -template struct is_optional_argument> : std::true_type {}; - -// A type of argument with multiple address(target) mappings -template using variadic_argument = std::vector>; - -template struct is_variadic_argument : std::false_type {}; -template struct is_variadic_argument> : std::true_type {}; - -struct default_argument_retriever { - static constexpr bool is_variadic = false; - static constexpr bool is_optional = false; -}; - -template std::optional convert(const ddwaf_object *obj) -{ - if constexpr (std::is_same_v) { - return obj; - } - - if constexpr (std::is_same_v || std::is_same_v) { - if (obj->type == DDWAF_OBJ_STRING) { - return T{obj->stringValue, static_cast(obj->nbEntries)}; - } - } - - if constexpr (std::is_same_v || std::is_same_v) { - using limits = std::numeric_limits; - if (obj->type == DDWAF_OBJ_UNSIGNED && obj->uintValue <= limits::max()) { - return static_cast(obj->uintValue); - } - } - - if constexpr (std::is_same_v || std::is_same_v) { - using limits = std::numeric_limits; - if (obj->type == DDWAF_OBJ_SIGNED && obj->intValue >= limits::min() && - obj->intValue <= limits::max()) { - return static_cast(obj->intValue); - } - } - - if constexpr (std::is_same_v) { - if (obj->type == DDWAF_OBJ_BOOL) { - return static_cast(obj->boolean); - } - } - - return {}; -} - -template struct argument_retriever : default_argument_retriever {}; - -template struct argument_retriever> : default_argument_retriever { - static std::optional> retrieve(const object_store &store, - const exclusion::object_set_ref &objects_excluded, const target_definition &target) - { - auto [object, attr] = store.get_target(target.root); - if (object == nullptr || objects_excluded.contains(object)) { - return std::nullopt; - } - - auto converted = convert(object); - if (!converted.has_value()) { - return std::nullopt; - } - - return unary_argument{target.name, target.key_path, - attr == object_store::attribute::ephemeral, std::move(converted.value())}; - } -}; - -template struct argument_retriever> : default_argument_retriever { - static constexpr bool is_optional = true; - static optional_argument retrieve(const object_store &store, - const exclusion::object_set_ref &objects_excluded, const target_definition &target) - { - return argument_retriever>::retrieve(store, objects_excluded, target); - } -}; - -template struct argument_retriever> : default_argument_retriever { - static constexpr bool is_variadic = true; - static variadic_argument retrieve(const object_store &store, - const exclusion::object_set_ref &objects_excluded, - const std::vector &targets) - { - variadic_argument args; - for (const auto &target : targets) { - auto arg = - argument_retriever>::retrieve(store, objects_excluded, target); - if (!arg.has_value()) { - continue; - } - args.emplace_back(std::move(arg.value())); - } - return args; - } -}; - template function_traits make_eval_traits( eval_result (Class::*)(Args...) const); template class base_impl : public base_condition { public: - explicit base_impl(std::vector args, const object_limits &limits) + explicit base_impl(std::vector args, const object_limits &limits) : arguments_(std::move(args)), limits_(limits) {} ~base_impl() override = default; @@ -195,7 +83,7 @@ template class base_impl : public base_condition { void get_addresses(std::unordered_map &addresses) const override { for (const auto &arg : arguments_) { - for (const auto &target : arg.targets) { addresses.emplace(target.root, target.name); } + for (const auto &target : arg.targets) { addresses.emplace(target.index, target.name); } } } @@ -243,6 +131,16 @@ template class base_impl : public base_condition { return target_type{}; } return retriever::retrieve(store, objects_excluded, arguments_[I].targets); + } else if constexpr (retriever::is_optional) { + if (arguments_.size() <= I) { + return target_type{}; + } + + const auto &arg = arguments_[I]; + if (arg.targets.empty()) { + return target_type{}; + } + return retriever::retrieve(store, objects_excluded, arg.targets.at(0)); } else { if (arguments_.size() <= I) { return std::optional{}; @@ -256,7 +154,7 @@ template class base_impl : public base_condition { } } - std::vector arguments_; + std::vector arguments_; const object_limits limits_; }; diff --git a/src/context.cpp b/src/context.cpp index f71a9b3a7..9e0617c7c 100644 --- a/src/context.cpp +++ b/src/context.cpp @@ -110,7 +110,7 @@ void context::eval_preprocessors(optional_ref &derived, ddwaf::tim auto it = processor_cache_.find(preproc.get()); if (it == processor_cache_.end()) { - auto [new_it, res] = processor_cache_.emplace(preproc.get(), processor::cache_type{}); + auto [new_it, res] = processor_cache_.emplace(preproc.get(), processor_cache{}); it = new_it; } @@ -130,7 +130,7 @@ void context::eval_postprocessors(optional_ref &derived, ddwaf::ti auto it = processor_cache_.find(postproc.get()); if (it == processor_cache_.end()) { - auto [new_it, res] = processor_cache_.emplace(postproc.get(), processor::cache_type{}); + auto [new_it, res] = processor_cache_.emplace(postproc.get(), processor_cache{}); it = new_it; } diff --git a/src/context.hpp b/src/context.hpp index 12b6bef3a..df6dcc2bf 100644 --- a/src/context.hpp +++ b/src/context.hpp @@ -80,7 +80,7 @@ class context { using input_filter = exclusion::input_filter; using rule_filter = exclusion::rule_filter; - memory::unordered_map processor_cache_; + memory::unordered_map processor_cache_; // Caches of filters and conditions memory::unordered_map rule_filter_cache_{}; diff --git a/src/generator/base.hpp b/src/generator/base.hpp deleted file mode 100644 index 331476e97..000000000 --- a/src/generator/base.hpp +++ /dev/null @@ -1,34 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - -#pragma once - -#include -#include -#include -#include -#include - -#include "clock.hpp" -#include "scanner.hpp" -#include "utils.hpp" - -namespace ddwaf::generator { - -class base { -public: - base() = default; - virtual ~base() = default; - base(const base &) = default; - base(base &&) = default; - base &operator=(const base &) = default; - base &operator=(base &&) = default; - - virtual ddwaf_object generate(const ddwaf_object *input, - const std::set &scanners, ddwaf::timer &deadline) = 0; -}; - -} // namespace ddwaf::generator diff --git a/src/generator/extract_schema.hpp b/src/generator/extract_schema.hpp deleted file mode 100644 index d1744cf60..000000000 --- a/src/generator/extract_schema.hpp +++ /dev/null @@ -1,37 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - -#pragma once - -#include -#include -#include -#include -#include - -#include "generator/base.hpp" -#include "utils.hpp" - -namespace ddwaf::generator { - -class extract_schema : public base { -public: - static constexpr std::size_t max_container_depth = 18; - static constexpr std::size_t max_array_nodes = 10; - static constexpr std::size_t max_record_nodes = 255; - - extract_schema() = default; - ~extract_schema() override = default; - extract_schema(const extract_schema &) = delete; - extract_schema(extract_schema &&) = default; - extract_schema &operator=(const extract_schema &) = delete; - extract_schema &operator=(extract_schema &&) = default; - - ddwaf_object generate(const ddwaf_object *input, const std::set &scanners, - ddwaf::timer &deadline) override; -}; - -} // namespace ddwaf::generator diff --git a/src/parser/common.cpp b/src/parser/common.cpp index 49978a66e..c13f5b513 100644 --- a/src/parser/common.cpp +++ b/src/parser/common.cpp @@ -59,4 +59,27 @@ std::optional transformer_from_string(std::string_view str) return std::nullopt; } +reference_spec parse_reference(const parameter::map &target) +{ + auto ref_id = at(target, "rule_id", {}); + if (!ref_id.empty()) { + return {reference_type::id, std::move(ref_id), {}}; + } + + ref_id = at(target, "id", {}); + if (!ref_id.empty()) { + return {reference_type::id, std::move(ref_id), {}}; + } + + auto tag_map = at(target, "tags", {}); + if (!tag_map.empty()) { + std::unordered_map tags; + for (auto &[key, value] : tag_map) { tags.emplace(key, value); } + + return {reference_type::tags, {}, std::move(tags)}; + } + + return {reference_type::none, {}, {}}; +} + } // namespace ddwaf::parser diff --git a/src/parser/common.hpp b/src/parser/common.hpp index 40d173c7c..3f0f3cb8c 100644 --- a/src/parser/common.hpp +++ b/src/parser/common.hpp @@ -11,8 +11,12 @@ #include "exception.hpp" #include "parameter.hpp" +#include "parser/specification.hpp" +#include "ruleset_info.hpp" #include "transformer/base.hpp" +using base_section_info = ddwaf::base_ruleset_info::base_section_info; + namespace ddwaf::parser { struct address_container { @@ -46,4 +50,12 @@ std::optional transformer_from_string(std::string_view str); inline std::string index_to_id(unsigned idx) { return "index:" + to_string(idx); } +reference_spec parse_reference(const parameter::map &target); + +inline void add_addresses_to_info(const address_container &addresses, base_section_info &info) +{ + for (const auto &address : addresses.required) { info.add_required_address(address); } + for (const auto &address : addresses.optional) { info.add_optional_address(address); } +} + } // namespace ddwaf::parser diff --git a/src/parser/exclusion_parser.cpp b/src/parser/exclusion_parser.cpp new file mode 100644 index 000000000..6f68ed221 --- /dev/null +++ b/src/parser/exclusion_parser.cpp @@ -0,0 +1,135 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "exclusion/object_filter.hpp" +#include "parameter.hpp" +#include "parser/common.hpp" +#include "parser/parser.hpp" +#include "parser/specification.hpp" +#include "utils.hpp" + +namespace ddwaf::parser::v2 { + +namespace { + +input_filter_spec parse_input_filter( + const parameter::map &filter, address_container &addresses, const object_limits &limits) +{ + // Check for conditions first + auto conditions_array = at(filter, "conditions", {}); + auto expr = parse_simplified_expression(conditions_array, addresses, limits); + + std::vector rules_target; + auto rules_target_array = at(filter, "rules_target", {}); + if (!rules_target_array.empty()) { + rules_target.reserve(rules_target_array.size()); + + for (const auto &target : rules_target_array) { + rules_target.emplace_back(parse_reference(static_cast(target))); + } + } + + auto obj_filter = std::make_shared(limits); + auto inputs_array = at(filter, "inputs"); + + // TODO: add empty method to object filter and check after + if (expr->empty() && inputs_array.empty() && rules_target.empty()) { + throw ddwaf::parsing_error("empty exclusion filter"); + } + + for (const auto &input_param : inputs_array) { + auto input_map = static_cast(input_param); + auto address = at(input_map, "address"); + + auto target = get_target_index(address); + auto key_path = at>(input_map, "key_path", {}); + + addresses.optional.emplace(address); + obj_filter->insert(target, std::move(address), key_path); + } + + return {std::move(expr), std::move(obj_filter), std::move(rules_target)}; +} + +rule_filter_spec parse_rule_filter( + const parameter::map &filter, address_container &addresses, const object_limits &limits) +{ + // Check for conditions first + auto conditions_array = at(filter, "conditions", {}); + auto expr = parse_simplified_expression(conditions_array, addresses, limits); + + std::vector rules_target; + auto rules_target_array = at(filter, "rules_target", {}); + if (!rules_target_array.empty()) { + rules_target.reserve(rules_target_array.size()); + + for (const auto &target : rules_target_array) { + rules_target.emplace_back(parse_reference(static_cast(target))); + } + } + + exclusion::filter_mode on_match; + auto on_match_str = at(filter, "on_match", "bypass"); + if (on_match_str == "bypass") { + on_match = exclusion::filter_mode::bypass; + } else if (on_match_str == "monitor") { + on_match = exclusion::filter_mode::monitor; + } else { + throw ddwaf::parsing_error("unsupported on_match value: " + std::string(on_match_str)); + } + + if (expr->empty() && rules_target.empty()) { + throw ddwaf::parsing_error("empty exclusion filter"); + } + + return {std::move(expr), std::move(rules_target), on_match}; +} + +} // namespace + +filter_spec_container parse_filters( + parameter::vector &filter_array, base_section_info &info, const object_limits &limits) +{ + filter_spec_container filters; + for (unsigned i = 0; i < filter_array.size(); i++) { + const auto &node_param = filter_array[i]; + auto node = static_cast(node_param); + std::string id; + try { + address_container addresses; + id = at(node, "id"); + if (filters.ids.find(id) != filters.ids.end()) { + DDWAF_WARN("Duplicate filter: {}", id); + info.add_failed(id, "duplicate filter"); + continue; + } + + if (node.find("inputs") != node.end()) { + auto filter = parse_input_filter(node, addresses, limits); + filters.ids.emplace(id); + filters.input_filters.emplace(id, std::move(filter)); + } else { + auto filter = parse_rule_filter(node, addresses, limits); + filters.ids.emplace(id); + filters.rule_filters.emplace(id, std::move(filter)); + } + DDWAF_DEBUG("Parsed exclusion filter {}", id); + + info.add_loaded(id); + add_addresses_to_info(addresses, info); + } catch (const std::exception &e) { + if (id.empty()) { + id = index_to_id(i); + } + DDWAF_WARN("Failed to parse filter '{}': {}", id, e.what()); + info.add_failed(id, e.what()); + } + } + + return filters; +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/expression_parser.cpp b/src/parser/expression_parser.cpp new file mode 100644 index 000000000..f33966e92 --- /dev/null +++ b/src/parser/expression_parser.cpp @@ -0,0 +1,131 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "condition/lfi_detector.hpp" +#include "condition/scalar_condition.hpp" +#include "condition/sqli_detector.hpp" +#include "condition/ssrf_detector.hpp" +#include "expression.hpp" +#include "parser/common.hpp" +#include "parser/parser.hpp" +#include + +namespace ddwaf::parser::v2 { + +namespace { + +template +std::vector parse_arguments(const parameter::map ¶ms, data_source source, + const std::vector &transformers, address_container &addresses) +{ + const auto &specification = T::arguments(); + std::vector definitions; + + definitions.reserve(specification.size()); + + for (const auto spec : specification) { + definitions.emplace_back(); + condition_parameter &def = definitions.back(); + + auto inputs = at(params, spec.name); + if (inputs.empty()) { + if (!spec.optional) { + throw ddwaf::parsing_error("empty non-optional argument"); + } + continue; + } + + if (!spec.variadic && inputs.size() > 1) { + throw ddwaf::parsing_error("multiple targets for non-variadic argument"); + } + + auto &targets = def.targets; + for (const auto &input_param : inputs) { + auto input = static_cast(input_param); + auto address = at(input, "address"); + + DDWAF_DEBUG("Found address {}", address); + + if (address.empty()) { + throw ddwaf::parsing_error("empty address"); + } + + auto kp = at>(input, "key_path", {}); + for (const auto &path : kp) { + if (path.empty()) { + throw ddwaf::parsing_error("empty key_path"); + } + } + + addresses.required.emplace(address); + auto it = input.find("transformers"); + if (it == input.end()) { + targets.emplace_back(condition_target{ + address, get_target_index(address), std::move(kp), transformers, source}); + } else { + auto input_transformers = static_cast(it->second); + source = data_source::values; + auto new_transformers = parse_transformers(input_transformers, source); + targets.emplace_back(condition_target{address, get_target_index(address), + std::move(kp), std::move(new_transformers), source}); + } + } + } + + return definitions; +} + +} // namespace + +std::shared_ptr parse_expression(const parameter::vector &conditions_array, + std::unordered_map &data_ids, data_source source, + const std::vector &transformers, address_container &addresses, + const object_limits &limits) +{ + std::vector> conditions; + for (const auto &cond_param : conditions_array) { + auto root = static_cast(cond_param); + + auto operator_name = at(root, "operator"); + auto params = at(root, "parameters"); + + if (operator_name == "lfi_detector") { + auto arguments = parse_arguments(params, source, transformers, addresses); + conditions.emplace_back(std::make_unique(std::move(arguments), limits)); + } else if (operator_name == "ssrf_detector") { + auto arguments = + parse_arguments(params, source, transformers, addresses); + conditions.emplace_back(std::make_unique(std::move(arguments), limits)); + } else if (operator_name == "sqli_detector") { + auto arguments = + parse_arguments(params, source, transformers, addresses); + conditions.emplace_back(std::make_unique(std::move(arguments), limits)); + } else { + auto [data_id, matcher] = parse_matcher(operator_name, params); + + if (!matcher && !data_id.empty()) { + data_ids.emplace(data_id, operator_name); + } + + auto arguments = + parse_arguments(params, source, transformers, addresses); + + conditions.emplace_back(std::make_unique( + std::move(matcher), data_id, std::move(arguments), limits)); + } + } + + return std::make_shared(std::move(conditions)); +} + +std::shared_ptr parse_simplified_expression(const parameter::vector &conditions_array, + address_container &addresses, const object_limits &limits) +{ + std::unordered_map data_ids; + return parse_expression(conditions_array, data_ids, data_source::values, {}, addresses, limits); +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/matcher_parser.cpp b/src/parser/matcher_parser.cpp new file mode 100644 index 000000000..3461ade73 --- /dev/null +++ b/src/parser/matcher_parser.cpp @@ -0,0 +1,113 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2024 Datadog, Inc. + +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include + +#include "matcher/equals.hpp" +#include "matcher/exact_match.hpp" +#include "matcher/ip_match.hpp" +#include "matcher/is_sqli.hpp" +#include "matcher/is_xss.hpp" +#include "matcher/phrase_match.hpp" +#include "matcher/regex_match.hpp" +#include "parameter.hpp" +#include "parser/common.hpp" + +namespace ddwaf::parser::v2 { + +std::pair> parse_matcher( + std::string_view name, const parameter::map ¶ms) +{ + parameter::map options; + std::unique_ptr matcher; + std::string rule_data_id; + + if (name == "phrase_match") { + auto list = at(params, "list"); + options = at(params, "options", options); + auto word_boundary = at(options, "enforce_word_boundary", false); + + std::vector patterns; + std::vector lengths; + + patterns.reserve(list.size()); + lengths.reserve(list.size()); + + for (auto &pattern : list) { + if (pattern.type != DDWAF_OBJ_STRING) { + throw ddwaf::parsing_error("phrase_match list item not a string"); + } + + patterns.push_back(pattern.stringValue); + lengths.push_back((uint32_t)pattern.nbEntries); + } + + matcher = std::make_unique(patterns, lengths, word_boundary); + } else if (name == "match_regex") { + auto regex = at(params, "regex"); + options = at(params, "options", options); + + auto case_sensitive = at(options, "case_sensitive", false); + auto min_length = at(options, "min_length", 0); + if (min_length < 0) { + throw ddwaf::parsing_error("min_length is a negative number"); + } + + matcher = std::make_unique(regex, min_length, case_sensitive); + } else if (name == "is_xss") { + matcher = std::make_unique(); + } else if (name == "is_sqli") { + matcher = std::make_unique(); + } else if (name == "ip_match") { + auto it = params.find("list"); + if (it == params.end()) { + rule_data_id = at(params, "data"); + } else { + matcher = std::make_unique( + static_cast>(it->second)); + } + } else if (name == "exact_match") { + auto it = params.find("list"); + if (it == params.end()) { + rule_data_id = at(params, "data"); + } else { + matcher = std::make_unique( + static_cast>(it->second)); + } + } else if (name == "equals") { + auto value_type = at(params, "type"); + if (value_type == "string") { + auto value = at(params, "value"); + matcher = std::make_unique>(std::move(value)); + } else if (value_type == "boolean") { + auto value = at(params, "value"); + matcher = std::make_unique>(value); + } else if (value_type == "unsigned") { + auto value = at(params, "value"); + matcher = std::make_unique>(value); + } else if (value_type == "signed") { + auto value = at(params, "value"); + matcher = std::make_unique>(value); + } else if (value_type == "float") { + auto value = at(params, "value"); + auto delta = at(params, "delta", 0.01); + matcher = std::make_unique>(value, delta); + } else { + throw ddwaf::parsing_error("invalid type for matcher equals " + value_type); + } + } else { + throw ddwaf::parsing_error("unknown matcher: " + std::string(name)); + } + + return {std::move(rule_data_id), std::move(matcher)}; +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/parser.hpp b/src/parser/parser.hpp index 5867be33f..80b19d9d8 100644 --- a/src/parser/parser.hpp +++ b/src/parser/parser.hpp @@ -10,15 +10,15 @@ #include #include +#include "builder/processor_builder.hpp" #include "indexer.hpp" #include "parameter.hpp" +#include "parser/common.hpp" #include "parser/specification.hpp" #include "rule.hpp" #include "ruleset.hpp" #include "ruleset_info.hpp" -using base_section_info = ddwaf::base_ruleset_info::base_section_info; - namespace ddwaf::parser { unsigned parse_schema_version(parameter::map &ruleset); @@ -50,5 +50,18 @@ indexer parse_scanners(parameter::vector &scanner_array, base_sec std::shared_ptr parse_actions( parameter::vector &actions_array, base_section_info &info); +std::shared_ptr parse_expression(const parameter::vector &conditions_array, + std::unordered_map &data_ids, data_source source, + const std::vector &transformers, address_container &addresses, + const object_limits &limits); + +std::shared_ptr parse_simplified_expression(const parameter::vector &conditions_array, + address_container &addresses, const object_limits &limits); + +std::vector parse_transformers(const parameter::vector &root, data_source &source); + +std::pair> parse_matcher( + std::string_view name, const parameter::map ¶ms); + } // namespace v2 } // namespace ddwaf::parser diff --git a/src/parser/parser_v1.cpp b/src/parser/parser_v1.cpp index 786104269..13d3f1ad0 100644 --- a/src/parser/parser_v1.cpp +++ b/src/parser/parser_v1.cpp @@ -78,9 +78,9 @@ std::shared_ptr parse_expression(parameter::vector &conditions_array throw ddwaf::parsing_error("unknown matcher: " + std::string(matcher_name)); } - std::vector definitions; + std::vector definitions; definitions.emplace_back(); - parameter_definition &def = definitions.back(); + condition_parameter &def = definitions.back(); auto inputs = at(params, "inputs"); for (const auto &input_param : inputs) { @@ -99,7 +99,7 @@ std::shared_ptr parse_expression(parameter::vector &conditions_array key_path.emplace_back(input.substr(pos + 1, input.size())); } - def.targets.emplace_back(target_definition{root, get_target_index(root), + def.targets.emplace_back(condition_target{root, get_target_index(root), std::move(key_path), transformers, data_source::values}); } diff --git a/src/parser/parser_v2.cpp b/src/parser/parser_v2.cpp deleted file mode 100644 index ab34fd039..000000000 --- a/src/parser/parser_v2.cpp +++ /dev/null @@ -1,808 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - -#include -#include -#include -#include -#include - -#include "condition/lfi_detector.hpp" -#include "condition/scalar_condition.hpp" -#include "condition/sqli_detector.hpp" -#include "condition/ssrf_detector.hpp" -#include "exception.hpp" -#include "exclusion/object_filter.hpp" -#include "generator/extract_schema.hpp" -#include "indexer.hpp" -#include "log.hpp" -#include "matcher/equals.hpp" -#include "matcher/exact_match.hpp" -#include "matcher/ip_match.hpp" -#include "matcher/is_sqli.hpp" -#include "matcher/is_xss.hpp" -#include "matcher/phrase_match.hpp" -#include "matcher/regex_match.hpp" -#include "parameter.hpp" -#include "parser/common.hpp" -#include "parser/parser.hpp" -#include "parser/rule_data_parser.hpp" -#include "parser/specification.hpp" -#include "rule.hpp" -#include "ruleset.hpp" -#include "ruleset_info.hpp" -#include "utils.hpp" - -namespace ddwaf::parser::v2 { - -namespace { - -std::pair> parse_matcher( - std::string_view name, const parameter::map ¶ms) -{ - parameter::map options; - std::unique_ptr matcher; - std::string rule_data_id; - - if (name == "phrase_match") { - auto list = at(params, "list"); - options = at(params, "options", options); - auto word_boundary = at(options, "enforce_word_boundary", false); - - std::vector patterns; - std::vector lengths; - - patterns.reserve(list.size()); - lengths.reserve(list.size()); - - for (auto &pattern : list) { - if (pattern.type != DDWAF_OBJ_STRING) { - throw ddwaf::parsing_error("phrase_match list item not a string"); - } - - patterns.push_back(pattern.stringValue); - lengths.push_back((uint32_t)pattern.nbEntries); - } - - matcher = std::make_unique(patterns, lengths, word_boundary); - } else if (name == "match_regex") { - auto regex = at(params, "regex"); - options = at(params, "options", options); - - auto case_sensitive = at(options, "case_sensitive", false); - auto min_length = at(options, "min_length", 0); - if (min_length < 0) { - throw ddwaf::parsing_error("min_length is a negative number"); - } - - matcher = std::make_unique(regex, min_length, case_sensitive); - } else if (name == "is_xss") { - matcher = std::make_unique(); - } else if (name == "is_sqli") { - matcher = std::make_unique(); - } else if (name == "ip_match") { - auto it = params.find("list"); - if (it == params.end()) { - rule_data_id = at(params, "data"); - } else { - matcher = std::make_unique( - static_cast>(it->second)); - } - } else if (name == "exact_match") { - auto it = params.find("list"); - if (it == params.end()) { - rule_data_id = at(params, "data"); - } else { - matcher = std::make_unique( - static_cast>(it->second)); - } - } else if (name == "equals") { - auto value_type = at(params, "type"); - if (value_type == "string") { - auto value = at(params, "value"); - matcher = std::make_unique>(std::move(value)); - } else if (value_type == "boolean") { - auto value = at(params, "value"); - matcher = std::make_unique>(value); - } else if (value_type == "unsigned") { - auto value = at(params, "value"); - matcher = std::make_unique>(value); - } else if (value_type == "signed") { - auto value = at(params, "value"); - matcher = std::make_unique>(value); - } else if (value_type == "float") { - auto value = at(params, "value"); - auto delta = at(params, "delta", 0.01); - matcher = std::make_unique>(value, delta); - } else { - throw ddwaf::parsing_error("invalid type for matcher equals " + value_type); - } - } else { - throw ddwaf::parsing_error("unknown matcher: " + std::string(name)); - } - - return {std::move(rule_data_id), std::move(matcher)}; -} - -std::vector parse_transformers(const parameter::vector &root, data_source &source) -{ - if (root.empty()) { - return {}; - } - - std::vector transformers; - transformers.reserve(root.size()); - - for (const auto &transformer_param : root) { - auto transformer = static_cast(transformer_param); - auto id = transformer_from_string(transformer); - if (id.has_value()) { - transformers.emplace_back(id.value()); - } else if (transformer == "keys_only") { - source = ddwaf::data_source::keys; - } else if (transformer == "values_only") { - source = ddwaf::data_source::values; - } else { - throw ddwaf::parsing_error("invalid transformer " + std::string(transformer)); - } - } - return transformers; -} - -template -std::vector parse_arguments(const parameter::map ¶ms, data_source source, - const std::vector &transformers, address_container &addresses) -{ - const auto &specification = T::arguments(); - std::vector definitions; - - definitions.reserve(specification.size()); - - for (const auto spec : specification) { - definitions.emplace_back(); - parameter_definition &def = definitions.back(); - - auto inputs = at(params, spec.name); - if (inputs.empty()) { - if (!spec.optional) { - throw ddwaf::parsing_error("empty non-optional argument"); - } - continue; - } - - if (!spec.variadic && inputs.size() > 1) { - throw ddwaf::parsing_error("multiple targets for non-variadic argument"); - } - - auto &targets = def.targets; - for (const auto &input_param : inputs) { - auto input = static_cast(input_param); - auto address = at(input, "address"); - - DDWAF_DEBUG("Found address {}", address); - - if (address.empty()) { - throw ddwaf::parsing_error("empty address"); - } - - auto kp = at>(input, "key_path", {}); - for (const auto &path : kp) { - if (path.empty()) { - throw ddwaf::parsing_error("empty key_path"); - } - } - - addresses.required.emplace(address); - auto it = input.find("transformers"); - if (it == input.end()) { - targets.emplace_back(target_definition{ - address, get_target_index(address), std::move(kp), transformers, source}); - } else { - auto input_transformers = static_cast(it->second); - source = data_source::values; - auto new_transformers = parse_transformers(input_transformers, source); - targets.emplace_back(target_definition{address, get_target_index(address), - std::move(kp), std::move(new_transformers), source}); - } - } - } - - return definitions; -} - -std::shared_ptr parse_expression(const parameter::vector &conditions_array, - std::unordered_map &data_ids, data_source source, - const std::vector &transformers, address_container &addresses, - const object_limits &limits) -{ - std::vector> conditions; - for (const auto &cond_param : conditions_array) { - auto root = static_cast(cond_param); - - auto operator_name = at(root, "operator"); - auto params = at(root, "parameters"); - - if (operator_name == "lfi_detector") { - auto arguments = parse_arguments(params, source, transformers, addresses); - conditions.emplace_back(std::make_unique(std::move(arguments), limits)); - } else if (operator_name == "ssrf_detector") { - auto arguments = - parse_arguments(params, source, transformers, addresses); - conditions.emplace_back(std::make_unique(std::move(arguments), limits)); - } else if (operator_name == "sqli_detector") { - auto arguments = - parse_arguments(params, source, transformers, addresses); - conditions.emplace_back(std::make_unique(std::move(arguments), limits)); - } else { - auto [data_id, matcher] = parse_matcher(operator_name, params); - - if (!matcher && !data_id.empty()) { - data_ids.emplace(data_id, operator_name); - } - - auto arguments = - parse_arguments(params, source, transformers, addresses); - - conditions.emplace_back(std::make_unique( - std::move(matcher), data_id, std::move(arguments), limits)); - } - } - - return std::make_shared(std::move(conditions)); -} - -rule_spec parse_rule(parameter::map &rule, - std::unordered_map &rule_data_ids, const object_limits &limits, - rule::source_type source, address_container &addresses) -{ - std::vector rule_transformers; - auto data_source = ddwaf::data_source::values; - auto transformers = at(rule, "transformers", {}); - rule_transformers = parse_transformers(transformers, data_source); - - auto conditions_array = at(rule, "conditions"); - auto expr = parse_expression( - conditions_array, rule_data_ids, data_source, rule_transformers, addresses, limits); - if (expr->empty()) { - // This is likely unreachable - throw ddwaf::parsing_error("rule has no valid conditions"); - } - - std::unordered_map tags; - for (auto &[key, value] : at(rule, "tags")) { - try { - tags.emplace(key, std::string(value)); - } catch (const bad_cast &e) { - throw invalid_type(std::string(key), e); - } - } - - if (tags.find("type") == tags.end()) { - throw ddwaf::parsing_error("missing key 'type'"); - } - - return {at(rule, "enabled", true), source, at(rule, "name"), std::move(tags), - std::move(expr), at>(rule, "on_match", {})}; -} - -reference_spec parse_reference(const parameter::map &target) -{ - auto ref_id = at(target, "rule_id", {}); - if (!ref_id.empty()) { - return {reference_type::id, std::move(ref_id), {}}; - } - - ref_id = at(target, "id", {}); - if (!ref_id.empty()) { - return {reference_type::id, std::move(ref_id), {}}; - } - - auto tag_map = at(target, "tags", {}); - if (!tag_map.empty()) { - std::unordered_map tags; - for (auto &[key, value] : tag_map) { tags.emplace(key, value); } - - return {reference_type::tags, {}, std::move(tags)}; - } - - return {reference_type::none, {}, {}}; -} - -std::pair parse_override(const parameter::map &node) -{ - // Note that ID is a duplicate field and will be deprecated at some point - override_spec current; - - auto it = node.find("enabled"); - if (it != node.end()) { - current.enabled = static_cast(it->second); - } - - it = node.find("on_match"); - if (it != node.end()) { - auto actions = static_cast>(it->second); - current.actions = std::move(actions); - } - - reference_type type = reference_type::none; - - auto rules_target_array = at(node, "rules_target", {}); - if (!rules_target_array.empty()) { - current.targets.reserve(rules_target_array.size()); - - for (const auto &target : rules_target_array) { - auto target_spec = parse_reference(static_cast(target)); - if (type == reference_type::none) { - type = target_spec.type; - } else if (type != target_spec.type) { - throw ddwaf::parsing_error("rule override targets rules and tags"); - } - - current.targets.emplace_back(std::move(target_spec)); - } - } else { - // Since the rules_target array is empty, the ID is mandatory - current.targets.emplace_back( - reference_spec{reference_type::id, at(node, "id"), {}}); - type = reference_type::id; - } - - if (!current.actions.has_value() && !current.enabled.has_value()) { - throw ddwaf::parsing_error("rule override without side-effects"); - } - - return {current, type}; -} - -std::shared_ptr parse_simplified_expression(const parameter::vector &conditions_array, - address_container &addresses, const object_limits &limits) -{ - std::unordered_map data_ids; - return parse_expression(conditions_array, data_ids, data_source::values, {}, addresses, limits); -} - -input_filter_spec parse_input_filter( - const parameter::map &filter, address_container &addresses, const object_limits &limits) -{ - // Check for conditions first - auto conditions_array = at(filter, "conditions", {}); - auto expr = parse_simplified_expression(conditions_array, addresses, limits); - - std::vector rules_target; - auto rules_target_array = at(filter, "rules_target", {}); - if (!rules_target_array.empty()) { - rules_target.reserve(rules_target_array.size()); - - for (const auto &target : rules_target_array) { - rules_target.emplace_back(parse_reference(static_cast(target))); - } - } - - auto obj_filter = std::make_shared(limits); - auto inputs_array = at(filter, "inputs"); - - // TODO: add empty method to object filter and check after - if (expr->empty() && inputs_array.empty() && rules_target.empty()) { - throw ddwaf::parsing_error("empty exclusion filter"); - } - - for (const auto &input_param : inputs_array) { - auto input_map = static_cast(input_param); - auto address = at(input_map, "address"); - - auto target = get_target_index(address); - auto key_path = at>(input_map, "key_path", {}); - - addresses.optional.emplace(address); - obj_filter->insert(target, std::move(address), key_path); - } - - return {std::move(expr), std::move(obj_filter), std::move(rules_target)}; -} - -rule_filter_spec parse_rule_filter( - const parameter::map &filter, address_container &addresses, const object_limits &limits) -{ - // Check for conditions first - auto conditions_array = at(filter, "conditions", {}); - auto expr = parse_simplified_expression(conditions_array, addresses, limits); - - std::vector rules_target; - auto rules_target_array = at(filter, "rules_target", {}); - if (!rules_target_array.empty()) { - rules_target.reserve(rules_target_array.size()); - - for (const auto &target : rules_target_array) { - rules_target.emplace_back(parse_reference(static_cast(target))); - } - } - - exclusion::filter_mode on_match; - auto on_match_str = at(filter, "on_match", "bypass"); - if (on_match_str == "bypass") { - on_match = exclusion::filter_mode::bypass; - } else if (on_match_str == "monitor") { - on_match = exclusion::filter_mode::monitor; - } else { - throw ddwaf::parsing_error("unsupported on_match value: " + std::string(on_match_str)); - } - - if (expr->empty() && rules_target.empty()) { - throw ddwaf::parsing_error("empty exclusion filter"); - } - - return {std::move(expr), std::move(rules_target), on_match}; -} - -std::vector parse_processor_mappings( - const parameter::vector &root, address_container &addresses) -{ - if (root.empty()) { - throw ddwaf::parsing_error("empty mappings"); - } - - std::vector mappings; - for (const auto &node : root) { - auto mapping = static_cast(node); - - // TODO support n:1 mappings and key paths - auto inputs = at(mapping, "inputs"); - if (inputs.empty()) { - throw ddwaf::parsing_error("empty processor input mapping"); - } - - auto input = static_cast(inputs[0]); - auto input_address = at(input, "address"); - auto output = at(mapping, "output"); - - addresses.optional.emplace(input_address); - - mappings.emplace_back(processor::target_mapping{get_target_index(input_address), - std::move(input_address), get_target_index(output), std::move(output)}); - } - - return mappings; -} - -std::unique_ptr parse_scanner_matcher(const parameter::map &root) -{ - auto matcher_name = at(root, "operator"); - auto matcher_params = at(root, "parameters"); - - auto [rule_data_id, matcher] = parse_matcher(matcher_name, matcher_params); - if (!rule_data_id.empty()) { - throw ddwaf::parsing_error("dynamic data on scanner condition"); - } - - return std::move(matcher); -} - -void add_addresses_to_info(const address_container &addresses, base_section_info &info) -{ - for (const auto &address : addresses.required) { info.add_required_address(address); } - - for (const auto &address : addresses.optional) { info.add_optional_address(address); } -} - -} // namespace - -rule_spec_container parse_rules(parameter::vector &rule_array, base_section_info &info, - std::unordered_map &rule_data_ids, const object_limits &limits, - rule::source_type source) -{ - rule_spec_container rules; - for (unsigned i = 0; i < rule_array.size(); ++i) { - const auto &rule_param = rule_array[i]; - auto rule_map = static_cast(rule_param); - std::string id; - try { - address_container addresses; - - id = at(rule_map, "id"); - if (rules.find(id) != rules.end()) { - DDWAF_WARN("Duplicate rule {}", id); - info.add_failed(id, "duplicate rule"); - continue; - } - - auto rule = parse_rule(rule_map, rule_data_ids, limits, source, addresses); - DDWAF_DEBUG("Parsed rule {}", id); - info.add_loaded(id); - add_addresses_to_info(addresses, info); - - rules.emplace(std::move(id), std::move(rule)); - } catch (const std::exception &e) { - if (id.empty()) { - id = index_to_id(i); - } - DDWAF_WARN("Failed to parse rule '{}': {}", id, e.what()); - info.add_failed(id, e.what()); - } - } - - return rules; -} - -rule_data_container parse_rule_data(parameter::vector &rule_data, base_section_info &info, - std::unordered_map &rule_data_ids) -{ - rule_data_container matchers; - for (unsigned i = 0; i < rule_data.size(); ++i) { - const ddwaf::parameter object = rule_data[i]; - std::string id; - try { - const auto entry = static_cast(object); - - id = at(entry, "id"); - - auto type = at(entry, "type"); - auto data = at(entry, "data"); - - std::string_view matcher_name; - auto it = rule_data_ids.find(id); - if (it == rule_data_ids.end()) { - // Infer matcher from data type - if (type == "ip_with_expiration") { - matcher_name = "ip_match"; - } else if (type == "data_with_expiration") { - matcher_name = "exact_match"; - } else { - DDWAF_DEBUG("Failed to process rule data id '{}", id); - info.add_failed(id, "failed to infer matcher"); - continue; - } - } else { - matcher_name = it->second; - } - - std::shared_ptr matcher; - if (matcher_name == "ip_match") { - using rule_data_type = matcher::ip_match::rule_data_type; - auto parsed_data = parser::parse_rule_data(type, data); - matcher = std::make_shared(parsed_data); - } else if (matcher_name == "exact_match") { - using rule_data_type = matcher::exact_match::rule_data_type; - auto parsed_data = parser::parse_rule_data(type, data); - matcher = std::make_shared(parsed_data); - } else { - DDWAF_WARN("Matcher {} doesn't support dynamic rule data", matcher_name.data()); - info.add_failed(id, - "matcher " + std::string(matcher_name) + " doesn't support dynamic rule data"); - continue; - } - - DDWAF_DEBUG("Parsed rule data {}", id); - info.add_loaded(id); - matchers.emplace(std::move(id), std::move(matcher)); - } catch (const ddwaf::exception &e) { - if (id.empty()) { - id = index_to_id(i); - } - - DDWAF_ERROR("Failed to parse data id '{}': {}", id, e.what()); - info.add_failed(id, e.what()); - } - } - - return matchers; -} - -override_spec_container parse_overrides(parameter::vector &override_array, base_section_info &info) -{ - override_spec_container overrides; - - for (unsigned i = 0; i < override_array.size(); ++i) { - auto id = index_to_id(i); - const auto &node_param = override_array[i]; - auto node = static_cast(node_param); - try { - auto [spec, type] = parse_override(node); - if (type == reference_type::id) { - overrides.by_ids.emplace_back(std::move(spec)); - } else if (type == reference_type::tags) { - overrides.by_tags.emplace_back(std::move(spec)); - } else { - // This code is likely unreachable - DDWAF_WARN("Rule override with no targets"); - info.add_failed(id, "rule override with no targets"); - continue; - } - DDWAF_DEBUG("Parsed override {}", id); - info.add_loaded(id); - } catch (const std::exception &e) { - DDWAF_WARN("Failed to parse rule override: {}", e.what()); - info.add_failed(id, e.what()); - } - } - - return overrides; -} - -filter_spec_container parse_filters( - parameter::vector &filter_array, base_section_info &info, const object_limits &limits) -{ - filter_spec_container filters; - for (unsigned i = 0; i < filter_array.size(); i++) { - const auto &node_param = filter_array[i]; - auto node = static_cast(node_param); - std::string id; - try { - address_container addresses; - id = at(node, "id"); - if (filters.ids.find(id) != filters.ids.end()) { - DDWAF_WARN("Duplicate filter: {}", id); - info.add_failed(id, "duplicate filter"); - continue; - } - - if (node.find("inputs") != node.end()) { - auto filter = parse_input_filter(node, addresses, limits); - filters.ids.emplace(id); - filters.input_filters.emplace(id, std::move(filter)); - } else { - auto filter = parse_rule_filter(node, addresses, limits); - filters.ids.emplace(id); - filters.rule_filters.emplace(id, std::move(filter)); - } - DDWAF_DEBUG("Parsed exclusion filter {}", id); - - info.add_loaded(id); - add_addresses_to_info(addresses, info); - } catch (const std::exception &e) { - if (id.empty()) { - id = index_to_id(i); - } - DDWAF_WARN("Failed to parse filter '{}': {}", id, e.what()); - info.add_failed(id, e.what()); - } - } - - return filters; -} - -processor_container parse_processors( - parameter::vector &processor_array, base_section_info &info, const object_limits &limits) -{ - processor_container processors; - std::unordered_set known_processors; - - for (unsigned i = 0; i < processor_array.size(); i++) { - const auto &node_param = processor_array[i]; - auto node = static_cast(node_param); - std::string id; - try { - address_container addresses; - - id = at(node, "id"); - if (known_processors.contains(id)) { - DDWAF_WARN("Duplicate processor: {}", id); - info.add_failed(id, "duplicate processor"); - continue; - } - - std::shared_ptr generator; - auto generator_id = at(node, "generator"); - if (generator_id == "extract_schema") { - generator = std::make_shared(); - } else { - DDWAF_WARN("Unknown generator: {}", generator_id); - info.add_failed(id, "unknown generator '" + generator_id + "'"); - continue; - } - - auto conditions_array = at(node, "conditions", {}); - auto expr = parse_simplified_expression(conditions_array, addresses, limits); - - auto params = at(node, "parameters"); - auto mappings_vec = at(params, "mappings"); - auto mappings = parse_processor_mappings(mappings_vec, addresses); - - std::vector scanners; - auto scanners_ref_array = at(params, "scanners", {}); - if (!scanners_ref_array.empty()) { - scanners.reserve(scanners_ref_array.size()); - for (const auto &ref : scanners_ref_array) { - scanners.emplace_back(parse_reference(static_cast(ref))); - } - } - - auto eval = at(node, "evaluate", true); - auto output = at(node, "output", false); - - if (!eval && !output) { - DDWAF_WARN("Processor {} not used for evaluation or output", id); - info.add_failed(id, "processor not used for evaluation or output"); - continue; - } - - DDWAF_DEBUG("Parsed processor {}", id); - known_processors.emplace(id); - info.add_loaded(id); - add_addresses_to_info(addresses, info); - - if (eval) { - processors.pre.emplace( - std::move(id), processor_spec{std::move(generator), std::move(expr), - std::move(mappings), std::move(scanners), eval, output}); - } else { - processors.post.emplace( - std::move(id), processor_spec{std::move(generator), std::move(expr), - std::move(mappings), std::move(scanners), eval, output}); - } - - } catch (const std::exception &e) { - if (id.empty()) { - id = index_to_id(i); - } - DDWAF_WARN("Failed to parse processor '{}': {}", id, e.what()); - info.add_failed(id, e.what()); - } - } - return processors; -} - -indexer parse_scanners(parameter::vector &scanner_array, base_section_info &info) -{ - indexer scanners; - for (unsigned i = 0; i < scanner_array.size(); i++) { - const auto &node_param = scanner_array[i]; - auto node = static_cast(node_param); - std::string id; - try { - id = at(node, "id"); - if (scanners.find_by_id(id) != nullptr) { - DDWAF_WARN("Duplicate scanner: {}", id); - info.add_failed(id, "duplicate scanner"); - continue; - } - - std::unordered_map tags; - for (auto &[key, value] : at(node, "tags")) { - try { - tags.emplace(key, std::string(value)); - } catch (const bad_cast &e) { - throw invalid_type(std::string(key), e); - } - } - - std::unique_ptr key_matcher{}; - std::unique_ptr value_matcher{}; - - auto it = node.find("key"); - if (it != node.end()) { - auto matcher_node = parameter::map(it->second); - key_matcher = parse_scanner_matcher(matcher_node); - } - - it = node.find("value"); - if (it != node.end()) { - auto matcher_node = parameter::map(it->second); - value_matcher = parse_scanner_matcher(matcher_node); - } - - if (!key_matcher && !value_matcher) { - DDWAF_WARN("Scanner {} has no key or value matcher", id); - info.add_failed(id, "scanner has no key or value matcher"); - continue; - } - - DDWAF_DEBUG("Parsed scanner {}", id); - auto scnr = std::make_shared(scanner{ - std::move(id), std::move(tags), std::move(key_matcher), std::move(value_matcher)}); - scanners.emplace(scnr); - info.add_loaded(scnr->get_id()); - } catch (const std::exception &e) { - if (id.empty()) { - id = index_to_id(i); - } - DDWAF_WARN("Failed to parse scanner '{}': {}", id, e.what()); - info.add_failed(id, e.what()); - } - } - return scanners; -} - -} // namespace ddwaf::parser::v2 diff --git a/src/parser/processor_parser.cpp b/src/parser/processor_parser.cpp new file mode 100644 index 000000000..abac7e45d --- /dev/null +++ b/src/parser/processor_parser.cpp @@ -0,0 +1,129 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "builder/processor_builder.hpp" +#include "parser/common.hpp" +#include "parser/parser.hpp" +#include "processor/base.hpp" +#include + +namespace ddwaf::parser::v2 { + +namespace { +std::vector parse_processor_mappings( + const parameter::vector &root, address_container &addresses) +{ + if (root.empty()) { + throw ddwaf::parsing_error("empty mappings"); + } + + std::vector mappings; + for (const auto &node : root) { + auto mapping = static_cast(node); + + // TODO support n:1 mappings and key paths + auto inputs = at(mapping, "inputs"); + if (inputs.empty()) { + throw ddwaf::parsing_error("empty processor input mapping"); + } + + auto input = static_cast(inputs[0]); + auto input_address = at(input, "address"); + auto output = at(mapping, "output"); + + addresses.optional.emplace(input_address); + + mappings.emplace_back(processor_mapping{ + {processor_parameter{ + {processor_target{get_target_index(input_address), std::move(input_address), {}}}}}, + {get_target_index(output), std::move(output), {}}}); + } + + return mappings; +} + +} // namespace + +processor_container parse_processors( + parameter::vector &processor_array, base_section_info &info, const object_limits &limits) +{ + processor_container processors; + std::unordered_set known_processors; + + for (unsigned i = 0; i < processor_array.size(); i++) { + const auto &node_param = processor_array[i]; + auto node = static_cast(node_param); + std::string id; + try { + address_container addresses; + + id = at(node, "id"); + if (known_processors.contains(id)) { + DDWAF_WARN("Duplicate processor: {}", id); + info.add_failed(id, "duplicate processor"); + continue; + } + + processor_type type; + auto generator_id = at(node, "generator"); + if (generator_id == "extract_schema") { + type = processor_type::extract_schema; + } else { + DDWAF_WARN("Unknown generator: {}", generator_id); + info.add_failed(id, "unknown generator '" + generator_id + "'"); + continue; + } + + auto conditions_array = at(node, "conditions", {}); + auto expr = parse_simplified_expression(conditions_array, addresses, limits); + + auto params = at(node, "parameters"); + auto mappings_vec = at(params, "mappings"); + auto mappings = parse_processor_mappings(mappings_vec, addresses); + + std::vector scanners; + auto scanners_ref_array = at(params, "scanners", {}); + if (!scanners_ref_array.empty()) { + scanners.reserve(scanners_ref_array.size()); + for (const auto &ref : scanners_ref_array) { + scanners.emplace_back(parse_reference(static_cast(ref))); + } + } + + auto eval = at(node, "evaluate", true); + auto output = at(node, "output", false); + + if (!eval && !output) { + DDWAF_WARN("Processor {} not used for evaluation or output", id); + info.add_failed(id, "processor not used for evaluation or output"); + continue; + } + + DDWAF_DEBUG("Parsed processor {}", id); + known_processors.emplace(id); + info.add_loaded(id); + add_addresses_to_info(addresses, info); + + if (eval) { + processors.pre.emplace_back(processor_builder{type, std::move(id), std::move(expr), + std::move(mappings), std::move(scanners), eval, output}); + } else { + processors.post.emplace_back(processor_builder{type, std::move(id), std::move(expr), + std::move(mappings), std::move(scanners), eval, output}); + } + + } catch (const std::exception &e) { + if (id.empty()) { + id = index_to_id(i); + } + DDWAF_WARN("Failed to parse processor '{}': {}", id, e.what()); + info.add_failed(id, e.what()); + } + } + return processors; +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/rule_data_parser.cpp b/src/parser/rule_data_parser.cpp index 9a09296ed..853ee0229 100644 --- a/src/parser/rule_data_parser.cpp +++ b/src/parser/rule_data_parser.cpp @@ -4,15 +4,23 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. -#include "parser/rule_data_parser.hpp" -#include "exception.hpp" +#include +#include -namespace ddwaf::parser { +#include "matcher/exact_match.hpp" +#include "matcher/ip_match.hpp" +#include "parser/common.hpp" +#include "parser/specification.hpp" +namespace ddwaf::parser::v2 { + +namespace { using data_with_expiration = std::vector>; +template T parse_data(std::string_view type, parameter &input); + template <> -data_with_expiration parse_rule_data(std::string_view type, parameter &input) +data_with_expiration parse_data(std::string_view type, parameter &input) { if (type != "ip_with_expiration" && type != "data_with_expiration") { return {}; @@ -31,4 +39,70 @@ data_with_expiration parse_rule_data(std::string_view type return data; } -} // namespace ddwaf::parser +} // namespace + +rule_data_container parse_rule_data(parameter::vector &rule_data, base_section_info &info, + std::unordered_map &rule_data_ids) +{ + rule_data_container matchers; + for (unsigned i = 0; i < rule_data.size(); ++i) { + const ddwaf::parameter object = rule_data[i]; + std::string id; + try { + const auto entry = static_cast(object); + + id = at(entry, "id"); + + auto type = at(entry, "type"); + auto data = at(entry, "data"); + + std::string_view matcher_name; + auto it = rule_data_ids.find(id); + if (it == rule_data_ids.end()) { + // Infer matcher from data type + if (type == "ip_with_expiration") { + matcher_name = "ip_match"; + } else if (type == "data_with_expiration") { + matcher_name = "exact_match"; + } else { + DDWAF_DEBUG("Failed to process rule data id '{}", id); + info.add_failed(id, "failed to infer matcher"); + continue; + } + } else { + matcher_name = it->second; + } + + std::shared_ptr matcher; + if (matcher_name == "ip_match") { + using rule_data_type = matcher::ip_match::rule_data_type; + auto parsed_data = parse_data(type, data); + matcher = std::make_shared(parsed_data); + } else if (matcher_name == "exact_match") { + using rule_data_type = matcher::exact_match::rule_data_type; + auto parsed_data = parse_data(type, data); + matcher = std::make_shared(parsed_data); + } else { + DDWAF_WARN("Matcher {} doesn't support dynamic rule data", matcher_name.data()); + info.add_failed(id, + "matcher " + std::string(matcher_name) + " doesn't support dynamic rule data"); + continue; + } + + DDWAF_DEBUG("Parsed rule data {}", id); + info.add_loaded(id); + matchers.emplace(std::move(id), std::move(matcher)); + } catch (const ddwaf::exception &e) { + if (id.empty()) { + id = index_to_id(i); + } + + DDWAF_ERROR("Failed to parse data id '{}': {}", id, e.what()); + info.add_failed(id, e.what()); + } + } + + return matchers; +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/rule_data_parser.hpp b/src/parser/rule_data_parser.hpp deleted file mode 100644 index 2d36a39c8..000000000 --- a/src/parser/rule_data_parser.hpp +++ /dev/null @@ -1,18 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - -#pragma once - -#include - -#include "parameter.hpp" -#include "parser/common.hpp" - -namespace ddwaf::parser { - -template T parse_rule_data(std::string_view type, parameter &input); - -} diff --git a/src/parser/rule_override_parser.cpp b/src/parser/rule_override_parser.cpp new file mode 100644 index 000000000..ca4fd4dc3 --- /dev/null +++ b/src/parser/rule_override_parser.cpp @@ -0,0 +1,94 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "parser/common.hpp" +#include "parser/specification.hpp" +#include + +namespace ddwaf::parser::v2 { + +namespace { + +std::pair parse_override(const parameter::map &node) +{ + // Note that ID is a duplicate field and will be deprecated at some point + override_spec current; + + auto it = node.find("enabled"); + if (it != node.end()) { + current.enabled = static_cast(it->second); + } + + it = node.find("on_match"); + if (it != node.end()) { + auto actions = static_cast>(it->second); + current.actions = std::move(actions); + } + + reference_type type = reference_type::none; + + auto rules_target_array = at(node, "rules_target", {}); + if (!rules_target_array.empty()) { + current.targets.reserve(rules_target_array.size()); + + for (const auto &target : rules_target_array) { + auto target_spec = parse_reference(static_cast(target)); + if (type == reference_type::none) { + type = target_spec.type; + } else if (type != target_spec.type) { + throw ddwaf::parsing_error("rule override targets rules and tags"); + } + + current.targets.emplace_back(std::move(target_spec)); + } + } else { + // Since the rules_target array is empty, the ID is mandatory + current.targets.emplace_back( + reference_spec{reference_type::id, at(node, "id"), {}}); + type = reference_type::id; + } + + if (!current.actions.has_value() && !current.enabled.has_value()) { + throw ddwaf::parsing_error("rule override without side-effects"); + } + + return {current, type}; +} + +} // namespace + +override_spec_container parse_overrides(parameter::vector &override_array, base_section_info &info) +{ + override_spec_container overrides; + + for (unsigned i = 0; i < override_array.size(); ++i) { + auto id = index_to_id(i); + const auto &node_param = override_array[i]; + auto node = static_cast(node_param); + try { + auto [spec, type] = parse_override(node); + if (type == reference_type::id) { + overrides.by_ids.emplace_back(std::move(spec)); + } else if (type == reference_type::tags) { + overrides.by_tags.emplace_back(std::move(spec)); + } else { + // This code is likely unreachable + DDWAF_WARN("Rule override with no targets"); + info.add_failed(id, "rule override with no targets"); + continue; + } + DDWAF_DEBUG("Parsed override {}", id); + info.add_loaded(id); + } catch (const std::exception &e) { + DDWAF_WARN("Failed to parse rule override: {}", e.what()); + info.add_failed(id, e.what()); + } + } + + return overrides; +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/rule_parser.cpp b/src/parser/rule_parser.cpp new file mode 100644 index 000000000..e351777cb --- /dev/null +++ b/src/parser/rule_parser.cpp @@ -0,0 +1,88 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "parser/common.hpp" +#include "parser/parser.hpp" +#include "parser/specification.hpp" + +namespace ddwaf::parser::v2 { + +namespace { + +rule_spec parse_rule(parameter::map &rule, + std::unordered_map &rule_data_ids, const object_limits &limits, + rule::source_type source, address_container &addresses) +{ + std::vector rule_transformers; + auto data_source = ddwaf::data_source::values; + auto transformers = at(rule, "transformers", {}); + rule_transformers = parse_transformers(transformers, data_source); + + auto conditions_array = at(rule, "conditions"); + auto expr = parse_expression( + conditions_array, rule_data_ids, data_source, rule_transformers, addresses, limits); + if (expr->empty()) { + // This is likely unreachable + throw ddwaf::parsing_error("rule has no valid conditions"); + } + + std::unordered_map tags; + for (auto &[key, value] : at(rule, "tags")) { + try { + tags.emplace(key, std::string(value)); + } catch (const bad_cast &e) { + throw invalid_type(std::string(key), e); + } + } + + if (tags.find("type") == tags.end()) { + throw ddwaf::parsing_error("missing key 'type'"); + } + + return {at(rule, "enabled", true), source, at(rule, "name"), std::move(tags), + std::move(expr), at>(rule, "on_match", {})}; +} + +} // namespace + +rule_spec_container parse_rules(parameter::vector &rule_array, base_section_info &info, + std::unordered_map &rule_data_ids, const object_limits &limits, + rule::source_type source) +{ + rule_spec_container rules; + for (unsigned i = 0; i < rule_array.size(); ++i) { + const auto &rule_param = rule_array[i]; + auto rule_map = static_cast(rule_param); + std::string id; + try { + address_container addresses; + + id = at(rule_map, "id"); + if (rules.find(id) != rules.end()) { + DDWAF_WARN("Duplicate rule {}", id); + info.add_failed(id, "duplicate rule"); + continue; + } + + auto rule = parse_rule(rule_map, rule_data_ids, limits, source, addresses); + DDWAF_DEBUG("Parsed rule {}", id); + info.add_loaded(id); + add_addresses_to_info(addresses, info); + + rules.emplace(std::move(id), std::move(rule)); + } catch (const std::exception &e) { + if (id.empty()) { + id = index_to_id(i); + } + DDWAF_WARN("Failed to parse rule '{}': {}", id, e.what()); + info.add_failed(id, e.what()); + } + } + + return rules; +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/scanner_parser.cpp b/src/parser/scanner_parser.cpp new file mode 100644 index 000000000..7172b29d0 --- /dev/null +++ b/src/parser/scanner_parser.cpp @@ -0,0 +1,96 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include + +#include "exception.hpp" +#include "indexer.hpp" +#include "matcher/base.hpp" +#include "parser/common.hpp" +#include "parser/parser.hpp" +#include "scanner.hpp" + +namespace ddwaf::parser::v2 { + +namespace { + +std::unique_ptr parse_scanner_matcher(const parameter::map &root) +{ + auto matcher_name = at(root, "operator"); + auto matcher_params = at(root, "parameters"); + + auto [rule_data_id, matcher] = parse_matcher(matcher_name, matcher_params); + if (!rule_data_id.empty()) { + throw ddwaf::parsing_error("dynamic data on scanner condition"); + } + + return std::move(matcher); +} + +} // namespace + +indexer parse_scanners(parameter::vector &scanner_array, base_section_info &info) +{ + indexer scanners; + for (unsigned i = 0; i < scanner_array.size(); i++) { + const auto &node_param = scanner_array[i]; + auto node = static_cast(node_param); + std::string id; + try { + id = at(node, "id"); + if (scanners.find_by_id(id) != nullptr) { + DDWAF_WARN("Duplicate scanner: {}", id); + info.add_failed(id, "duplicate scanner"); + continue; + } + + std::unordered_map tags; + for (auto &[key, value] : at(node, "tags")) { + try { + tags.emplace(key, std::string(value)); + } catch (const bad_cast &e) { + throw invalid_type(std::string(key), e); + } + } + + std::unique_ptr key_matcher{}; + std::unique_ptr value_matcher{}; + + auto it = node.find("key"); + if (it != node.end()) { + auto matcher_node = parameter::map(it->second); + key_matcher = parse_scanner_matcher(matcher_node); + } + + it = node.find("value"); + if (it != node.end()) { + auto matcher_node = parameter::map(it->second); + value_matcher = parse_scanner_matcher(matcher_node); + } + + if (!key_matcher && !value_matcher) { + DDWAF_WARN("Scanner {} has no key or value matcher", id); + info.add_failed(id, "scanner has no key or value matcher"); + continue; + } + + DDWAF_DEBUG("Parsed scanner {}", id); + auto scnr = std::make_shared(scanner{ + std::move(id), std::move(tags), std::move(key_matcher), std::move(value_matcher)}); + scanners.emplace(scnr); + info.add_loaded(scnr->get_id()); + } catch (const std::exception &e) { + if (id.empty()) { + id = index_to_id(i); + } + DDWAF_WARN("Failed to parse scanner '{}': {}", id, e.what()); + info.add_failed(id, e.what()); + } + } + return scanners; +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/specification.hpp b/src/parser/specification.hpp index ac39f11fc..83b60bf06 100644 --- a/src/parser/specification.hpp +++ b/src/parser/specification.hpp @@ -13,7 +13,7 @@ #include "exclusion/rule_filter.hpp" #include "expression.hpp" #include "parameter.hpp" -#include "processor.hpp" +#include "processor/base.hpp" #include "rule.hpp" #include "scanner.hpp" @@ -54,33 +54,11 @@ struct input_filter_spec { std::vector targets; }; -struct processor_spec { - std::shared_ptr generator; - std::shared_ptr expr; - std::vector mappings; - std::vector scanners; - bool evaluate{false}; - bool output{true}; -}; - // Containers using rule_spec_container = std::unordered_map; using rule_data_container = std::unordered_map>; using scanner_container = std::unordered_map>; -struct processor_container { - [[nodiscard]] bool empty() const { return pre.empty() && post.empty(); } - [[nodiscard]] std::size_t size() const { return pre.size() + post.size(); } - void clear() - { - pre.clear(); - post.clear(); - } - - std::unordered_map pre; - std::unordered_map post; -}; - struct override_spec_container { [[nodiscard]] bool empty() const { return by_ids.empty() && by_tags.empty(); } void clear() diff --git a/src/parser/transformer_parser.cpp b/src/parser/transformer_parser.cpp new file mode 100644 index 000000000..f8159fe5c --- /dev/null +++ b/src/parser/transformer_parser.cpp @@ -0,0 +1,42 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2024 Datadog, Inc. + +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "parser/common.hpp" +#include + +namespace ddwaf::parser::v2 { + +std::vector parse_transformers(const parameter::vector &root, data_source &source) +{ + if (root.empty()) { + return {}; + } + + std::vector transformers; + transformers.reserve(root.size()); + + for (const auto &transformer_param : root) { + auto transformer = static_cast(transformer_param); + auto id = transformer_from_string(transformer); + if (id.has_value()) { + transformers.emplace_back(id.value()); + } else if (transformer == "keys_only") { + source = ddwaf::data_source::keys; + } else if (transformer == "values_only") { + source = ddwaf::data_source::values; + } else { + throw ddwaf::parsing_error("invalid transformer " + std::string(transformer)); + } + } + return transformers; +} + +} // namespace ddwaf::parser::v2 diff --git a/src/processor.cpp b/src/processor.cpp deleted file mode 100644 index d3d2a29f3..000000000 --- a/src/processor.cpp +++ /dev/null @@ -1,69 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - -#include "processor.hpp" -#include "ddwaf.h" -#include "exception.hpp" - -namespace ddwaf { - -void processor::eval(object_store &store, optional_ref &derived, cache_type &cache, - ddwaf::timer &deadline) const -{ - // No result structure, but this processor only produces derived objects - // so it makes no sense to evaluate. - if (!derived.has_value() && !evaluate_ && output_) { - return; - } - - DDWAF_DEBUG("Evaluating processor '{}'", id_); - - if (!expr_->eval(cache.expr_cache, store, {}, {}, deadline).outcome) { - return; - } - - for (const auto &mapping : mappings_) { - if (deadline.expired()) { - throw ddwaf::timeout_exception(); - } - - if (store.has_target(mapping.output) || - cache.generated.find(mapping.output) != cache.generated.end()) { - continue; - } - - auto [input, attr] = store.get_target(mapping.input); - if (input == nullptr) { - continue; - } - - if (attr != object_store::attribute::ephemeral) { - // Whatever the outcome, we don't want to try and generate it again - cache.generated.emplace(mapping.output); - } - - auto object = generator_->generate(input, scanners_, deadline); - if (object.type == DDWAF_OBJ_INVALID) { - continue; - } - - if (evaluate_) { - store.insert(mapping.output, mapping.output_address, object, attr); - } - - if (output_ && derived.has_value()) { - ddwaf_object &output = derived.value(); - if (evaluate_) { - auto copy = ddwaf::object::clone(&object); - ddwaf_object_map_add(&output, mapping.output_address.c_str(), ©); - } else { - ddwaf_object_map_add(&output, mapping.output_address.c_str(), &object); - } - } - } -} - -} // namespace ddwaf diff --git a/src/processor.hpp b/src/processor.hpp deleted file mode 100644 index 27176e454..000000000 --- a/src/processor.hpp +++ /dev/null @@ -1,71 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - -#pragma once - -#include -#include -#include - -#include "expression.hpp" -#include "generator/base.hpp" -#include "object_store.hpp" -#include "utils.hpp" - -namespace ddwaf { - -class processor { -public: - struct target_mapping { - // TODO implement n:1 support - target_index input; - std::string input_address; - target_index output; - std::string output_address; - }; - - struct cache_type { - expression::cache_type expr_cache; - std::unordered_set generated; - }; - - processor(std::string id, std::shared_ptr generator, - std::shared_ptr expr, std::vector mappings, - std::set scanners, bool evaluate, bool output) - : id_(std::move(id)), generator_(std::move(generator)), expr_(std::move(expr)), - mappings_(std::move(mappings)), scanners_(std::move(scanners)), evaluate_(evaluate), - output_(output) - {} - - processor(const processor &) = delete; - processor &operator=(const processor &) = delete; - - processor(processor &&rhs) noexcept = default; - processor &operator=(processor &&rhs) noexcept = default; - virtual ~processor() = default; - - virtual void eval(object_store &store, optional_ref &derived, cache_type &cache, - ddwaf::timer &deadline) const; - - [[nodiscard]] const std::string &get_id() const { return id_; } - - void get_addresses(std::unordered_map &addresses) const - { - expr_->get_addresses(addresses); - for (auto mapping : mappings_) { addresses.emplace(mapping.input, mapping.input_address); } - } - -protected: - std::string id_; - std::shared_ptr generator_; - std::shared_ptr expr_; - std::vector mappings_; - std::set scanners_; - bool evaluate_{false}; - bool output_{true}; -}; - -} // namespace ddwaf diff --git a/src/processor/base.hpp b/src/processor/base.hpp new file mode 100644 index 000000000..e13a2eadf --- /dev/null +++ b/src/processor/base.hpp @@ -0,0 +1,249 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include + +#include "argument_retriever.hpp" +#include "exception.hpp" +#include "expression.hpp" +#include "object_store.hpp" +#include "utils.hpp" + +namespace ddwaf { + +struct processor_target { + target_index index; + std::string name; + std::vector key_path; +}; + +struct processor_parameter { + std::vector targets; +}; + +struct processor_mapping { + std::vector inputs; + processor_target output; +}; + +struct processor_cache { + expression::cache_type expr_cache; + std::unordered_set generated; +}; + +template +function_traits make_eval_traits( + std::pair (Class::*)(Args...) const); + +class base_processor { +public: + base_processor() = default; + base_processor(const base_processor &) = delete; + base_processor &operator=(const base_processor &) = delete; + + base_processor(base_processor &&rhs) noexcept = default; + base_processor &operator=(base_processor &&rhs) noexcept = default; + virtual ~base_processor() = default; + + virtual void eval(object_store &store, optional_ref &derived, + processor_cache &cache, ddwaf::timer &deadline) const = 0; + + virtual void get_addresses(std::unordered_map &addresses) const = 0; + + [[nodiscard]] virtual const std::string &get_id() const = 0; +}; + +template class structured_processor : public base_processor { +public: + structured_processor(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : id_(std::move(id)), expr_(std::move(expr)), mappings_(std::move(mappings)), + evaluate_(evaluate), output_(output) + {} + + structured_processor(const structured_processor &) = delete; + structured_processor &operator=(const structured_processor &) = delete; + + structured_processor(structured_processor &&rhs) noexcept = default; + structured_processor &operator=(structured_processor &&rhs) noexcept = default; + ~structured_processor() override = default; + + void eval(object_store &store, optional_ref &derived, processor_cache &cache, + ddwaf::timer &deadline) const override + { + // No result structure, but this processor only produces derived objects + // so it makes no sense to evaluate. + if (!derived.has_value() && !evaluate_ && output_) { + return; + } + + DDWAF_DEBUG("Evaluating processor '{}'", id_); + + if (!expr_->eval(cache.expr_cache, store, {}, {}, deadline).outcome) { + return; + } + + using func_traits = decltype(make_eval_traits(&Self::eval_impl)); + static_assert(func_traits::nargs == Self::param_names.size()); + + for (const auto &mapping : mappings_) { + if (deadline.expired()) { + throw ddwaf::timeout_exception(); + } + + if (store.has_target(mapping.output.index) || + cache.generated.find(mapping.output.index) != cache.generated.end()) { + continue; + } + + typename func_traits::tuple_type args; + if (!resolve_arguments( + mapping, store, args, std::make_index_sequence{})) { + continue; + } + + auto [object, attr] = std::apply( + [&](auto &&...args) { + return static_cast(this)->eval_impl( + std::forward(args)..., deadline); + }, + std::move(args)); + + if (attr != object_store::attribute::ephemeral) { + // Whatever the outcome, we don't want to try and generate it again + cache.generated.emplace(mapping.output.index); + } + + if (object.type == DDWAF_OBJ_INVALID) { + continue; + } + + if (evaluate_) { + store.insert(mapping.output.index, mapping.output.name, object, attr); + } + + if (output_ && derived.has_value()) { + ddwaf_object &output = derived.value(); + if (evaluate_) { + auto copy = ddwaf::object::clone(&object); + ddwaf_object_map_add(&output, mapping.output.name.c_str(), ©); + } else { + ddwaf_object_map_add(&output, mapping.output.name.c_str(), &object); + } + } + } + } + [[nodiscard]] const std::string &get_id() const override { return id_; } + + void get_addresses(std::unordered_map &addresses) const override + { + expr_->get_addresses(addresses); + for (const auto &mapping : mappings_) { + for (const auto &input : mapping.inputs) { + for (const auto &target : input.targets) { + addresses.emplace(target.index, target.name); + } + } + } + } + + static constexpr auto arguments() + { + return generate_argument_spec(std::make_index_sequence()); + } + + template + static constexpr auto generate_argument_spec(std::index_sequence) // NOLINT + { + constexpr auto param_names = Self::param_names; + using func_traits = decltype(make_eval_traits(&Self::eval_impl)); + static_assert(param_names.size() <= func_traits::nargs); + + return std::array{{ + { + param_names[Is], + argument_retriever>::is_variadic, + argument_retriever>::is_optional, + }..., + }}; + } + +protected: + template + bool resolve_arguments(const processor_mapping &mapping, const object_store &store, Args &args, + std::index_sequence /*unused*/) const + { + using TupleElement = std::tuple_element_t; + auto arg = resolve_argument(mapping, store); + if constexpr (is_unary_argument::value) { + if (!arg.has_value()) { + return false; + } + + std::get(args) = std::move(arg.value()); + } else if constexpr (is_variadic_argument::value) { + if (arg.empty()) { + return false; + } + + std::get(args) = std::move(arg); + } else { + std::get(args) = std::move(arg); + } + + if constexpr (sizeof...(Is) > 0) { + return resolve_arguments(mapping, store, args, std::index_sequence{}); + } else { + return true; + } + } + + template + auto resolve_argument(const processor_mapping &mapping, const object_store &store) const + { + using func_traits = decltype(make_eval_traits(&Self::eval_impl)); + using target_type = typename func_traits::template arg_type; + + using retriever = argument_retriever; + if constexpr (retriever::is_variadic) { + if (mapping.inputs.size() <= I) { + return target_type{}; + } + return retriever::retrieve(store, {}, mapping.inputs[I].targets); + } else if constexpr (retriever::is_optional) { + if (mapping.inputs.size() <= I) { + return target_type{}; + } + + const auto &arg = mapping.inputs[I]; + if (arg.targets.empty()) { + return target_type{}; + } + return retriever::retrieve(store, {}, arg.targets.at(0)); + } else { + if (mapping.inputs.size() <= I) { + return std::optional{}; + } + + const auto &arg = mapping.inputs[I]; + if (arg.targets.empty()) { + return std::optional{}; + } + return retriever::retrieve(store, {}, arg.targets.at(0)); + } + } + std::string id_; + std::shared_ptr expr_; + std::vector mappings_; + bool evaluate_{false}; + bool output_{true}; +}; + +} // namespace ddwaf diff --git a/src/generator/extract_schema.cpp b/src/processor/extract_schema.cpp similarity index 95% rename from src/generator/extract_schema.cpp rename to src/processor/extract_schema.cpp index 73e0faf4c..85f2faa67 100644 --- a/src/generator/extract_schema.cpp +++ b/src/processor/extract_schema.cpp @@ -14,10 +14,10 @@ #include #include "exception.hpp" -#include "generator/extract_schema.hpp" #include "log.hpp" +#include "processor/extract_schema.hpp" -namespace ddwaf::generator { +namespace ddwaf { namespace schema { struct node_record; @@ -338,14 +338,16 @@ ddwaf_object generate( } // namespace schema -ddwaf_object extract_schema::generate( - const ddwaf_object *input, const std::set &scanners, ddwaf::timer &deadline) +std::pair extract_schema::eval_impl( + const unary_argument &input, ddwaf::timer &deadline) const { - if (input == nullptr) { + if (input.value == nullptr) { return {}; } - return schema::generate(input, scanners, deadline); + object_store::attribute attr = + input.ephemeral ? object_store::attribute::ephemeral : object_store::attribute::none; + return {schema::generate(input.value, scanners_, deadline), attr}; } -} // namespace ddwaf::generator +} // namespace ddwaf diff --git a/src/processor/extract_schema.hpp b/src/processor/extract_schema.hpp new file mode 100644 index 000000000..bb4312c26 --- /dev/null +++ b/src/processor/extract_schema.hpp @@ -0,0 +1,39 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include + +#include "processor/base.hpp" +#include "scanner.hpp" + +namespace ddwaf { + +class extract_schema : public structured_processor { +public: + static constexpr std::array param_names{"inputs"}; + + static constexpr std::size_t max_container_depth = 18; + static constexpr std::size_t max_array_nodes = 10; + static constexpr std::size_t max_record_nodes = 255; + + extract_schema(std::string id, std::shared_ptr expr, + std::vector mappings, std::set scanners, bool evaluate, + bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output), + scanners_(std::move(scanners)) + {} + + std::pair eval_impl( + const unary_argument &input, ddwaf::timer &deadline) const; + +protected: + std::set scanners_; +}; + +} // namespace ddwaf diff --git a/src/ruleset.hpp b/src/ruleset.hpp index cffbaa07a..3bddc62ce 100644 --- a/src/ruleset.hpp +++ b/src/ruleset.hpp @@ -15,7 +15,7 @@ #include "exclusion/input_filter.hpp" #include "exclusion/rule_filter.hpp" #include "obfuscator.hpp" -#include "processor.hpp" +#include "processor/base.hpp" #include "rule.hpp" #include "scanner.hpp" @@ -126,8 +126,8 @@ struct ruleset { ddwaf_object_free_fn free_fn{ddwaf_object_free}; std::shared_ptr event_obfuscator; - std::unordered_map> preprocessors; - std::unordered_map> postprocessors; + std::unordered_map> preprocessors; + std::unordered_map> postprocessors; std::unordered_map> rule_filters; std::unordered_map> input_filters; diff --git a/src/ruleset_builder.cpp b/src/ruleset_builder.cpp index 571898cd5..b238486ae 100644 --- a/src/ruleset_builder.cpp +++ b/src/ruleset_builder.cpp @@ -57,25 +57,6 @@ std::set references_to_rules( return rule_refs; } -std::set references_to_scanners( - const std::vector &references, const indexer &scanners) -{ - std::set scanner_refs; - for (const auto &ref : references) { - if (ref.type == parser::reference_type::id) { - const auto *scanner = scanners.find_by_id(ref.ref_id); - if (scanner == nullptr) { - continue; - } - scanner_refs.emplace(scanner); - } else if (ref.type == parser::reference_type::tags) { - auto current_refs = scanners.find_by_tags(ref.tags); - scanner_refs.merge(current_refs); - } - } - return scanner_refs; -} - } // namespace std::shared_ptr ruleset_builder::build(parameter::map &root, base_ruleset_info &info) @@ -175,17 +156,13 @@ std::shared_ptr ruleset_builder::build(parameter::map &root, base_rules preprocessors_.clear(); postprocessors_.clear(); - for (auto &[id, spec] : processors_.pre) { - auto scanners = references_to_scanners(spec.scanners, scanners_); - auto proc = std::make_shared(id, spec.generator, spec.expr, spec.mappings, - std::move(scanners), spec.evaluate, spec.output); + for (auto &builder : processors_.pre) { + auto proc = builder.build(scanners_); preprocessors_.emplace(proc->get_id(), std::move(proc)); } - for (auto &[id, spec] : processors_.post) { - auto scanners = references_to_scanners(spec.scanners, scanners_); - auto proc = std::make_shared(id, spec.generator, spec.expr, spec.mappings, - std::move(scanners), spec.evaluate, spec.output); + for (auto &builder : processors_.post) { + auto proc = builder.build(scanners_); postprocessors_.emplace(proc->get_id(), std::move(proc)); } } diff --git a/src/ruleset_builder.hpp b/src/ruleset_builder.hpp index 4b1a5bbeb..ac7fed445 100644 --- a/src/ruleset_builder.hpp +++ b/src/ruleset_builder.hpp @@ -11,6 +11,7 @@ #include #include +#include "builder/processor_builder.hpp" #include "indexer.hpp" #include "parameter.hpp" #include "parser/specification.hpp" @@ -87,7 +88,7 @@ class ruleset_builder { // Obtained from 'exclusions' parser::filter_spec_container exclusions_; // Obtained from 'processors' - parser::processor_container processors_; + processor_container processors_; // These are the contents of the latest generated ruleset // Rules @@ -99,8 +100,8 @@ class ruleset_builder { std::unordered_map> input_filters_; // Processors - std::unordered_map> preprocessors_; - std::unordered_map> postprocessors_; + std::unordered_map> preprocessors_; + std::unordered_map> postprocessors_; // Scanners indexer scanners_; diff --git a/src/transformer/lowercase.cpp b/src/transformer/lowercase.cpp index 5362f7fe2..af8bd57e1 100644 --- a/src/transformer/lowercase.cpp +++ b/src/transformer/lowercase.cpp @@ -41,13 +41,13 @@ bool lowercase::needs_transform(std::string_view str) const char *input = str.data(); - const __m128i sse_mask_lower_bound = _mm_set1_epi8('A'); - const __m128i sse_mask_upper_bound = _mm_set1_epi8('Z'); + const __m128i sse_mask_lower_bound = _mm_set1_epi8('A' - 1); + const __m128i sse_mask_upper_bound = _mm_set1_epi8('Z' + 1); const std::size_t aligned_size = str.size() & ~0xF; - __m128i cmp_result_final = _mm_setzero_si128(); - for (std::size_t i = 0; i < aligned_size; i += 16) { + bool has_uppercase = false; + for (std::size_t i = 0; i < aligned_size && !has_uppercase; i += 16) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast) const __m128i input_data = _mm_loadu_si128((__m128i *)(input + i)); @@ -56,15 +56,15 @@ bool lowercase::needs_transform(std::string_view str) const __m128i cmp_lower = _mm_cmpgt_epi8(sse_mask_upper_bound, input_data); // Less than 'Z' const __m128i cmp_result = _mm_and_si128(cmp_upper, cmp_lower); // Combine the two comparison results - cmp_result_final = _mm_or_si128(cmp_result_final, cmp_result); + has_uppercase = + _mm_movemask_epi8(_mm_cmpeq_epi8(cmp_result, _mm_setzero_si128())) != 0xFFFF; } - bool is_lowercase = - _mm_movemask_epi8(_mm_cmpeq_epi8(cmp_result_final, _mm_setzero_si128())) == 0xFFFF; - - for (std::size_t i = aligned_size; i < str.size(); i++) { is_lowercase &= !isupper(input[i]); } + for (std::size_t i = aligned_size; i < str.size() && !has_uppercase; i++) { + has_uppercase = isupper(input[i]); + } - return !is_lowercase; + return has_uppercase; } bool lowercase::transform_impl(cow_string &str) @@ -72,7 +72,7 @@ bool lowercase::transform_impl(cow_string &str) auto size = str.length(); char *input = str.modifiable_data(); - const __m128i sse_mask_upper_bound = _mm_set1_epi8('Z'); + const __m128i sse_mask_upper_bound = _mm_set1_epi8('Z' + 1); const __m128i sse_mask_lower_bound = _mm_set1_epi8('A' - 1); const __m128i sse_addition_value = _mm_set1_epi8(0x20); // value to add to convert up to lc @@ -83,7 +83,7 @@ bool lowercase::transform_impl(cow_string &str) const __m128i input_data = _mm_loadu_si128((__m128i *)(input + i)); const __m128i cmp_upper = _mm_cmpgt_epi8(input_data, sse_mask_lower_bound); // > 'A' - 1 - const __m128i cmp_lower = _mm_cmpgt_epi8(sse_mask_upper_bound, input_data); // < 'Z' + const __m128i cmp_lower = _mm_cmpgt_epi8(sse_mask_upper_bound, input_data); // < 'Z' + 1 const __m128i cmp_result = _mm_and_si128(cmp_upper, cmp_lower); const __m128i result = diff --git a/src/transformer/remove_nulls.cpp b/src/transformer/remove_nulls.cpp index d3086c7ec..77d9c1251 100644 --- a/src/transformer/remove_nulls.cpp +++ b/src/transformer/remove_nulls.cpp @@ -24,10 +24,8 @@ bool remove_nulls::transform_impl(cow_string &str) std::size_t write = read; for (; read < str.length(); ++read) { auto c = str.at(read); - if (c == 0) { - continue; - } - str[write++] = c; + str[write] = c; + write += !!c; } str.truncate(write); diff --git a/tests/context_test.cpp b/tests/context_test.cpp index ee70b8fee..b8144528a 100644 --- a/tests/context_test.cpp +++ b/tests/context_test.cpp @@ -92,15 +92,18 @@ class input_filter : public ddwaf::exclusion::input_filter { (const object_store &store, cache_type &cache, ddwaf::timer &deadline), (const override)); }; -class processor : public ddwaf::processor { +class processor : public ddwaf::base_processor { public: - processor() : ddwaf::processor({}, {}, {}, {}, {}, true, true) {} + processor() = default; ~processor() override = default; + MOCK_METHOD( + void, get_addresses, ((std::unordered_map &)), (const override)); MOCK_METHOD(void, eval, - (object_store & store, optional_ref &, processor::cache_type &, + (object_store & store, optional_ref &, processor_cache &, ddwaf::timer &deadline), (const override)); + MOCK_METHOD(const std::string &, get_id, (), (const override)); }; } // namespace mock diff --git a/tests/lfi_detector_test.cpp b/tests/lfi_detector_test.cpp index aefd7bbcc..ba5bd201f 100644 --- a/tests/lfi_detector_test.cpp +++ b/tests/lfi_detector_test.cpp @@ -13,7 +13,7 @@ using namespace std::literals; namespace { -template std::vector gen_param_def(Args... addresses) +template std::vector gen_param_def(Args... addresses) { return {{{{std::string{addresses}, get_target_index(addresses)}}}...}; } diff --git a/tests/generator/extract_schema_test.cpp b/tests/processor/extract_schema_test.cpp similarity index 79% rename from tests/generator/extract_schema_test.cpp rename to tests/processor/extract_schema_test.cpp index 74560a9dc..1274e9d41 100644 --- a/tests/generator/extract_schema_test.cpp +++ b/tests/processor/extract_schema_test.cpp @@ -5,8 +5,8 @@ // (https://www.datadoghq.com/). Copyright 2023 Datadog, Inc. #include "../test_utils.hpp" -#include "generator/extract_schema.hpp" #include "matcher/regex_match.hpp" +#include "processor/extract_schema.hpp" using namespace ddwaf; using namespace std::literals; @@ -18,10 +18,10 @@ TEST(TestExtractSchema, UnknownScalarSchema) ddwaf_object input; ddwaf_object_invalid(&input); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([0])"); ddwaf_object_free(&output); @@ -32,10 +32,10 @@ TEST(TestExtractSchema, NullScalarSchema) ddwaf_object input; ddwaf_object_null(&input); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([1])"); ddwaf_object_free(&output); @@ -46,10 +46,10 @@ TEST(TestExtractSchema, BoolScalarSchema) ddwaf_object input; ddwaf_object_bool(&input, true); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([2])"); ddwaf_object_free(&output); @@ -61,10 +61,10 @@ TEST(TestExtractSchema, IntScalarSchema) { ddwaf_object_unsigned(&input, 5); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([4])"); ddwaf_object_free(&output); @@ -72,10 +72,10 @@ TEST(TestExtractSchema, IntScalarSchema) { ddwaf_object_signed(&input, -5); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([4])"); ddwaf_object_free(&output); @@ -87,10 +87,10 @@ TEST(TestExtractSchema, StringScalarSchema) ddwaf_object input; ddwaf_object_string(&input, "string"); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([8])"); ddwaf_object_free(&output); @@ -102,10 +102,10 @@ TEST(TestExtractSchema, FloatScalarSchema) ddwaf_object input; ddwaf_object_float(&input, 1.5); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([16])"); ddwaf_object_free(&output); @@ -116,10 +116,10 @@ TEST(TestExtractSchema, EmptyArraySchema) ddwaf_object input; ddwaf_object_array(&input); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[],{"len":0}])"); ddwaf_object_free(&output); @@ -135,10 +135,10 @@ TEST(TestExtractSchema, ArraySchema) ddwaf_object_array_add(&input, ddwaf_object_invalid(&tmp)); ddwaf_object_array_add(&input, ddwaf_object_null(&tmp)); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[[1],[0],[8],[4]],{"len":4}])"); ddwaf_object_free(&output); @@ -155,10 +155,10 @@ TEST(TestExtractSchema, ArrayWithDuplicateScalarSchema) ddwaf_object_array_add(&input, ddwaf_object_string(&tmp, "string")); ddwaf_object_array_add(&input, ddwaf_object_string(&tmp, "string")); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[[8]],{"len":4}])"); ddwaf_object_free(&output); @@ -191,10 +191,10 @@ TEST(TestExtractSchema, ArrayWithDuplicateMapsSchema) ddwaf_object_map_add(&child, "string", ddwaf_object_string(&tmp, "wahtever")); ddwaf_object_array_add(&input, &child); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[[{"unsigned":[4]}],[{"signed":[4]}],[{"string":[8],"unsigned":[4]}]],{"len":4}])"); @@ -228,10 +228,10 @@ TEST(TestExtractSchema, ArrayWithDuplicateArraysSchema) ddwaf_object_array_add(&child, ddwaf_object_string(&tmp, "wahtever")); ddwaf_object_array_add(&input, &child); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[[[[4]],{"len":1}],[[[8],[4]],{"len":2}]],{"len":4}])"); ddwaf_object_free(&output); @@ -264,10 +264,10 @@ TEST(TestExtractSchema, ArrayWithDuplicateContainersSchema) ddwaf_object_map_add(&child, "unsigned", ddwaf_object_unsigned(&tmp, 109)); ddwaf_object_array_add(&input, &child); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[[[[4]],{"len":1}],[{"string":[8],"unsigned":[4]}]],{"len":4}])"); ddwaf_object_free(&output); @@ -279,10 +279,10 @@ TEST(TestExtractSchema, EmptyMapSchema) ddwaf_object input; ddwaf_object_map(&input); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([{}])"); ddwaf_object_free(&output); @@ -308,10 +308,10 @@ TEST(TestExtractSchema, MapSchema) ddwaf_object_array_add(&child, ddwaf_object_signed(&tmp, -5)); ddwaf_object_map_add(&input, "array", &child); - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([{"array":[[[4]],{"len":1}],"invalid":[0],"map":[{"unsigned":[4],"string":[8]}],"null":[1],"string":[8],"unsigned":[4]}])"); ddwaf_object_free(&output); @@ -324,16 +324,17 @@ TEST(TestExtractSchema, DepthLimit) ddwaf_object_array(&input); ddwaf_object *parent = &input; - for (unsigned i = 0; i < generator::extract_schema::max_container_depth + 10; ++i) { + for (unsigned i = 0; i < extract_schema::max_container_depth + 10; ++i) { ddwaf_object child; ddwaf_object_array(&child); ddwaf_object_array_add(parent, &child); parent = &parent->array[0]; } - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; + ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}]],{"len":1}])"); @@ -345,15 +346,15 @@ TEST(TestExtractSchema, ArrayNodesLimit) { ddwaf_object input; ddwaf_object_array(&input); - for (unsigned i = 0; i < generator::extract_schema::max_array_nodes + 10; ++i) { + for (unsigned i = 0; i < extract_schema::max_array_nodes + 10; ++i) { ddwaf_object child; ddwaf_object_array(&child); ddwaf_object_array_add(&input, &child); } - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[[[],{"len":0}]],{"len":20,"truncated":true}])"); ddwaf_object_free(&output); @@ -364,15 +365,15 @@ TEST(TestExtractSchema, RecordNodesLimit) { ddwaf_object input; ddwaf_object_map(&input); - for (unsigned i = 0; i < generator::extract_schema::max_record_nodes + 10; ++i) { + for (unsigned i = 0; i < extract_schema::max_record_nodes + 10; ++i) { ddwaf_object child; ddwaf_object_array(&child); ddwaf_object_map_add(&input, "child", &child); } - generator::extract_schema gen; + extract_schema gen{"id", {}, {}, {}, false, true}; ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([{"child":[[],{"len":0}]},{"truncated":true}])"); ddwaf_object_free(&output); @@ -384,13 +385,13 @@ TEST(TestExtractSchema, SchemaWithSingleScanner) ddwaf_object input; ddwaf_object_string(&input, "string"); - generator::extract_schema gen; - scanner scnr{"0", {{"type", "PII"}, {"category", "IP"}}, nullptr, std::make_unique("string", 6, true)}; + extract_schema gen{"id", {}, {}, {&scnr}, false, true}; + ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {&scnr}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([8,{"type":"PII","category":"IP"}])"); ddwaf_object_free(&output); @@ -402,8 +403,6 @@ TEST(TestExtractSchema, SchemaWithMultipleScanners) ddwaf_object input; ddwaf_object_string(&input, "string"); - generator::extract_schema gen; - scanner scnr0{"0", {{"type", "PII"}, {"category", "first"}}, nullptr, std::make_unique("strong", 6, true)}; scanner scnr1{"1", {{"type", "PII"}, {"category", "second"}}, nullptr, @@ -411,8 +410,10 @@ TEST(TestExtractSchema, SchemaWithMultipleScanners) scanner scnr2{"2", {{"type", "PII"}, {"category", "third"}}, nullptr, std::make_unique("stng", 4, true)}; + extract_schema gen{"id", {}, {}, {&scnr0, &scnr1, &scnr2}, false, true}; + ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {&scnr0, &scnr1, &scnr2}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([8,{"type":"PII","category":"second"}])"); ddwaf_object_free(&output); @@ -424,8 +425,6 @@ TEST(TestExtractSchema, SchemaWithScannerNoMatch) ddwaf_object input; ddwaf_object_string(&input, "string"); - generator::extract_schema gen; - scanner scnr0{"0", {{"type", "PII"}, {"category", "first"}}, nullptr, std::make_unique("strong", 6, true)}; scanner scnr1{"1", {{"type", "PII"}, {"category", "second"}}, nullptr, @@ -433,8 +432,10 @@ TEST(TestExtractSchema, SchemaWithScannerNoMatch) scanner scnr2{"2", {{"type", "PII"}, {"category", "third"}}, nullptr, std::make_unique("what", 4, true)}; + extract_schema gen{"id", {}, {}, {&scnr0, &scnr1, &scnr2}, false, true}; + ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {&scnr0, &scnr1, &scnr2}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([8])"); ddwaf_object_free(&output); @@ -446,14 +447,14 @@ TEST(TestExtractSchema, SchemaWithScannerSingleValueNoKey) ddwaf_object input; ddwaf_object_string(&input, "string"); - generator::extract_schema gen; - scanner scnr{"0", {{"type", "PII"}, {"category", "IP"}}, std::make_unique("string", 6, true), std::make_unique("string", 6, true)}; + extract_schema gen{"id", {}, {}, {&scnr}, false, true}; + ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {&scnr}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([8])"); ddwaf_object_free(&output); @@ -467,14 +468,14 @@ TEST(TestExtractSchema, SchemaWithScannerArrayNoKey) ddwaf_object_array(&input); ddwaf_object_array_add(&input, ddwaf_object_string(&tmp, "string")); - generator::extract_schema gen; - scanner scnr{"0", {{"type", "PII"}, {"category", "IP"}}, std::make_unique("string", 6, true), std::make_unique("string", 6, true)}; + extract_schema gen{"id", {}, {}, {&scnr}, false, true}; + ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {&scnr}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([[[8]],{"len":1}])"); ddwaf_object_free(&output); @@ -493,14 +494,14 @@ TEST(TestExtractSchema, SchemaWithScannerArrayWithKey) ddwaf_object_map_add(&input, "string", &array); - generator::extract_schema gen; - scanner scnr{"0", {{"type", "PII"}, {"category", "IP"}}, std::make_unique("string", 6, true), std::make_unique("string", 6, true)}; + extract_schema gen{"id", {}, {}, {&scnr}, false, true}; + ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {&scnr}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ(output, R"([{"string":[[[8,{"category":"IP","type":"PII"}]],{"len":1}]}])"); ddwaf_object_free(&output); @@ -523,14 +524,14 @@ TEST(TestExtractSchema, SchemaWithScannerNestedArrayWithKey) ddwaf_object_map_add(&input, "string", &array0); - generator::extract_schema gen; - scanner scnr{"0", {{"type", "PII"}, {"category", "IP"}}, std::make_unique("string", 6, true), std::make_unique("string", 6, true)}; + extract_schema gen{"id", {}, {}, {&scnr}, false, true}; + ddwaf::timer deadline{2s}; - auto output = gen.generate(&input, {&scnr}, deadline); + auto [output, attr] = gen.eval_impl({{}, {}, false, &input}, deadline); EXPECT_SCHEMA_EQ( output, R"([{"string":[[[[[8,{"category":"IP","type":"PII"}]],{"len":1}]],{"len":1}]}])"); diff --git a/tests/processor_test.cpp b/tests/processor/processor_test.cpp similarity index 67% rename from tests/processor_test.cpp rename to tests/processor/processor_test.cpp index d2c814b3d..c7cab5ddc 100644 --- a/tests/processor_test.cpp +++ b/tests/processor/processor_test.cpp @@ -5,13 +5,12 @@ // Copyright 2021 Datadog, Inc. #include "exception.hpp" -#include "generator/base.hpp" #include "matcher/equals.hpp" -#include "processor.hpp" +#include "processor/base.hpp" #include -#include "test_utils.hpp" +#include "../test_utils.hpp" using ::testing::_; using ::testing::Return; @@ -22,10 +21,18 @@ using namespace std::literals; namespace { namespace mock { -class generator : public ddwaf::generator::base { +class processor : public ddwaf::structured_processor { public: - MOCK_METHOD(ddwaf_object, generate, - (const ddwaf_object *, const std::set &, ddwaf::timer &), (override)); + static constexpr std::array param_names{"inputs"}; + + processor(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + MOCK_METHOD((std::pair), eval_impl, + (const unary_argument &, ddwaf::timer &), (const)); }; } // namespace mock @@ -35,9 +42,6 @@ TEST(TestProcessor, SingleMappingOutputNoEvalUnconditional) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).WillOnce(Return(output)); - ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -48,17 +52,22 @@ TEST(TestProcessor, SingleMappingOutputNoEvalUnconditional) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; + + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; + + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, false, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); ddwaf_object output_map; ddwaf_object_map(&output_map); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{output_map}; @@ -80,11 +89,6 @@ TEST(TestProcessor, MultiMappingOutputNoEvalUnconditional) ddwaf_object_string(&first_output, "first_output_string"); ddwaf_object_string(&second_output, "second_output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)) - .WillOnce(Return(first_output)) - .WillOnce(Return(second_output)); - ddwaf_object first_input; ddwaf_object second_input; ddwaf_object_string(&first_input, "first_input_string"); @@ -98,20 +102,25 @@ TEST(TestProcessor, MultiMappingOutputNoEvalUnconditional) object_store store; store.insert(input_map); - std::vector mappings{ - {get_target_index("input_address.first"), "input_address.first", - get_target_index("output_address.first"), "output_address.first"}, - {get_target_index("input_address.second"), "input_address.second", - get_target_index("output_address.second"), "output_address.second"}}; + std::vector mappings{ + {{{{{get_target_index("input_address.first"), "input_address.first", {}}}}}, + {get_target_index("output_address.first"), "output_address.first", {}}}, + {{{{{get_target_index("input_address.second"), "input_address.second", {}}}}}, + {get_target_index("output_address.second"), "output_address.second", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, false, true}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair( + first_output, object_store::attribute::none))) + .WillOnce(Return(std::pair( + second_output, object_store::attribute::none))); + ddwaf_object output_map; ddwaf_object_map(&output_map); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{output_map}; @@ -139,9 +148,6 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalTrue) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).WillOnce(Return(output)); - ddwaf_object tmp; ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -154,8 +160,9 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalTrue) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; test::expression_builder builder(1); builder.start_condition(); @@ -163,13 +170,17 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalTrue) builder.add_target("enabled?"); builder.end_condition>(true); - processor proc{"id", std::move(gen), builder.build(), std::move(mappings), {}, false, true}; + mock::processor proc{"id", builder.build(), std::move(mappings), false, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); + ddwaf_object output_map; ddwaf_object_map(&output_map); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{output_map}; @@ -189,9 +200,6 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalCached) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).WillOnce(Return(output)); - ddwaf_object tmp; ddwaf_object input_map; ddwaf_object_map(&input_map); @@ -200,8 +208,9 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalCached) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; test::expression_builder builder(1); builder.start_condition(); @@ -209,13 +218,17 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalCached) builder.add_target("enabled?"); builder.end_condition>(true); - processor proc{"id", std::move(gen), builder.build(), std::move(mappings), {}, false, true}; + mock::processor proc{"id", builder.build(), std::move(mappings), false, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); + ddwaf_object output_map; ddwaf_object_map(&output_map); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{output_map}; @@ -247,8 +260,6 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalFalse) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - ddwaf_object tmp; ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -261,8 +272,9 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalFalse) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; test::expression_builder builder(1); builder.start_condition(); @@ -270,13 +282,13 @@ TEST(TestProcessor, SingleMappingOutputNoEvalConditionalFalse) builder.add_target("enabled?"); builder.end_condition>(true); - processor proc{"id", std::move(gen), builder.build(), std::move(mappings), {}, false, true}; + mock::processor proc{"id", builder.build(), std::move(mappings), false, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); ddwaf_object output_map; ddwaf_object_map(&output_map); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{output_map}; @@ -294,9 +306,6 @@ TEST(TestProcessor, SingleMappingNoOutputEvalUnconditional) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).WillOnce(Return(output)); - ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -307,14 +316,18 @@ TEST(TestProcessor, SingleMappingNoOutputEvalUnconditional) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, true, false}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), true, false}; EXPECT_STREQ(proc.get_id().c_str(), "id"); - processor::cache_type cache; + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); + + processor_cache cache; timer deadline{2s}; optional_ref derived{std::nullopt}; @@ -338,9 +351,6 @@ TEST(TestProcessor, SingleMappingNoOutputEvalConditionalTrue) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).WillOnce(Return(output)); - ddwaf_object tmp; ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -353,8 +363,9 @@ TEST(TestProcessor, SingleMappingNoOutputEvalConditionalTrue) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; test::expression_builder builder(1); builder.start_condition(); @@ -362,10 +373,14 @@ TEST(TestProcessor, SingleMappingNoOutputEvalConditionalTrue) builder.add_target("enabled?"); builder.end_condition>(true); - processor proc{"id", std::move(gen), builder.build(), std::move(mappings), {}, true, false}; + mock::processor proc{"id", builder.build(), std::move(mappings), true, false}; EXPECT_STREQ(proc.get_id().c_str(), "id"); - processor::cache_type cache; + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); + processor_cache cache; + timer deadline{2s}; optional_ref derived{std::nullopt}; @@ -386,8 +401,6 @@ TEST(TestProcessor, SingleMappingNoOutputEvalConditionalFalse) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - ddwaf_object tmp; ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -400,8 +413,9 @@ TEST(TestProcessor, SingleMappingNoOutputEvalConditionalFalse) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; test::expression_builder builder(1); builder.start_condition(); @@ -409,10 +423,10 @@ TEST(TestProcessor, SingleMappingNoOutputEvalConditionalFalse) builder.add_target("enabled?"); builder.end_condition>(true); - processor proc{"id", std::move(gen), builder.build(), std::move(mappings), {}, true, false}; + mock::processor proc{"id", builder.build(), std::move(mappings), true, false}; EXPECT_STREQ(proc.get_id().c_str(), "id"); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{std::nullopt}; @@ -432,11 +446,6 @@ TEST(TestProcessor, MultiMappingNoOutputEvalUnconditional) ddwaf_object_string(&first_output, "first_output_string"); ddwaf_object_string(&second_output, "second_output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)) - .WillOnce(Return(first_output)) - .WillOnce(Return(second_output)); - ddwaf_object first_input; ddwaf_object second_input; ddwaf_object_string(&first_input, "first_input_string"); @@ -450,17 +459,22 @@ TEST(TestProcessor, MultiMappingNoOutputEvalUnconditional) object_store store; store.insert(input_map); - std::vector mappings{ - {get_target_index("input_address.first"), "input_address.first", - get_target_index("output_address.first"), "output_address.first"}, - {get_target_index("input_address.second"), "input_address.second", - get_target_index("output_address.second"), "output_address.second"}}; + std::vector mappings{ + {{{{{get_target_index("input_address.first"), "input_address.first", {}}}}}, + {get_target_index("output_address.first"), "output_address.first", {}}}, + {{{{{get_target_index("input_address.second"), "input_address.second", {}}}}}, + {get_target_index("output_address.second"), "output_address.second", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, true, false}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), true, false}; EXPECT_STREQ(proc.get_id().c_str(), "id"); - processor::cache_type cache; + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair( + first_output, object_store::attribute::none))) + .WillOnce(Return(std::pair( + second_output, object_store::attribute::none))); + + processor_cache cache; timer deadline{2s}; optional_ref derived{std::nullopt}; @@ -487,9 +501,6 @@ TEST(TestProcessor, SingleMappingOutputEvalUnconditional) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).WillOnce(Return(output)); - ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -500,17 +511,21 @@ TEST(TestProcessor, SingleMappingOutputEvalUnconditional) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, true, true}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), true, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); + ddwaf_object output_map; ddwaf_object_map(&output_map); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{output_map}; @@ -540,9 +555,6 @@ TEST(TestProcessor, SingleMappingOutputEvalUnconditional) TEST(TestProcessor, OutputAlreadyAvailableInStore) { - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).Times(0); - ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -554,17 +566,19 @@ TEST(TestProcessor, OutputAlreadyAvailableInStore) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, false, true}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); + EXPECT_CALL(proc, eval_impl(_, _)).Times(0); + ddwaf_object output_map; ddwaf_object_map(&output_map); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{output_map}; @@ -576,9 +590,6 @@ TEST(TestProcessor, OutputAlreadyAvailableInStore) TEST(TestProcessor, OutputAlreadyGenerated) { - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).Times(1); - ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -589,17 +600,19 @@ TEST(TestProcessor, OutputAlreadyGenerated) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, false, true}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); + EXPECT_CALL(proc, eval_impl(_, _)).Times(1); + ddwaf_object output_map; ddwaf_object_map(&output_map); - processor::cache_type cache; + processor_cache cache; timer deadline{2s}; optional_ref derived{output_map}; @@ -612,9 +625,6 @@ TEST(TestProcessor, OutputAlreadyGenerated) TEST(TestProcessor, EvalAlreadyAvailableInStore) { - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).Times(0); - ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -626,14 +636,16 @@ TEST(TestProcessor, EvalAlreadyAvailableInStore) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, true, false}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), true, false}; EXPECT_STREQ(proc.get_id().c_str(), "id"); - processor::cache_type cache; + EXPECT_CALL(proc, eval_impl(_, _)).Times(0); + + processor_cache cache; timer deadline{2s}; optional_ref derived{}; @@ -642,9 +654,6 @@ TEST(TestProcessor, EvalAlreadyAvailableInStore) TEST(TestProcessor, OutputWithoutDerivedMap) { - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).Times(0); - ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -655,14 +664,16 @@ TEST(TestProcessor, OutputWithoutDerivedMap) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, false, true}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); - processor::cache_type cache; + EXPECT_CALL(proc, eval_impl(_, _)).Times(0); + + processor_cache cache; timer deadline{2s}; optional_ref derived{}; @@ -674,9 +685,6 @@ TEST(TestProcessor, OutputEvalWithoutDerivedMap) ddwaf_object output; ddwaf_object_string(&output, "output_string"); - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).WillOnce(Return(output)); - ddwaf_object input; ddwaf_object_string(&input, "input_string"); @@ -687,14 +695,18 @@ TEST(TestProcessor, OutputEvalWithoutDerivedMap) object_store store; store.insert(input_map); - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, true, true}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), true, true}; EXPECT_STREQ(proc.get_id().c_str(), "id"); - processor::cache_type cache; + EXPECT_CALL(proc, eval_impl(_, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); + + processor_cache cache; timer deadline{2s}; optional_ref derived{}; @@ -715,19 +727,18 @@ TEST(TestProcessor, OutputEvalWithoutDerivedMap) TEST(TestProcessor, Timeout) { - auto gen = std::make_unique(); - EXPECT_CALL(*gen, generate(_, _, _)).Times(0); - object_store store; - std::vector mappings{{get_target_index("input_address"), - "input_address", get_target_index("output_address"), "output_address"}}; + std::vector mappings{ + {{{{{get_target_index("input_address"), "input_address", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; - processor proc{ - "id", std::move(gen), std::make_shared(), std::move(mappings), {}, true, false}; + mock::processor proc{"id", std::make_shared(), std::move(mappings), true, false}; EXPECT_STREQ(proc.get_id().c_str(), "id"); - processor::cache_type cache; + EXPECT_CALL(proc, eval_impl(_, _)).Times(0); + + processor_cache cache; timer deadline{0s}; optional_ref derived{}; diff --git a/tests/processor/structured_processor_test.cpp b/tests/processor/structured_processor_test.cpp new file mode 100644 index 000000000..936fe7356 --- /dev/null +++ b/tests/processor/structured_processor_test.cpp @@ -0,0 +1,222 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "exception.hpp" +#include "matcher/equals.hpp" +#include "processor/base.hpp" + +#include + +#include "../test_utils.hpp" + +using ::testing::_; +using ::testing::Return; + +using namespace ddwaf; +using namespace std::literals; + +namespace { + +namespace mock { +class processor : public ddwaf::structured_processor { +public: + static constexpr std::array param_names{"unary", "optional", "variadic"}; + + processor(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + MOCK_METHOD((std::pair), eval_impl, + (const unary_argument &unary, + const optional_argument &optional, + const variadic_argument &variadic, ddwaf::timer &), + (const)); +}; + +} // namespace mock + +TEST(TestStructuredProcessor, AllParametersAvailable) +{ + ddwaf_object output; + ddwaf_object_string(&output, "output_string"); + + ddwaf_object tmp; + + ddwaf_object input_map; + ddwaf_object_map(&input_map); + ddwaf_object_map_add(&input_map, "unary_address", ddwaf_object_string(&tmp, "unary_string")); + ddwaf_object_map_add( + &input_map, "optional_address", ddwaf_object_string(&tmp, "optional_string")); + ddwaf_object_map_add(&input_map, "variadic_address_1", ddwaf_object_unsigned(&tmp, 1)); + ddwaf_object_map_add(&input_map, "variadic_address_2", ddwaf_object_unsigned(&tmp, 1)); + + object_store store; + store.insert(input_map); + + std::vector mappings{ + {{{{{get_target_index("unary_address"), "unary_address", {}}}}, + {{{get_target_index("optional_address"), "optional_address", {}}}}, + {{{get_target_index("variadic_address_1"), "variadic_address_1", {}}, + {get_target_index("variadic_address_2"), "variadic_address_2", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; + + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; + + EXPECT_CALL(proc, eval_impl(_, _, _, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); + + EXPECT_STREQ(proc.get_id().c_str(), "id"); + + ddwaf_object output_map; + ddwaf_object_map(&output_map); + + processor_cache cache; + timer deadline{2s}; + optional_ref derived{output_map}; + + EXPECT_EQ(ddwaf_object_size(&output_map), 0); + proc.eval(store, derived, cache, deadline); + + EXPECT_EQ(ddwaf_object_size(&output_map), 1); + const auto *obtained = ddwaf_object_get_index(&output_map, 0); + EXPECT_STREQ(obtained->parameterName, "output_address"); + EXPECT_STREQ(obtained->stringValue, "output_string"); + + ddwaf_object_free(&output_map); +} + +TEST(TestStructuredProcessor, OptionalParametersNotAvailable) +{ + ddwaf_object output; + ddwaf_object_string(&output, "output_string"); + + ddwaf_object tmp; + + ddwaf_object input_map; + ddwaf_object_map(&input_map); + ddwaf_object_map_add(&input_map, "unary_address", ddwaf_object_string(&tmp, "unary_string")); + ddwaf_object_map_add(&input_map, "variadic_address_1", ddwaf_object_unsigned(&tmp, 1)); + ddwaf_object_map_add(&input_map, "variadic_address_2", ddwaf_object_unsigned(&tmp, 1)); + + object_store store; + store.insert(input_map); + + std::vector mappings{ + {{{{{get_target_index("unary_address"), "unary_address", {}}}}, + {{{get_target_index("optional_address"), "optional_address", {}}}}, + {{{get_target_index("variadic_address_1"), "variadic_address_1", {}}, + {get_target_index("variadic_address_2"), "variadic_address_2", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; + + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; + + EXPECT_CALL(proc, eval_impl(_, _, _, _)) + .WillOnce(Return(std::pair{ + output, object_store::attribute::none})); + + EXPECT_STREQ(proc.get_id().c_str(), "id"); + + ddwaf_object output_map; + ddwaf_object_map(&output_map); + + processor_cache cache; + timer deadline{2s}; + optional_ref derived{output_map}; + + EXPECT_EQ(ddwaf_object_size(&output_map), 0); + proc.eval(store, derived, cache, deadline); + + EXPECT_EQ(ddwaf_object_size(&output_map), 1); + const auto *obtained = ddwaf_object_get_index(&output_map, 0); + EXPECT_STREQ(obtained->parameterName, "output_address"); + EXPECT_STREQ(obtained->stringValue, "output_string"); + + ddwaf_object_free(&output_map); +} + +TEST(TestStructuredProcessor, RequiredParameterNotAvailable) +{ + ddwaf_object tmp; + ddwaf_object input_map; + ddwaf_object_map(&input_map); + ddwaf_object_map_add( + &input_map, "optional_address", ddwaf_object_string(&tmp, "optional_string")); + ddwaf_object_map_add(&input_map, "variadic_address_1", ddwaf_object_unsigned(&tmp, 1)); + ddwaf_object_map_add(&input_map, "variadic_address_2", ddwaf_object_unsigned(&tmp, 1)); + + object_store store; + store.insert(input_map); + + std::vector mappings{ + {{{{{get_target_index("unary_address"), "unary_address", {}}}}, + {{{get_target_index("optional_address"), "optional_address", {}}}}, + {{{get_target_index("variadic_address_1"), "variadic_address_1", {}}, + {get_target_index("variadic_address_2"), "variadic_address_2", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; + + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; + + EXPECT_CALL(proc, eval_impl(_, _, _, _)).Times(0); + + EXPECT_STREQ(proc.get_id().c_str(), "id"); + + ddwaf_object output_map; + ddwaf_object_map(&output_map); + + processor_cache cache; + timer deadline{2s}; + optional_ref derived{output_map}; + + EXPECT_EQ(ddwaf_object_size(&output_map), 0); + proc.eval(store, derived, cache, deadline); + EXPECT_EQ(ddwaf_object_size(&output_map), 0); + + ddwaf_object_free(&output_map); +} + +TEST(TestStructuredProcessor, NoVariadocParametersAvailable) +{ + ddwaf_object tmp; + ddwaf_object input_map; + ddwaf_object_map(&input_map); + ddwaf_object_map_add(&input_map, "unary_address", ddwaf_object_string(&tmp, "unary_string")); + ddwaf_object_map_add( + &input_map, "optional_address", ddwaf_object_string(&tmp, "optional_string")); + + object_store store; + store.insert(input_map); + + std::vector mappings{ + {{{{{get_target_index("unary_address"), "unary_address", {}}}}, + {{{get_target_index("optional_address"), "optional_address", {}}}}, + {{{get_target_index("variadic_address_1"), "variadic_address_1", {}}, + {get_target_index("variadic_address_2"), "variadic_address_2", {}}}}}, + {get_target_index("output_address"), "output_address", {}}}}; + + mock::processor proc{"id", std::make_shared(), std::move(mappings), false, true}; + + EXPECT_CALL(proc, eval_impl(_, _, _, _)).Times(0); + + EXPECT_STREQ(proc.get_id().c_str(), "id"); + + ddwaf_object output_map; + ddwaf_object_map(&output_map); + + processor_cache cache; + timer deadline{2s}; + optional_ref derived{output_map}; + + EXPECT_EQ(ddwaf_object_size(&output_map), 0); + proc.eval(store, derived, cache, deadline); + EXPECT_EQ(ddwaf_object_size(&output_map), 0); + + ddwaf_object_free(&output_map); +} + +} // namespace diff --git a/tests/sqli_detector_test.cpp b/tests/sqli_detector_test.cpp index a0939f11c..213a429c9 100644 --- a/tests/sqli_detector_test.cpp +++ b/tests/sqli_detector_test.cpp @@ -17,7 +17,7 @@ class DialectTestFixture : public ::testing::TestWithParam {}; INSTANTIATE_TEST_SUITE_P( TestSqliDetector, DialectTestFixture, ::testing::Values("mysql", "sqlite", "postgresql")); -template std::vector gen_param_def(Args... addresses) +template std::vector gen_param_def(Args... addresses) { return {{{{std::string{addresses}, get_target_index(addresses)}}}...}; } diff --git a/tests/ssrf_detector_test.cpp b/tests/ssrf_detector_test.cpp index 4aef9f870..1e0cf21f4 100644 --- a/tests/ssrf_detector_test.cpp +++ b/tests/ssrf_detector_test.cpp @@ -13,7 +13,7 @@ using namespace std::literals; namespace { -template std::vector gen_param_def(Args... addresses) +template std::vector gen_param_def(Args... addresses) { return {{{{std::string{addresses}, get_target_index(addresses)}}}...}; } diff --git a/tests/test_utils.hpp b/tests/test_utils.hpp index 4f7f935c1..f95acc6c0 100644 --- a/tests/test_utils.hpp +++ b/tests/test_utils.hpp @@ -95,7 +95,7 @@ class expression_builder { std::vector transformers = {}, data_source source = data_source::values) { auto &argument = arguments_.back(); - argument.targets.emplace_back(target_definition{ + argument.targets.emplace_back(condition_target{ name, get_target_index(name), std::move(key_path), std::move(transformers), source}); } @@ -105,7 +105,7 @@ class expression_builder { } protected: - std::vector arguments_{}; + std::vector arguments_{}; std::vector> conditions_{}; }; diff --git a/tests/transformer/lowercase_test.cpp b/tests/transformer/lowercase_test.cpp index 040ab66e2..2af78719a 100644 --- a/tests/transformer/lowercase_test.cpp +++ b/tests/transformer/lowercase_test.cpp @@ -22,6 +22,8 @@ TEST(TestLowercase, EmptyString) { EXPECT_NO_TRANSFORM(lowercase, ""); } TEST(TestLowercase, ValidTransform) { EXPECT_TRANSFORM(lowercase, "L", "l"); + EXPECT_TRANSFORM(lowercase, "zzzzzzzzzzzzzzzZ", "zzzzzzzzzzzzzzzz"); + EXPECT_TRANSFORM(lowercase, "aaaaaaaaaaaaaaaA", "aaaaaaaaaaaaaaaa"); EXPECT_TRANSFORM(lowercase, "LE", "le"); EXPECT_TRANSFORM(lowercase, "LoWeRCase", "lowercase"); EXPECT_TRANSFORM(lowercase, "LowercasE", "lowercase");