Merge branch 'master' into anilm3/v2

.clang-tidy

@@ -4,7 +4,6 @@
 Checks:          '*,misc-const-correctness,-bugprone-reserved-identifier,-hicpp-signed-bitwise,-llvmlibc-restrict-system-libc-headers,-altera-unroll-loops,-hicpp-named-parameter,-cert-dcl37-c,-cert-dcl51-cpp,-read,-cppcoreguidelines-init-variables,-cppcoreguidelines-avoid-non-const-global-variables,-altera-id-dependent-backward-branch,-performance-no-int-to-ptr,-altera-struct-pack-align,-google-readability-casting,-modernize-use-trailing-return-type,-llvmlibc-implementation-in-namespace,-llvmlibc-callee-namespace,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-fuchsia-default-arguments-declarations,-fuchsia-overloaded-operator,-cppcoreguidelines-pro-type-union-access,-fuchsia-default-arguments-calls,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-google-readability-todo,-llvm-header-guard,-readability-function-cognitive-complexity,-readability-identifier-length,-cppcoreguidelines-owning-memory,-cert-err58-cpp,-fuchsia-statically-constructed-objects,-google-build-using-namespace,-hicpp-avoid-goto,-cppcoreguidelines-avoid-goto,-hicpp-no-array-decay,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-bounds-constant-array-index,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers,-abseil-string-find-str-contains,-bugprone-unchecked-optional-access,-readability-use-anyofallof'
 WarningsAsErrors: '*'
 HeaderFilterRegex: ''
-AnalyzeTemporaryDtors: false
 CheckOptions:
   - key:             readability-function-cognitive-complexity.Threshold
     value:           35

cmake/objects.cmake

@@ -12,7 +12,6 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/expression.cpp
     ${libddwaf_SOURCE_DIR}/src/ruleset_info.cpp
     ${libddwaf_SOURCE_DIR}/src/ip_utils.cpp
-    ${libddwaf_SOURCE_DIR}/src/processor.cpp
     ${libddwaf_SOURCE_DIR}/src/iterator.cpp
     ${libddwaf_SOURCE_DIR}/src/log.cpp
     ${libddwaf_SOURCE_DIR}/src/obfuscator.cpp
@@ -22,6 +21,7 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/platform.cpp
     ${libddwaf_SOURCE_DIR}/src/uuid.cpp
     ${libddwaf_SOURCE_DIR}/src/action_mapper.cpp
+    ${libddwaf_SOURCE_DIR}/src/builder/processor_builder.cpp
     ${libddwaf_SOURCE_DIR}/src/tokenizer/sql_base.cpp
     ${libddwaf_SOURCE_DIR}/src/tokenizer/pgsql.cpp
     ${libddwaf_SOURCE_DIR}/src/tokenizer/mysql.cpp
@@ -30,13 +30,20 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/exclusion/input_filter.cpp
     ${libddwaf_SOURCE_DIR}/src/exclusion/object_filter.cpp
     ${libddwaf_SOURCE_DIR}/src/exclusion/rule_filter.cpp
-    ${libddwaf_SOURCE_DIR}/src/generator/extract_schema.cpp
     ${libddwaf_SOURCE_DIR}/src/parser/actions_parser.cpp
     ${libddwaf_SOURCE_DIR}/src/parser/common.cpp
     ${libddwaf_SOURCE_DIR}/src/parser/parser.cpp
     ${libddwaf_SOURCE_DIR}/src/parser/parser_v1.cpp
-    ${libddwaf_SOURCE_DIR}/src/parser/parser_v2.cpp
     ${libddwaf_SOURCE_DIR}/src/parser/rule_data_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/processor_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/expression_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/matcher_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/transformer_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/rule_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/rule_override_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/scanner_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp
     ${libddwaf_SOURCE_DIR}/src/condition/lfi_detector.cpp
     ${libddwaf_SOURCE_DIR}/src/condition/sqli_detector.cpp
     ${libddwaf_SOURCE_DIR}/src/condition/ssrf_detector.cpp

fuzzer/lfi_detector/src/main.cpp

@@ -20,7 +20,7 @@ extern "C" int LLVMFuzzerInitialize(const int * /*argc*/, char *** /*argv*/)
     return 0;
 }
 
-template <typename... Args> std::vector<parameter_definition> gen_param_def(Args... addresses)
+template <typename... Args> std::vector<condition_parameter> gen_param_def(Args... addresses)
 {
     return {{{{std::string{addresses}, get_target_index(addresses)}}}...};
 }

fuzzer/sqli_detector/src/main.cpp

@@ -31,7 +31,7 @@ extern "C" int LLVMFuzzerInitialize(const int *argc, char ***argv)
     return 0;
 }
 
-template <typename... Args> std::vector<parameter_definition> gen_param_def(Args... addresses)
+template <typename... Args> std::vector<condition_parameter> gen_param_def(Args... addresses)
 {
     return {{{{std::string{addresses}, get_target_index(addresses)}}}...};
 }

fuzzer/ssrf_detector/src/main.cpp

@@ -20,7 +20,7 @@ extern "C" int LLVMFuzzerInitialize(const int * /*argc*/, char *** /*argv*/)
     return 0;
 }
 
-template <typename... Args> std::vector<parameter_definition> gen_param_def(Args... addresses)
+template <typename... Args> std::vector<condition_parameter> gen_param_def(Args... addresses)
 {
     return {{{{std::string{addresses}, get_target_index(addresses)}}}...};
 }

src/argument_retriever.hpp

@@ -0,0 +1,141 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+#pragma once
+
+#include <memory>
+#include <span>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include "condition/base.hpp"
+#include "traits.hpp"
+#include "utils.hpp"
+
+namespace ddwaf {
+
+// A type of argument with a single address (target) mapping
+template <typename T> struct unary_argument {
+    // The memory associated with the address and the key path is owned
+    // by either the condition (condition_target) or the processor (processor_target).
+    std::string_view address{};
+    std::span<const std::string> key_path;
+    bool ephemeral{false};
+    T value;
+};
+
+template <typename T, typename = void> struct is_unary_argument : std::false_type {};
+template <typename T> struct is_unary_argument<unary_argument<T>> : std::true_type {};
+
+// A type of argument which is considered to be optional
+template <typename T> using optional_argument = std::optional<unary_argument<T>>;
+
+template <typename T, typename = void> struct is_optional_argument : std::false_type {};
+template <typename T> struct is_optional_argument<optional_argument<T>> : std::true_type {};
+
+// A type of argument with multiple address(target) mappings
+template <typename T> using variadic_argument = std::vector<unary_argument<T>>;
+
+template <typename T, typename = void> struct is_variadic_argument : std::false_type {};
+template <typename T> struct is_variadic_argument<variadic_argument<T>> : std::true_type {};
+
+template <typename T> std::optional<T> convert(const ddwaf_object *obj)
+{
+    if constexpr (std::is_same_v<T, decltype(obj)>) {
+        return obj;
+    }
+
+    if constexpr (std::is_same_v<T, std::string_view> || std::is_same_v<T, std::string>) {
+        if (obj->type == DDWAF_OBJ_STRING) {
+            return T{obj->stringValue, static_cast<std::size_t>(obj->nbEntries)};
+        }
+    }
+
+    if constexpr (std::is_same_v<T, uint64_t> || std::is_same_v<T, unsigned>) {
+        using limits = std::numeric_limits<T>;
+        if (obj->type == DDWAF_OBJ_UNSIGNED && obj->uintValue <= limits::max()) {
+            return static_cast<T>(obj->uintValue);
+        }
+    }
+
+    if constexpr (std::is_same_v<T, int64_t> || std::is_same_v<T, int>) {
+        using limits = std::numeric_limits<T>;
+        if (obj->type == DDWAF_OBJ_SIGNED && obj->intValue >= limits::min() &&
+            obj->intValue <= limits::max()) {
+            return static_cast<T>(obj->intValue);
+        }
+    }
+
+    if constexpr (std::is_same_v<T, bool>) {
+        if (obj->type == DDWAF_OBJ_BOOL) {
+            return static_cast<T>(obj->boolean);
+        }
+    }
+
+    return {};
+}
+
+struct default_argument_retriever {
+    static constexpr bool is_variadic = false;
+    static constexpr bool is_optional = false;
+};
+
+template <typename T> struct argument_retriever : default_argument_retriever {};
+
+template <typename T> struct argument_retriever<unary_argument<T>> : default_argument_retriever {
+    template <typename TargetType>
+    static std::optional<unary_argument<T>> retrieve(const object_store &store,
+        const exclusion::object_set_ref &objects_excluded, const TargetType &target)
+    {
+        auto [object, attr] = store.get_target(target.index);
+        if (object == nullptr || objects_excluded.contains(object)) {
+            return std::nullopt;
+        }
+
+        auto converted = convert<T>(object);
+        if (!converted.has_value()) {
+            return std::nullopt;
+        }
+
+        return unary_argument<T>{target.name, target.key_path,
+            attr == object_store::attribute::ephemeral, std::move(converted.value())};
+    }
+};
+
+template <typename T> struct argument_retriever<optional_argument<T>> : default_argument_retriever {
+    static constexpr bool is_optional = true;
+
+    template <typename TargetType>
+    static optional_argument<T> retrieve(const object_store &store,
+        const exclusion::object_set_ref &objects_excluded, const TargetType &target)
+    {
+        return argument_retriever<unary_argument<T>>::retrieve(store, objects_excluded, target);
+    }
+};
+
+template <typename T> struct argument_retriever<variadic_argument<T>> : default_argument_retriever {
+    static constexpr bool is_variadic = true;
+
+    template <typename TargetType>
+    static variadic_argument<T> retrieve(const object_store &store,
+        const exclusion::object_set_ref &objects_excluded, const std::vector<TargetType> &targets)
+    {
+        variadic_argument<T> args;
+        for (const auto &target : targets) {
+            auto arg =
+                argument_retriever<unary_argument<T>>::retrieve(store, objects_excluded, target);
+            if (!arg.has_value()) {
+                continue;
+            }
+            args.emplace_back(std::move(arg.value()));
+        }
+        return args;
+    }
+};
+
+} // namespace ddwaf

src/builder/processor_builder.cpp

@@ -0,0 +1,78 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+#include <concepts>
+
+#include "builder/processor_builder.hpp"
+#include "processor/extract_schema.hpp"
+
+namespace ddwaf {
+
+namespace {
+std::set<const scanner *> references_to_scanners(
+    const std::vector<parser::reference_spec> &references, const indexer<const scanner> &scanners)
+{
+    std::set<const scanner *> scanner_refs;
+    for (const auto &ref : references) {
+        if (ref.type == parser::reference_type::id) {
+            const auto *scanner = scanners.find_by_id(ref.ref_id);
+            if (scanner == nullptr) {
+                continue;
+            }
+            scanner_refs.emplace(scanner);
+        } else if (ref.type == parser::reference_type::tags) {
+            auto current_refs = scanners.find_by_tags(ref.tags);
+            scanner_refs.merge(current_refs);
+        }
+    }
+    return scanner_refs;
+}
+
+template <typename T> struct typed_processor_builder;
+
+template <> struct typed_processor_builder<extract_schema> {
+    std::shared_ptr<base_processor> build(const auto &spec, const auto &scanners)
+    {
+        auto ref_scanners = references_to_scanners(spec.scanners, scanners);
+        return std::make_shared<extract_schema>(
+            spec.id, spec.expr, spec.mappings, std::move(ref_scanners), spec.evaluate, spec.output);
+    }
+};
+
+template <typename T, typename Spec, typename Scanners>
+concept has_build_with_scanners =
+    requires(typed_processor_builder<T> b, Spec spec, Scanners scanners) {
+        {
+            b.build(spec, scanners)
+        } -> std::same_as<std::shared_ptr<base_processor>>;
+    };
+
+template <typename T>
+[[nodiscard]] std::shared_ptr<base_processor> build_with_type(
+    const auto &spec, const auto &scanners)
+    requires std::is_base_of_v<base_processor, T>
+{
+    typed_processor_builder<T> typed_builder;
+    if constexpr (has_build_with_scanners<T, decltype(spec), decltype(scanners)>) {
+        return typed_builder.build(spec, scanners);
+    } else {
+        return typed_builder.build(spec);
+    }
+}
+} // namespace
+
+[[nodiscard]] std::shared_ptr<base_processor> processor_builder::build(
+    const indexer<const scanner> &scanners) const
+{
+    switch (type) {
+    case processor_type::extract_schema:
+        return build_with_type<extract_schema>(*this, scanners);
+    default:
+        break;
+    }
+    return {};
+}
+} // namespace ddwaf

src/builder/processor_builder.hpp

@@ -0,0 +1,54 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "indexer.hpp"
+#include "parser/specification.hpp"
+#include "processor/base.hpp"
+
+namespace ddwaf {
+
+enum class processor_type : unsigned {
+    extract_schema,
+    // Reserved
+    http_fingerprint,
+    session_fingerprint,
+    network_fingerprint,
+    header_fingerprint,
+};
+
+struct processor_builder {
+    [[nodiscard]] std::shared_ptr<base_processor> build(
+        const indexer<const scanner> &scanners) const;
+
+    processor_type type;
+    std::string id;
+    std::shared_ptr<expression> expr;
+    std::vector<processor_mapping> mappings;
+    std::vector<parser::reference_spec> scanners;
+    bool evaluate{false};
+    bool output{true};
+};
+
+struct processor_container {
+    [[nodiscard]] bool empty() const { return pre.empty() && post.empty(); }
+    [[nodiscard]] std::size_t size() const { return pre.size() + post.size(); }
+    void clear()
+    {
+        pre.clear();
+        post.clear();
+    }
+
+    std::vector<processor_builder> pre;
+    std::vector<processor_builder> post;
+};
+
+} // namespace ddwaf

src/condition/base.hpp

@@ -56,18 +56,18 @@ struct parameter_specification {
 
 // Provides the definition of an individual address(target) to parameter mapping.
 // Each target must satisfy the associated parameter specification.
-struct target_definition {
+struct condition_target {
     std::string name;
-    target_index root{};
+    target_index index{};
     std::vector<std::string> key_path{};
     std::vector<transformer_id> transformers{};
     data_source source{data_source::values};
 };
 
 // Provides the list of targets mapped to the given condition parameter. If the
 // parameter is non-variadic, only one mapping should be present.
-struct parameter_definition {
-    std::vector<target_definition> targets;
+struct condition_parameter {
+    std::vector<condition_target> targets;
 };
 
 class base_condition {

src/condition/lfi_detector.hpp

@@ -14,7 +14,7 @@ class lfi_detector : public base_impl<lfi_detector> {
 public:
     static constexpr std::array<std::string_view, 2> param_names{"resource", "params"};
 
-    explicit lfi_detector(std::vector<parameter_definition> args, const object_limits &limits = {})
+    explicit lfi_detector(std::vector<condition_parameter> args, const object_limits &limits = {})
         : base_impl<lfi_detector>(std::move(args), limits)
     {}
 

src/condition/scalar_condition.cpp

@@ -122,7 +122,7 @@ eval_result scalar_condition::eval(condition_cache &cache, const object_store &s
         }
 
         const auto &target = targets_[i];
-        auto [object, attr] = store.get_target(target.root);
+        auto [object, attr] = store.get_target(target.index);
         if (object == nullptr || object == cache.targets[i]) {
             continue;
         }