Server-side request forgery (SSRF) detection operator (#268)

.clang-tidy

@@ -1,7 +1,7 @@
 ---
 # readability-function-cognitive-complexity temporarily disabled until clang-tidy is fixed
 # right now emalloc causes it to misbehave
-Checks:          '*,misc-const-correctness,-bugprone-reserved-identifier,-hicpp-signed-bitwise,-llvmlibc-restrict-system-libc-headers,-altera-unroll-loops,-hicpp-named-parameter,-cert-dcl37-c,-cert-dcl51-cpp,-read,-cppcoreguidelines-init-variables,-cppcoreguidelines-avoid-non-const-global-variables,-altera-id-dependent-backward-branch,-performance-no-int-to-ptr,-altera-struct-pack-align,-google-readability-casting,-modernize-use-trailing-return-type,-llvmlibc-implementation-in-namespace,-llvmlibc-callee-namespace,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-fuchsia-default-arguments-declarations,-fuchsia-overloaded-operator,-cppcoreguidelines-pro-type-union-access,-fuchsia-default-arguments-calls,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-google-readability-todo,-llvm-header-guard,-readability-function-cognitive-complexity,-readability-identifier-length,-cppcoreguidelines-owning-memory,-cert-err58-cpp,-fuchsia-statically-constructed-objects,-google-build-using-namespace,-hicpp-avoid-goto,-cppcoreguidelines-avoid-goto,-hicpp-no-array-decay,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-bounds-constant-array-index,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers,-abseil-string-find-str-contains'
+Checks:          '*,misc-const-correctness,-bugprone-reserved-identifier,-hicpp-signed-bitwise,-llvmlibc-restrict-system-libc-headers,-altera-unroll-loops,-hicpp-named-parameter,-cert-dcl37-c,-cert-dcl51-cpp,-read,-cppcoreguidelines-init-variables,-cppcoreguidelines-avoid-non-const-global-variables,-altera-id-dependent-backward-branch,-performance-no-int-to-ptr,-altera-struct-pack-align,-google-readability-casting,-modernize-use-trailing-return-type,-llvmlibc-implementation-in-namespace,-llvmlibc-callee-namespace,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-fuchsia-default-arguments-declarations,-fuchsia-overloaded-operator,-cppcoreguidelines-pro-type-union-access,-fuchsia-default-arguments-calls,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-google-readability-todo,-llvm-header-guard,-readability-function-cognitive-complexity,-readability-identifier-length,-cppcoreguidelines-owning-memory,-cert-err58-cpp,-fuchsia-statically-constructed-objects,-google-build-using-namespace,-hicpp-avoid-goto,-cppcoreguidelines-avoid-goto,-hicpp-no-array-decay,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-cppcoreguidelines-pro-bounds-constant-array-index,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers,-abseil-string-find-str-contains,-bugprone-unchecked-optional-access'
 WarningsAsErrors: '*'
 HeaderFilterRegex: ''
 AnalyzeTemporaryDtors: false

.github/workflows/fuzz.yml

@@ -17,7 +17,8 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 jobs:
-  fuzzing:
+  # TODO: build all fuzzers first, then run independently
+  global-fuzzer:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -31,25 +32,58 @@ jobs:
           DEBIAN_FRONTEND="noninteractive" sudo apt-get -y install libfuzzer-17-dev
 
       - name: Build
-        run: ./fuzzing/build.sh
+        run: ./fuzzer/global/build.sh
 
       - name: Run fuzzer
-        run: ./fuzzing/run.sh ${{ github.event.inputs.duration }}
+        run: ./fuzzer/global/run.sh ${{ github.event.inputs.duration }}
 
       - name: Log
         if: ${{ always() }}
-        run: grep -v -f fuzzing/scripts/report-negative-patterns.txt fuzzing/fuzz-*.log
+        run: grep -v -f fuzzer/global/scripts/report-negative-patterns.txt fuzzer/global/fuzz-*.log
 
       - name: Show coverage
-        run: ./fuzzing/scripts/show_coverage.sh 40
+        run: ./fuzzer/global/scripts/show_coverage.sh 40
 
       - name: Compress artifact
         if: ${{ always() }}
-        run: tar -czvf fuzzing.tar.gz fuzzing/
+        run: tar -czvf fuzzing.tar.gz fuzzer/global/
 
       - name: Artifact
         uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: fuzzing-data
           path: fuzzing.tar.gz
+  local-fuzzer:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        variant:
+          - uri_parse
+          - ssrf_detector
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install deps
+        run: |
+          DEBIAN_FRONTEND="noninteractive" sudo apt-get -y remove python3-lldb-14
+          sudo .github/workflows/scripts/llvm.sh 17
+          DEBIAN_FRONTEND="noninteractive" sudo apt-get -y install libfuzzer-17-dev
+
+      - name: Build
+        env:
+          CC: clang-17
+          CXX: clang++-17
+        run: |
+          mkdir build ; cd build
+          cmake -DCMAKE_VERBOSE_MAKEFILE=1 -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
+          make -j $(nproc) ${{ matrix.variant }}_fuzzer
+          cp fuzzer/${{ matrix.variant }}_fuzzer ../fuzzer/${{ matrix.variant }}
+
+      - name: Run fuzzer
+        run: |
+          cd fuzzer/${{ matrix.variant }}
+          ./${{ matrix.variant }}_fuzzer -max_total_time=${{ github.event.inputs.duration || 60 }} corpus/

.gitignore

@@ -12,20 +12,23 @@ tests/default.profraw
 perf/test_files/parsed_*
 perf/test_files/breakdown.numbers
 
-/fuzzing/build
-/fuzzing/venv
-/fuzzing/corpus
+/fuzzer/global/build
+/fuzzer/global/venv
+/fuzzer/global/corpus
 
-/fuzzing/sample_rules.yml
-/fuzzing/sample_dict.txt
-/fuzzing/fuzzer
-/fuzzing/libddwaf.a
-/fuzzing/fuzzer.dSYM
-/fuzzing/default.profdata
-/fuzzing/default.profraw
-/fuzzing/coverage.html
-/fuzzing/fuzz-*.log
-/fuzzing/results/crash-*
+/fuzzer/global/sample_rules.yml
+/fuzzer/global/sample_dict.txt
+/fuzzer/global/fuzzer
+/fuzzer/global/libddwaf.a
+/fuzzer/global/fuzzer.dSYM
+/fuzzer/global/default.profdata
+/fuzzer/global/default.profraw
+/fuzzer/global/coverage.html
+/fuzzer/global/fuzz-*.log
+/fuzzer/global/results/crash-*
+
+/fuzzer/*/corpus/*
+!/fuzzer/*/corpus/corpus-*
 
 /src/version.hpp
 

CMakeLists.txt

@@ -112,7 +112,7 @@ if (LIBDDWAF_TESTING)
     add_subdirectory(tests EXCLUDE_FROM_ALL)
     add_subdirectory(validator EXCLUDE_FROM_ALL)
     add_subdirectory(benchmark EXCLUDE_FROM_ALL)
-    add_subdirectory(fuzzing EXCLUDE_FROM_ALL)
+    add_subdirectory(fuzzer EXCLUDE_FROM_ALL)
     add_subdirectory(tools EXCLUDE_FROM_ALL)
 
     include(cmake/clang-tidy.cmake)

cmake/clang-format.cmake

@@ -5,7 +5,7 @@ if(CLANG_FORMAT STREQUAL CLANG_FORMAT-NOTFOUND)
 endif()
 
 set(FILE_LIST "")
-foreach(DIR IN ITEMS src tests validator benchmark fuzzing)
+foreach(DIR IN ITEMS src tests validator benchmark fuzzer)
     file(GLOB_RECURSE SOURCE_FILES ${DIR}/*.hpp ${DIR}/*.cpp)
     list(APPEND FILE_LIST ${SOURCE_FILES})
 endforeach()

cmake/objects.cmake

@@ -16,6 +16,7 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/iterator.cpp
     ${libddwaf_SOURCE_DIR}/src/log.cpp
     ${libddwaf_SOURCE_DIR}/src/obfuscator.cpp
+    ${libddwaf_SOURCE_DIR}/src/uri_utils.cpp
     ${libddwaf_SOURCE_DIR}/src/utils.cpp
     ${libddwaf_SOURCE_DIR}/src/waf.cpp
     ${libddwaf_SOURCE_DIR}/src/platform.cpp
@@ -29,6 +30,7 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/parser/parser_v2.cpp
     ${libddwaf_SOURCE_DIR}/src/parser/rule_data_parser.cpp
     ${libddwaf_SOURCE_DIR}/src/condition/lfi_detector.cpp
+    ${libddwaf_SOURCE_DIR}/src/condition/ssrf_detector.cpp
     ${libddwaf_SOURCE_DIR}/src/condition/scalar_condition.cpp
     ${libddwaf_SOURCE_DIR}/src/matcher/phrase_match.cpp
     ${libddwaf_SOURCE_DIR}/src/matcher/regex_match.cpp

fuzzer/CMakeLists.txt

@@ -0,0 +1,39 @@
+MACRO(GET_DIRS subdirs parent)
+  FILE(GLOB children RELATIVE ${parent} ${parent}/*)
+  SET(subdirs "")
+  FOREACH(child ${children})
+    IF(IS_DIRECTORY ${parent}/${child})
+      LIST(APPEND subdirs ${child})
+    ENDIF()
+  ENDFOREACH()
+ENDMACRO()
+
+GET_DIRS(subdirs ${CMAKE_CURRENT_SOURCE_DIR})
+
+set(LINK_COMPILE_FLAGS "-fsanitize=fuzzer,address,undefined,leak -fprofile-instr-generate -fcoverage-mapping")
+
+add_library(fuzzer-common OBJECT ${LIBDDWAF_SOURCE})
+set_target_properties(fuzzer-common PROPERTIES
+    CXX_STANDARD 20
+    CXX_STANDARD_REQUIRED YES
+    CXX_EXTENSIONS NO
+    COMPILE_FLAGS ${LINK_COMPILE_FLAGS}
+    LINK_FLAGS ${LINK_COMPILE_FLAGS})
+target_include_directories(fuzzer-common PRIVATE ${LIBDDWAF_PUBLIC_INCLUDES} ${LIBDDWAF_PRIVATE_INCLUDES})
+
+foreach(dir ${subdirs})
+    set(FUZZER_NAME "${dir}_fuzzer")
+    file(GLOB_RECURSE FUZZER_SOURCE ${dir}/src/*.cpp)
+    add_executable(${FUZZER_NAME} ${FUZZER_SOURCE})
+
+    set_target_properties(${FUZZER_NAME} PROPERTIES
+        CXX_STANDARD 20
+        CXX_STANDARD_REQUIRED YES
+        CXX_EXTENSIONS NO
+        COMPILE_FLAGS ${LINK_COMPILE_FLAGS}
+        LINK_FLAGS ${LINK_COMPILE_FLAGS})
+
+    target_include_directories(${FUZZER_NAME} PRIVATE ${LIBDDWAF_PUBLIC_INCLUDES} ${LIBDDWAF_PRIVATE_INCLUDES})
+    target_link_libraries(${FUZZER_NAME} PRIVATE fuzzer-common lib_yamlcpp)
+endforeach()
+

fuzzing/build.sh → fuzzer/global/build.sh

@@ -8,6 +8,6 @@ rm -rf build && mkdir build && cd build
 
 cmake -DCMAKE_VERBOSE_MAKEFILE=1 -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
 
-make -j $(nproc) fuzzer
+make -j $(nproc) global_fuzzer
 
-cp fuzzing/fuzzer ../fuzzing/
+cp fuzzer/global_fuzzer ../fuzzer/global/

fuzzer/global/run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -eu
+
+cp build/fuzzer/global_fuzzer fuzzer/global/global_fuzzer
+python3 fuzzer/global/scripts/build_corpus.py
+
+cd fuzzer/global
+
+export ASAN_OPTIONS=detect_leaks=1
+
+rm -f fuzz-*.log
+
+echo "Run global fuzzer for ${1:-60} seconds"
+./global_fuzzer -timeout=0.1 -report_slow_units=0.01 -max_total_time=${1:-60} -max_len=1000 -rss_limit_mb=4096 -use_value_profile=1 -dict=sample_dict.txt -artifact_prefix=results/ -jobs=4 -workers=4 -reload=0 corpus

fuzzing/scripts/build_corpus.py → fuzzer/global/scripts/build_corpus.py

@@ -12,8 +12,8 @@
 printable_chars='0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\x0b\x0c'
 
 class data():
-    re2_regexs_with_metadata = json.load(open("fuzzing/data/regex.json", "r"))
-    blns = json.load(open("fuzzing/data/blns.json", "r"))
+    re2_regexs_with_metadata = json.load(open("fuzzer/global/data/regex.json", "r"))
+    blns = json.load(open("fuzzer/global/data/blns.json", "r"))
 
 
 class cached_property(object):
@@ -484,9 +484,9 @@ def to_fuzz_dict(c):
         return "".join("\\x{:02x}".format(i) for i in c.encode("utf-8"))
 
     payload = generator.get_payload()
-    yaml.dump(payload["init_payload"], open("fuzzing/sample_rules.yml", "w"), default_flow_style=False)
+    yaml.dump(payload["init_payload"], open("fuzzer/global/sample_rules.yml", "w"), default_flow_style=False)
 
-    with open("fuzzing/sample_dict.txt", "w") as f:
+    with open("fuzzer/global/sample_dict.txt", "w") as f:
         libfuzz_magics = [
             "\x06\x06\x06",
         ]
@@ -498,7 +498,7 @@ def to_fuzz_dict(c):
                 f.write("\n")
 
     try:
-        os.mkdir('fuzzing/corpus')
+        os.mkdir('fuzzer/global/corpus')
     except FileExistsError:
         pass
 
@@ -548,7 +548,7 @@ def build_payload_corpus(data):
 def write_corpus_file(filename, data, log_byte=0, reload_rules=False):
     reload_rules = 66 if reload_rules else 0
 
-    with open(f"fuzzing/corpus/{filename}", "wb") as f:
+    with open(f"fuzzer/global/corpus/{filename}", "wb") as f:
         f.write(bytearray([log_byte, 0, reload_rules]))
         f.write(bytearray(build_payload_corpus(data)))
 

fuzzing/scripts/build_dict.py → fuzzer/global/scripts/build_dict.py

@@ -21,7 +21,7 @@ def to_fuzz_dict(c):
 
 
 def load_values():
-    data = yaml.safe_load(open("fuzzing/sample_rules.yml", "r"))
+    data = yaml.safe_load(open("fuzzer/global/sample_rules.yml", "r"))
 
     results = set()
     for rule in data["rules"]:
@@ -49,7 +49,7 @@ def load_values():
 def write_values(values):
     values.add("\x06\x06\x06")  # fuzzer magics
 
-    with open("fuzzing/sample_dict.txt", "w") as f:
+    with open("fuzzer/global/sample_dict.txt", "w") as f:
 
         for value in values:
             f.write("# " + repr(value) + "\n")

fuzzing/scripts/clean.sh → fuzzer/global/scripts/clean.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 set -eu
 
-cd fuzzing
+cd fuzzer/global
 
 rm -rf corpus/
 rm -f fuzz-*.log
 rm -f sample_dict.txt sample_rules.yml
 rm -f default.profdata default.profraw coverage.html
-rm -rf fuzzer.dSYM
-rm -rf fuzzer
+rm -rf global_fuzzer.dSYM
+rm -rf global_fuzzer

fuzzer/global/scripts/show_coverage.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eu
+
+cd fuzzer/global
+
+llvm-profdata-17 merge -sparse *.profraw -o default.profdata
+llvm-cov-17 show global_fuzzer -instr-profile=default.profdata -ignore-filename-regex="(vendor|fuzzer|third_party)" -format=html > coverage.html
+llvm-cov-17 report -instr-profile default.profdata global_fuzzer -ignore-filename-regex="(vendor|fuzzer|third_party)" -show-region-summary=false
+
+if [ ! -z ${1:-} ]; then 
+    THRESHOLD=$1
+    TOTAL=$(llvm-cov-17 report -instr-profile default.profdata global_fuzzer -ignore-filename-regex="(vendor|fuzzer|third_party)" -show-region-summary=false | grep TOTAL)
+    ARRAY=($TOTAL)
+    COVERAGE=$(echo ${ARRAY[3]} | sed -e "s/\.[[:digit:]]*%//g")
+
+    if (( $COVERAGE < $THRESHOLD )); then
+        echo "Sorry, the fuzzer found no bug, but the coverage is below $THRESHOLD%. Can't call it a success." 1>&2
+        exit 1
+    fi
+fi