Update to yara 4.5.0

CHANGELOG.md

@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+Yara 4.5:
+
+- New Warning on unknown escape sequences in regexes. See [PR #68](https://github.com/vthib/boreal/pull/68).
+  This warning is however more broad than the YARA one.
+- Do not report strings whose name starts with `_` as unused.
+- Add `pe.export_details[*].rva` field.
+- `math.count` and `math.percentage` now returns an undefined value when given a
+  value outside the `[0; 255]` range.
+- Imported dlls are ignored if the dll name is longer than 255 bytes.
+- Fix endianness issue in `macho.magic` field, see the [Yara fix](https://github.com/VirusTotal/yara/pull/2041).
+- Always expose `pe.is_signed` as long as the `authenticode` feature is enabled.
+
 ## [0.4.0] - 2024-02-11
 
 This release introduces process memory scanning, implemented on Windows, Linux and macOS. In addition,

Cargo.toml

@@ -20,12 +20,3 @@ members = [
 # - Handle VirtualSize=0 when searching through sections
 #   - f7e5d82c5f6fa4ca9d6ef5a39e83b6109fe2c18c
 object = { git = 'https://github.com/vthib/boreal-object', branch = "version-0.32" }
-
-# yara-rust:
-# - Add CallbackMsg::ConsoleLog: https://github.com/Hugal31/yara-rust/pull/139
-# yara:
-# - Add base address to elf entrypoint: https://github.com/VirusTotal/yara/pull/1989
-# - Fixes for macho module and process memory: https://github.com/VirusTotal/yara/pull/1995
-# - Fix for elf module and process memory: https://github.com/VirusTotal/yara/pull/1996
-# - Fix for macos process scanning: https://github.com/VirusTotal/yara/pull/2033
-yara = { git = 'https://github.com/vthib/yara-rust', branch = 'version-0.24.0' }

boreal-parser/README.md

@@ -14,4 +14,4 @@ evaluation of YARA rules.
 
 ### YARA version supported
 
-All features available in the 4.2 version of YARA are handled.
+All features available in the 4.5 version of YARA are handled.

boreal/Cargo.toml

@@ -66,7 +66,7 @@ tlsh2 = { version = "0.3", optional = true }
 object = { version = "0.32", optional = true, default-features = false, features = ["read"] }
 
 # "authenticode" feature
-authenticode-parser = { version = "0.3", optional = true }
+authenticode-parser = { version = "0.5", optional = true }
 
 # "memmap" feature
 memmap2 = { version = "0.9", optional = true }
@@ -95,7 +95,7 @@ windows = { version = "0.52", optional = true, features = [
 base64 = "0.21"
 glob = "0.3.1"
 tempfile = "3.9"
-yara = { version = "0.24", features = ["vendored"] }
+yara = { version = "0.26", features = ["vendored"] }
 
 [package.metadata.docs.rs]
 features = ["authenticode", "memmap"]

boreal/README.md

@@ -19,7 +19,7 @@ The main goals of the project are:
 
 ## Features
 
-* Full compatibility with YARA 4.3 and [most modules](#modules). Any existing rule can be used as is.
+* Full compatibility with YARA 4.5 and [most modules](#modules). Any existing rule can be used as is.
 * Avoid scanning for strings when not required, greatly reducing execution time on carefully crafted
   rules. See [no scan optimization](#no-scan-optimization).
 * Protection against any untrusted inputs, be it rules or scanned bytes. Ill-crafted rules or inputs should never
@@ -82,9 +82,6 @@ on boreal and YARA to guarantee the exact same behavior.
 There are however, some exceptions to this compatibility:
 
 * Evaluation bugs. Boreal may not suffer from some of them, or may has already fixed some of them.
-  For example, there are many bugs that are already fixed in boreal and YARA, but have not yet
-  been released by YARA, so even though boreal supports YARA 4.3, some bugs are fixed in boreal
-  and not in YARA.
 
 * Overflows or underflows. Those are not specified by YARA and in fact, signed overflows is UB in
   itself. Behavior of evaluations on overflows/underflows is no longer UB in boreal, but is

boreal/src/compiler/rule.rs

@@ -226,7 +226,7 @@ pub(super) fn compile_rule(
     let mut variables_statistics = Vec::new();
 
     for (i, var) in rule.variables.into_iter().enumerate() {
-        if !compiler.variables[i].used {
+        if !compiler.variables[i].used && !var.name.starts_with('_') {
             return Err(CompilationError::UnusedVariable {
                 name: var.name,
                 span: var.span,

boreal/src/module/macho.rs

@@ -833,7 +833,7 @@ fn parse_header<Mach: MachHeader<Endian = Endianness>>(
     }
 
     [
-        ("magic", header.magic().to_be().into()),
+        ("magic", header.magic().into()),
         ("cputype", cputype.into()),
         ("cpusubtype", cpusubtype.into()),
         ("filetype", header.filetype(e).into()),

boreal/src/module/math.rs

@@ -284,10 +284,7 @@ impl Math {
     fn count(ctx: &mut EvalContext, args: Vec<Value>) -> Option<Value> {
         let mut args = args.into_iter();
         let byte: i64 = args.next()?.try_into().ok()?;
-        // libyara type cast this to a byte directly.
-        #[allow(clippy::cast_possible_truncation)]
-        #[allow(clippy::cast_sign_loss)]
-        let byte: u8 = byte as u8;
+        let byte: usize = byte.try_into().ok()?;
 
         let dist = match (args.next(), args.next()) {
             (Some(Value::Integer(offset)), Some(Value::Integer(length))) => {
@@ -300,7 +297,7 @@ impl Math {
         };
 
         dist.counters
-            .get(byte as usize)
+            .get(byte)
             .and_then(|v| i64::try_from(*v).ok())
             .map(Value::Integer)
     }

boreal/src/module/pe.rs

@@ -24,8 +24,9 @@ mod version_info;
 
 const MAX_PE_SECTIONS: usize = 96;
 const MAX_PE_IMPORTS: usize = 16384;
-const MAX_PE_EXPORTS: usize = 8192;
+const MAX_PE_EXPORTS: usize = 16384;
 const MAX_EXPORT_NAME_LENGTH: usize = 512;
+const MAX_IMPORT_DLL_NAME_LENGTH: usize = 256;
 const MAX_RESOURCES: usize = 65536;
 const MAX_NB_DATA_DIRECTORIES: usize = 32768;
 const MAX_NB_VERSION_INFOS: usize = 32768;
@@ -954,6 +955,7 @@ impl Module for Pe {
                     ("name", Type::Bytes),
                     ("forward_name", Type::Bytes),
                     ("ordinal", Type::Integer),
+                    ("rva", Type::Integer),
                 ])),
             ),
             (
@@ -1345,6 +1347,7 @@ impl Pe {
                 let _r = map.insert("signatures", Value::Array(signatures));
             } else {
                 let _r = map.insert("number_of_signatures", Value::Integer(0));
+                let _r = map.insert("is_signed", Value::Integer(0));
             }
         }
 
@@ -1413,8 +1416,11 @@ fn add_imports<Pe: ImageNtHeaders>(
                 Ok(name) => name.to_vec(),
                 Err(_) => continue,
             };
-            let mut data_functions = Vec::new();
+            if library.len() >= MAX_IMPORT_DLL_NAME_LENGTH || !dll_name_is_valid(&library) {
+                continue;
+            }
 
+            let mut data_functions = Vec::new();
             let import_thunk_list = table
                 .thunks(import_desc.original_first_thunk.get(LE))
                 .or_else(|_| table.thunks(import_desc.first_thunk.get(LE)));
@@ -1429,6 +1435,9 @@ fn add_imports<Pe: ImageNtHeaders>(
                     &mut nb_functions_total,
                 )
             });
+            if functions.as_ref().map_or(true, Vec::is_empty) {
+                continue;
+            }
 
             data.imports.push(DataImport {
                 dll_name: library.clone(),
@@ -1477,16 +1486,17 @@ where
         if *nb_functions_total >= MAX_PE_IMPORTS {
             break;
         }
-        *nb_functions_total += 1;
 
-        add_thunk::<Pe, _>(
+        if add_thunk::<Pe, _>(
             thunk,
             dll_name,
             rva,
             &hint_name,
             &mut functions,
             data_functions,
-        );
+        ) {
+            *nb_functions_total += 1;
+        }
         rva += if is_32 { 4 } else { 8 };
     }
     functions
@@ -1499,7 +1509,8 @@ fn add_thunk<Pe: ImageNtHeaders, F>(
     hint_name: F,
     functions: &mut Vec<Value>,
     data_functions: &mut Vec<DataFunction>,
-) where
+) -> bool
+where
     F: Fn(u32) -> object::Result<Vec<u8>>,
 {
     if thunk.is_ordinal() {
@@ -1517,17 +1528,36 @@ fn add_thunk<Pe: ImageNtHeaders, F>(
             ("ordinal", ordinal.into()),
             ("rva", rva.into()),
         ]));
+        true
     } else {
         let Ok(name) = hint_name(thunk.address()) else {
-            return;
+            return false;
         };
 
+        if !is_import_name_valid(&name) {
+            return false;
+        }
+
         data_functions.push(DataFunction {
             name: name.clone(),
             ordinal: None,
             rva,
         });
         functions.push(Value::object([("name", name.into()), ("rva", rva.into())]));
+        true
+    }
+}
+
+// This mirrors what pefile does.
+// See https://github.com/erocarrera/pefile/blob/593d094e35198dad92aaf040bef17eb800c8a373/pefile.py#L2326-L2348
+fn is_import_name_valid(name: &[u8]) -> bool {
+    if name.is_empty() {
+        false
+    } else {
+        name.iter().all(|b| {
+            b.is_ascii_alphanumeric()
+                || [b'.', b'_', b'?', b'@', b'$', b'(', b')', b'<', b'>'].contains(b)
+        })
     }
 }
 
@@ -1552,6 +1582,10 @@ fn add_delay_load_imports<Pe: ImageNtHeaders>(
                 Ok(name) => name.to_vec(),
                 Err(_) => continue,
             };
+            if !dll_name_is_valid(&library) {
+                continue;
+            }
+
             let mut data_functions = Vec::new();
             let functions = table
                 .thunks(import_desc.import_name_table_rva.get(LE))
@@ -1600,6 +1634,18 @@ fn add_delay_load_imports<Pe: ImageNtHeaders>(
     ]);
 }
 
+fn dll_name_is_valid(dll_name: &[u8]) -> bool {
+    dll_name.iter().all(|c| {
+        *c >= b' '
+            && *c != b'\"'
+            && *c != b'*'
+            && *c != b'<'
+            && *c != b'>'
+            && *c != b'?'
+            && *c != b'|'
+    })
+}
+
 fn add_exports(
     data_dirs: &DataDirectories,
     mem: &[u8],
@@ -1643,6 +1689,7 @@ fn add_exports(
                         },
                     ),
                     ("forward_name", forward_name.into()),
+                    ("rva", address.into()),
                 ])
             })
             .collect();

boreal/tests/assets/pe/README

@@ -59,3 +59,11 @@ Then compile with:
 
 This generates a DLL with no imports/exports, and some custom resources with names and not just IDs.
 I haven't found a way to get names for languages however, not sure it is possible.
+
+* `long_name_exporter.exe` and `long_name_importer.exe` are very simple PE files generated from C.
+
+* `long_dll_name.exe` is `long_name_importer.exe` but with a dll name modified in a hex editor to be
+  over 256 characters.
+
+* `invalid_dll_names.exe` is `pe_imports` from libyara assets, modified in a hex editor
+  to make a standard and delayed imported dll name invalid.

boreal/tests/it/libyara_compat/macho.rs

@@ -284,6 +284,14 @@ fn test_macho() {
         true,
     );
 
+    check_file(
+        "import \"macho\" rule test { condition:
+    macho.file[0].magic == 0xcefaedfe and
+    macho.file[1].magic == 0xcffaedfe }",
+        "tests/assets/libyara/data/tiny-universal",
+        true,
+    );
+
     // Entry points for files (LC_MAIN)
 
     check_file(