Version 0.2.0 (#55)

* Add tests/ and stub files Signed-off-by: stickie <[email protected]> * Add pytest to dependencies Signed-off-by: stickie <[email protected]> * Add test for fct Signed-off-by: stickie <[email protected]> * Add test for blankobf2 Signed-off-by: stickie <[email protected]> * Add hyperion tests (skip for now since full deobf not available) Signed-off-by: stickie <[email protected]> * Add structure for pyobfuscate v1/v2 tests (Need to recreate v1 obf since it's not available anymore) Signed-off-by: stickie <[email protected]> * Add tests for vare v1, setup for v2 Signed-off-by: stickie <[email protected]> * Add testing workflow Signed-off-by: stickie <[email protected]> * Add actions/setup-python Signed-off-by: GitHub <[email protected]> * Remove unused imports Signed-off-by: stickie <[email protected]> * Reduce clutter in cli.py - Move current utils.py -> deobf_utils.py - Move logging and color code in cli.py -> utils.py Signed-off-by: stickie <[email protected]> * Add basic structure for deobfs Signed-off-by: stickie <[email protected]> * Update base code for Deobfuscators Signed-off-by: stickie <[email protected]> * Update existing deobfs to use new format Signed-off-by: stickie <[email protected]> * Add isort to deps Signed-off-by: stickie <[email protected]> * Update logging for renamed argument Signed-off-by: stickie <[email protected]> * Bump version to 0.2.0 Signed-off-by: stickie <[email protected]> * Update README.md Signed-off-by: stickie <[email protected]> * Update CLI to provide list of existing deobfs Signed-off-by: stickie <[email protected]> * Fix pipeline Signed-off-by: stickie <[email protected]> --------- Signed-off-by: stickie <[email protected]> Signed-off-by: GitHub <[email protected]> Co-authored-by: Bradley Reynolds <[email protected]>
vipyrsec · Sep 26, 2024 · ffa1f66 · ffa1f66
1 parent a59013f
commit ffa1f66
Show file tree

Hide file tree

Showing 41 changed files with 3,994 additions and 575 deletions.
diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
@@ -0,0 +1,29 @@
+name: "Tests"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+
+      - name: Set up Python 3.x
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: 3.x
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install dependencies
+        run: python -m pip install .[dev]
+
+      - name: "Run tests"
+        run: pytest
+
diff --git a/README.md b/README.md
@@ -12,7 +12,7 @@ pip install "vipyr-deobf[argcomplete] @ git+https://github.com/vipyrsec/vipyrsec
 ## Supported Obfuscation Types
 
 - Vare
-- FreeCodingTools
+- FCT (FreeCodingTools)
 - BlankOBF v2
 - Hyperion (Incomplete)
 - PyObfuscate
@@ -26,4 +26,36 @@ py -m vipyr-deobf mal.py
 By default, the deobfuscator will make a 'best attempt' at discerning the obfuscation. If it is unable to detect the obfuscation type,
 one can be manually supplied with the `-t` or `--type` switch.
 
+Multiple obfuscation types and versions can be provided, separated by a comma. For example, `vipyr-deobf mal.py -t foov1,foov2,bar` will run the deobfuscator with version 1, 2 of `foo` and version 1 of `bar`.
+
 The deobfuscator also supports writing an output to a file with the `-o` or `--output` switch.
+
+## Adding Deobfuscators
+
+If you want to add your own deobfuscators, you can simply add a file to the `deobfuscators` folder and `vipyr-deobf` will detect it
+automatically. 
+
+The format is `**/deobfuscators/DeobfName/deobfname.py`. `deobfname` should equal `DeobfName` with all characters lowercased
+and spaces removed, and versioning is also supported by adding `_v(version number)` to the file name (if not provided, version defaults to 1). 
+For example,
+```
+Foo/foo_v1.py
+Eggs Bacon/eggsbacon_v2.py
+Eggs Bacon/eggsbacon.py
+```
+are all valid, but
+```
+Foo/bar.py (different file name)
+Foo/barv1.py (no _ before v)
+```
+are not.
+
+After you've added your code to the file, add the following lines:
+```py
+from vipyr_deobf.deobf_base import Deobfuscator, register
+
+blankobf_v2_deobf = Deobfuscator(deobf, format_results, scan)
+register(blankobf_v2_deobf)
+```
+Look at the type hints for the `Deobfuscator` class to determine what the three functions
+should look like and wrap your code into those three functions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "vipyr-deobf"
-version = "0.1.0"
+version = "0.2.0"
 description = "Rewrapping FieryIceStickie's Deobfuscation Tools"
 readme = "README.md"
 authors = [
@@ -13,6 +13,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+dev = ["pytest", "isort"]
 argcomplete = ["argcomplete"]
 
 [project.urls]
@@ -24,3 +25,7 @@ vipyr-deobf = "vipyr_deobf.cli:run"
 [build-system]
 requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"
+
+[tool.isort]
+multi_line_output = 5
+balanced_wrapping = true
diff --git a/src/vipyr_deobf/cli.py b/src/vipyr_deobf/cli.py
@@ -1,165 +1,83 @@
 # PYTHON_ARGCOMPLETE_OK
 import argparse
-import logging
-import logging.config
 import importlib.util
-from typing import Callable, TypeVar
-
-from .deobfuscators.blankobf2 import deobf_blankobf2, format_blankobf2
-from .deobfuscators.fct import deobf_fct, format_fct
-from .deobfuscators.hyperion import deobf_hyperion, format_hyperion
-from .deobfuscators.lzmaspam import deobf_lzma_b64, format_lzma_b64
-from .deobfuscators.pyobfuscate import deobf_pyobfuscate, format_pyobfuscate
-from .deobfuscators.vare import deobf_vare, format_vare
-from .exceptions import DeobfuscationFailError
-from .scanners.blankobf2_scan import scan_blankobf2
-from .scanners.fct_scan import scan_fct
-from .scanners.hyperion_scan import scan_hyperion
-from .scanners.lzmaspam_scan import scan_lzma
-from .scanners.pyobfuscate_scan import scan_pyobfuscate
-from .scanners.vare_scan import scan_vare
-
-R = TypeVar('R')
-
-supported_obfuscators: dict[str, tuple[Callable[[str], R], Callable[[R], str]]] = {
-    'hyperion': (deobf_hyperion, format_hyperion),
-    'lzmaspam': (deobf_lzma_b64, format_lzma_b64),
-    'vare': (deobf_vare, format_vare),
-    'fct': (deobf_fct, format_fct),
-    'blankobf2': (deobf_blankobf2, format_blankobf2),
-    'pyobfuscate': (deobf_pyobfuscate, format_pyobfuscate),
-}
-
-scanners: dict[str, Callable[[str], bool]] = {
-    'hyperion': scan_hyperion,
-    'lzmaspam': scan_lzma,
-    'vare': scan_vare,
-    'fct': scan_fct,
-    'blankobf2': scan_blankobf2,
-    'pyobfuscate': scan_pyobfuscate,
-}
-
-alias_dict: dict[str, str] = {
-    'vore': 'vare',
-    'hyperd': 'hyperion',
-    'fct_obfuscate': 'fct',
-    'not_pyobfuscate': 'fct',
-    'blankobfv2': 'blankobf2',
-}
-
-
-class Color:
-    clear = '\x1b[0m'
-    red = '\x1b[0;31m'
-    green = '\x1b[0;32m'
-    yellow = '\x1b[0;33m'
-    blue = '\x1b[0;34m'
-    white = '\x1b[0;37m'
-    bold_red = '\x1b[1;31m'
-    bold_green = '\x1b[1;32m'
-    bold_yellow = '\x1b[1;33m'
-    bold_blue = '\x1b[1;34m'
-    bold_white = '\x1b[1;37m'
-
-
-class NoSoftWarning(logging.Filter):
-    def filter(self, record: logging.LogRecord) -> bool:
-        return not record.msg.endswith('(Expected)')
-
-
-def run_deobf(code: str, deobf_type: str) -> str:
-    deobf_func, format_func = supported_obfuscators[deobf_type]
-    results = deobf_func(code)
-    return format_func(*results) if isinstance(results, tuple) else format_func(results)
-
+import logging
 
-def setup_logging(args: argparse.Namespace):
-    logging_config = {
-        'version': 1,
-        'disable_existing_loggers': False,
-        'formatters': {
-            'default': {
-                'format': f'{Color.bold_yellow}[{Color.blue}%(asctime)s{Color.bold_yellow}]'
-                          f'{Color.bold_white}:{Color.green}%(levelname)s'
-                          f'{Color.bold_white}:{Color.red}%(message)s{Color.clear}'
-            }
-        },
-        'handlers': {
-            'stdout': {
-                'class': 'logging.StreamHandler',
-                'formatter': 'default',
-                'stream': 'ext://sys.stdout',
-                'filters': [] if args.soft else ['no_soft_warning']
-            }
-        },
-        'filters': {
-            'no_soft_warning': {'()': 'vipyr_deobf.cli.NoSoftWarning'}
-        },
-        'loggers': {
-            'root': {
-                'level': 'DEBUG' if args.debug else 'INFO',
-                'handlers': ['stdout']
-            }
-        }
-    }
-    logging.config.dictConfig(logging_config)
+from vipyr_deobf.deobf_base import (
+    get_available_deobfs, iter_deobfs, load_all_deobfs, load_deobfs
+)
+from vipyr_deobf.exceptions import DeobfuscationFailError
+from vipyr_deobf.utils import Color, setup_logging
 
 
-def run():
+def get_parser():
     parser = argparse.ArgumentParser(
         prog='Vipyr Deobfuscator',
         description='Deobfuscates obfuscated scripts',
+        epilog='Available deobfuscators:\n' + '\n'.join(
+            f'  - {deobf_name} v{version}'
+            for deobf_name, versions in get_available_deobfs().items()
+            for version in versions
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     parser.add_argument('path', help='path to obfuscated file')
     parser.add_argument('-t', '--type', default='auto', type=str,
-                        choices=list(supported_obfuscators.keys())+['auto'],
-                        help='type of obfuscation used (defaults to auto)')
-    parser.add_argument('-o', '--output', help='file to output deobf result to, defaults to stdout')
-    parser.add_argument('-d', '--debug', action='store_true', help='display debug logs (defaults to false)')
-    parser.add_argument('-s', '--soft', action='store_true', help='display expected warnings (defaults to false)')
+                        help='type of obfuscation used, see help for options (defaults to auto)')
+    parser.add_argument('-o', '--output', help='file to output deobf result to (defaults to stdout)')
+    parser.add_argument('-s', '--skip-scan', action='store_true', help='skip scanning phase to identify schema')
+    parser.add_argument('-d', '--debug', action='store_true', help='display debug logs')
+    parser.add_argument('--show-expected', action='store_true', help='display expected warnings')
+    return parser
+
+
+def run():
+    parser = get_parser()
     if importlib.util.find_spec('argcomplete') is not None:
         import argcomplete
         argcomplete.autocomplete(parser)
+
     args = parser.parse_args()
 
     logger = logging.getLogger('deobf')
     setup_logging(args)
+    logger.info('Logging setup finished')
 
+    logger.info(f'Opening file at {args.path}')
     try:
         with open(args.path, 'r') as file:
             data = file.read()
     except FileNotFoundError:
         logger.error(f'{args.path} is not a valid path.')
         return
+    logger.info('Data successfully read from file')
 
-    schemas = []
+    logger.info('Loading deobfuscators...')
     if args.type == 'auto':
-        logger.info('Scanning file to identify schema')
-        for schema, scanner in scanners.items():
-            if scanner(data):
-                schemas.append(schema)
-        if not schemas:
-            logger.error('Could not identify obfuscation schema')
-            return
-        logger.info(f'Schemas matched: {", ".join(schemas)}')
+        load_all_deobfs()
+    else:
+        load_deobfs(args.type)
+
+    if args.skip_scan:
+        deobfs = [*iter_deobfs()]
     else:
-        deobf_type = args.type.replace('-', '_')
-        deobf_type = alias_dict.get(deobf_type, deobf_type)
-        if deobf_type not in supported_obfuscators:
-            logger.error(
-                f'Unsupported obfuscation schema.\n'
-                f'Supported obfuscation schemes include:\n'
-                f'{", ".join(supported_obfuscators)}'
-            )
-            return
-        schemas.append(deobf_type)
+        logger.info('Running scanners...')
+        deobfs = []
+        for deobf in iter_deobfs():
+            logger.info(f'Scanning with schema {deobf.name}v{deobf.version}')
+            if deobf.scan(data):
+                logger.info('Scan succeeded, adding to schema list')
+                deobfs.append(deobf)
+            else:
+                logger.info('Scan failed, skipping')
+    logger.info(f'Schema list: {", ".join([deobf.name for deobf in deobfs])}')
 
-    for schema in schemas:
+    for deobf in deobfs:
         try:
-            logger.info(f'Running deobf of {args.path} with schema <{schema}>')
-            output = run_deobf(data, schema)
+            logger.info(f'Running deobf of {args.path} with schema {deobf.name}')
+            output = deobf.deobf(data)
         except DeobfuscationFailError as exc:
-            logger.exception(f'Deobfuscation of {args.path} with schema <{schema}> failed:')
+            logger.exception(f'Deobfuscation of {args.path} with schema {deobf.name} failed:')
             for var, data in exc.env_vars.items():
                 print(f'{Color.bold_red}{var}{Color.clear}', data, sep='\n', end='\n\n')
         else:
@@ -169,7 +87,4 @@ def run():
                 logger.info(f'Writing results of deobf to file {args.output}')
                 with open(args.output, 'w') as file:
                     file.write(output)
-
-
-if __name__ == '__main__':
-    run()
+            break