tool: add cached mode

Signed-off-by: Konrad Weihmann <[email protected]>

README.md

@@ -87,6 +87,9 @@ options:
   --exit-zero           Always return a 0 (non-error) status code, even if lint errors are found
   --release {...}
                         Run against a specific Yocto release
+  --cached              Use caches (default: off)
+  --cachedir CACHEDIR   Cache directory (default $HOME/.oelint/caches)
+  --clear-caches        Clear cache directory and exit
   --version             show program's version number and exit
 ```
 
@@ -495,6 +498,15 @@ bitbake-getvar --quiet --value DISTROOVERRIDES | tr ':' '\n' | jq -Rn  '{replace
 
 (***) `bitbake-getvar` command is available since `kirkstone` release. For older release you can use `bitbake core-image-minimal -e | grep ^MACHINEOVERRIDES` resp. `bitbake core-image-minimal -e | grep ^DISTROOVERRIDES` and pass them into the rest of the pipe.
 
+## cached mode
+
+When run with ``--cached`` the tool will store resuls into a local caching directory and reuse the results, if
+nothing has changed in the input tree and configuration.
+
+By default caches are stored to ``OELINT_CACHE_DIR`` environment variable (or `~/.oelint/caches` if not set).
+
+To clear the local caches run ``--clear-caches``
+
 ## vscode extension
 
 Find the extension in the [marketplace](https://marketplace.visualstudio.com/items?itemName=kweihmann.oelint-vscode), or search for `oelint-vscode`.

oelint_adv/__main__.py

@@ -11,6 +11,7 @@
 from oelint_adv.core import TypeSafeAppendAction, arguments_post, parse_configfile, run
 from oelint_adv.tweaks import Tweaks
 from oelint_adv.version import __version__
+from oelint_adv.caches import __default_cache_dir
 
 sys.path.append(os.path.abspath(os.path.join(__file__, '..')))
 
@@ -71,6 +72,10 @@ def create_argparser() -> argparse.ArgumentParser:
                         help='Always return a 0 (non-error) status code, even if lint errors are found')
     parser.add_argument('--release', default=Tweaks.DEFAULT_RELEASE, choices=Tweaks._map.keys(),
                         help='Run against a specific Yocto release')
+    parser.add_argument('--cached', action='store_true', help='Use caches (default: off)')
+    parser.add_argument('--cachedir', default=os.environ.get('OELINT_CACHE_DIR', __default_cache_dir),
+                        help=f'Cache directory (default {__default_cache_dir})')
+    parser.add_argument('--clear-caches', action='store_true', help='Clear cache directory and exit')
     # Override the defaults with the values from the config file
     parser.set_defaults(**parse_configfile())
 
@@ -104,6 +109,10 @@ def main() -> int:  # pragma: no cover
         print_rulefile(args)
         sys.exit(0)
 
+    if args.clear_caches:
+        args.state._caches.ClearCaches()
+        sys.exit(0)
+
     try:
         issues = run(args)
     except Exception as e:  # pragma: no cover - that shouldn't be covered anyway

oelint_adv/caches.py

@@ -0,0 +1,71 @@
+import argparse
+import hashlib
+import os
+import pickle  # noqa: S403
+import shutil
+from typing import Any, List, Tuple, Union
+
+__default_cache_dir = os.path.join(os.environ.get('HOME'), '.oelint/caches')
+
+
+class Caches():
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        self.__directory = args.cachedir
+        self.__enabled = args.cached
+        self.__quiet = args.quiet
+        self.__arg_fingerprint = self.__calculate_args_fingerprint(args)
+
+        if self.__enabled:
+            os.makedirs(self.__directory, exist_ok=True)
+
+    @property
+    def FingerPrint(self):
+        return self.__arg_fingerprint
+
+    def __calculate_args_fingerprint(self, args: argparse.Namespace) -> object:
+        _hash = hashlib.sha1(usedforsecurity=False)
+        for item in [
+                # list all hashing relevant arguments
+                args.suppress,
+                args.relpaths,
+                args.state.hide,
+                args.release,
+                args.color,
+        ]:
+            _hash.update(f'{item}'.encode())
+        return _hash.hexdigest()
+
+    def AddToFingerPrint(self, input_: Union[str, bytes]) -> None:
+        _hash = hashlib.sha1(self.__arg_fingerprint.encode(), usedforsecurity=False)  # noqa: DUO130
+        if isinstance(input_, str):
+            input_ = input_.encode()
+        _hash.update(input_)
+        self.__arg_fingerprint = _hash.hexdigest()
+
+    def ClearCaches(self) -> None:
+        shutil.rmtree(self.__directory, ignore_errors=True)
+
+    def GetFromCache(self, rule_ids: List[str], stash_fingerprint: str) -> Union[None, Tuple[Tuple[str, int], List[str], str]]:
+        if not self.__enabled:
+            return None
+        _hash = hashlib.sha1(f'{self.__arg_fingerprint}{rule_ids}{stash_fingerprint}'.encode(), usedforsecurity=False)  # noqa: DUO130
+        _hash_path = os.path.join(self.__directory, _hash.hexdigest())
+        try:
+            with open(_hash_path, 'rb') as i:  # pragma: no cover
+                if not self.__quiet:
+                    print(f'Using cached item {_hash_path}')
+                return pickle.load(i)  # noqa: DUO103, S301
+        except (pickle.PickleError, FileNotFoundError):
+            return None
+
+    def SaveToCache(self, rule_ids: List[str], stash_fingerprint: str, content: Any) -> None:
+        if not self.__enabled:
+            return
+        _hash = hashlib.sha1(f'{self.__arg_fingerprint}{rule_ids}{stash_fingerprint}'.encode(), usedforsecurity=False)  # noqa: DUO130
+        _hash_path = os.path.join(self.__directory, _hash.hexdigest())
+        try:
+            with open(_hash_path, 'wb') as o:
+                return pickle.dump(content, o, protocol=0)
+        except (pickle.PicklingError, FileNotFoundError):  # pragma: no cover
+            pass  # pragma: no cover

oelint_adv/core.py

@@ -15,6 +15,7 @@
 from oelint_parser.constants import CONSTANTS
 from oelint_parser.rpl_regex import RegexRpl
 
+from oelint_adv.caches import Caches, __default_cache_dir
 from oelint_adv.cls_rule import Rule, load_rules
 from oelint_adv.rule_base.rule_file_inlinesuppress_na import (
     FileNotApplicableInlineSuppression,
@@ -190,6 +191,10 @@ def group_run(group: List[Tuple],
 
     stash.Finalize()
 
+    cached_res = state._caches.GetFromCache([x.ID for x in rules], stash.FingerPrint)
+    if cached_res is not None:
+        return cached_res
+
     inline_supp_map = {}
     for item in stash.GetItemsFor(classifier=Comment.CLASSIFIER):
         for line in item.get_items():
@@ -239,6 +244,8 @@ def group_run(group: List[Tuple],
                         obj = FileNotApplicableInlineSuppression(state)
                         issues += obj.finding(_file, _line, override_msg=obj.Msg.format(id=_id))
 
+    state._caches.SaveToCache([x.ID for x in rules], stash.FingerPrint, issues)
+
     return issues
 
 
@@ -267,7 +274,9 @@ def create_lib_arguments(files: List[str],
                          messageformat: str = None,
                          constantmods: List[str] = None,
                          release: str = None,
-                         mode: str = 'fast') -> argparse.Namespace:
+                         mode: str = 'fast',
+                         cached: bool = False,
+                         cachedir: str = __default_cache_dir) -> argparse.Namespace:
     """Create runtime arguments in library mode
 
     Args:
@@ -288,6 +297,8 @@ def create_lib_arguments(files: List[str],
         constantmods (List[str], optional): Constant mods. Defaults to None.
         release (str, optional): Release to check against. Defaults to None.
         mode (str, optional): Level of testing. Defaults to fast.
+        cached (bool, optional): Use caching
+        cachedir (str, optional): Path to cache directory
 
     Returns:
         argparse.Namespace: runtime arguments
@@ -312,6 +323,10 @@ def create_lib_arguments(files: List[str],
     parser.add_argument('--constantmods', default=[], nargs='+')
     parser.add_argument('--release', default=Tweaks.DEFAULT_RELEASE, choices=Tweaks._map.keys())
     parser.add_argument('--mode', default='fast', choices=['fast', 'all'])
+    parser.add_argument('--cached', action='store_true', help='Use caches')
+    parser.add_argument('--cachedir', default=os.environ.get('OELINT_CACHE_DIR', __default_cache_dir),
+                        help=f'Cache directory (default {__default_cache_dir})')
+    parser.add_argument('--clear-caches', action='store_true', help='Clear cache directory and exit')
     # Override the defaults with the values from the config file
     parser.set_defaults(**parse_configfile())
 
@@ -334,7 +349,9 @@ def create_lib_arguments(files: List[str],
         *['--constantmods={x}' for x in (constantmods or ())],
         '--release={release}' if release else '',
         f'--mode={mode}',
-        *files,
+        '--cached' if cached else '',
+        f'--cachedir={cachedir}',
+        * files,
     ] if y != '']
 
     return arguments_post(parser.parse_args(dummy_args))
@@ -378,7 +395,7 @@ def arguments_post(args: argparse.Namespace) -> argparse.Namespace:  # noqa: C90
         except AttributeError:  # pragma: no cover
             pass  # pragma: no cover
 
-    if args.files == [] and not args.print_rulefile:
+    if args.files == [] and not args.print_rulefile and not args.clear_caches:
         raise argparse.ArgumentTypeError('no input files')
 
     if args.rulefile:
@@ -399,23 +416,30 @@ def arguments_post(args: argparse.Namespace) -> argparse.Namespace:  # noqa: C90
     if args.nowarn:
         args.state.hide['warning'] = True
 
+    args.state._caches = Caches(args)
+
     for mod in args.constantmods:
         if isinstance(mod, str):
             try:
                 with open(mod.lstrip('+-')) as _in:
                     _cnt = json.load(_in)
                 if mod.startswith('+'):
                     CONSTANTS.AddConstants(_cnt)
+                    args.state._caches.AddToFingerPrint(f'+{_cnt}')
                 elif mod.startswith('-'):
                     CONSTANTS.RemoveConstants(_cnt)
+                    args.state._caches.AddToFingerPrint(f'-{_cnt}')
                 else:
                     CONSTANTS.OverrideConstants(_cnt)
+                    args.state._caches.AddToFingerPrint(str(_cnt))
             except (FileNotFoundError, json.JSONDecodeError):
                 raise argparse.ArgumentTypeError(
                     'mod file \'{file}\' is not a valid file'.format(file=mod))
         else:
             CONSTANTS.AddConstants(mod.get('+', {}))
             CONSTANTS.RemoveConstants(mod.get('-', {}))
+            args.state._caches.AddToFingerPrint(f'mod{mod.get("+", {})}')
+            args.state._caches.AddToFingerPrint(f'mod{mod.get("-", {})}')
 
     args.state.color = args.color
     args.state.nobackup = args.nobackup

oelint_adv/state.py

@@ -2,6 +2,8 @@
 from colorama import Fore
 from colorama.ansi import AnsiCodes
 
+from oelint_adv.caches import Caches
+
 
 class State():
     """State/Configuration shared between processes."""
@@ -24,6 +26,7 @@ def __init__(self) -> None:
         }
 
         self._seen_inline_suppressions = []
+        self._caches: Caches = None
 
     def get_colorize(self) -> bool:
         """Returns weather or not the terminal output is to be colorized"""

tests/test_cached.py

@@ -0,0 +1,103 @@
+import pytest  # noqa: I900
+import os
+import glob
+
+from .base import TestBaseClass
+
+
+# flake8: noqa S101 - n.a. for test files
+class TestCached(TestBaseClass):
+
+    @pytest.mark.parametrize('input_',
+                             [
+                                 {
+                                     'test.bb':
+                                     '''
+                                     INSANE_SKIP:${PN} = "foo"
+                                     ''',
+                                 },
+                             ],
+                             )
+    def test_cached(self, capsys, input_):
+
+        from oelint_adv.core import run
+        from oelint_adv.__main__ import arguments_post
+
+        tmpdir = os.path.dirname(self._create_tempfile('tmppath/.marker', ''))
+        files = [self._create_tempfile(k, v) for k, v in input_.items()]
+
+        args = arguments_post(self._create_args_parser().parse_args(
+            ['--jobs=1', '--cached', f'--cachedir={tmpdir}', *files]
+        ))
+
+        run(args)
+
+        captured = capsys.readouterr()
+
+        assert 'Using cached item ' not in captured.out
+
+        assert any(glob.glob(f'{tmpdir}/*'))
+
+        run(args)
+
+    @pytest.mark.parametrize('input_',
+                             [
+                                 {
+                                     'test.bb':
+                                     '''
+                                     INSANE_SKIP:${PN} = "foo"
+                                     ''',
+                                 },
+                             ],
+                             )
+    def test_no_cached(self, capsys, input_):
+
+        from oelint_adv.core import run
+        from oelint_adv.__main__ import arguments_post
+
+        tmpdir = os.path.dirname(self._create_tempfile('tmppath/.marker', ''))
+        files = [self._create_tempfile(k, v) for k, v in input_.items()]
+
+        args = arguments_post(self._create_args_parser().parse_args(
+            ['--jobs=1', *files]
+        ))
+
+        run(args)
+
+        captured = capsys.readouterr()
+
+        assert 'Using cached item ' not in captured.out
+
+        assert not any(glob.glob(f'{tmpdir}/*'))
+
+    def test_clear_cached(self):
+
+        from oelint_adv.__main__ import arguments_post
+
+        tmpdir = os.path.dirname(self._create_tempfile('tmppath/.marker', ''))
+
+        args = arguments_post(self._create_args_parser().parse_args(
+            ['--jobs=1', '--cached', f'--cachedir={tmpdir}', '--clear-caches']
+        ))
+
+        args.state._caches.ClearCaches()
+
+    def test_arg_fingerprint(self):
+
+        from oelint_adv.__main__ import arguments_post
+
+        tmpdir = os.path.dirname(self._create_tempfile('tmppath/.marker', ''))
+
+        args = arguments_post(self._create_args_parser().parse_args(
+            ['--jobs=1', '--cached', f'--cachedir={tmpdir}', self._create_tempfile('test.bb', '')]
+        ))
+
+        fp = args.state._caches.FingerPrint
+
+        args.state._caches.AddToFingerPrint(b'1234')
+
+        assert args.state._caches.FingerPrint != fp
+
+        args.state._caches.AddToFingerPrint('1234')
+
+        assert args.state._caches.FingerPrint != fp