Merge pull request #138 from jscotka/guess_default_deco

guess proper return object type based on  return value

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

requre/helpers/guess_object.py

@@ -0,0 +1,83 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+import yaml
+import logging
+import pickle
+import warnings
+
+from typing import Any, Dict, Optional
+from requre.objects import ObjectStorage
+from requre.helpers.simple_object import Simple, Void
+
+logger = logging.getLogger(__name__)
+GUESS_STR = "guess_type"
+
+
+class Guess(Simple):
+    """
+    This class is able to store/read all simple types in requre
+    Void, Simple, ObjectStorage (pickle)
+    It select proper type if possible.
+
+    Warning: when it uses ObjectStorage, it may lead some secrets to pickled objects
+    and it could be hidden inside object representation.
+    """
+
+    @staticmethod
+    def guess_type(value):
+        try:
+            # Try to use type for storing simple output (list, dict, str, nums, etc...)
+            yaml.safe_dump(value)
+            return Simple
+        except Exception:
+            try:
+                # Try to store anything serializable via pickle module
+                pickle.dumps(value)
+                return ObjectStorage
+            except Exception:
+                # do not store anything if not possible directly
+                warnings.warn(
+                    "Guess class - nonserializable return object - "
+                    f"Using supressed output, are you sure? {value}"
+                )
+                return Void
+
+    def write(self, obj: Any, metadata: Optional[Dict] = None) -> Any:
+        """
+        Write the object representation to storage
+        Internally it will use self.to_serializable
+        method to get serializable object representation
+
+        :param obj: some object
+        :param metadata: store metedata to object
+        :return: same obj
+        """
+        object_serialization_type = self.guess_type(obj)
+        metadata[GUESS_STR] = object_serialization_type.__name__
+        instance = object_serialization_type(
+            store_keys=self.store_keys,
+            cassette=self.get_cassette(),
+            storage_object_kwargs=self.storage_object_kwargs,
+        )
+        return instance.write(obj, metadata=metadata)
+
+    def read(self):
+        """
+        Crete object representation of serialized data in persistent storage
+        Internally it will use self.from_serializable method transform object
+
+        :return: proper object
+        """
+        data = self.get_cassette()[self.store_keys]
+        guess_type = self.get_cassette().data_miner.metadata[GUESS_STR]
+        if guess_type == ObjectStorage.__name__:
+            return pickle.loads(data)
+        elif guess_type == Simple.__name__:
+            return data
+        elif guess_type == Void.__name__:
+            return data
+        else:
+            raise ValueError(
+                f"Unsupported type of stored object inside cassette: {guess_type}"
+            )

requre/online_replacing.py

@@ -15,8 +15,8 @@
     TEST_METHOD_REGEXP,
 )
 from requre.helpers.requests_response import RequestResponseHandling
-from requre.objects import ObjectStorage
 from requre.utils import get_datafile_filename
+from requre.helpers.guess_object import Guess
 
 logger = logging.getLogger(__name__)
 
@@ -287,9 +287,10 @@ def replace_module_match(
     :param storage_keys_strategy: you can change key strategy for storing data
                                   default simple one avoid to store stack information
     """
-    if (decorate is None and replace is None) or (
-        decorate is not None and replace is not None
-    ):
+    if decorate is None and replace is None:
+        logger.info(f"Using default decorator for {what}")
+        decorate = Guess.decorator_plain(cassette=cassette)
+    elif decorate is not None and replace is not None:
         raise ValueError("right one from [decorate, replace] parameter has to be set.")
 
     def decorator_cover(func):
@@ -357,9 +358,7 @@ def record(
     cassette.storage_file = storage_file
 
     def _record_inner(func):
-        return replace_module_match(
-            what=what, cassette=cassette, decorate=ObjectStorage.decorator_all_keys
-        )(func)
+        return replace_module_match(what=what, cassette=cassette)(func)
 
     return _record_inner
 
@@ -443,6 +442,12 @@ def recording(
     cassette.data_miner.key_stategy_cls = storage_keys_strategy
     # ensure that directory structure exists already
     os.makedirs(os.path.dirname(cassette.storage_file), exist_ok=True)
+    # use default decorator for context manager if not given.
+    if decorate is None and replace is None:
+        logger.info(f"Using default decorator for {what}")
+        decorate = Guess.decorator_plain(cassette=cassette)
+    elif decorate is not None and replace is not None:
+        raise ValueError("right one from [decorate, replace] parameter has to be set.")
     # Store values and their replacements for modules to be able to revert changes back
     original_module_items = _parse_and_replace_sys_modules(
         what=what, cassette=cassette, decorate=decorate, replace=replace

tests/test_guess_objects.py

@@ -0,0 +1,92 @@
+import unittest
+import math
+import os
+from requre.objects import ObjectStorage
+from requre.utils import StorageMode
+from requre.helpers.guess_object import Guess, GUESS_STR
+from requre.helpers.simple_object import Void, Simple
+from requre.online_replacing import apply_decorator_to_all_methods, replace_module_match
+from requre.cassette import Cassette
+from tests.testbase import BaseClass
+from tests.test_object import OwnClass
+import sys
+
+
+class Unit(BaseClass):
+    def testVoid(self):
+        self.assertEqual(Void, Guess.guess_type(sys.__stdout__))
+
+    def testSimple(self):
+        self.assertEqual(Simple, Guess.guess_type("abc"))
+        self.assertEqual(Simple, Guess.guess_type({1: 2, "a": "b"}))
+        self.assertNotEqual(ObjectStorage, Guess.guess_type({1: 2, "a": "b"}))
+
+    def testPickle(self):
+        self.assertEqual(ObjectStorage, Guess.guess_type(OwnClass(1)))
+        self.assertNotEqual(Void, Guess.guess_type(OwnClass(1)))
+        self.assertNotEqual(Simple, Guess.guess_type(OwnClass(1)))
+
+
+def obj_return(num, obj):
+    _ = num
+    return obj
+
+
+class Store(BaseClass):
+    def testFunctionDecorator(self):
+        """
+        Check if it is able to guess and store/restore proper values with types
+        """
+        decorated_own = Guess.decorator(cassette=self.cassette, item_list=[0])(
+            obj_return
+        )
+        before1 = decorated_own(1, "abc")
+        before2 = decorated_own(2, OwnClass(2))
+        before3 = decorated_own(2, OwnClass(3))
+        before4 = decorated_own(3, sys.__stdin__)
+        self.cassette.dump()
+        self.cassette.mode = StorageMode.read
+
+        after2 = decorated_own(2, OwnClass(2))
+        self.assertEqual(
+            self.cassette.data_miner.metadata[GUESS_STR], ObjectStorage.__name__
+        )
+        after3 = decorated_own(2, OwnClass(3))
+        self.assertEqual(
+            self.cassette.data_miner.metadata[GUESS_STR], ObjectStorage.__name__
+        )
+        after1 = decorated_own(1, "abc")
+        self.assertEqual(self.cassette.data_miner.metadata[GUESS_STR], Simple.__name__)
+        after4 = decorated_own(3, sys.__stdin__)
+        self.assertEqual(self.cassette.data_miner.metadata[GUESS_STR], Void.__name__)
+
+        self.assertEqual(before1, after1)
+        self.assertEqual(before2.__class__.__name__, after2.__class__.__name__)
+        self.assertEqual(before3.__class__.__name__, after3.__class__.__name__)
+        self.assertEqual(after2.__class__.__name__, "OwnClass")
+        self.assertEqual(before4.__class__.__name__, "TextIOWrapper")
+        self.assertEqual(after4.__class__.__name__, "str")
+
+
+@apply_decorator_to_all_methods(replace_module_match(what="math.sin"))
+class ApplyDefaultDecorator(unittest.TestCase):
+    SIN_OUTPUT = 0.9974949866040544
+
+    def cassette_setup(self, cassette):
+        self.assertEqual(cassette.storage_object, {})
+
+    def cassette_teardown(self, cassette):
+        os.remove(cassette.storage_file)
+
+    def test(self, cassette: Cassette):
+        math.sin(1.5)
+        self.assertEqual(len(cassette.storage_object["math"]["sin"]), 1)
+        self.assertAlmostEqual(
+            self.SIN_OUTPUT,
+            cassette.storage_object["math"]["sin"][0]["output"],
+            delta=0.0005,
+        )
+        self.assertEqual(
+            cassette.storage_object["math"]["sin"][0]["metadata"][GUESS_STR],
+            "Simple",
+        )