diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..2bcd70e --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 88 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..feb30c3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,36 @@ +language: python + +python: + - '3.4' + - '3.5' + - '3.6' + - nightly + +matrix: + allow_failures: + - python: nightly + +services: + - redis-server + +install: + - pip install -U pip setuptools + - pip install -U -e ".[dev]" + +script: + - make test + +deploy: + + - provider: pypi + + on: + tags: true + repo: Overseas-Student-Living/ddebounce + condition: $TRAVIS_PYTHON_VERSION = "3.5" + + distributions: sdist bdist_wheel + + user: iky + password: + secure: daXlSd4bxnpSFT0mcbMWMOLgTV6lKrfwTeaA9SCoKtqlUFAxQ2xKBej30qLx6TBWm2G7yILESRSeRoClsqTfO13Vx8pXU6Xf8Ei9F2RyZZ0WNZV5uxGxfMA8RDhLURDyoyhSWQIDXaAGeWAQhmMZsAWwzIhd6ozjWjmT4z0KU9zxLCzRUrE8+GOUyT+PhG6xA0AlSUlrKueAr0TAUnwLAaLiHaklkFlFqSnZzjdOtOdULnfskckU0zPLlXLOLJ1OHyZLQdkUCBlYkLhBb8Qr4+tHuCrDQaOMA2uJtbi/WK52UpMHZQeM3JhU7XVgKcliLhkSnO5n8HP5J6C0DXgQQAVhapsEHMTB32+GpLm7v/9VkqEr4mPWyxXB5VXdwp0aIdDujS9y/ONzhJGO43bhNNpQxr1Q/QqiURE8deltvTUfQp1Z8jxhJnEWMqq1i3XmAg9N+khjWOf05B31ag2IvoQ8emDRVbJCQRMpIDPa6qQ54dtX9yjUGDGK6EzLAXSu3/XWjuYllj9TlbOCEosgMxLVAYCFthbc1KFMYap0ZA0b9+VEaILYtiLelRLLpyzbEAvQFMpT0CGOE6xPo9Hql/EPd6D1YfOX9FOJPOpD1yRxQgagZ34QG3pVrImw0I3zjmnlrOI+NReKBRBPdL2w73uEyIP0glaD1yv0nSXWfxg= diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..52c5782 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +test: flake8 pylint pytest + +flake8: + flake8 ddebounce tests + +pylint: + pylint ddebounce -E --disable=no-value-for-parameter + +pytest: + coverage run --concurrency=eventlet --source ddebounce --branch -m pytest tests + coverage report --show-missing --fail-under=100 diff --git a/ddebounce/__init__.py b/ddebounce/__init__.py new file mode 100644 index 0000000..1c9af42 --- /dev/null +++ b/ddebounce/__init__.py @@ -0,0 +1,2 @@ +from .lock import Lock # noqa: F401 +from .api import debounce, skip_duplicates # noqa: F401 diff --git a/ddebounce/api.py b/ddebounce/api.py new file mode 100644 index 0000000..8b44ee9 --- /dev/null +++ b/ddebounce/api.py @@ -0,0 +1,39 @@ +import operator + +import wrapt + +from .lock import Lock + + +def debounce(client, wrapped=None, key=None, repeat=False, callback=None, ttl=None): + @wrapt.decorator + def wrapper(wrapped, instance, args, kwargs): + if instance and isinstance(client, operator.attrgetter): + decorated = Lock(client(instance), ttl).debounce( + wrapped, key, repeat, callback + ) + else: + decorated = Lock(client, ttl).debounce(wrapped, key, repeat, callback) + return decorated(*args, **kwargs) + + def logger(func): + func.debounce_applied = (key, repeat, callback, ttl) + return wrapper(func) + + return logger + + +def skip_duplicates(client, wrapped=None, key=None, ttl=None): + @wrapt.decorator + def wrapper(wrapped, instance, args, kwargs): + if instance and isinstance(client, operator.attrgetter): + decorated = Lock(client(instance), ttl).skip_duplicates(wrapped, key) + else: + decorated = Lock(client, ttl).skip_duplicates(wrapped, key) + return decorated(*args, **kwargs) + + def logger(func): + func.skip_duplicates_applied = (key, ttl) + return wrapper(func) + + return logger diff --git a/ddebounce/lock.py b/ddebounce/lock.py new file mode 100644 index 0000000..48aca92 --- /dev/null +++ b/ddebounce/lock.py @@ -0,0 +1,71 @@ +import functools + +import wrapt + + +class Lock: + def __init__(self, client, default_ttl=None): + self.client = client + self.default_ttl = default_ttl or 30 + + format_key = "lock:{}".format + + def acquire(self, key): + key = self.format_key(key) + pipe = self.client.pipeline() + pipe.incr(key) + pipe.expire(key, self.default_ttl) + count, _ = pipe.execute() + return count <= 1 + + def release(self, key): + key = self.format_key(key) + pipe = self.client.pipeline() + pipe.getset(key, 0) + pipe.expire(key, self.default_ttl) + count, _ = pipe.execute() + count = int(count) if count else 0 + return count > 1 + + def debounce(self, wrapped=None, key=None, repeat=False, callback=None): + + if wrapped is None: + return functools.partial( + self.debounce, key=key, repeat=repeat, callback=callback + ) + + vars(wrapped)["debounced"] = (key, repeat, callback) + + format_key = key or "{0}({{0}})".format(wrapped.__name__).format + + @wrapt.decorator + def wrapper(wrapped, instance, args, kwargs): + key = format_key(*args, **kwargs) + if self.acquire(key): + try: + result = wrapped(*args, **kwargs) + finally: + turns = self.release(key) + if turns: + if callback: + callback(*args, **kwargs) + if repeat: + return wrapper(wrapped)(*args, **kwargs) + return result + + return wrapper(wrapped) + + def skip_duplicates(self, wrapped=None, key=None): + + if wrapped is None: + return functools.partial(self.skip_duplicates, key=key) + + format_key = key or "{0}({{0}})".format(wrapped.__name__).format + + @wrapt.decorator + def wrapper(wrapped, instance, args, kwargs): + key = format_key(*args, **kwargs) + if self.acquire(key): + return wrapped(*args, **kwargs) + + return wrapper(wrapped) diff --git a/ddebounce/pytest.py b/ddebounce/pytest.py new file mode 100644 index 0000000..ae82861 --- /dev/null +++ b/ddebounce/pytest.py @@ -0,0 +1,23 @@ +import pytest + + +@pytest.fixture +def debounce_applied(): + def _debounce_applied(func, key=None, repeat=False, callback=None, ttl=None): + try: + return (key, repeat, callback, ttl) == getattr(func, "debounce_applied") + except AttributeError: + return False + + return _debounce_applied + + +@pytest.fixture +def skip_duplicates_applied(): + def _skip_duplicates_applied(func, key=None, ttl=None): + try: + return (key, ttl) == getattr(func, "skip_duplicates_applied") + except AttributeError: + return False + + return _skip_duplicates_applied diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..bb4d7f0 --- /dev/null +++ b/setup.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +from setuptools import setup + +setup( + name='ddebounce', + version='0.1.0-rc0', + author='Student.com', + url='http://github.com/iky/ddebounce', + packages=['ddebounce'], + install_requires=[ + "redis>=2.10.5", + 'wrapt>=1.10.8', + ], + extras_require={ + 'dev': [ + "coverage==4.5.2", + "eventlet==0.21.0", + "flake8==3.6.0", + "mock==2.0.0", + "pylint==2.2.2", + "pytest==4.1.1", + ], + }, + entry_points={'pytest11': ['ddebounce=ddebounce.pytest']}, + dependency_links=[], + zip_safe=True, + license='Apache License, Version 2.0', + classifiers=[ + "Programming Language :: Python", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Internet", + "Topic :: Software Development :: Libraries :: Python Modules", + "Intended Audience :: Developers", + ] +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d14daae --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ +import pytest +import redis + + +def pytest_addoption(parser): + parser.addoption( + "--test-redis-uri", + action="store", + dest="TEST_REDIS_URI", + default="redis://localhost:6379/11", + help='Redis uri for testing (e.g. "redis://localhost:6379/11")', + ) + + +@pytest.fixture +def redis_(request): + client = redis.StrictRedis.from_url(request.config.getoption("TEST_REDIS_URI")) + yield client + client.flushdb() diff --git a/tests/test_debounce.py b/tests/test_debounce.py new file mode 100644 index 0000000..1bb7c65 --- /dev/null +++ b/tests/test_debounce.py @@ -0,0 +1,369 @@ +import eventlet +from eventlet.event import Event +from mock import call, Mock, patch +import operator +import pytest + +from ddebounce import debounce + + +@pytest.fixture +def tracker(): + return Mock() + + +@pytest.fixture +def release(): + return Event() + + +class TestDebounce: + @pytest.fixture(params=("func", "meth", "meth_using_instance_client")) + def debounced(self, request, redis_, release, tracker): + @debounce(redis_) + def spam(*args, **kwargs): + release.wait() + tracker(*args, **kwargs) + return tracker + + class Spam: + @debounce(redis_) + def spam(self, *args, **kwargs): + release.wait() + tracker(*args, **kwargs) + return tracker + + class SpamWithClientOnInstance: + + redis = redis_ + + @debounce(operator.attrgetter("redis")) + def spam(self, *args, **kwargs): + release.wait() + tracker(*args, **kwargs) + return tracker + + samples = { + "func": spam, + "meth": Spam().spam, + "meth_using_instance_client": SpamWithClientOnInstance().spam, + } + + return samples[request.param] + + def test_debounce(self, debounced, debounce_applied, redis_, release, tracker): + def coroutine(): + return debounced("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:spam(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:spam(egg)") + + assert tracker == thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + assert debounce_applied(debounced) + + def test_debounce_failing_on_execution(self, debounced, redis_, release, tracker): + class Whoops(Exception): + pass + + tracker.side_effect = Whoops("Yo!") + + def coroutine(): + with pytest.raises(Whoops): + debounced("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:spam(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:spam(egg)") + + thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +class TestDebounceWithCustomKey: + @pytest.fixture(params=("func", "meth", "meth_using_instance_client")) + def debounced(self, request, redis_, release, tracker): + def key(_, spam): + return "yo:{}".format(spam.upper()) + + @debounce(redis_, key=key) + def spam(*args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + class SomeClass: + @debounce(redis_, key=key) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + class SpamWithClientOnInstance: + + redis = redis_ + + @debounce(operator.attrgetter("redis"), key=key) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + samples = { + "func": spam, + "meth": SomeClass().spam, + "meth_using_instance_client": SpamWithClientOnInstance().spam, + } + + return samples[request.param] + + def test_debounce(self, debounced, redis_, release, tracker): + def coroutine(): + return debounced("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:yo:HAM") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:yo:HAM") + + assert tracker == thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +class TestDebounceWithRepeat: + @pytest.fixture(params=("func", "meth", "meth_using_instance_client")) + def debounced(self, request, redis_, release, tracker): + @debounce(redis_, repeat=True) + def spam(*args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + class SomeClass: + @debounce(redis_, repeat=True) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + class SpamWithClientOnInstance: + + redis = redis_ + + @debounce(operator.attrgetter("redis"), repeat=True) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + samples = { + "func": spam, + "meth": SomeClass().spam, + "meth_using_instance_client": SpamWithClientOnInstance().spam, + } + + return samples[request.param] + + def test_debounce(self, debounced, redis_, release, tracker): + def coroutine(): + return debounced("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:spam(egg)") + + # simulate locking attempt + redis_.incr("lock:spam(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:spam(egg)") + + assert tracker == thread.wait() + + # must be called twice with the same args + assert 2 == tracker.call_count + assert [ + call("egg", spam="ham"), + call("egg", spam="ham"), + ] == tracker.call_args_list + + def test_debounce_failing_on_repeat_execution( + self, debounced, redis_, release, tracker + ): + class Whoops(Exception): + pass + + tracker.side_effect = [None, Whoops("Yo!")] + + def coroutine(): + with pytest.raises(Whoops): + debounced("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:spam(egg)") + + # simulate locking attempt + redis_.incr("lock:spam(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:spam(egg)") + + thread.wait() + + # must be called twice with the same args + assert 2 == tracker.call_count + assert [ + call("egg", spam="ham"), + call("egg", spam="ham"), + ] == tracker.call_args_list + + +class TestDebounceWithCallback: + @pytest.fixture + def callback_tracker(self): + return Mock() + + @pytest.fixture(params=("func", "meth", "meth_using_instance_client")) + def debounced(self, callback_tracker, request, redis_, release, tracker): + def callback(*args, **kwargs): + callback_tracker(*args, **kwargs) + + @debounce(redis_, callback=callback) + def spam(*args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + class SomeClass: + @debounce(redis_, callback=callback) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + class SpamWithClientOnInstance: + + redis = redis_ + + @debounce(operator.attrgetter("redis"), callback=callback) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + samples = { + "func": spam, + "meth": SomeClass().spam, + "meth_using_instance_client": SpamWithClientOnInstance().spam, + } + + return samples[request.param] + + def test_debounce(self, callback_tracker, debounced, redis_, release, tracker): + def coroutine(): + return debounced("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:spam(egg)") + + # simulate locking attempt + redis_.incr("lock:spam(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:spam(egg)") + + assert tracker == thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + # test callback call + assert 1 == callback_tracker.call_count + assert call("egg", spam="ham") == callback_tracker.call_args + + def test_debounce_failing_on_callback_execution( + self, callback_tracker, debounced, redis_, release, tracker + ): + class Whoops(Exception): + pass + + callback_tracker.side_effect = Whoops("Yo!") + + def callback(*args, **kwargs): + callback_tracker(*args, **kwargs) + + def coroutine(): + with pytest.raises(Whoops): + debounced("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:spam(egg)") + + # simulate locking attempt + redis_.incr("lock:spam(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:spam(egg)") + + thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + # test callback call + assert 1 == callback_tracker.call_count + assert call("egg", spam="ham") == callback_tracker.call_args + + +@patch("ddebounce.api.Lock") +def test_custom_ttl(Lock): + + redis_ = Mock() + + @debounce(redis_, ttl=60) + def spam(): + pass + + spam() + + assert Lock.call_args == call(redis_, 60) diff --git a/tests/test_lock.py b/tests/test_lock.py new file mode 100644 index 0000000..d630bb4 --- /dev/null +++ b/tests/test_lock.py @@ -0,0 +1,436 @@ +import eventlet +from eventlet.event import Event +from mock import call, Mock +import pytest + +from ddebounce import Lock + + +def test_debounce(redis_): + + lock = Lock(redis_) + + tracker = Mock() + release = Event() + + @lock.debounce + def func(*args, **kwargs): + release.wait() + tracker(*args, **kwargs) + return tracker + + def coroutine(): + return func("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:func(egg)") + + another_thread = eventlet.spawn(coroutine) + assert another_thread.wait() is None + + assert tracker.call_count == 0 + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:func(egg)") + + assert tracker == thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +def test_debounce_with_custom_key(redis_): + + lock = Lock(redis_) + + tracker = Mock() + release = Event() + + @lock.debounce(key=lambda _, spam: "yo:{}".format(spam.upper())) + def func(*args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + def coroutine(): + return func("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:yo:HAM") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:yo:HAM") + + assert tracker == thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +def test_debounce_with_repeat(redis_): + + lock = Lock(redis_) + + tracker = Mock() + release = Event() + + @lock.debounce(repeat=True) + def func(*args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + def coroutine(): + return func("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:func(egg)") + + # simulate locking attempt + redis_.incr("lock:func(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:func(egg)") + + assert tracker == thread.wait() + + # must be called twice with the same args + assert 2 == tracker.call_count + assert [call("egg", spam="ham"), call("egg", spam="ham")] == tracker.call_args_list + + +def test_debounce_with_callback(redis_): + + lock = Lock(redis_) + + tracker, callback_tracker = Mock(), Mock() + release = Event() + + def callback(*args, **kwargs): + callback_tracker(*args, **kwargs) + + @lock.debounce(callback=callback) + def func(*args, **kwargs): + tracker(*args, **kwargs) + release.wait() + return tracker + + def coroutine(): + return func("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:func(egg)") + + # simulate locking attempt + redis_.incr("lock:func(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:func(egg)") + + assert tracker == thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + # test callback call + assert 1 == callback_tracker.call_count + assert call("egg", spam="ham") == callback_tracker.call_args + + +def test_debounce_failing_on_execution(redis_): + + lock = Lock(redis_) + + tracker = Mock() + release = Event() + + class Whoops(Exception): + pass + + tracker.side_effect = Whoops("Yo!") + + @lock.debounce() + def func(*args, **kwargs): + release.wait() + tracker(*args, **kwargs) + + def coroutine(): + with pytest.raises(Whoops): + func("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:func(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:func(egg)") + + thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +def test_debounce_failing_on_repeat_execution(redis_): + + lock = Lock(redis_) + + tracker = Mock() + release = Event() + + class Whoops(Exception): + pass + + tracker.side_effect = [None, Whoops("Yo!")] + + @lock.debounce(repeat=True) + def func(*args, **kwargs): + tracker(*args, **kwargs) + release.wait() + + def coroutine(): + with pytest.raises(Whoops): + func("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:func(egg)") + + # simulate locking attempt + redis_.incr("lock:func(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:func(egg)") + + thread.wait() + + # must be called twice with the same args + assert 2 == tracker.call_count + assert [call("egg", spam="ham"), call("egg", spam="ham")] == tracker.call_args_list + + +def test_debounce_failing_on_callback_execution(redis_): + + lock = Lock(redis_) + + tracker, callback_tracker = Mock(), Mock() + release = Event() + + class Whoops(Exception): + pass + + callback_tracker.side_effect = Whoops("Yo!") + + def callback(*args, **kwargs): + callback_tracker(*args, **kwargs) + + @lock.debounce(callback=callback) + def func(*args, **kwargs): + tracker(*args, **kwargs) + release.wait() + + def coroutine(): + with pytest.raises(Whoops): + func("egg", spam="ham") + + thread = eventlet.spawn(coroutine) + eventlet.sleep(0.1) + + assert b"1" == redis_.get("lock:func(egg)") + + # simulate locking attempt + redis_.incr("lock:func(egg)") + + release.send() + eventlet.sleep(0.1) + + assert b"0" == redis_.get("lock:func(egg)") + + thread.wait() + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + # test callback call + assert 1 == callback_tracker.call_count + assert call("egg", spam="ham") == callback_tracker.call_args + + +def test_skip_duplicates_success(redis_): + + lock = Lock(redis_) + + tracker = Mock() + + @lock.skip_duplicates + def func(*args, **kwargs): + tracker(*args, **kwargs) + return tracker + + func("egg", spam="ham") + + assert b"1" == redis_.get("lock:func(egg)") + + func("egg", spam="ham") + + assert b"2" == redis_.get("lock:func(egg)") + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +def test_skip_duplicates_with_custom_key(redis_): + + lock = Lock(redis_) + + tracker = Mock() + + @lock.skip_duplicates(key=lambda _, spam: "yo:{}".format(spam.upper())) + def func(*args, **kwargs): + tracker(*args, **kwargs) + return tracker + + func("egg", spam="ham") + + assert b"1" == redis_.get("lock:yo:HAM") + + func("egg", spam="ham") + + assert b"2" == redis_.get("lock:yo:HAM") + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +def test_simple_acquire_and_release(redis_): + + lock = Lock(redis_) + + assert lock.acquire("101") is True + assert lock.acquire("101") is False + assert lock.acquire("102") is True + assert lock.acquire("101") is False + assert lock.acquire("102") is False + + assert lock.acquire("100") is True + + assert lock.release("101") is True + assert lock.release("102") is True + + assert lock.release("102") is False + + assert lock.release("wat") is False # never acquired + + +def test_default_expiration(redis_): + + ttl = 1 + + lock = Lock(redis_, ttl) + + key = "101" + + assert lock.acquire(key) is True + assert lock.acquire(key) is False + # wait for TTL plus 0.1 to avoid hitting it at the point it just + # expired but still returns + eventlet.sleep(ttl + 0.1) + assert lock.acquire(key) is True + + +def test_key_expires_after_release(redis_): + + ttl = 1 + + lock = Lock(redis_, ttl) + + key = "101" + + formatted_key = lock.format_key(key) + + assert lock.acquire(key) is True + + assert redis_.ttl(formatted_key) > 0 + + lock.release(key) + + # expect a ttl to still be set on the released key + assert redis_.ttl(formatted_key) > 0 + + # sleep until released key expires + eventlet.sleep(ttl + 0.1) + + # ttl = -2 means that the key has gone + assert redis_.ttl(formatted_key) == -2 + + +def test_complex_scenario(redis_): + + lock = Lock(redis_) + + key = "101" + + # P1 acquires and starts processing the task + assert lock.acquire(key) is True + + # P2 and P3 must not acquire - they should not process the task + assert lock.acquire(key) is False + assert lock.acquire(key) is False + + # P1 finishes the task processing and releases the lock + # as there were others trying to acquire the same lock during + # the time P1 was holding it, P1 will will retry the process + # and acquire the lock again + should_retry = lock.release(key) + assert should_retry is True + assert lock.acquire(key) is True + + # P4, P5, P6 kick in - all ignoring the task processing + assert lock.acquire(key) is False + assert lock.acquire(key) is False + assert lock.acquire(key) is False + + # P1 finishes the task processing and releases the lock + # as there were others trying to acquire the same lock during + # the time P1 was holding it, P1 will will retry the process + # and acquire the lock again + should_retry = lock.release(key) + assert should_retry is True + # BUT P7 kicks in before P1 acquires the lock again + assert lock.acquire(key) is True + # then P1 will fail acquiring the lock + assert lock.acquire(key) is False + + # P7 finishes and releases, as P1 tried to acquire, P7 will retry + should_retry = lock.release(key) + assert should_retry is True + assert lock.acquire(key) is True + + # P7 finishes retry and leaves + should_retry = lock.release(key) + assert should_retry is False + + # P8 ... + assert lock.acquire(key) is True diff --git a/tests/test_pytest.py b/tests/test_pytest.py new file mode 100644 index 0000000..cb4171b --- /dev/null +++ b/tests/test_pytest.py @@ -0,0 +1,74 @@ +from mock import Mock +import pytest + +from ddebounce import debounce, skip_duplicates + + +@pytest.fixture +def redis_(): + return Mock() + + +def test_debounce_not_applied(debounce_applied): + def spam(): + pass + + assert not debounce_applied(spam) + + +def test_debounce_applied(debounce_applied, redis_): + @debounce(redis_) + def spam(): + pass + + assert debounce_applied(spam) + + +def test_debounce_applied_with_exact_attributes(debounce_applied, redis_): + + key, another_key = Mock(), Mock() + callback = Mock() + + @debounce(redis_, key=key, repeat=True, callback=callback, ttl=60) + def spam(): + pass + + assert not debounce_applied(spam) + assert not debounce_applied(spam, key=key) + assert not debounce_applied(spam, repeat=True) + assert not debounce_applied(spam, callback=callback) + assert not debounce_applied(spam, ttl=60) + + assert debounce_applied(spam, key=key, repeat=True, callback=callback, ttl=60) + + assert not debounce_applied( + spam, key=another_key, repeat=True, callback=callback, ttl=60 + ) + + +def test_skip_duplicates_not_applied(skip_duplicates_applied): + def spam(): + pass + + assert not skip_duplicates_applied(spam) + + +def test_skip_duplicates_applied(skip_duplicates_applied, redis_): + @skip_duplicates(redis_) + def spam(): + pass + + assert skip_duplicates_applied(spam) + + +def test_skip_duplicates_applied_with_exact_attributes(skip_duplicates_applied, redis_): + key, another_key = Mock(), Mock() + + @skip_duplicates(redis_, key=key, ttl=60) + def spam(): + pass + + assert not skip_duplicates_applied(spam) + assert not skip_duplicates_applied(spam, key=another_key) + + assert skip_duplicates_applied(spam, key=key, ttl=60) diff --git a/tests/test_skip_duplicates.py b/tests/test_skip_duplicates.py new file mode 100644 index 0000000..113d5a2 --- /dev/null +++ b/tests/test_skip_duplicates.py @@ -0,0 +1,117 @@ +from mock import call, patch, Mock +import operator +import pytest + +from ddebounce import skip_duplicates + + +@pytest.fixture +def tracker(): + return Mock() + + +class TestSkipDuplicates: + @pytest.fixture(params=("func", "meth", "meth_using_instance_client")) + def decorated(self, request, redis_, tracker): + @skip_duplicates(redis_) + def spam(*args, **kwargs): + tracker(*args, **kwargs) + return tracker + + class Spam: + @skip_duplicates(redis_) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + return tracker + + class SpamWithClientOnInstance: + + redis = redis_ + + @skip_duplicates(operator.attrgetter("redis")) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + return tracker + + samples = { + "func": spam, + "meth": Spam().spam, + "meth_using_instance_client": SpamWithClientOnInstance().spam, + } + + return samples[request.param] + + def test_skip_duplicates(self, decorated, redis_, tracker): + + decorated("egg", spam="ham") + + assert b"1" == redis_.get("lock:spam(egg)") + + decorated("egg", spam="ham") + + assert b"2" == redis_.get("lock:spam(egg)") + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +class TestSkipDuplicatesWithCustomKey: + @pytest.fixture(params=("func", "meth", "meth_using_instance_client")) + def decorated(self, request, redis_, tracker): + def key(_, spam): + return "yo:{}".format(spam.upper()) + + @skip_duplicates(redis_, key=key) + def spam(*args, **kwargs): + tracker(*args, **kwargs) + return tracker + + class Spam: + @skip_duplicates(redis_, key=key) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + return tracker + + class SpamWithClientOnInstance: + + redis = redis_ + + @skip_duplicates(operator.attrgetter("redis"), key=key) + def spam(self, *args, **kwargs): + tracker(*args, **kwargs) + return tracker + + samples = { + "func": spam, + "meth": Spam().spam, + "meth_using_instance_client": SpamWithClientOnInstance().spam, + } + + return samples[request.param] + + def test_skip_duplicates(self, decorated, redis_, tracker): + + decorated("egg", spam="ham") + + assert b"1" == redis_.get("lock:yo:HAM") + + decorated("egg", spam="ham") + + assert b"2" == redis_.get("lock:yo:HAM") + + assert 1 == tracker.call_count + assert call("egg", spam="ham") == tracker.call_args + + +@patch("ddebounce.api.Lock") +def test_custom_ttl(Lock): + + redis_ = Mock() + + @skip_duplicates(redis_, ttl=60) + def spam(): + pass + + spam() + + assert Lock.call_args == call(redis_, 60) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..6c27fe8 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = {py34,py35,p36,p37}-test +skipsdist = True + +[testenv] +whitelist_externals = make + +commands = + pip install -U pip setuptools + pip install --editable .[dev] + make test