From b5509336a629bbe83c8e8b785885b6e913612e81 Mon Sep 17 00:00:00 2001 From: SergioSim Date: Fri, 8 Dec 2023 09:26:05 +0100 Subject: [PATCH] fixup! make max_statements=0 mean no limit --- src/ralph/backends/data/async_es.py | 2 +- src/ralph/backends/data/async_mongo.py | 2 +- src/ralph/backends/data/base.py | 14 ++++---------- src/ralph/backends/data/clickhouse.py | 2 +- src/ralph/backends/data/es.py | 2 +- src/ralph/backends/data/fs.py | 2 +- src/ralph/backends/data/ldp.py | 2 +- src/ralph/backends/data/mongo.py | 2 +- src/ralph/backends/data/s3.py | 2 +- src/ralph/backends/data/swift.py | 2 +- tests/backends/data/test_base.py | 14 ++++++++------ 11 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/ralph/backends/data/async_es.py b/src/ralph/backends/data/async_es.py index 7603ae31d..79fe9b75f 100644 --- a/src/ralph/backends/data/async_es.py +++ b/src/ralph/backends/data/async_es.py @@ -134,7 +134,7 @@ async def read( # noqa: PLR0913 prefetch (int): The number of records to prefetch (queue) while yielding. If `prefetch` is `None` it defaults to `1` - no records are prefetched. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: bytes: The next raw document if `raw_output` is True. diff --git a/src/ralph/backends/data/async_mongo.py b/src/ralph/backends/data/async_mongo.py index a0070ab72..529c58756 100644 --- a/src/ralph/backends/data/async_mongo.py +++ b/src/ralph/backends/data/async_mongo.py @@ -141,7 +141,7 @@ async def read( # noqa: PLR0913 prefetch (int): The number of records to prefetch (queue) while yielding. If `prefetch` is `None` it defaults to `1` - no records are prefetched. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: bytes: The next raw document if `raw_output` is True. diff --git a/src/ralph/backends/data/base.py b/src/ralph/backends/data/base.py index 7287ab922..03b100818 100644 --- a/src/ralph/backends/data/base.py +++ b/src/ralph/backends/data/base.py @@ -300,7 +300,7 @@ def read( # noqa: PLR0913 will be ignored and logged. If `False` (default), a `BackendException` is raised on any error. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: dict: If `raw_output` is False. @@ -315,11 +315,8 @@ def read( # noqa: PLR0913 query = validate_backend_query(query, self.query_class, self.logger) reader = self._read_bytes if raw_output else self._read_dicts statements = reader(query, target, chunk_size, ignore_errors) - if max_statements is None: - yield from statements - return - if not max_statements: + yield from statements return max_statements -= 1 @@ -535,7 +532,7 @@ async def read( # noqa: PLR0913 prefetch (int): The number of records to prefetch (queue) while yielding. If `prefetch` is `None` it defaults to `1` - no records are prefetched. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: dict: If `raw_output` is False. @@ -579,14 +576,11 @@ async def read( # noqa: PLR0913 query = validate_backend_query(query, self.query_class, self.logger) reader = self._read_bytes if raw_output else self._read_dicts statements = reader(query, target, chunk_size, ignore_errors) - if max_statements is None: + if not max_statements: async for statement in statements: yield statement return - if not max_statements: - return - i = 0 async for statement in statements: yield statement diff --git a/src/ralph/backends/data/clickhouse.py b/src/ralph/backends/data/clickhouse.py index 1841b44c2..704fadf14 100755 --- a/src/ralph/backends/data/clickhouse.py +++ b/src/ralph/backends/data/clickhouse.py @@ -226,7 +226,7 @@ def read( # noqa: PLR0913 will be ignored and logged. If `False` (default), a `BackendException` is raised on any error. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: bytes: The next raw document if `raw_output` is True. diff --git a/src/ralph/backends/data/es.py b/src/ralph/backends/data/es.py index 0020a24fd..18b0beba2 100644 --- a/src/ralph/backends/data/es.py +++ b/src/ralph/backends/data/es.py @@ -215,7 +215,7 @@ def read( # noqa: PLR0913 ignore_errors (bool): No impact as encoding errors are not expected in Elasticsearch results. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: bytes: The next raw document if `raw_output` is True. diff --git a/src/ralph/backends/data/fs.py b/src/ralph/backends/data/fs.py index f111fbff1..228ec0e5f 100644 --- a/src/ralph/backends/data/fs.py +++ b/src/ralph/backends/data/fs.py @@ -172,7 +172,7 @@ def read( # noqa: PLR0913 will be ignored and logged. If `False` (default), a `BackendException` is raised on any error. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: bytes: The next chunk of the read files if `raw_output` is True. diff --git a/src/ralph/backends/data/ldp.py b/src/ralph/backends/data/ldp.py index 53b2e6d9b..3d86b52ea 100644 --- a/src/ralph/backends/data/ldp.py +++ b/src/ralph/backends/data/ldp.py @@ -170,7 +170,7 @@ def read( # noqa: PLR0913 raw_output (bool): Should always be set to `True`. ignore_errors (bool): No impact as no encoding operation is performed. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: bytes: The content of the archive matching the query. diff --git a/src/ralph/backends/data/mongo.py b/src/ralph/backends/data/mongo.py index 3f960694f..b93ed1dda 100644 --- a/src/ralph/backends/data/mongo.py +++ b/src/ralph/backends/data/mongo.py @@ -193,7 +193,7 @@ def read( # noqa: PLR0913 will be ignored and logged. If `False` (default), a `BackendException` is raised on any error. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: dict: If `raw_output` is False. diff --git a/src/ralph/backends/data/s3.py b/src/ralph/backends/data/s3.py index 3643b8ff1..15307fd0b 100644 --- a/src/ralph/backends/data/s3.py +++ b/src/ralph/backends/data/s3.py @@ -186,7 +186,7 @@ def read( # noqa: PLR0913 will be ignored and logged. If `False` (default), a `BackendException` is raised on any error. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: dict: If `raw_output` is False. diff --git a/src/ralph/backends/data/swift.py b/src/ralph/backends/data/swift.py index 006a8cc92..88245aedf 100644 --- a/src/ralph/backends/data/swift.py +++ b/src/ralph/backends/data/swift.py @@ -200,7 +200,7 @@ def read( # noqa: PLR0913 will be ignored and logged. If `False` (default), a `BackendException` is raised on any error. max_statements (int): The maximum number of statements to yield. - If `None` (default), there is no maximum. + If `None` (default) or `0`, there is no maximum. Yield: dict: If `raw_output` is False. diff --git a/tests/backends/data/test_base.py b/tests/backends/data/test_base.py index 012fdb3eb..e7f8f355a 100644 --- a/tests/backends/data/test_base.py +++ b/tests/backends/data/test_base.py @@ -147,7 +147,9 @@ def close(self): backend = MockBaseDataBackend() assert list(backend.read()) == [{}, {}, {}, {}] + assert list(backend.read(max_statements=0)) == [{}, {}, {}, {}] assert list(backend.read(raw_output=True)) == [b"", b"", b""] + assert list(backend.read(max_statements=0, raw_output=True)) == [b"", b"", b""] assert list(backend.read(max_statements=9)) == [{}, {}, {}, {}] assert list(backend.read(max_statements=9, raw_output=True)) == [b"", b"", b""] @@ -155,9 +157,6 @@ def close(self): assert list(backend.read(max_statements=3)) == [{}, {}, {}] assert list(backend.read(max_statements=3, raw_output=True)) == [b"", b"", b""] - assert not list(backend.read(max_statements=0)) - assert not list(backend.read(max_statements=0, raw_output=True)) - @pytest.mark.anyio @pytest.mark.parametrize( @@ -311,7 +310,13 @@ async def close(self): backend = MockAsyncBaseDataBackend() assert [_ async for _ in backend.read()] == [{}, {}, {}, {}] + assert [_ async for _ in backend.read(max_statements=0)] == [{}, {}, {}, {}] assert [_ async for _ in backend.read(raw_output=True)] == [b"", b"", b""] + assert [_ async for _ in backend.read(max_statements=0, raw_output=True)] == [ + b"", + b"", + b"", + ] assert [_ async for _ in backend.read(max_statements=9)] == [{}, {}, {}, {}] assert [_ async for _ in backend.read(max_statements=9, raw_output=True)] == [ @@ -327,9 +332,6 @@ async def close(self): b"", ] - assert not [_ async for _ in backend.read(max_statements=0)] - assert not [_ async for _ in backend.read(max_statements=0, raw_output=True)] - @pytest.mark.anyio @pytest.mark.parametrize(