Skip to content

Commit

Permalink
fixup! make max_statements=0 mean no limit
Browse files Browse the repository at this point in the history
  • Loading branch information
SergioSim committed Dec 8, 2023
1 parent 3942dd2 commit b550933
Show file tree
Hide file tree
Showing 11 changed files with 21 additions and 25 deletions.
2 changes: 1 addition & 1 deletion src/ralph/backends/data/async_es.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ async def read( # noqa: PLR0913
prefetch (int): The number of records to prefetch (queue) while yielding.
If `prefetch` is `None` it defaults to `1` - no records are prefetched.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
bytes: The next raw document if `raw_output` is True.
Expand Down
2 changes: 1 addition & 1 deletion src/ralph/backends/data/async_mongo.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ async def read( # noqa: PLR0913
prefetch (int): The number of records to prefetch (queue) while yielding.
If `prefetch` is `None` it defaults to `1` - no records are prefetched.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
bytes: The next raw document if `raw_output` is True.
Expand Down
14 changes: 4 additions & 10 deletions src/ralph/backends/data/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ def read( # noqa: PLR0913
will be ignored and logged.
If `False` (default), a `BackendException` is raised on any error.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
dict: If `raw_output` is False.
Expand All @@ -315,11 +315,8 @@ def read( # noqa: PLR0913
query = validate_backend_query(query, self.query_class, self.logger)
reader = self._read_bytes if raw_output else self._read_dicts
statements = reader(query, target, chunk_size, ignore_errors)
if max_statements is None:
yield from statements
return

if not max_statements:
yield from statements
return

max_statements -= 1
Expand Down Expand Up @@ -535,7 +532,7 @@ async def read( # noqa: PLR0913
prefetch (int): The number of records to prefetch (queue) while yielding.
If `prefetch` is `None` it defaults to `1` - no records are prefetched.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
dict: If `raw_output` is False.
Expand Down Expand Up @@ -579,14 +576,11 @@ async def read( # noqa: PLR0913
query = validate_backend_query(query, self.query_class, self.logger)
reader = self._read_bytes if raw_output else self._read_dicts
statements = reader(query, target, chunk_size, ignore_errors)
if max_statements is None:
if not max_statements:
async for statement in statements:
yield statement
return

if not max_statements:
return

i = 0
async for statement in statements:
yield statement
Expand Down
2 changes: 1 addition & 1 deletion src/ralph/backends/data/clickhouse.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ def read( # noqa: PLR0913
will be ignored and logged.
If `False` (default), a `BackendException` is raised on any error.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
bytes: The next raw document if `raw_output` is True.
Expand Down
2 changes: 1 addition & 1 deletion src/ralph/backends/data/es.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ def read( # noqa: PLR0913
ignore_errors (bool): No impact as encoding errors are not expected in
Elasticsearch results.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
bytes: The next raw document if `raw_output` is True.
Expand Down
2 changes: 1 addition & 1 deletion src/ralph/backends/data/fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ def read( # noqa: PLR0913
will be ignored and logged.
If `False` (default), a `BackendException` is raised on any error.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
bytes: The next chunk of the read files if `raw_output` is True.
Expand Down
2 changes: 1 addition & 1 deletion src/ralph/backends/data/ldp.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ def read( # noqa: PLR0913
raw_output (bool): Should always be set to `True`.
ignore_errors (bool): No impact as no encoding operation is performed.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
bytes: The content of the archive matching the query.
Expand Down
2 changes: 1 addition & 1 deletion src/ralph/backends/data/mongo.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ def read( # noqa: PLR0913
will be ignored and logged.
If `False` (default), a `BackendException` is raised on any error.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
dict: If `raw_output` is False.
Expand Down
2 changes: 1 addition & 1 deletion src/ralph/backends/data/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ def read( # noqa: PLR0913
will be ignored and logged.
If `False` (default), a `BackendException` is raised on any error.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
dict: If `raw_output` is False.
Expand Down
2 changes: 1 addition & 1 deletion src/ralph/backends/data/swift.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ def read( # noqa: PLR0913
will be ignored and logged.
If `False` (default), a `BackendException` is raised on any error.
max_statements (int): The maximum number of statements to yield.
If `None` (default), there is no maximum.
If `None` (default) or `0`, there is no maximum.
Yield:
dict: If `raw_output` is False.
Expand Down
14 changes: 8 additions & 6 deletions tests/backends/data/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,17 +147,16 @@ def close(self):

backend = MockBaseDataBackend()
assert list(backend.read()) == [{}, {}, {}, {}]
assert list(backend.read(max_statements=0)) == [{}, {}, {}, {}]
assert list(backend.read(raw_output=True)) == [b"", b"", b""]
assert list(backend.read(max_statements=0, raw_output=True)) == [b"", b"", b""]

assert list(backend.read(max_statements=9)) == [{}, {}, {}, {}]
assert list(backend.read(max_statements=9, raw_output=True)) == [b"", b"", b""]

assert list(backend.read(max_statements=3)) == [{}, {}, {}]
assert list(backend.read(max_statements=3, raw_output=True)) == [b"", b"", b""]

assert not list(backend.read(max_statements=0))
assert not list(backend.read(max_statements=0, raw_output=True))


@pytest.mark.anyio
@pytest.mark.parametrize(
Expand Down Expand Up @@ -311,7 +310,13 @@ async def close(self):

backend = MockAsyncBaseDataBackend()
assert [_ async for _ in backend.read()] == [{}, {}, {}, {}]
assert [_ async for _ in backend.read(max_statements=0)] == [{}, {}, {}, {}]
assert [_ async for _ in backend.read(raw_output=True)] == [b"", b"", b""]
assert [_ async for _ in backend.read(max_statements=0, raw_output=True)] == [
b"",
b"",
b"",
]

assert [_ async for _ in backend.read(max_statements=9)] == [{}, {}, {}, {}]
assert [_ async for _ in backend.read(max_statements=9, raw_output=True)] == [
Expand All @@ -327,9 +332,6 @@ async def close(self):
b"",
]

assert not [_ async for _ in backend.read(max_statements=0)]
assert not [_ async for _ in backend.read(max_statements=0, raw_output=True)]


@pytest.mark.anyio
@pytest.mark.parametrize(
Expand Down

0 comments on commit b550933

Please sign in to comment.