Skip to content

Commit

Permalink
get rid of stream_state interpolation in sentry
Browse files Browse the repository at this point in the history
  • Loading branch information
brianjlai committed Oct 18, 2024
1 parent 99bee1e commit 15cb6dd
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,13 @@ def select_state(self, stream_slice: Optional[StreamSlice] = None) -> Optional[S
def _calculate_earliest_possible_value(self, end_datetime: datetime.datetime) -> datetime.datetime:
lookback_delta = self._parse_timedelta(self._lookback_window.eval(self.config) if self._lookback_window else "P0D")
earliest_possible_start_datetime = min(self._start_datetime.get_datetime(self.config), end_datetime)
cursor_datetime = self._calculate_cursor_datetime_from_state(self.get_stream_state())
return max(earliest_possible_start_datetime, cursor_datetime) - lookback_delta
try:
cursor_datetime = self._calculate_cursor_datetime_from_state(self.get_stream_state()) - lookback_delta
except OverflowError:
# cursor_datetime defers to the minimum date if it does not exist in the state. Trying to subtract
# a timedelta from the minimum datetime results in an OverflowError
cursor_datetime = self._calculate_cursor_datetime_from_state(self.get_stream_state())
return max(earliest_possible_start_datetime, cursor_datetime)

def select_best_end_datetime(self) -> datetime.datetime:
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,18 +247,18 @@ def mock_datetime_now(monkeypatch):
"test_with_lookback_window_from_start_date",
NO_STATE,
MinMaxDatetime(datetime="2021-01-05", datetime_format="%Y-%m-%d", parameters={}),
MinMaxDatetime(datetime="2021-01-05", datetime_format="%Y-%m-%d", parameters={}),
MinMaxDatetime(datetime="2021-01-08", datetime_format="%Y-%m-%d", parameters={}),
"P1D",
cursor_field,
"P3D",
datetime_format,
cursor_granularity,
None,
[
{"start_time": "2021-01-02T00:00:00.000000+0000", "end_time": "2021-01-02T23:59:59.999999+0000"},
{"start_time": "2021-01-03T00:00:00.000000+0000", "end_time": "2021-01-03T23:59:59.999999+0000"},
{"start_time": "2021-01-04T00:00:00.000000+0000", "end_time": "2021-01-04T23:59:59.999999+0000"},
{"start_time": "2021-01-05T00:00:00.000000+0000", "end_time": "2021-01-05T00:00:00.000000+0000"},
{"start_time": "2021-01-05T00:00:00.000000+0000", "end_time": "2021-01-05T23:59:59.999999+0000"},
{"start_time": "2021-01-06T00:00:00.000000+0000", "end_time": "2021-01-06T23:59:59.999999+0000"},
{"start_time": "2021-01-07T00:00:00.000000+0000", "end_time": "2021-01-07T23:59:59.999999+0000"},
{"start_time": "2021-01-08T00:00:00.000000+0000", "end_time": "2021-01-08T00:00:00.000000+0000"},
],
),
(
Expand Down
6 changes: 3 additions & 3 deletions airbyte-integrations/connectors/source-sentry/poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ definitions:
record_selector:
$ref: "#/definitions/record_selector"
record_filter:
condition: "{{ record[parameters['cursor_field']] > stream_state.get(parameters['cursor_field'], '') }}"
condition: "{{ record[parameters['cursor_field']] > stream_interval.get('start_time', '') }}"
paginator:
$ref: "#/definitions/paginator"
incremental_sync:
Expand All @@ -117,7 +117,7 @@ definitions:
requester:
$ref: "#/definitions/requester"
request_parameters:
query: "lastSeen:>{{ stream_state.get(parameters['cursor_field']) or '1900-01-01T00:00:00.0Z' if stream_state else '1900-01-01T00:00:00.0Z' }}"
query: "lastSeen:>{{ stream_interval.get('start_time', '') or '1900-01-01T00:00:00.0Z' if stream_interval else '1900-01-01T00:00:00.0Z' }}"
record_selector:
$ref: "#/definitions/record_selector"
paginator:
Expand Down

0 comments on commit 15cb6dd

Please sign in to comment.