Skip to content

Commit

Permalink
PLAYWRIGHT_REQUEST_RESPONSE_LOGGER_ENABLED setting
Browse files Browse the repository at this point in the history
  • Loading branch information
elacuesta committed Sep 10, 2024
1 parent f87154a commit 5de5a52
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 2 deletions.
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,14 @@ def should_abort_request(request):
PLAYWRIGHT_ABORT_REQUEST = should_abort_request
```

### `PLAYWRIGHT_REQUEST_RESPONSE_LOGGER_ENABLED`
Type `bool`, default `True`

Whether or not to emit debug-level log lines for Playwright request & responses.
```python
PLAYWRIGHT_REQUEST_RESPONSE_LOGGER_ENABLED = False
```

### General note about settings
For settings that accept object paths as strings, passing callable objects is
only supported when using Scrapy>=2.4. With prior versions, only strings are
Expand Down
9 changes: 7 additions & 2 deletions scrapy_playwright/handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ class Config:
restart_disconnected_browser: bool
target_closed_max_retries: int = 3
use_threaded_loop: bool = False
request_response_logger_enabled: bool = True

@classmethod
def from_settings(cls, settings: Settings) -> "Config":
Expand All @@ -122,6 +123,9 @@ def from_settings(cls, settings: Settings) -> "Config":
),
use_threaded_loop=platform.system() == "Windows"
or settings.getbool("_PLAYWRIGHT_THREADED_LOOP", False),
request_response_logger_enabled=settings.getbool(
"PLAYWRIGHT_REQUEST_RESPONSE_LOGGER_ENABLED", default=True
),
)
cfg.cdp_kwargs.pop("endpoint_url", None)
cfg.connect_kwargs.pop("ws_endpoint", None)
Expand Down Expand Up @@ -323,10 +327,11 @@ async def _create_page(self, request: Request, spider: Spider) -> Page:

page.on("close", self._make_close_page_callback(context_name))
page.on("crash", self._make_close_page_callback(context_name))
page.on("request", _make_request_logger(context_name, spider))
page.on("response", _make_response_logger(context_name, spider))
page.on("request", self._increment_request_stats)
page.on("response", self._increment_response_stats)
if self.config.request_response_logger_enabled:
page.on("request", _make_request_logger(context_name, spider))
page.on("response", _make_response_logger(context_name, spider))

return page

Expand Down

0 comments on commit 5de5a52

Please sign in to comment.