pyproject.toml

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "crawlee"
version = "0.3.8"
description = "Crawlee for Python"
authors = ["Apify Technologies s.r.o. <support@apify.com>"]
license = "Apache-2.0"
readme = "README.md"
packages = [{ include = "crawlee", from = "src" }]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: Apache Software License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Software Development :: Libraries",
]
keywords = [
    "apify",
    "automation",
    "chrome",
    "crawlee",
    "crawler",
    "headless",
    "scraper",
    "scraping",
]

[tool.poetry.urls]
"Homepage" = "https://crawlee.dev/python"
"Apify Homepage" = "https://apify.com"
"Changelog" = "https://crawlee.dev/python/docs/changelog"
"Documentation" = "https://crawlee.dev/python/docs/quick-start"
"Issue Tracker" = "https://github.com/apify/crawlee-python/issues"
"Repository" = "https://github.com/apify/crawlee-python"

# We use inclusive ordered comparison clauses for external packages intentionally in order to enhance Crawlee's
# compatibility with external packages. This decision was discussed in detail in the following PR:
# https://github.com/apify/apify-sdk-python/pull/154.
[tool.poetry.dependencies]
python = "^3.9"
apify = { version = ">=2.0.0", optional = true }
beautifulsoup4 = { version = ">=4.12.0", optional = true }
colorama = ">=0.4.0"
cookiecutter = ">=2.6.0"
curl-cffi = { version = ">=0.7.2", optional = true }
docutils = ">=0.21.0"
eval-type-backport = ">=0.2.0"
html5lib = { version = ">=1.0", optional = true }
httpx = { version = ">=0.27.0", extras = ["brotli", "http2"] }
inquirer = ">=3.3.0"
lxml = { version = ">=5.2.0", optional = true }
more_itertools = ">=10.2.0"
parsel = { version = ">=1.9.0", optional = true }
playwright = { version = ">=1.27.0", optional = true }
psutil = ">=6.0.0"
pydantic = ">=2.6.0"
pydantic-settings = ">=2.2.0"
pyee = ">=9.0.0"
python-dateutil = ">=2.9.0"
sortedcollections = ">=2.1.0"
tldextract = ">=5.1.0"
typer = ">=0.12.0"
typing-extensions = ">=4.1.0"

[tool.poetry.group.dev.dependencies]
build = "~1.2.0"
filelock = "~3.16.0"
ipdb = "~0.13.0"
mypy = "~1.11.0"
pre-commit = "~3.8.0"
proxy-py = "~2.4.0"
pydoc-markdown = "~4.8.0"
pytest = "~8.3.0"
pytest-asyncio = "~0.24.0"
pytest-cov = "~5.0.0"
pytest-only = "~2.1.0"
pytest-timeout = "~2.3.0"
pytest-xdist = "~3.6.0"
respx = "~0.21.0"
ruff = "~0.6.0"
setuptools = "~73.0.0"                    # setuptools are used by pytest, but not explicitly required
types-beautifulsoup4 = "~4.12.0.20240229"
types-colorama = "~0.4.15.20240106"
types-psutil = "~5.9.5.20240205"
types-python-dateutil = "~2.9.0.20240316"

[tool.poetry.extras]
all = ["apify", "beautifulsoup4", "lxml", "html5lib", "curl-cffi", "playwright"]
apify = ["apify"]
beautifulsoup = ["beautifulsoup4", "lxml", "html5lib"]
curl-impersonate = ["curl-cffi"]
playwright = ["playwright"]
parsel = ["parsel"]

[tool.poetry.scripts]
crawlee = "crawlee._cli:cli"

[tool.ruff]
line-length = 120

[tool.ruff.lint]
select = ["ALL"]
ignore = [
    "ANN101",   # Missing type annotation for `self` in method
    "ANN102",   # Missing type annotation for `{name}` in classmethod
    "ANN401",   # Dynamically typed expressions (typing.Any) are disallowed in {filename}
    "ASYNC109", # Async function definition with a `timeout` parameter
    "BLE001",   # Do not catch blind exception
    "C901",     # `{name}` is too complex
    "COM812",   # This rule may cause conflicts when used with the formatter
    "D100",     # Missing docstring in public module
    "D104",     # Missing docstring in public package
    "D107",     # Missing docstring in `__init__`
    "EM",       # flake8-errmsg
    "G004",     # Logging statement uses f-string
    "ISC001",   # This rule may cause conflicts when used with the formatter
    "FIX",      # flake8-fixme
    "PGH003",   # Use specific rule codes when ignoring type issues
    "PLR0911",  # Too many return statements
    "PLR0913",  # Too many arguments in function definition
    "PLR0915",  # Too many statements
    "PTH",      # flake8-use-pathlib
    "PYI034",   # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
    "PYI036",   # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
    "S102",     # Use of `exec` detected
    "S105",     # Possible hardcoded password assigned to
    "S106",     # Possible hardcoded password assigned to argument: "{name}"
    "S301",     # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
    "S303",     # Use of insecure MD2, MD4, MD5, or SHA1 hash function
    "S311",     # Standard pseudo-random generators are not suitable for cryptographic purposes
    "TD002",    # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
    "TRY003",   # Avoid specifying long messages outside the exception class
]

[tool.ruff.format]
quote-style = "single"
indent-style = "space"

[tool.ruff.lint.per-file-ignores]
"**/__init__.py" = [
    "F401", # Unused imports
]
"**/{scripts}/*" = [
    "D",       # Everything from the pydocstyle
    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
    "T20",     # flake8-print
]
"**/{tests}/*" = [
    "D",       # Everything from the pydocstyle
    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
    "S101",    # Use of assert detected
    "SLF001",  # Private member accessed: `{name}`
    "T20",     # flake8-print
    "TRY301",  # Abstract `raise` to an inner function
]
"**/{docs}/**" = [
    "D",       # Everything from the pydocstyle
    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
    "F841",    # Local variable {variable} is assigned to but never used
    "N999",    # Invalid module name
]

[tool.ruff.lint.flake8-quotes]
docstring-quotes = "double"
inline-quotes = "single"

[tool.ruff.lint.flake8-builtins]
builtins-ignorelist = ["id"]

[tool.ruff.lint.pydocstyle]
convention = "google"

[tool.ruff.lint.isort]
known-first-party = ["crawlee"]

[tool.ruff.lint.pylint]
max-branches = 18

[tool.pytest.ini_options]
addopts = "-ra"
asyncio_default_fixture_loop_scope = "function"
asyncio_mode = "auto"
timeout = 1200

[tool.mypy]
python_version = "3.9"
plugins = ["pydantic.mypy"]
files = ["scripts", "src", "tests"]
check_untyped_defs = true
disallow_incomplete_defs = true
disallow_untyped_calls = true
disallow_untyped_decorators = true
disallow_untyped_defs = true
no_implicit_optional = true
warn_redundant_casts = true
warn_return_any = true
warn_unreachable = true
warn_unused_ignores = true
exclude = [
    "docs/introduction/code/08_main.py",
]

[tool.mypy-scrapy]
ignore_missing_imports = true

[tool.mypy-sortedcollections]
ignore_missing_imports = true

[tool.coverage.report]
exclude_lines = [
    "pragma: no cover",
    "if TYPE_CHECKING:",
    "assert_never()"
]

[tool.basedpyright]
typeCheckingMode = "standard"

[tool.ipdb]
context = 7