Add Support for Dynamic Dependencies in pyproject.toml (#351)

* Parse tool.setuptools.dynamic section in pyprojecttoml

* Fix lint line-too-long

* Fix format

* Add unit tests for dynamic dependencies

* Check if dynamic is in "[poject]" first in pyproject.toml

* Apply reviews

* Put codes in a separate helper parse_dynamic_pyproject_contents

* Parse dynamic fields after parsing pep621 pyproject contents

* Remove is_dynamic in create_one_fake_project and hardcoding in tests

* Add tests for parsing regular and dynamic sources from pyprojecttoml

* Fix logic of parsing pep621 and dynamic codes and update tests

* Add two more tests for parsing dynamic deps and opt deps

fawltydeps/extract_declared_dependencies.py

@@ -233,7 +233,61 @@ def parse_optional(contents: TomlData, src: Location) -> NamedLocations:
                 yield req, src
 
     fields_parsers = [("main", parse_main), ("optional", parse_optional)]
-    return parse_pyproject_elements(parsed_contents, source, "PEP621", fields_parsers)
+
+    if "dynamic" in parsed_contents.get("project", {}):
+        yield from parse_dynamic_pyproject_contents(parsed_contents, source)
+        if "dependencies" in parsed_contents["project"]["dynamic"]:
+            if "optional-dependencies" in parsed_contents["project"]["dynamic"]:
+                fields_parsers = []
+            else:
+                fields_parsers = [("optional", parse_optional)]
+        else:
+            if "optional-dependencies" in parsed_contents["project"]["dynamic"]:
+                fields_parsers = [("main", parse_main)]
+
+    yield from parse_pyproject_elements(
+        parsed_contents, source, "PEP621", fields_parsers
+    )
+
+
+def parse_dynamic_pyproject_contents(
+    parsed_contents: TomlData, source: Location
+) -> Iterator[DeclaredDependency]:
+    """Extract dynamic dependencies from a pyproject.toml using the PEP 621 fields"""
+
+    dynamic = parsed_contents["project"]["dynamic"]
+
+    deps_files = []
+    try:
+        if "dependencies" in dynamic:
+            deps_files = parsed_contents["tool"]["setuptools"]["dynamic"][
+                "dependencies"
+            ]["file"]
+    except KeyError:
+        pass
+
+    optional_deps_files = []
+    try:
+        if "optional-dependencies" in dynamic:
+            optional_deps = parsed_contents["tool"]["setuptools"]["dynamic"][
+                "optional-dependencies"
+            ]
+            # Extract the file paths and flatten them into a single list
+            optional_deps_files = [
+                file_path
+                for file_path_list in [v["file"] for v in optional_deps.values()]
+                for file_path in file_path_list
+            ]
+    except KeyError:
+        pass
+
+    dynamic_files = deps_files + optional_deps_files
+    for req_file in dynamic_files:
+        req_file_path = Path(source.path).parent / req_file
+        if req_file_path.exists():
+            yield from parse_requirements_txt(req_file_path)
+        else:
+            logger.error("%s does not exist. Skipping.", req_file_path)
 
 
 def parse_pyproject_elements(
@@ -272,7 +326,7 @@ def parse_pyproject_toml(path: Path) -> Iterator[DeclaredDependency]:
 
     There are multiple ways to declare dependencies inside a pyproject.toml.
     We currently handle:
-    - PEP 621 core metadata fields
+    - PEP 621 core and dynamic metadata fields.
     - Poetry-specific metadata in `tool.poetry` sections.
     """
     source = Location(path)

tests/test_extract_declared_dependencies_success.py

@@ -525,6 +525,153 @@ def test_find_and_parse_sources__project_with_pyproject__returns_list(fake_proje
     assert_unordered_equivalence(actual, expect)
 
 
+def test_find_and_parse_dynamic_sources__project_with_pyproject__returns_list(
+    write_tmp_files,
+    fake_project,
+):
+    # Write requirements files into a place where files should be initially ignored
+    # but will be included when the dynamic sections in pyproject.toml are parsed.
+    tmp_path = fake_project(
+        files_with_declared_deps={
+            ".subdir/requirements.txt": ["pandas"],
+            ".subdir/requirements-test.txt": ["pylint >= 2.15.8"],
+        },
+        extra_file_contents={
+            "pyproject.toml": """\
+            [project]
+            name = "MyLib"
+            dynamic = ["dependencies", "optional-dependencies"]
+            [tool.setuptools.dynamic]
+            dependencies = { file = [".subdir/requirements.txt"] }
+            optional-dependencies.test = { file = [".subdir/requirements-test.txt"] } """,
+        },
+    )
+    expect = [
+        "pandas",
+        "pylint",
+    ]
+    settings = Settings(code=set(), deps={tmp_path})
+    deps_sources = list(find_sources(settings, {DepsSource}))
+    actual = collect_dep_names(parse_sources(deps_sources))
+    assert_unordered_equivalence(actual, expect)
+
+
+def test_find_and_parse_static_and_dynamic_sources__project_with_pyproject__returns_list(
+    write_tmp_files,
+    fake_project,
+):
+    # Write requirements files into a place where files should be initially ignored
+    # but will be included when the dynamic sections in pyproject.toml are parsed.
+
+    # If dependencies or optional dependencies are declared dynamic, they can
+    # no longer be declared static. Therefore, the static [project.dependencies]
+    # and [project.optional-dependencies] sections will not be parsed since
+    # "dependencies" and "optional-dependencies" are declared in [project.dynamic].
+    tmp_path = fake_project(
+        files_with_declared_deps={
+            ".subdir/requirements.txt": ["pandas"],
+            ".subdir/requirements-test.txt": ["pylint >= 2.15.8"],
+        },
+        extra_file_contents={
+            "pyproject.toml": """\
+            [project]
+            name = "MyLib"
+            dynamic = ["dependencies", "optional-dependencies"]
+            dependencies = ["django"]
+            optional-dependencies = {"dev" = ["black"]}
+            [tool.setuptools.dynamic]
+            dependencies = { file = [".subdir/requirements.txt"] }
+            optional-dependencies.test = { file = [".subdir/requirements-test.txt"] } """,
+        },
+    )
+    expect = [
+        "pandas",
+        "pylint",
+    ]
+    settings = Settings(code=set(), deps={tmp_path})
+    deps_sources = list(find_sources(settings, {DepsSource}))
+    actual = collect_dep_names(parse_sources(deps_sources))
+    assert_unordered_equivalence(actual, expect)
+
+
+def test_find_and_parse_static_and_dynamic_dependencies__project_with_pyproject__returns_list(
+    write_tmp_files,
+    fake_project,
+):
+    # Write requirements files into a place where files should be initially ignored
+    # but will be included when the dynamic sections in pyproject.toml are parsed.
+
+    # If dependencies or optional dependencies are declared dynamic, they can no longer
+    # be declared as static. As a result, the [project.dependencies] section won't be parsed,
+    # since "dependencies" is declared in [project.dynamic]. However, the static
+    # [project.optional-dependencies] section will still be parsed, as "optional-dependencies"
+    # is not marked as dynamic.
+    tmp_path = fake_project(
+        files_with_declared_deps={
+            ".subdir/requirements.txt": ["pandas"],
+            ".subdir/requirements-test.txt": ["pylint >= 2.15.8"],
+        },
+        extra_file_contents={
+            "pyproject.toml": """\
+            [project]
+            name = "MyLib"
+            dynamic = ["dependencies"]
+            dependencies = ["django"]
+            optional-dependencies = {"dev" = ["black"]}
+            [tool.setuptools.dynamic]
+            dependencies = { file = [".subdir/requirements.txt"] }
+            optional-dependencies.test = { file = [".subdir/requirements-test.txt"] } """,
+        },
+    )
+    expect = [
+        "pandas",
+        "black",
+    ]
+    settings = Settings(code=set(), deps={tmp_path})
+    deps_sources = list(find_sources(settings, {DepsSource}))
+    actual = collect_dep_names(parse_sources(deps_sources))
+    assert_unordered_equivalence(actual, expect)
+
+
+def test_find_and_parse_static_and_dynamic_opt_dependencies__project_with_pyproject__returns_list(
+    write_tmp_files,
+    fake_project,
+):
+    # Write requirements files into a place where files should be initially ignored
+    # but will be included when the dynamic sections in pyproject.toml are parsed.
+
+    # If dependencies or optional dependencies are declared dynamic, they can no longer
+    # be declared as static. As a result, the [project.optional-dependencies] section
+    # won't be parsed, since "optional-dependencies" is declared in [project.dynamic].
+    # However, the static [project.dependencies] section will still be parsed,
+    # as "dependencies" is not marked as dynamic.
+    tmp_path = fake_project(
+        files_with_declared_deps={
+            ".subdir/requirements.txt": ["pandas"],
+            ".subdir/requirements-test.txt": ["pylint >= 2.15.8"],
+        },
+        extra_file_contents={
+            "pyproject.toml": """\
+            [project]
+            name = "MyLib"
+            dynamic = ["optional-dependencies"]
+            dependencies = ["django"]
+            optional-dependencies = {"dev" = ["black"]}
+            [tool.setuptools.dynamic]
+            dependencies = { file = [".subdir/requirements.txt"] }
+            optional-dependencies.test = { file = [".subdir/requirements-test.txt"] } """,
+        },
+    )
+    expect = [
+        "django",
+        "pylint",
+    ]
+    settings = Settings(code=set(), deps={tmp_path})
+    deps_sources = list(find_sources(settings, {DepsSource}))
+    actual = collect_dep_names(parse_sources(deps_sources))
+    assert_unordered_equivalence(actual, expect)
+
+
 def test_find_and_parse_sources__project_with_setup_cfg__returns_list(fake_project):
     tmp_path = fake_project(
         files_with_declared_deps={