Skip to content

Commit

Permalink
Add parser for swift-show-dependencies.deplock (#3829)
Browse files Browse the repository at this point in the history
* Add parser for swift-show-dependencies.deplock
* Add package assembly for swift-show-dependencies.deplock

Reference: Reference: aboutcode-org/scancode.io#1278
Signed-off-by: Keshav Priyadarshi <[email protected]>
  • Loading branch information
keshav-space authored Jun 28, 2024
1 parent 89f8851 commit 60b0e0f
Show file tree
Hide file tree
Showing 10 changed files with 1,735 additions and 31 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ v33.0.0 (next next, roadmap)
from cocoapods lockfile `Podfile.lock`.
See https://github.com/nexB/scancode-toolkit/pull/3827

- Add support for parsing packages and dependency relationships
from swift `swift-show-dependencies.deplock` generated by DepLock.
See https://github.com/nexB/scancode-toolkit/pull/3829

v32.2.0 - 2024-06-19
----------------------

Expand Down
9 changes: 8 additions & 1 deletion docs/source/reference/available_package_parsers.rst
Original file line number Diff line number Diff line change
Expand Up @@ -766,8 +766,9 @@ parsers in scancode-toolkit during documentation builds.
- ``squashfs_disk_image``
- None
- https://en.wikipedia.org/wiki/SquashFS
* - JSON dump of Package.swift created with ``swift package dump-package &gt; Package.swift.json``
* - JSON dump of Package.swift created by DepLock or with ``swift package dump-package &gt; Package.swift.json``
- ``*/Package.swift.json``
``*/Package.swift.deplock``
- ``swift``
- ``swift_package_manifest_json``
- Swift
Expand All @@ -779,6 +780,12 @@ parsers in scancode-toolkit during documentation builds.
- ``swift_package_resolved``
- swift
- https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html#package-dependency
* - Swift dependency graph created by DepLock
- ``*/swift-show-dependencies.deplock``
- ``swift``
- ``swift_package_show_dependencies``
- Swift
- https://forums.swift.org/t/swiftpm-show-dependencies-without-fetching-dependencies/51154
* - Java Web Application Archive
- ``*.war``
- ``war``
Expand Down
1 change: 1 addition & 0 deletions src/packagedcode/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@

swift.SwiftManifestJsonHandler,
swift.SwiftPackageResolvedHandler,
swift.SwiftShowDependenciesDepLockHandler,

windows.MicrosoftUpdateManifestHandler,

Expand Down
256 changes: 228 additions & 28 deletions src/packagedcode/swift.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,76 @@ def logger_debug(*args):
return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args))


class SwiftShowDependenciesDepLockHandler(models.DatafileHandler):
datasource_id = "swift_package_show_dependencies"
path_patterns = ("*/swift-show-dependencies.deplock",)
default_package_type = "swift"
default_primary_language = "Swift"
description = "Swift dependency graph created by DepLock"
documentation_url = "https://forums.swift.org/t/swiftpm-show-dependencies-without-fetching-dependencies/51154"

@classmethod
def _parse(cls, swift_dependency_relation, package_only=False):

if TRACE:
logger_debug(
f"SwiftShowDependenciesDepLockHandler: deplock: package: {swift_dependency_relation}"
)

dependencies = get_flatten_dependencies(
dependency_tree=swift_dependency_relation.get("dependencies")
)

package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
# ``namespace`` is derived from repo URL and same is not available in dependency graph
# See related issue: https://github.com/nexB/scancode-toolkit/issues/3793
name=swift_dependency_relation.get("name"),
dependencies=dependencies,
)

return models.PackageData.from_data(package_data, package_only)

@classmethod
def parse(cls, location, package_only=False):
with io.open(location, encoding="utf-8") as loc:
swift_dependency_relation = json.load(loc)

yield cls._parse(swift_dependency_relation, package_only)

@classmethod
def assemble(
cls, package_data, resource, codebase, package_adder=models.add_to_package
):
siblings = resource.siblings(codebase)
swift_manifest_resource = [
r
for r in siblings
if r.name in ("Package.swift.json", "Package.swift.deplock")
]

# Skip the assembly if the Swift manifest is present.
# SwiftManifestJsonHandler's assembly will take care of the
# dependencies from swift-show-dependencies.deplock file.
if swift_manifest_resource:
return []

yield from super(SwiftShowDependenciesDepLockHandler, cls).assemble(
package_data=package_data,
resource=resource,
codebase=codebase,
package_adder=package_adder,
)


class SwiftManifestJsonHandler(models.DatafileHandler):
datasource_id = "swift_package_manifest_json"
path_patterns = ("*/Package.swift.json",)
path_patterns = ("*/Package.swift.json", "*/Package.swift.deplock")
default_package_type = "swift"
default_primary_language = "Swift"
description = "JSON dump of Package.swift created with ``swift package dump-package > Package.swift.json``"
description = "JSON dump of Package.swift created by DepLock or with ``swift package dump-package > Package.swift.json``"
documentation_url = "https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html"

@classmethod
Expand Down Expand Up @@ -98,41 +162,64 @@ def assemble(
top-level package with resolved dependencies.
"""
siblings = resource.siblings(codebase)
processed_dependencies = []
swift_resolved_package_resource = [
r for r in siblings if r.name == "Package.resolved"
]
dependencies_from_manifest = package_data.dependencies

processed_dependencies = []
if swift_resolved_package_resource:
swift_resolved_package_resource = swift_resolved_package_resource[0]
swift_resolved_package_data = swift_resolved_package_resource.package_data
swift_show_dependencies_resources = [
r for r in siblings if r.name == "swift-show-dependencies.deplock"
]

for package in swift_resolved_package_data:
version = package.get("version")
name = package.get("name")
if swift_show_dependencies_resources:
swift_show_dependencies_resource = swift_show_dependencies_resources[0]
swift_show_dependencies_package_data = (
swift_show_dependencies_resource.package_data
)

# Dependencies from `swift-show-dependencies.deplock` supersede dependencies from other datafiles.
processed_dependencies = swift_show_dependencies_package_data[0][
"dependencies"
]
processed_dependencies = [
models.DependentPackage.from_dict(i) for i in processed_dependencies
]

# Use dependencies from `Package.resolved` when `swift-show-dependencies.deplock` is not present.
else:
dependencies_from_manifest = package_data.dependencies

purl = PackageURL(
type=cls.default_package_type, name=name, version=version
if swift_resolved_package_resource:
swift_resolved_package_resource = swift_resolved_package_resource[0]
swift_resolved_package_data = (
swift_resolved_package_resource.package_data
)
processed_dependencies.append(
models.DependentPackage(
purl=purl.to_string(),
scope="dependencies",
is_runtime=True,
is_optional=False,
is_resolved=True,
extracted_requirement=version,

for package in swift_resolved_package_data:
version = package.get("version")
name = package.get("name")

purl = PackageURL(
type=cls.default_package_type, name=name, version=version
)
processed_dependencies.append(
models.DependentPackage(
purl=purl.to_string(),
scope="dependencies",
is_runtime=True,
is_optional=False,
is_resolved=True,
extracted_requirement=version,
)
)
)

for dependency in dependencies_from_manifest[:]:
dependency_purl = PackageURL.from_string(dependency.purl)
for dependency in dependencies_from_manifest[:]:
dependency_purl = PackageURL.from_string(dependency.purl)

if dependency_purl.name == name:
dependencies_from_manifest.remove(dependency)
if dependency_purl.name == name:
dependencies_from_manifest.remove(dependency)

processed_dependencies.extend(dependencies_from_manifest)
processed_dependencies.extend(dependencies_from_manifest)

datafile_path = resource.path
if package_data.purl:
Expand All @@ -141,7 +228,12 @@ def assemble(
datafile_path=datafile_path,
)

if swift_resolved_package_resource:
if swift_show_dependencies_resources:
package.datafile_paths.append(swift_show_dependencies_resource.path)
package.datasource_ids.append(
SwiftShowDependenciesDepLockHandler.datasource_id
)
elif swift_resolved_package_resource:
package.datafile_paths.append(swift_resolved_package_resource.path)
package.datasource_ids.append(SwiftPackageResolvedHandler.datasource_id)

Expand Down Expand Up @@ -196,7 +288,9 @@ def assemble(
):
siblings = resource.siblings(codebase)
swift_manifest_resource = [
r for r in siblings if r.name == "Package.swift.json"
r
for r in siblings
if r.name in ("Package.swift.json", "Package.swift.deplock")
]

# Skip the assembly if the ``Package.swift.json`` manifest is present.
Expand Down Expand Up @@ -328,3 +422,109 @@ def get_namespace_and_name(url):
canonical_name = hostname + path

return canonical_name.rsplit("/", 1)


def get_flatten_dependencies(dependency_tree):
"""
Get the list of dependencies from the dependency graph where each
element is a DependentPackage containing its 1st order dependencies.
"""
dependencies = []
transitive_dependencies = []

# process direct dependency
for dependency in dependency_tree:
transitives = dependency.get("dependencies", [])
transitive_dependencies.append(transitives)
parent_child_dep = get_dependent_package_from_subtree(
dependency=dependency,
is_top_level_dependency=True,
)
dependencies.append(parent_child_dep)

# process all transitive dependencies
while transitive_dependencies:
transitive_dependency_tree = transitive_dependencies.pop(0)
if not transitive_dependency_tree:
continue

for transitive in transitive_dependency_tree:
dependencies_of_transitive_dependency = transitive.get("dependencies", [])
# add nested dependencies in transitive_dependencies queue for processing
transitive_dependencies.append(dependencies_of_transitive_dependency)

parent_child_dep = get_dependent_package_from_subtree(
dependency=transitive,
is_top_level_dependency=False,
)

dependencies.append(parent_child_dep)

return dependencies


def get_dependent_package_from_subtree(dependency, is_top_level_dependency):
"""
Get the DependentPackage for a ``dependency`` subtree along with its 1st
order dependencies. Set `is_direct` to True if the subtree is a direct
dependency for the top-level package.
"""
dependencies_of_parent = []
repository_url = dependency.get("url")
version = dependency.get("version")
transitives = dependency.get("dependencies", [])
namespace, name = get_namespace_and_name(repository_url)
purl = PackageURL(
type="swift",
namespace=namespace,
name=name,
version=version,
)

for transitive in transitives:
transitive_repository_url = transitive.get("url")
transitive_version = transitive.get("version")
transitive_namespace, transitive_name = get_namespace_and_name(
transitive_repository_url
)
transitive_purl = PackageURL(
type="swift",
namespace=transitive_namespace,
name=transitive_name,
version=transitive_version,
)

child_dependency = models.DependentPackage(
purl=transitive_purl.to_string(),
scope="dependencies",
extracted_requirement=transitive_version,
is_runtime=False,
is_optional=False,
is_resolved=True,
is_direct=True,
).to_dict()

dependencies_of_parent.append(child_dependency)

parent_package_data_mapping = dict(
datasource_id=SwiftShowDependenciesDepLockHandler.datasource_id,
type=SwiftShowDependenciesDepLockHandler.default_package_type,
primary_language=SwiftShowDependenciesDepLockHandler.default_primary_language,
namespace=namespace,
name=name,
version=version,
dependencies=dependencies_of_parent,
is_virtual=True,
)
parent_dependency = models.PackageData.from_data(parent_package_data_mapping)

return models.DependentPackage(
purl=purl.to_string(),
scope="dependencies",
extracted_requirement=version,
is_runtime=False,
is_optional=False,
is_resolved=True,
is_direct=is_top_level_dependency,
resolved_package=parent_dependency,
)
11 changes: 9 additions & 2 deletions tests/packagedcode/data/plugin/help.txt
Original file line number Diff line number Diff line change
Expand Up @@ -836,8 +836,8 @@ Package type: swift
datasource_id: swift_package_manifest_json
documentation URL: https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html
primary language: Swift
description: JSON dump of Package.swift created with ``swift package dump-package > Package.swift.json``
path_patterns: '*/Package.swift.json'
description: JSON dump of Package.swift created by DepLock or with ``swift package dump-package > Package.swift.json``
path_patterns: '*/Package.swift.json', '*/Package.swift.deplock'
--------------------------------------------
Package type: swift
datasource_id: swift_package_resolved
Expand All @@ -846,6 +846,13 @@ Package type: swift
description: Resolved full dependency lockfile for Package.swift created with ``swift package resolve``
path_patterns: '*/Package.resolved', '*/.package.resolved'
--------------------------------------------
Package type: swift
datasource_id: swift_package_show_dependencies
documentation URL: https://forums.swift.org/t/swiftpm-show-dependencies-without-fetching-dependencies/51154
primary language: Swift
description: Swift dependency graph created by DepLock
path_patterns: '*/swift-show-dependencies.deplock'
--------------------------------------------
Package type: war
datasource_id: java_war_archive
documentation URL: https://en.wikipedia.org/wiki/WAR_(file_format)
Expand Down
Loading

0 comments on commit 60b0e0f

Please sign in to comment.