Update package assembly to consider other manifests

Previously we used to yeild resources for the entire subtree after
we find a package manifest, and ingore and skip creating package
and dependencies from other manifests present there. The package
assembly process is updated to only yeild and ignore package manifest
resources used already to create package/dependencies.

Reference: #3604
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

src/packagedcode/about.py

@@ -79,9 +79,11 @@ def parse(cls, location, package_only=False):
         declared_license_expression = package_data.get('license_expression')
 
         owner = package_data.get('owner')
-        if not isinstance(owner, str):
-            owner = repr(owner)
-        parties = [models.Party(type=models.party_person, name=owner, role='owner')]
+        parties = None
+        if owner:
+            if not isinstance(owner, str):
+                owner = repr(owner)
+            parties = [models.Party(type=models.party_person, name=owner, role='owner')]
 
         # FIXME: also include notice_file and license_file(s) as file_references
         file_references = []
@@ -157,7 +159,8 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                     missing = sorted(file_references_by_path.values(), key=lambda r: r.path)
                     package.extra_data['missing_file_references'] = missing
             else:
-                package.extra_data['missing_file_references'] = package_data.file_references[:]
+                if package.file_references:
+                    package.extra_data['missing_file_references'] = package_data.file_references[:]
 
         # we yield this as we do not want this further processed
         yield resource

src/packagedcode/cargo.py

@@ -39,13 +39,18 @@ def logger_debug(*args):
 
 
 class CargoBaseHandler(models.DatafileHandler):
+
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble Cargo.toml and possible Cargo.lock datafiles. Also
         support cargo workspaces where we have multiple packages from
         a repository and some shared information present at top-level.
         """
+        datafile_name_patterns = (
+            CargoLockHandler.path_patterns + CargoTomlHandler.path_patterns
+        )
+
         workspace = package_data.extra_data.get('workspace', {})
         workspace_members = workspace.get("members", [])
         workspace_package_data = workspace.get("package", {})
@@ -89,14 +94,14 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                         resource.save(codebase)
 
                 yield from cls.assemble_from_many_datafiles(
-                    datafile_name_patterns=('Cargo.toml', 'cargo.toml', 'Cargo.lock', 'cargo.lock'),
+                    datafile_name_patterns=datafile_name_patterns,
                     directory=workspace_directory,
                     codebase=codebase,
                     package_adder=package_adder,
                 )
         else:
             yield from cls.assemble_from_many_datafiles(
-                datafile_name_patterns=('Cargo.toml', 'cargo.toml', 'Cargo.lock', 'cargo.lock'),
+                datafile_name_patterns=datafile_name_patterns,
                 directory=resource.parent(codebase),
                 codebase=codebase,
                 package_adder=package_adder,

src/packagedcode/chef.py

@@ -146,8 +146,11 @@ def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble Package from Chef metadata.rb, then from metadata.json files.
         """
+        datafile_name_patterns = (
+            ChefMetadataRbHandler.path_patterns + ChefMetadataJsonHandler.path_patterns
+        )
         yield from cls.assemble_from_many_datafiles(
-            datafile_name_patterns=('metadata.rb', 'metadata.json',),
+            datafile_name_patterns=datafile_name_patterns,
             directory=resource.parent(codebase),
             codebase=codebase,
             package_adder=package_adder,

src/packagedcode/cocoapods.py

@@ -13,6 +13,7 @@
 import logging
 import sys
 from functools import partial
+from fnmatch import fnmatchcase
 
 import saneyaml
 from packageurl import PackageURL
@@ -142,18 +143,18 @@ def assemble(cls, package_data, resource, codebase, package_adder):
         else:
             # do we have more than one podspec?
             parent = resource.parent(codebase)
+            podspec_path_pattern = PodspecHandler.path_patterns[0] 
             sibling_podspecs = [
                 r for r in parent.children(codebase)
-                if r.name.endswith('.podspec')
+                if fnmatchcase(r.name, podspec_path_pattern)
             ]
 
             siblings_counts = len(sibling_podspecs)
             has_single_podspec = siblings_counts == 1
             has_multiple_podspec = siblings_counts > 1
 
             datafile_name_patterns = (
-                'Podfile.lock',
-                'Podfile',
+                PodfileHandler.path_patterns + PodfileLockHandler.path_patterns
             )
 
             if has_single_podspec:

src/packagedcode/golang.py

@@ -32,8 +32,11 @@ def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Always use go.mod first then go.sum
         """
+        datafile_name_patterns = (
+            GoModHandler.path_patterns + GoSumHandler.path_patterns
+        )
         yield from cls.assemble_from_many_datafiles(
-            datafile_name_patterns=('go.mod', 'go.sum',),
+            datafile_name_patterns=datafile_name_patterns,
             directory=resource.parent(codebase),
             codebase=codebase,
             package_adder=package_adder,

src/packagedcode/maven.py

@@ -72,7 +72,7 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_
 
         # This order is important as we want pom.xml to be used for package
         # creation and then to update from MANIFEST later 
-        manifest_path_pattern = '*/META-INF/MANIFEST.MF'
+        manifest_path_pattern = JavaJarManifestHandler.path_patterns[0]
         nested_pom_xml_path_pattern = '*/META-INF/maven/**/pom.xml'
         datafile_name_patterns = (nested_pom_xml_path_pattern, manifest_path_pattern)
 

src/packagedcode/models.py

@@ -1279,7 +1279,7 @@ def assemble_from_many(
             else:
                 # FIXME: What is the package_data is NOT for the same package as package?
                 # FIXME: What if the update did not do anything? (it does return True or False)
-                # FIXME: There we would be missing out packges AND/OR errors
+                # FIXME: There we would be missing out packages AND/OR errors
                 package.update(
                     package_data=package_data,
                     datafile_path=resource.path,
@@ -1309,7 +1309,7 @@ def assemble_from_many(
             yield package
         yield from dependencies
 
-        # Associate Package to Resources and yield them
+        # Associate Package to the manifest resources and yield them
         for resource in resources:
             package_adder(package_uid, resource, codebase)
             yield resource
@@ -1318,11 +1318,12 @@ def assemble_from_many(
             package_adder(package_uid, resource, codebase)
             yield resource
 
-        # the whole parent subtree of the base_resource is for this package
+        # the whole parent subtree of the base_resource is for this package,
+        # so assign resources to package
         if package_uid:
             for res in base_resource.walk(codebase):
                 package_adder(package_uid, res, codebase)
-                yield res
+
             if parent_resource:
                 package_adder(package_uid, parent_resource, codebase)
                 yield parent_resource
@@ -1368,7 +1369,10 @@ def assemble_from_many_datafiles(
         # we iterate on datafile_name_patterns because their order matters
         for datafile_name_pattern in datafile_name_patterns:
             for sibling in siblings:
-                if fnmatchcase(sibling.name, datafile_name_pattern):
+                if (
+                    fnmatchcase(sibling.name, datafile_name_pattern) or 
+                    fnmatchcase(sibling.location, datafile_name_pattern)
+                ):
                     for package_data in sibling.package_data:
                         package_data = PackageData.from_dict(package_data)
                         pkgdata_resources.append((package_data, sibling,))

src/packagedcode/npm.py

@@ -161,7 +161,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                 for npm_res in cls.walk_npm(resource=workspace_root, codebase=codebase):
                     if package_uid and package_uid not in npm_res.for_packages:
                         package_adder(package_uid, npm_res, codebase)
-                    yield npm_res
+
             yield package_resource
 
         elif workspaces:
@@ -190,7 +190,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                     for npm_res in cls.walk_npm(resource=workspace_root, codebase=codebase):
                         if package_uid and not npm_res.for_packages:
                             package_adder(package_uid, npm_res, codebase)
-                        yield npm_res
+
                 yield package_resource
 
         else:
@@ -257,7 +257,6 @@ def create_packages_from_workspaces(
             for npm_res in cls.walk_npm(resource=member_root, codebase=codebase):
                 if package_uid and package_uid not in npm_res.for_packages:
                     package_adder(package_uid, npm_res, codebase)
-                yield npm_res
 
             yield from cls.yield_npm_dependencies_and_resources(
                 package_resource=workspace_member,

src/packagedcode/phpcomposer.py

@@ -26,10 +26,8 @@ class BasePhpComposerHandler(models.DatafileHandler):
 
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):
-        datafile_name_patterns = (
-            'composer.json',
-            'composer.lock',
-        )
+        datafile_name_patterns = \
+            PhpComposerJsonHandler.path_patterns + PhpComposerLockHandler.path_patterns
 
         if resource.has_parent():
             dir_resource = resource.parent(codebase)

src/packagedcode/plugin_package.py

@@ -376,15 +376,15 @@ def get_package_and_deps(codebase, package_adder=add_to_package, strip_root=Fals
     packages = []
     dependencies = []
 
-    seen_resource_paths = set()
+    seen_package_manifest_paths = set()
 
     has_single_resource = codebase.has_single_resource
     # track resource ids that have been already processed
     for resource in codebase.walk(topdown=False):
         if not resource.package_data:
             continue
 
-        if resource.path in seen_resource_paths:
+        if resource.path in seen_package_manifest_paths:
             continue
 
         if TRACE_ASSEMBLY:
@@ -430,12 +430,12 @@ def get_package_and_deps(codebase, package_adder=add_to_package, strip_root=Fals
                         dependencies.append(item)
 
                     elif isinstance(item, Resource):
-                        seen_resource_paths.add(item.path)
+                        seen_package_manifest_paths.add(item.path)
 
                         if TRACE_ASSEMBLY:
                             logger_debug(
-                                '    get_package_and_deps: seen_resource_path:',
-                                seen_resource_paths,
+                                '    get_package_and_deps: seen_package_manifest_paths:',
+                                seen_package_manifest_paths,
                             )
 
                     else:

src/packagedcode/rubygems.py

@@ -56,11 +56,9 @@ def assemble_extracted_gem(cls, package_data, resource, codebase, package_adder)
     An assemble implementation shared by handlers for manifests found in an
     extracted gem using extractcode.
     """
-    datafile_name_patterns = (
-        'metadata.gz-extract/metadata.gz-extract',
-        'data.gz-extract/*.gemspec',
-        'data.gz-extract/Gemfile',
-        'data.gz-extract/Gemfile.lock',
+    datafile_name_patterns = ('metadata.gz-extract/metadata.gz-extract',) + (
+        GemspecHandler.path_patterns + GemfileHandler.path_patterns
+        + GemfileLockHandler.path_patterns
     )
 
     gemroot = get_ancestor(levels_up=2, resource=resource, codebase=codebase)
@@ -105,11 +103,9 @@ class BaseGemProjectHandler(models.DatafileHandler):
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):
         datafile_name_patterns = (
-            '*.gemspec',
-            'Gemfile',
-            'Gemfile.lock',
+            GemspecHandler.path_patterns + GemfileHandler.path_patterns
+            + GemfileLockHandler.path_patterns
         )
-
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=datafile_name_patterns,
             directory=resource.parent(codebase),