diff --git a/perceval/backend.py b/perceval/backend.py index 47db68543..5ac9f7be1 100644 --- a/perceval/backend.py +++ b/perceval/backend.py @@ -33,6 +33,7 @@ import os import pkgutil import sys +import io from grimoirelab_toolkit.introspect import find_signature_parameters from grimoirelab_toolkit.datetime import (datetime_utcnow, @@ -801,6 +802,8 @@ def run(self): fetch_archive = self.archive_manager and self.parsed_args.fetch_archive archived_since = backend_args.pop('archived_since', None) + backend_args = {k: v for k, v in backend_args.items() if not isinstance(v, io.IOBase)} + with BackendItemsGenerator(self.BACKEND, backend_args, category, filter_classified=filter_classified, manager=self.archive_manager, @@ -914,6 +917,7 @@ class BackendItemsGenerator: :param fetch_archive: If enabled, items are fetched from archives :param archived_after: return items archived after this date """ + def __init__(self, backend_class, backend_args, category, filter_classified=False, manager=None, fetch_archive=False, archived_after=None): @@ -1027,6 +1031,7 @@ class Summary: Finally, the summary also includes some extra fields, which can be used by any backend to include fetch-specific information. """ + def __init__(self): self.fetched = 0 self.skipped = 0 diff --git a/perceval/backends/core/dockerhub.py b/perceval/backends/core/dockerhub.py index e444959ac..6a0c5e460 100644 --- a/perceval/backends/core/dockerhub.py +++ b/perceval/backends/core/dockerhub.py @@ -80,22 +80,6 @@ def __init__(self, owner, repository, tag=None, archive=None, ssl_verify=True): self.repository = repository self.client = None - def fetch(self, category=CATEGORY_DOCKERHUB_DATA): - """Fetch data from a Docker Hub repository. - - The method retrieves, from a repository stored in Docker Hub, - its data which includes number of pulls, stars, description, - among other data. - - :param category: the category of items to fetch - - :returns: a generator of data - """ - kwargs = {} - items = super().fetch(category, **kwargs) - - return items - def fetch_items(self, category, **kwargs): """Fetch the Dockher Hub items @@ -227,4 +211,6 @@ def setup_cmd_parser(cls): parser.parser.add_argument('repository', help="Docker Hub repository") + parser.parser.set_defaults(category=CATEGORY_DOCKERHUB_DATA) + return parser diff --git a/tests/test_dockerhub.py b/tests/test_dockerhub.py index 0f97af765..6351306ed 100644 --- a/tests/test_dockerhub.py +++ b/tests/test_dockerhub.py @@ -38,7 +38,8 @@ from perceval.backend import BackendCommandArgumentParser from perceval.backends.core.dockerhub import (DockerHub, DockerHubClient, - DockerHubCommand) + DockerHubCommand, + CATEGORY_DOCKERHUB_DATA) from base import TestCaseBackendArchive DOCKERHUB_URL = "https://hub.docker.com/" @@ -122,7 +123,7 @@ def test_fetch(self, mock_utcnow): setup_http_server() dockerhub = DockerHub('grimoirelab', 'perceval') - items = [item for item in dockerhub.fetch()] + items = [item for item in dockerhub.fetch(category=CATEGORY_DOCKERHUB_DATA)] self.assertEqual(len(items), 1) item = items[0] @@ -146,7 +147,7 @@ def test_search_fields(self, mock_utcnow): setup_http_server() dockerhub = DockerHub('grimoirelab', 'perceval') - items = [item for item in dockerhub.fetch()] + items = [item for item in dockerhub.fetch(category=CATEGORY_DOCKERHUB_DATA)] self.assertEqual(len(items), 1) item = items[0] @@ -187,6 +188,23 @@ def test_fetch_from_archive(self, mock_utcnow): setup_http_server() self._test_fetch_from_archive() + def _test_fetch_from_archive(self, **kwargs): + """Test whether the method fetch_from_archive works properly""" + + items = [items for items in self.backend_write_archive.fetch(category=CATEGORY_DOCKERHUB_DATA, **kwargs)] + items_archived = [item for item in self.backend_read_archive.fetch_from_archive()] + + self.assertEqual(len(items), len(items_archived)) + + for i in range(len(items)): + item = items[i] + archived_item = items_archived[i] + + del item['timestamp'] + del archived_item['timestamp'] + + self.assertEqual(item, archived_item) + class TestDockerHubClient(unittest.TestCase): """DockerHub API client tests.