diff --git a/blueos_repository/consolidate-old.py b/blueos_repository/consolidate-old.py new file mode 100755 index 0000000..4672791 --- /dev/null +++ b/blueos_repository/consolidate-old.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +import asyncio +import dataclasses +import json +from enum import Enum +from pathlib import Path +from typing import Any, AsyncIterable, Dict, List, Optional, Union + +import aiohttp +import json5 +import semver +from registry import Registry + +REPO_ROOT = "https://raw.githubusercontent.com/bluerobotics/BlueOS-Extensions-Repository/master/repos" + + +class StrEnum(str, Enum): + """Temporary filler until Python 3.11 available.""" + + def __str__(self) -> str: + return self.value # type: ignore + + +class EnhancedJSONEncoder(json.JSONEncoder): + """ + Custom json encoder for dataclasses, + see https://docs.python.org/3/library/json.html#json.JSONEncoder.default + Returns a serializable type + """ + + def default(self, o: Any) -> Union[Any, Dict[str, Any]]: + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return super().default(o) + + +@dataclasses.dataclass +class Author: + name: str + email: str + + @staticmethod + def from_json(json_dict: Dict[str, str]) -> "Author": + return Author(name=json_dict["name"], email=json_dict["email"]) + + +@dataclasses.dataclass +class Platform: + architecture: str + variant: Optional[str] = None + # pylint: disable=invalid-name + os: Optional[str] = None + + +@dataclasses.dataclass +class Image: + digest: Optional[str] + size: int + platform: Platform + + +@dataclasses.dataclass +class Company: + name: str + about: Optional[str] + email: Optional[str] + + @staticmethod + def from_json(json_dict: Dict[str, str]) -> "Company": + return Company(name=json_dict["name"], email=json_dict.get("email", None), about=json_dict.get("about", None)) + + +class ExtensionType(StrEnum): + DEVICE_INTEGRATION = "device-integration" + EXAMPLE = "example" + THEME = "theme" + OTHER = "other" + TOOL = "tool" + + +# pylint: disable=too-many-instance-attributes +@dataclasses.dataclass +class Version: + permissions: Optional[Dict[str, Any]] + requirements: Optional[str] + tag: Optional[str] + website: str + authors: List[Author] + docs: Optional[str] + readme: Optional[str] + company: Optional[Company] + support: Optional[str] + type: ExtensionType + filter_tags: List[str] + extra_links: Dict[str, str] + images: List[Image] + + @staticmethod + def validate_filter_tags(tags: List[str]) -> List[str]: + """Returns a list of up to 10 lower-case alpha-numeric tags (dashes allowed).""" + return [tag.lower() for tag in tags if tag.replace("-", "").isalnum()][:10] + + +@dataclasses.dataclass +class RepositoryEntry: + identifier: str + name: str + description: str + docker: str + website: str + versions: Dict[str, Version] + extension_logo: Optional[str] + company_logo: Optional[str] + + +class Consolidator: + consolidated_data: List[RepositoryEntry] = [] + + @staticmethod + def repo_folder() -> Path: + return Path(__file__).parent.parent.joinpath("repos") + + @staticmethod + async def fetch_readme(url: str) -> str: + if not url.startswith("http"): + print(f"Invalid Readme url: {url}") + return "Readme not provided." + async with aiohttp.ClientSession() as session: + async with session.get(url) as resp: + if resp.status != 200: + print(f"Error status {resp.status}") + raise Exception(f"Could not get readme {url}: status: {resp.status}") + if resp.content_type != "text/plain": + raise Exception(f"bad response type for readme: {resp.content_type}, expected text/plain") + return await resp.text() + + async def all_repositories(self) -> AsyncIterable[RepositoryEntry]: + repos = self.repo_folder() + for repo in repos.glob("**/metadata.json"): + with open(repo, "r", encoding="utf-8") as individual_file: + company, extension_name = repo.as_posix().split("/")[-3:-1] + identifier = ".".join([company, extension_name]) + try: + data = json5.load(individual_file) + except Exception as exc: + raise Exception(f"Unable to parse file {repo}") from exc + company_logo = (repo / "../../company_logo.png").resolve().relative_to(repos.resolve()) + extension_logo_file = (repo / "../extension_logo.png").resolve() + if extension_logo_file.exists(): + extension_logo = extension_logo_file.resolve().relative_to(repos.resolve()) + else: + extension_logo = company_logo + try: + new_repo = RepositoryEntry( + identifier=identifier, + name=data["name"], + docker=data["docker"], + description=data["description"], + website=data["website"], + extension_logo=f"{REPO_ROOT}/{extension_logo}" if extension_logo else None, + versions={}, + company_logo=f"{REPO_ROOT}/{company_logo}" if company_logo else None, + ) + yield new_repo + except Exception as error: + raise Exception(f"unable to read file {repo}: {error}") from error + + @staticmethod + def valid_semver(string: str) -> Optional[semver.VersionInfo]: + # We want to allow versions to be prefixed with a 'v'. + # This is up for discussion + if string.startswith("v"): + string = string[1:] + try: + return semver.VersionInfo.parse(string) + except ValueError: + return None # not valid + + def extract_images_from_tag(self, tag: Any) -> List[Image]: + active_images = [ + image + for image in tag["images"] + if (image["status"] == "active" and image["architecture"] != "unknown" and image["os"] != "unknown") + ] + + images = [ + Image( + digest=image.get("digest", None), + size=image["size"], + platform=Platform( + architecture=image["architecture"], + variant=image.get("variant", None), + os=image.get("os", None), + ), + ) + for image in active_images + ] + return images + + # pylint: disable=too-many-locals + async def run(self) -> None: + async for repository in self.all_repositories(): + for tag in await self.registry.fetch_remote_tags(repository.docker): + tag_name = tag["name"] + try: + if not self.valid_semver(tag_name): + print(f"{tag_name} is not valid SemVer, ignoring it...") + continue + raw_labels = await self.registry.fetch_labels(f"{repository.docker}:{tag_name}") + permissions = raw_labels.get("permissions", None) + links = json5.loads(raw_labels.get("links", "{}")) + website = links.pop("website", raw_labels.get("website", None)) + authors = json5.loads(raw_labels.get("authors", "[]")) + # documentation is just a URL for a link, but the old format had it as its own label + docs = links.pop("docs", links.pop("documentation", raw_labels.get("docs", None))) + readme = raw_labels.get("readme", None) + if readme is not None: + readme = readme.replace(r"{tag_name}", tag_name) + try: + readme = await self.fetch_readme(readme) + except Exception as error: # pylint: disable=broad-except + readme = str(error) + company_raw = raw_labels.get("company", None) + company = Company.from_json(json5.loads(company_raw)) if company_raw is not None else None + support = links.pop("support", raw_labels.get("support", None)) + type_ = raw_labels.get("type", ExtensionType.OTHER) + filter_tags = json5.loads(raw_labels.get("tags", "[]")) + + new_version = Version( + permissions=json5.loads(permissions) if permissions else None, + website=website, + authors=authors, + docs=json5.loads(docs) if docs else None, + readme=readme, + company=company, + support=support, + extra_links=links, + type=type_, + filter_tags=Version.validate_filter_tags(filter_tags), + requirements=raw_labels.get("requirements", None), + tag=tag_name, + images=self.extract_images_from_tag(tag), + ) + repository.versions[tag_name] = new_version + except KeyError as error: + raise Exception(f"unable to parse repository {repository}: {error}") from error + # sort the versions, with the highest version first + repository.versions = dict( + sorted(repository.versions.items(), key=lambda i: self.valid_semver(i[0]), reverse=True) # type: ignore + ) + if repository.versions: # only include if there's at least one valid version + self.consolidated_data.append(repository) + + with open("manifest.json", "w", encoding="utf-8") as manifest_file: + manifest_file.write(json.dumps(self.consolidated_data, indent=4, cls=EnhancedJSONEncoder)) + + +consolidator = Consolidator() +asyncio.run(consolidator.run()) diff --git a/blueos_repository/consolidate.py b/blueos_repository/consolidate.py old mode 100755 new mode 100644 index d6709c3..a18e8b9 --- a/blueos_repository/consolidate.py +++ b/blueos_repository/consolidate.py @@ -1,220 +1,111 @@ #!/usr/bin/env python3 import asyncio import dataclasses -import json -from enum import Enum +from typing import Dict, AsyncIterable from pathlib import Path -from typing import Any, AsyncIterable, Dict, List, Optional, Union - -import aiohttp +# Extensions +from extension.extension import Extension +from extension.models import ExtensionMetadata, ExtensionVersion +# Extra +import json import json5 -import semver -from registry import Registry - -REPO_ROOT = "https://raw.githubusercontent.com/bluerobotics/BlueOS-Extensions-Repository/master/repos" - - -class StrEnum(str, Enum): - """Temporary filler until Python 3.11 available.""" +from utils import EnhancedJSONEncoder - def __str__(self) -> str: - return self.value # type: ignore +# This is the name manifest file that will be generated +MANIFEST_FILE = "manifest.json" -class EnhancedJSONEncoder(json.JSONEncoder): - """ - Custom json encoder for dataclasses, - see https://docs.python.org/3/library/json.html#json.JSONEncoder.default - Returns a serializable type - """ - - def default(self, o: Any) -> Union[Any, Dict[str, Any]]: - if dataclasses.is_dataclass(o): - return dataclasses.asdict(o) - return super().default(o) +# This is the root URL for the public repository +REPO_ROOT_URL = "https://raw.githubusercontent.com/bluerobotics/BlueOS-Extensions-Repository/master/repos" @dataclasses.dataclass -class Author: - name: str - email: str - - @staticmethod - def from_json(json_dict: Dict[str, str]) -> "Author": - return Author(name=json_dict["name"], email=json_dict["email"]) - - -@dataclasses.dataclass -class Company: - name: str - about: Optional[str] - email: Optional[str] - - @staticmethod - def from_json(json_dict: Dict[str, str]) -> "Company": - return Company(name=json_dict["name"], email=json_dict.get("email", None), about=json_dict.get("about", None)) - - -class ExtensionType(StrEnum): - DEVICE_INTEGRATION = "device-integration" - EXAMPLE = "example" - THEME = "theme" - OTHER = "other" - TOOL = "tool" - - -# pylint: disable=too-many-instance-attributes -@dataclasses.dataclass -class Version: - permissions: Optional[Dict[str, Any]] - requirements: Optional[str] - tag: Optional[str] - website: str - authors: List[Author] - docs: Optional[str] - readme: Optional[str] - company: Optional[Company] - support: Optional[str] - type: ExtensionType - filter_tags: List[str] - extra_links: Dict[str, str] - - @staticmethod - def validate_filter_tags(tags: List[str]) -> List[str]: - """Returns a list of up to 10 lower-case alpha-numeric tags (dashes allowed).""" - return [tag.lower() for tag in tags if tag.replace("-", "").isalnum()][:10] +class RepositoryEntry(ExtensionMetadata): + """ + Represents a repository entry in the manifest output + Attributes: + versions (Dict[str, Version]): Available extension versions. + """ -@dataclasses.dataclass -class RepositoryEntry: - identifier: str - name: str - description: str - docker: str - website: str - versions: Dict[str, Version] - extension_logo: Optional[str] - company_logo: Optional[str] + versions: Dict[str, ExtensionVersion] class Consolidator: - registry = Registry() - consolidated_data: List[RepositoryEntry] = [] + """ + This class is used to consolidate the BlueOS extensions repository generating + a manifest file with all the extensions available and their versions. + """ @staticmethod def repo_folder() -> Path: return Path(__file__).parent.parent.joinpath("repos") - @staticmethod - async def fetch_readme(url: str) -> str: - if not url.startswith("http"): - print(f"Invalid Readme url: {url}") - return "Readme not provided." - async with aiohttp.ClientSession() as session: - async with session.get(url) as resp: - if resp.status != 200: - print(f"Error status {resp.status}") - raise Exception(f"Could not get readme {url}: status: {resp.status}") - if resp.content_type != "text/plain": - raise Exception(f"bad response type for readme: {resp.content_type}, expected text/plain") - return await resp.text() - - async def all_repositories(self) -> AsyncIterable[RepositoryEntry]: + + async def fetch_extensions_metadata(self) -> AsyncIterable[ExtensionMetadata]: + """ + Fetch the metadata for all the extensions in the repository. + + Returns: + List[ExtensionMetadata]: List of all the extensions metadata. + """ + repos = self.repo_folder() + for repo in repos.glob("**/metadata.json"): - with open(repo, "r", encoding="utf-8") as individual_file: + with open(repo, "r", encoding="utf-8") as metadata_file: company, extension_name = repo.as_posix().split("/")[-3:-1] identifier = ".".join([company, extension_name]) + try: - data = json5.load(individual_file) - except Exception as exc: - raise Exception(f"Unable to parse file {repo}") from exc + data = json5.load(metadata_file) + except Exception as error: + print(f" INVALID-EXTENSION - Skipping {repo}, unable to parse metadata file, exception: {error}") + continue + company_logo = (repo / "../../company_logo.png").resolve().relative_to(repos.resolve()) extension_logo_file = (repo / "../extension_logo.png").resolve() + if extension_logo_file.exists(): extension_logo = extension_logo_file.resolve().relative_to(repos.resolve()) else: extension_logo = company_logo + try: - new_repo = RepositoryEntry( + remote_extension_logo = f"{REPO_ROOT_URL}/{extension_logo}" if extension_logo else None + remote_company_logo = f"{REPO_ROOT_URL}/{company_logo}" if company_logo else None + + metadata = ExtensionMetadata( identifier=identifier, name=data["name"], docker=data["docker"], description=data["description"], website=data["website"], - extension_logo=f"{REPO_ROOT}/{extension_logo}" if extension_logo else None, - versions={}, - company_logo=f"{REPO_ROOT}/{company_logo}" if company_logo else None, + extension_logo=remote_extension_logo, + company_logo=remote_company_logo, ) - yield new_repo + + yield metadata except Exception as error: - raise Exception(f"unable to read file {repo}: {error}") from error + print(f"INVALID-EXTENSION - Skipping {repo}, invalid metadata file, exception: {error}") + continue + - @staticmethod - def valid_semver(string: str) -> Optional[semver.VersionInfo]: - # We want to allow versions to be prefixed with a 'v'. - # This is up for discussion - if string.startswith("v"): - string = string[1:] - try: - return semver.VersionInfo.parse(string) - except ValueError: - return None # not valid - - # pylint: disable=too-many-locals async def run(self) -> None: - async for repository in self.all_repositories(): - for tag in await self.registry.fetch_remote_tags(repository.docker): - try: - if not self.valid_semver(tag): - print(f"{tag} is not valid SemVer, ignoring it...") - continue - raw_labels = await self.registry.fetch_labels(f"{repository.docker}:{tag}") - permissions = raw_labels.get("permissions", None) - links = json5.loads(raw_labels.get("links", "{}")) - website = links.pop("website", raw_labels.get("website", None)) - authors = json5.loads(raw_labels.get("authors", "[]")) - # documentation is just a URL for a link, but the old format had it as its own label - docs = links.pop("docs", links.pop("documentation", raw_labels.get("docs", None))) - readme = raw_labels.get("readme", None) - if readme is not None: - readme = readme.replace(r"{tag}", tag) - try: - readme = await self.fetch_readme(readme) - except Exception as error: # pylint: disable=broad-except - readme = str(error) - company_raw = raw_labels.get("company", None) - company = Company.from_json(json5.loads(company_raw)) if company_raw is not None else None - support = links.pop("support", raw_labels.get("support", None)) - type_ = raw_labels.get("type", ExtensionType.OTHER) - filter_tags = json5.loads(raw_labels.get("tags", "[]")) - - new_version = Version( - permissions=json5.loads(permissions) if permissions else None, - website=website, - authors=authors, - docs=json5.loads(docs) if docs else None, - readme=readme, - company=company, - support=support, - extra_links=links, - type=type_, - filter_tags=Version.validate_filter_tags(filter_tags), - requirements=raw_labels.get("requirements", None), - tag=tag, - ) - repository.versions[tag] = new_version - except KeyError as error: - raise Exception(f"unable to parse repository {repository}: {error}") from error - # sort the versions, with the highest version first - repository.versions = dict( - sorted(repository.versions.items(), key=lambda i: self.valid_semver(i[0]), reverse=True) # type: ignore + extensions = [Extension(metadata) async for metadata in self.fetch_extensions_metadata()] + + await asyncio.gather(*(ext.inflate() for ext in extensions)) + + consolidated_data = [ + RepositoryEntry( + **ext.metadata, + versions=ext.versions ) - if repository.versions: # only include if there's at least one valid version - self.consolidated_data.append(repository) + for ext in extensions if ext.versions + ] - with open("manifest.json", "w", encoding="utf-8") as manifest_file: - manifest_file.write(json.dumps(self.consolidated_data, indent=4, cls=EnhancedJSONEncoder)) + with open(MANIFEST_FILE, "w", encoding="utf-8") as manifest_file: + manifest_file.write(json.dumps(consolidated_data, indent=4, cls=EnhancedJSONEncoder)) consolidator = Consolidator() diff --git a/blueos_repository/docker/auth.py b/blueos_repository/docker/auth.py new file mode 100644 index 0000000..1bfd69a --- /dev/null +++ b/blueos_repository/docker/auth.py @@ -0,0 +1,53 @@ +import aiohttp_retry +# API Models +from models.auth import AuthToken + + +class DockerAuthAPI: + """ + This class is used to interact with the Docker Auth API. + + More details in https://distribution.github.io/distribution/spec/auth/token/ + """ + + __api_url: str = "https://auth.docker.io" + + + def __init__(self, max_retries: int = 5) -> None: + """ + Constructor for the DockerAuthAPI class. + + Args: + max_retries: The maximum number of retries to be used in case of request failure. Defaults to 5. + + Returns: + None + """ + self.__retry_options = aiohttp_retry.ExponentialRetry(attempts=max_retries) + + + async def get_token(self, repo: str) -> AuthToken: + """ + Gets a token to be used in docker registry requests. + + Args: + repo: The repository name, for example "bluerobotics/core" + + Returns: + The token + """ + + payload = { + "service": "registry.docker.io", + "scope": f"repository:{repo}:pull", + } + + auth_url = f"{self.__api_url}/token?service=registry.docker.io&scope=repository:{repo}:pull" + async with aiohttp_retry.RetryClient(retry_options=self.__retry_options) as session: + async with session.get(auth_url + "/token", params=payload) as resp: + if resp.status != 200: + error_msg = f"Error on Docker Auth API with status {resp.status}" + print(error_msg) + raise Exception(error_msg) + + return AuthToken(**(await resp.json())) diff --git a/blueos_repository/docker/hub.py b/blueos_repository/docker/hub.py new file mode 100644 index 0000000..010f664 --- /dev/null +++ b/blueos_repository/docker/hub.py @@ -0,0 +1,79 @@ +import aiohttp_retry +from typing import Optional, Any +from models.tag import TagList + + +class DockerHub: + """ + This class is used to interact with the Docker Hub API. + + More details in https://docs.docker.com/docker-hub/api/latest/ + """ + + __api_base_url: str = "https://hub.docker.com" + __api_version: str = "v2" + __api_url: str = f"{__api_base_url}/{__api_version}" + + def __init__(self, repository: str, max_retries: int = 5) -> None: + """ + Constructor for the DockerHubAPI class. + + Args: + repository: Repository that this registry class will operate on + max_retries: The maximum number of retries to be used in case of request failure. Defaults to 5. + + Returns: + None + """ + + self.repository = repository + self.__retry_options = aiohttp_retry.ExponentialRetry(attempts=max_retries) + + + async def get(self, route: str, max_retries: Optional[int] = None, **kwargs) -> Any: + """ + Make a GET request to the Docker Hub API. + + Args: + route: The route to be used in the request. + max_retries: The maximum number of retries to be used in case of request failure. Defaults to None. + **kwargs: The keyword arguments to be used in the request like params, headers, etc. + + Returns: + The response from the request parsed as json. + + Raises: + Exception: The pretty error message in case of error status. + """ + + retry = self.__retry_options if not max_retries else aiohttp_retry.ExponentialRetry(attempts=max_retries) + + async with aiohttp_retry.RetryClient(retry_options=retry) as session: + async with session.get(f"{self.__api_url}/{route}", **kwargs) as resp: + if resp.status >= 400: + error_msg = f"Error on GET Docker HUB API with status {resp.status} on call to {resp.url}" + print(error_msg) + raise Exception(error_msg) + + return await resp.json(content_type=None) + + + async def get_tags(self) -> TagList: + """ + Get all tags for a given repository + + Args: + repo: The repository name, for example "bluerobotics/core" + + Returns: + A list of tags + """ + + route = f"/repositories/{self.repository}/tags" + params = { + "page_size": 25, + "page": 1, + "ordering": "last_updated", + } + + return TagList(**(await self.api.get(route, params))) diff --git a/blueos_repository/docker/models/auth.py b/blueos_repository/docker/models/auth.py new file mode 100644 index 0000000..528b8be --- /dev/null +++ b/blueos_repository/docker/models/auth.py @@ -0,0 +1,42 @@ +import datetime +import dataclasses +from typing import Optional + + +@dataclasses.dataclass +class AuthToken: + """ + Data structure to store the authorization token information + + Attributes: + token (str): An opaque Bearer token that clients should supply to subsequent requests in the Authorization header. + access_token (str): A token used for OAuth 2.0 compatibility. Equivalent to the `token` attribute. + expires_in (Optional[int]): The duration in seconds since the token was issued that it will remain valid. Defaults to 60 seconds. + issued_at (Optional[datetime.datetime]): The UTC time at which the token was issued, in RFC3339 format. + refresh_token (Optional[str]): A token that can be used to obtain new access tokens for different scopes. + """ + + token: Optional[str] = None + access_token: Optional[str] = None + expires_in: Optional[int] = 60 + issued_at: Optional[datetime.datetime] = None + refresh_token: Optional[str] = None + + + def __post_init__(self): + # Ensure that at least one of `token` or `access_token` is provided. + if not self.token and not self.access_token: + raise ValueError("Either 'token' or 'access_token' must be specified.") + + self.issued_at = datetime.datetime.fromisoformat(self.issued_at) if self.issued_at else datetime.datetime.now(datetime.timezone.utc) + + + @property + def is_expired(self) -> bool: + """ + Check if the token is expired. + + Returns: + bool: True if the token is expired, False otherwise. + """ + return (datetime.datetime.now(datetime.timezone.utc) - self.issued_at).total_seconds() > self.expires_in diff --git a/blueos_repository/docker/models/config.py b/blueos_repository/docker/models/config.py new file mode 100644 index 0000000..e69de29 diff --git a/blueos_repository/docker/models/manifest.py b/blueos_repository/docker/models/manifest.py new file mode 100644 index 0000000..d56a038 --- /dev/null +++ b/blueos_repository/docker/models/manifest.py @@ -0,0 +1,117 @@ +import dataclasses +from typing import Optional, List + + +@dataclasses.dataclass +class ManifestPlatform: + """ + Describes the platform which the image in the manifest runs on. + + Attributes: + architecture (str): The architecture field specifies the CPU architecture, for example amd64 or ppc64le + os (str): Operating system, for example linux or windows. + variant (Optional[str]): Variant of the CPU, for example v6 for ARM CPU. + features (Optional[List[str]]): The optional features field specifies an array of strings, each listing a required CPU feature (for example sse4 or aes). + """ + architecture: str + os: str + variant: Optional[str] = None + features: Optional[List[str]] = None + + +@dataclasses.dataclass +class Manifest: + """ + Represents a single image manifest within a manifest list. + + Attributes: + mediaType (str): The MIME type of the referenced object. + size (int): The size in bytes of the object. + digest (str): Digest of the content as defined by the Registry V2 HTTP API Specification. + platform (Platform): The platform object describes the platform which the image in the manifest runs on + """ + mediaType: str + size: int + digest: str + platform: ManifestPlatform + + +@dataclasses.dataclass +class ManifestList: + """ + The manifest list is the “fat manifest” which points to specific image manifests + for one or more platforms. Its use is optional, and relatively few images will use + one of these manifests. A client will distinguish a manifest list from an image + manifest based on the Content-Type returned in the HTTP response. + + Attributes: + schemaVersion (int): This field specifies the image manifest schema version as an integer. This schema uses the version 2. + mediaType (str): The MIME type of the manifest list. This should be set to application/vnd.docker.distribution.manifest.list.v2+json. + manifests (List[Manifest]): The manifests field contains a list of manifests for specific platforms. + """ + schemaVersion: int + mediaType: str + manifests: List[Manifest] + + +@dataclasses.dataclass +class ConfigReference: + """ + References a configuration object for a container, by digest. The configuration is a JSON blob used by the runtime. + + Attributes: + mediaType (str): MIME type of the referenced object. + size (int): Size in bytes of the object. + digest (str): Digest of the content, as defined by the Registry V2 HTTP API Specification. + """ + mediaType: str + size: int + digest: str + + +@dataclasses.dataclass +class ManifestLayer: + """ + Represents a single layer within the image manifest, specifying the content and its source. + + Attributes: + mediaType (str): MIME type of the referenced object. Expected to be application/vnd.docker.image.rootfs.diff.tar.gzip or application/vnd.docker.image.rootfs.foreign.diff.tar.gzip. + size (int): Size in bytes of the object. + digest (str): Digest of the content, as defined by the Registry V2 HTTP API Specification. + urls (Optional[List[str]]): List of URLs from which the content may be fetched. Optional field. + """ + mediaType: str + size: int + digest: str + urls: Optional[List[str]] = None + + +@dataclasses.dataclass +class ImageManifest: + """ + Provides a configuration and a set of layers for a container image, replacing the schema-1 manifest. + + Attributes: + schemaVersion (int): The image manifest schema version as an integer, version 2 is expected. + mediaType (str): MIME type of the manifest, expected to be application/vnd.docker.distribution.manifest.v2+json. + config (ConfigObject): The configuration object for a container by digest. + layers (List[Layer]): Ordered list of layers starting from the base image. + """ + schemaVersion: int + mediaType: str + config: ConfigReference + layers: List[ManifestLayer] + + +@dataclasses.dataclass +class ManifestFetch: + """ + Represents a manifest fetch response. + + Attributes: + is_image_manifest (bool): True if the manifest is an image manifest, False if is a ManifestList. + manifest (Manifest | ManifestList | ImageManifest): The manifest object. + """ + + is_image_manifest: bool + manifest: ManifestList | ImageManifest diff --git a/blueos_repository/docker/models/tag.py b/blueos_repository/docker/models/tag.py new file mode 100644 index 0000000..effccf0 --- /dev/null +++ b/blueos_repository/docker/models/tag.py @@ -0,0 +1,111 @@ +import dataclasses +from typing import Optional, List + + +@dataclasses.dataclass +class Layer: + """ + Represents a layer within a Docker image. + + Attributes: + digest (Optional[str]): The SHA256 digest of the layer, or None if not available. + size (int): The size of the layer in bytes. + instruction (str): The Dockerfile instruction that created this layer. + """ + digest: Optional[str] + size: int + instruction: str + + +@dataclasses.dataclass +class Image: + """ + Describes a Docker image and its characteristics. + + Attributes: + architecture (str): CPU architecture of the image. + features (str): Specific CPU features of the image. + variant (str): The variant of the CPU architecture. + digest (Optional[str]): The SHA256 digest of the image, or None if not available. + layers (Optional[List[Layer]]): The list of layers that compose the image. + os (str): The operating system on which the image is based. + os_features (str): Features or characteristics of the image's operating system. + os_version (Optional[str]): The version of the operating system. + size (int): The total size of the image in bytes. + status (str): The current status of the image. + last_pulled (Optional[str]): The timestamp of the last time the image was pulled. + last_pushed (Optional[str]): The timestamp of the last time the image was pushed. + """ + architecture: str + features: str + variant: str + digest: Optional[str] + layers: Optional[List[Layer]] + os: str + os_features: str + os_version: Optional[str] + size: int + status: str + last_pulled: Optional[str] + last_pushed: Optional[str] + + +@dataclasses.dataclass +class Tag: + """ + Represents a tag assigned to a set Docker image in a repository. + + Attributes: + id (int): The unique identifier for the tag. + images (List[Image]): The list of images associated with this tag. + creator (int): The user ID of the tag's creator. + last_updated (Optional[str]): The timestamp of the last update to the tag. + last_updater (int): The user ID of the last person to update the tag. + last_updater_username (str): The username of the last person to update the tag. + name (str): The name of the tag. + repository (int): The repository ID where the tag is located. + full_size (int): Compressed size (sum of all layers) of the tagged image. + v2 (str): Repository API version. + status (str): The current status of the tag can be "active" "inactive". + tag_last_pulled (Optional[str]): The timestamp of the last time the tag was pulled. + tag_last_pushed (Optional[str]): The timestamp of the last time the tag was pushed. + """ + id: int + images: List[Image] + creator: int + last_updated: Optional[str] + last_updater: int + last_updater_username: str + name: str + repository: int + full_size: int + v2: str + status: str + tag_last_pulled: Optional[str] + tag_last_pushed: Optional[str] + + + def __post_init__(self): + """ + Post-initialization processing to enforce constraints + """ + valid_statuses = {'active', 'inactive'} + if self.status not in valid_statuses: + raise ValueError(f"Status must be one of {valid_statuses}, not {self.status}.") + + +@dataclasses.dataclass +class TagList: + """ + Represents the result of fetching a list of tags from a Docker registry. + + Attributes: + count (int): The total number of tags available. + next (Optional[str]): The URL to the next page of tags, or None if this is the last page. + previous (Optional[str]): The URL to the previous page of tags, or None if this is the first page. + results (List[Tag]): The list of tags fetched. + """ + count: int + next: Optional[str] + previous: Optional[str] + results: List[Tag] diff --git a/blueos_repository/docker/registry.py b/blueos_repository/docker/registry.py new file mode 100644 index 0000000..5390b26 --- /dev/null +++ b/blueos_repository/docker/registry.py @@ -0,0 +1,140 @@ +import functools +import aiohttp_retry +from typing import Any, Optional, Callable +from auth import DockerAuthAPI, AuthToken +from models.manifest import ImageManifest, ManifestList, ManifestFetch + + +class DockerRegistry: + """ + This class is used to interact with the Docker Registry API. + + More details in https://distribution.github.io/distribution/spec/api/ + """ + + __api_base_url: str = "https://registry-1.docker.io" + __api_version: str = "v2" + __api_url: str = f"{__api_base_url}/{__api_version}" + + __token: Optional[AuthToken] = None + + + def __init__(self, repository: str, max_retries: int = 5) -> None: + """ + Constructor for the DockerHubAPI class. + + Args: + repository: Repository that this registry class will operate on + max_retries: The maximum number of retries to be used in case of request failure. Defaults to 5. + + Returns: + None + """ + self.repository = repository + self.__retry_options = aiohttp_retry.ExponentialRetry(attempts=max_retries) + + + async def __check_token(self) -> None: + """ + Checks if the token is set and not expired, otherwise renew it. + """ + + if not self.__token or self.__token.is_expired(): + auth = DockerAuthAPI() + self.__token = await auth.get_token(self.repository) + + + async def __raise_pretty(self, resp: Any) -> None: + """ + Throws a pretty error message in case of a error status conforming with + errors in the Docker Registry API. + + Args: + resp: The response from the request. + + Raises: + Exception: The pretty error message. + """ + + errors = (await resp.json()).get("errors", []) + error = errors[0] if errors else {} + + error_code = error.get("code", "Unknown Code") + error_message = error.get("message", "Unknown Message") + + error_msg = f"Error on Docker Registry API with status {resp.status} on call to {resp.url}: {error_code} - {error_message}" + + print(error_msg) + raise Exception(error_msg) + + + async def get(self, route: str, max_retries: Optional[int] = None, **kwargs) -> Any: + """ + Make a GET request to the Docker Hub API. + + Args: + route: The route to be used in the request. + params: The parameters to be used in the request. + max_retries: The maximum number of retries to be used in case of request failure. Defaults to None. + + Returns: + The response from the request parsed as json. + + Raises: + Exception: The pretty error message in case of error status. + """ + + retry = self.__retry_options if not max_retries else aiohttp_retry.ExponentialRetry(attempts=max_retries) + + async with aiohttp_retry.RetryClient(retry_options=retry) as session: + async with session.get(f"{self.__api_url}/{route}", **kwargs) as resp: + # In case of an error status, tries to make a better error message + if resp.status >= 400: + await self.__raise_pretty(resp) + + return await resp.json(content_type=None) + + + async def get_manifest(self, tag: str) -> ManifestFetch: + """ + Get the manifest for a given tag + + Args: + tag: The tag name + + Returns: + The manifest + """ + + await self.__check_token() + + route = f"{self.repository}/manifests/{tag}" + header = { + "Authorization": f"Bearer {self.__token}", + "Accept": "application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json", + } + + manifest = await self.api.get(route, headers=header) + + if "config" in manifest: + return ManifestFetch(is_image_manifest=True, manifest=ImageManifest(**manifest)) + + return ManifestFetch(is_image_manifest=False, manifest=ManifestList(**manifest)) + + + async def get_blob_manifes(self, digest: str) -> ManifestFetch: + """ + Get the blob config object for a given manifest digest + + + """ + + await self.__check_token() + + route = f"{self.repository}/blobs/{digest}" + header = { + "Authorization": f"Bearer {self.__token}", + "Accept": "application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json", + } + + blob = await self.api.get(route, headers=header) diff --git a/blueos_repository/extension/extension.py b/blueos_repository/extension/extension.py new file mode 100644 index 0000000..113c521 --- /dev/null +++ b/blueos_repository/extension/extension.py @@ -0,0 +1,163 @@ +import asyncio +from typing import Dict, Optional +# Docker +from docker.hub import DockerHub +from docker.registry import DockerRegistry +# Models +from docker.models.tag import Tag +from models import ExtensionMetadata, ExtensionVersion +from docker.models.manifest import ManifestFetch, ManifestPlatform +# Extra +from utils import valid_semver + + +class Extension: + """ + Represents an BlueOS extension. + + Attributes: + metadata (ExtensionMetadata): Metadata associated with the extension. + """ + + versions: Optional[Dict[str, ExtensionVersion]] = None + + + def __init__(self, metadata: ExtensionMetadata) -> None: + """ + Constructor for the Extension class. + + Args: + metadata (ExtensionMetadata): Metadata associated with the extension. + + Returns: + None + """ + + self.metadata = metadata + + # Docker API + self.hub: DockerHub = DockerHub(metadata.docker) + self.registry: DockerRegistry = DockerRegistry(metadata.docker) + + + def __is_compatible(self, platform: ManifestPlatform) -> bool: + """ + Checks if the platform is compatible with embedded devices BlueOS targets. + + Args: + platform (ManifestPlatform): Platform to check. + + Returns: + bool: True if compatible, False otherwise. + """ + + return platform.os == "linux" and platform.architecture == "arm" + + + async def __extract_valid_embedded_digest(self, fetch: ManifestFetch) -> str: + """ + Tries to find a valid embedded image digest in the extension manifest. + As this image should be able to run on the vehicle raspberry pi, it should be + a valid arm image with target os as linux. + + Args: + fetch (ManifestFetch): The manifest to extract the digest from. + + Returns: + str: The digest of the embedded image. + + Raises: + RuntimeError: If the manifest does not contain a valid embedded image. + """ + + # Regular images/ OCI images + if fetch.is_image_manifest: + return fetch.manifest.config.digest + + # Manifest list + valid_manifests = [ + entry for entry in fetch.manifest.manifests if self.__is_compatible(entry.platform) + ] + if len(valid_manifests) != 1: + raise RuntimeError(f"Expected one valid manifest for target embedded arch but found: {len(valid_manifests)}") + + return valid_manifests[0].digest + + + async def __process_tag_version(self, tag: Tag) -> None: + """ + Process a tag and create a version object for it and store it in the versions + dictionary property. + + Args: + tag (Tag): Tag to process. + """ + + tag_name = tag["name"] + + try: + if not valid_semver(tag_name): + raise ValueError(f"Invalid version naming: {tag_name}") + + manifest = await self.registry.get_manifest(tag_name) + + # Extract the digest of the embedded image + embedded_digest = await self.__extract_valid_embedded_digest(manifest) + + + except KeyError as error: + print(f" INVALID-EXTENSION - Failed to generate version for {tag_name}, error in key: {error}") + except Exception as error: + print(f" INVALID-EXTENSION - Failed to generate version for {tag_name}, exception: {error}") + + + + permissions = raw_labels.get("permissions", None) + links = json5.loads(raw_labels.get("links", "{}")) + website = links.pop("website", raw_labels.get("website", None)) + authors = json5.loads(raw_labels.get("authors", "[]")) + # documentation is just a URL for a link, but the old format had it as its own label + docs = links.pop("docs", links.pop("documentation", raw_labels.get("docs", None))) + readme = raw_labels.get("readme", None) + if readme is not None: + readme = readme.replace(r"{tag_name}", tag_name) + try: + readme = await self.fetch_readme(readme) + except Exception as error: # pylint: disable=broad-except + readme = str(error) + company_raw = raw_labels.get("company", None) + company = Company.from_json(json5.loads(company_raw)) if company_raw is not None else None + support = links.pop("support", raw_labels.get("support", None)) + type_ = raw_labels.get("type", ExtensionType.OTHER) + filter_tags = json5.loads(raw_labels.get("tags", "[]")) + + new_version = Version( + permissions=json5.loads(permissions) if permissions else None, + website=website, + authors=authors, + docs=json5.loads(docs) if docs else None, + readme=readme, + company=company, + support=support, + extra_links=links, + type=type_, + filter_tags=Version.validate_filter_tags(filter_tags), + requirements=raw_labels.get("requirements", None), + tag=tag_name, + images=self.extract_images_from_tag(tag), + ) + repository.versions[tag_name] = new_version + + + + async def inflate(self) -> None: + """ + Inflate extension data, this will fetch all the necessary data from docker hub, registry, etc. + And store it in the object to allow manifest formation after. + + Returns: + None + """ + tags = await self.hub.get_tags() + + await asyncio.gather(*(self.__process_tag_version(tag) for tag in tags)) diff --git a/blueos_repository/extension/models.py b/blueos_repository/extension/models.py new file mode 100644 index 0000000..f9a16b8 --- /dev/null +++ b/blueos_repository/extension/models.py @@ -0,0 +1,148 @@ +import dataclasses +from enum import StrEnum +from typing import Any, Dict, List, Optional + + +class ExtensionType(StrEnum): + """ + Represents the type of an extension. + + Attributes: + DEVICE_INTEGRATION (str): Device integration extension. + EXAMPLE (str): Example extension. + THEME (str): Theme extension. + OTHER (str): Other extension. + TOOL (str): Tool extension. + """ + + DEVICE_INTEGRATION = "device-integration" + EXAMPLE = "example" + THEME = "theme" + OTHER = "other" + TOOL = "tool" + + +@dataclasses.dataclass +class Author: + """ + Represents an author of an extension. + + Attributes: + name (str): Name of the author. + email (str): Email of the author. + """ + name: str + email: str + + +@dataclasses.dataclass +class Platform: + """ + Represents a platform supported by the extension. + + Attributes: + architecture (str): Architecture of the platform. + variant (Optional[str]): Variant of the platform. + os (Optional[str]): Operating system of the platform. + """ + + architecture: str + variant: Optional[str] = None + os: Optional[str] = None + + +@dataclasses.dataclass +class Image: + """ + Represents description of an image available for a given extension version. + + Attributes: + digest (Optional[str]): Digest of the image. + size (int): Uncompressed size of the image. + platform (Platform): Platform of the image. + """ + + digest: Optional[str] + size: int + platform: Platform + + +@dataclasses.dataclass +class Company: + """ + Represents a company associated with an extension. + + Attributes: + name (str): Name of the company. + about (Optional[str]): Description of the company. + email (Optional[str]): Email of the company. + """ + + name: str + about: Optional[str] + email: Optional[str] + + +@dataclasses.dataclass +class ExtensionVersion: + """ + Represents a version of an extension. + + Attributes: + permissions (Optional[Dict[str, Any]]): Permissions required by the extension. + requirements (Optional[str]): Requirements for the extension. + tag (Optional[str]): Tag for this extension version. + website (str): URL to the extension's website. + authors (List[Author]): List of authors of this extension version. + docs (Optional[str]): URL to the extension's documentation. + readme (Optional[str]): URL to the extension's readme. + company (Optional[Company]): Company associated with the extension. + support (Optional[str]): URL to the extension's support. + type (ExtensionType): Type of the extension. + filter_tags (List[str]): List of tags to be used in the extension filter. + extra_links (Dict[str, str]): Extra links to be used in the extension. + images (List[Image]): List of images associated with the extension. + """ + + permissions: Optional[Dict[str, Any]] + requirements: Optional[str] + tag: Optional[str] + website: str + authors: List[Author] + docs: Optional[str] + readme: Optional[str] + company: Optional[Company] + support: Optional[str] + type: ExtensionType + filter_tags: List[str] + extra_links: Dict[str, str] + images: List[Image] + + @staticmethod + def validate_filter_tags(tags: List[str]) -> List[str]: + """Returns a list of up to 10 lower-case alpha-numeric tags (dashes allowed).""" + return [tag.lower() for tag in tags if tag.replace("-", "").isalnum()][:10] + + +@dataclasses.dataclass +class ExtensionMetadata: + """ + Represents metadata associated with some extension. + + Attributes: + identifier (str): Identifier of the extension. + name (str): Name of the extension. + website (str): URL to the extension's website. + docker (str): Docker repository name. + description (str): Description of the extension. + extension_logo (Optional[str]): URL to the extension's logo. + company_logo (Optional[str]): URL to the company's logo. + """ + + identifier: str + name: str + website: str + docker: str + description: str + extension_logo: Optional[str] + company_logo: Optional[str] diff --git a/blueos_repository/registry.py b/blueos_repository/registry-old.py similarity index 100% rename from blueos_repository/registry.py rename to blueos_repository/registry-old.py diff --git a/blueos_repository/utils.py b/blueos_repository/utils.py new file mode 100644 index 0000000..0e147f2 --- /dev/null +++ b/blueos_repository/utils.py @@ -0,0 +1,38 @@ +import json +import dataclasses +from typing import Any, Dict, Union, Optional +# Extra +import semver + +class EnhancedJSONEncoder(json.JSONEncoder): + """ + Custom json encoder for dataclasses, + see https://docs.python.org/3/library/json.html#json.JSONEncoder.default + Returns a serializable type + """ + + def default(self, o: Any) -> Union[Any, Dict[str, Any]]: + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return super().default(o) + + +def valid_semver(string: str) -> Optional[semver.VersionInfo]: + """ + Check if a string is a valid SemVer version. + + Args: + string (str): The string to check. + + Returns: + Optional[semver.VersionInfo]: The version if it's valid, otherwise None. + """ + + # We want to allow versions to be prefixed with a 'v'. + # This is up for discussion + if string.startswith("v"): + string = string[1:] + try: + return semver.VersionInfo.parse(string) + except ValueError: + return None