From 86d2a39d961520b08b4330ddd5a5a2422fdb64d5 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 22 Jun 2023 16:17:36 +0200 Subject: [PATCH 01/18] Add schema file for the cloud API --- lib/galaxy/schema/cloud.py | 93 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 lib/galaxy/schema/cloud.py diff --git a/lib/galaxy/schema/cloud.py b/lib/galaxy/schema/cloud.py new file mode 100644 index 000000000000..9eeba707ec8f --- /dev/null +++ b/lib/galaxy/schema/cloud.py @@ -0,0 +1,93 @@ +from typing import ( + List, + Optional, + Union, +) + +from pydantic import ( + Field, + Required, +) +from typing_extensions import Literal + +from galaxy.schema.fields import DecodedDatabaseIdField +from galaxy.schema.schema import Model + + +class InputArguments(Model): + dbkey: Optional[str] = Field( + default="?", + title="Genome", + description="Sets the genome (e.g., `hg19`) of the objects being fetched to Galaxy.", + ) + file_type: Optional[str] = Field( + default="auto", + title="File Type", + description="Sets the Galaxy datatype (e.g., `bam`) for the objects being fetched to Galaxy. See the following link for a complete list of Galaxy data types: https://galaxyproject.org/learn/datatypes/.", + ) + to_posix_lines: Optional[Union[Literal["Yes"], bool]] = Field( + default="Yes", + title="POSIX line endings", + description="A boolean value ('true' or 'false'); if 'Yes', converts universal line endings to POSIX line endings. Set to 'False' if you upload a gzip, bz2 or zip archive containing a binary file.", + ) + space_to_tab: Optional[bool] = Field( + default=False, + title="Spaces to tabs", + description="A boolean value ('true' or 'false') that sets if spaces should be converted to tab in the objects being fetched to Galaxy. Applicable only if `to_posix_lines` is True", + ) + + +class CloudObjects(Model): + history_id: DecodedDatabaseIdField = Field( + default=Required, + title="History ID", + description="The ID of history to which the object should be received to.", + ) + bucket: str = Field( + default=Required, + title="Bucket", + description="The name of a bucket from which data should be fetched from (e.g., a bucket name on AWS S3).", + ) + objects: List[str] = Field( + default=Required, + title="Objects", + description="A list of the names of objects to be fetched.", + ) + authz_id: DecodedDatabaseIdField = Field( + default=Required, + title="Authentication ID", + description="The ID of CloudAuthz to be used for authorizing access to the resource provider. You may get a list of the defined authorizations via `/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a new authorization.", + ) + input_args: Optional[InputArguments] = Field( + default=None, + title="Input arguments", + description="A summary of the input arguments, which is optional and will default to {}.", + ) + + +class CloudDatasets(Model): + history_id: DecodedDatabaseIdField = Field( + default=Required, + title="History ID", + description="The ID of history from which the object should be downloaded", + ) + bucket: str = Field( + default=Required, + title="Bucket", + description="The name of a bucket to which data should be sent (e.g., a bucket name on AWS S3).", + ) + authz_id: DecodedDatabaseIdField = Field( + default=Required, + title="Authentication ID", + description="The ID of CloudAuthz to be used for authorizing access to the resource provider. You may get a list of the defined authorizations via `/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a new authorization.", + ) + dataset_ids: Optional[List[DecodedDatabaseIdField]] = Field( + default=None, + title="Objects", + description="A list of dataset IDs belonging to the specified history that should be sent to the given bucket. If not provided, Galaxy sends all the datasets belonging the specified history.", + ) + overwrite_existing: Optional[bool] = Field( + default=False, + title="Spaces to tabs", + description="A boolean value. If set to 'True', and an object with same name of the dataset to be sent already exist in the bucket, Galaxy replaces the existing object with the dataset to be sent. If set to 'False', Galaxy appends datetime to the dataset name to prevent overwriting an existing object.", + ) From f9a5101cc703394570c2fc87a062e1d808a62fe0 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 22 Jun 2023 16:18:23 +0200 Subject: [PATCH 02/18] Refactor index, get and send operation --- lib/galaxy/webapps/galaxy/api/cloud.py | 301 ++++++------------------- 1 file changed, 75 insertions(+), 226 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index e3fe3c881f49..4491fa6d97f3 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -3,255 +3,104 @@ """ import logging +from typing import ( + Any, + Dict, + List, + Union, +) -from galaxy import exceptions -from galaxy.exceptions import ActionInputError -from galaxy.managers import ( - cloud, - datasets, +from fastapi import ( + Body, + HTTPException, +) +from pydantic import Required + +from galaxy.managers.cloud import CloudManager +from galaxy.managers.context import ProvidesHistoryContext +from galaxy.managers.datasets import DatasetSerializer +from galaxy.schema.cloud import ( + CloudDatasets, + CloudObjects, + InputArguments, +) +from galaxy.webapps.galaxy.api import ( + depends, + Router, ) -from galaxy.structured_app import StructuredApp -from galaxy.web import expose_api -from . import BaseGalaxyAPIController +from . import DependsOnTrans log = logging.getLogger(__name__) +router = Router(tags=["cloud"]) -class CloudController(BaseGalaxyAPIController): - """ - RESTfull controller for interaction with Amazon S3. - """ - - def __init__( - self, app: StructuredApp, cloud_manager: cloud.CloudManager, datasets_serializer: datasets.DatasetSerializer - ): - super().__init__(app) - self.cloud_manager = cloud_manager - self.datasets_serializer = datasets_serializer - - @expose_api - def index(self, trans, **kwargs): - """ - GET /api/cloud/storage - Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. +@router.cbv +class FastAPICloudController: + cloud_manager: CloudManager = depends(CloudManager) + datasets_serializer: DatasetSerializer = depends(DatasetSerializer) - :return: A list of cloud-based buckets user has defined. - """ + @router.get( + "/api/cloud/storage", + summary="Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. Is not yet implemented", + ) + def index( + self, + ): # TODO: This can be implemented leveraging PluggedMedia objects (part of the user-based object store project) - trans.response.status = 501 - return "Not Implemented" - - @expose_api - def get(self, trans, payload, **kwargs): - """ - POST /api/cloud/storage/get - - gets given objects from a given cloud-based bucket to a Galaxy history. - - :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction - :param trans: Galaxy web transaction - - :type payload: dict - :param payload: A dictionary structure containing the following keys: - - * history_id: the (encoded) id of history to which the object should be received to. - * bucket: the name of a bucket from which data should be fetched from (e.g., a bucket name on AWS S3). - * objects: a list of the names of objects to be fetched. - * authz_id: the encoded ID of CloudAuthz to be used for authorizing access to the resource - provider. You may get a list of the defined authorizations via - `/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a - new authorization. - * input_args [Optional; default value is an empty dict] a dictionary containing the following keys: - - ** `dbkey`: [Optional; default value: is `?`] - Sets the genome (e.g., `hg19`) of the objects being - fetched to Galaxy. - - ** `file_type`: [Optional; default value is `auto`] - Sets the Galaxy datatype (e.g., `bam`) for the - objects being fetched to Galaxy. See the following - link for a complete list of Galaxy data types: - https://galaxyproject.org/learn/datatypes/ - - ** `space_to_tab`: [Optional; default value is `False`] - A boolean value ("true" or "false") that sets if spaces - should be converted to tab in the objects being - fetched to Galaxy. Applicable only if `to_posix_lines` - is True - - ** `to_posix_lines`: [Optional; default value is `Yes`] - A boolean value ("true" or "false"); if "Yes", converts - universal line endings to POSIX line endings. Set to - "False" if you upload a gzip, bz2 or zip archive - containing a binary file. - - :param kwargs: - - :rtype: dictionary - :return: a dictionary containing a `summary` view of the datasets copied from the given cloud-based storage. - - """ - if not isinstance(payload, dict): - raise ActionInputError( - "Invalid payload data type. The payload is expected to be a dictionary, " - "but received data of type `{}`.".format(str(type(payload))) - ) - - missing_arguments = [] - encoded_history_id = payload.get("history_id", None) - if encoded_history_id is None: - missing_arguments.append("history_id") - - bucket = payload.get("bucket", None) - if bucket is None: - missing_arguments.append("bucket") - - objects = payload.get("objects", None) - if objects is None: - missing_arguments.append("objects") - - encoded_authz_id = payload.get("authz_id", None) - if encoded_authz_id is None: - missing_arguments.append("authz_id") - - if len(missing_arguments) > 0: - raise ActionInputError(f"The following required arguments are missing in the payload: {missing_arguments}") - - try: - history_id = self.decode_id(encoded_history_id) - except exceptions.MalformedId as e: - raise ActionInputError(f"Invalid history ID. {e}") - - try: - authz_id = self.decode_id(encoded_authz_id) - except exceptions.MalformedId as e: - raise ActionInputError(f"Invalid authz ID. {e}") - - if not isinstance(objects, list): - raise ActionInputError( - f"The `objects` should be a list, but received an object of type {type(objects)} instead." - ) - + raise HTTPException(status_code=501) + + @router.post( + "/api/cloud/storage/get", + summary="Gets given objects from a given cloud-based bucket to a Galaxy history.", + ) + def get( + self, + payload: CloudObjects = Body(default=Required), + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> List[Dict[Any, Any]]: + input_args: Union[InputArguments, dict] = payload.input_args or {} datasets = self.cloud_manager.get( trans=trans, - history_id=history_id, - bucket_name=bucket, - objects=objects, - authz_id=authz_id, - input_args=payload.get("input_args", None), + history_id=payload.history_id, + bucket_name=payload.bucket, + objects=payload.objects, + authz_id=payload.authz_id, + input_args=input_args, ) rtv = [] for dataset in datasets: rtv.append(self.datasets_serializer.serialize_to_view(dataset, view="summary")) return rtv - @expose_api - def send(self, trans, payload, **kwargs): - """ - POST /api/cloud/storage/send - - Sends given dataset(s) in a given history to a given cloud-based bucket. Each dataset is named - using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`). - If no dataset ID is given, this API sends all the datasets belonging to a given history to a given - cloud-based bucket. - - :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction - :param trans: Galaxy web transaction - - :type payload: dictionary - :param payload: A dictionary structure containing the following keys: - - * history_id the (encoded) id of history from which the object should be downloaed. - * bucket: the name of a bucket to which data should be sent (e.g., a bucket name on AWS S3). - * authz_id: the encoded ID of CloudAuthz to be used for authorizing access to the resource - provider. You may get a list of the defined authorizations via - `/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a - new authorization. - * dataset_ids: [Optional; default: None] - A list of encoded dataset IDs belonging to the specified history - that should be sent to the given bucket. If not provided, Galaxy sends - all the datasets belonging the specified history. - * overwrite_existing: [Optional; default: False] - A boolean value. If set to "True", and an object with same name of the dataset - to be sent already exist in the bucket, Galaxy replaces the existing object - with the dataset to be sent. If set to "False", Galaxy appends datetime - to the dataset name to prevent overwriting an existing object. - - :rtype: dictionary - :return: Information about the (un)successfully submitted dataset send jobs, - containing the following keys: - - * `bucket_name`: The name of bucket to which the listed datasets are queued - to be sent. - * `sent_dataset_labels`: A list of JSON objects with the following key-value pair: - ** `object`: The name of object is queued to be created. - ** `job_id`: The id of the queued send job. - - * `failed_dataset_labels`: A list of JSON objects with the following key-value pair - representing the datasets Galaxy failed to create - (and queue) send job for: - - ** `object`: The name of object is queued to be created. - ** `error`: A descriptive error message. - - """ - missing_arguments = [] - encoded_history_id = payload.get("history_id", None) - if encoded_history_id is None: - missing_arguments.append("history_id") - - bucket = payload.get("bucket", None) - if bucket is None: - missing_arguments.append("bucket") - - encoded_authz_id = payload.get("authz_id", None) - if encoded_authz_id is None: - missing_arguments.append("authz_id") - - if len(missing_arguments) > 0: - raise ActionInputError(f"The following required arguments are missing in the payload: {missing_arguments}") - - try: - history_id = self.decode_id(encoded_history_id) - except exceptions.MalformedId as e: - raise ActionInputError(f"Invalid history ID. {e}") - - try: - authz_id = self.decode_id(encoded_authz_id) - except exceptions.MalformedId as e: - raise ActionInputError(f"Invalid authz ID. {e}") - - encoded_dataset_ids = payload.get("dataset_ids", None) - if encoded_dataset_ids is None: - dataset_ids = None - else: - dataset_ids = set() - invalid_dataset_ids = [] - for encoded_id in encoded_dataset_ids: - try: - dataset_ids.add(self.decode_id(encoded_id)) - except exceptions.MalformedId: - invalid_dataset_ids.append(encoded_id) - if len(invalid_dataset_ids) > 0: - raise ActionInputError( - "The following provided dataset IDs are invalid, please correct them and retry. " - "{}".format(invalid_dataset_ids) - ) - + @router.post( + "/api/cloud/storage/send", + summary="Sends given dataset(s) in a given history to a given cloud-based bucket. Each dataset is named using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`). If no dataset ID is given, this API sends all the datasets belonging to a given history to a given cloud-based bucket.", + ) + def send( + self, + payload: CloudDatasets, + trans: ProvidesHistoryContext = DependsOnTrans, + ) -> Dict[str, Union[List[Any], str]]: log.info( msg="Received api/send request for `{}` datasets using authnz with id `{}`, and history `{}`." "".format( - "all the dataset in the given history" if not dataset_ids else len(dataset_ids), authz_id, history_id + "all the dataset in the given history" if not payload.dataset_ids else len(payload.dataset_ids), + payload.authz_id, + payload.history_id, ) ) sent, failed = self.cloud_manager.send( trans=trans, - history_id=history_id, - bucket_name=bucket, - authz_id=authz_id, - dataset_ids=dataset_ids, - overwrite_existing=payload.get("overwrite_existing", False), + history_id=payload.history_id, + bucket_name=payload.bucket, + authz_id=payload.authz_id, + dataset_ids=payload.dataset_ids, + overwrite_existing=payload.overwrite_existing, ) - return {"sent_dataset_labels": sent, "failed_dataset_labels": failed, "bucket_name": bucket} + return { + "sent_dataset_labels": sent, + "failed_dataset_labels": failed, + "bucket_name": payload.bucket, + } From bc629dc226773466263d72c3340ed01a2631542f Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 22 Jun 2023 16:19:06 +0200 Subject: [PATCH 03/18] Remove the mapping to the legacy route --- lib/galaxy/webapps/galaxy/buildapp.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 672292bb6294..9e228b70215f 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -310,23 +310,6 @@ def populate_api_routes(webapp, app): _add_item_tags_controller( webapp, name_prefix="history_content_", path_prefix="/api/histories/{history_id}/contents/{history_content_id}" ) - webapp.mapper.connect( - "cloud_storage", "/api/cloud/storage/", controller="cloud", action="index", conditions=dict(method=["GET"]) - ) - webapp.mapper.connect( - "cloud_storage_get", - "/api/cloud/storage/get", - controller="cloud", - action="get", - conditions=dict(method=["POST"]), - ) - webapp.mapper.connect( - "cloud_storage_send", - "/api/cloud/storage/send", - controller="cloud", - action="send", - conditions=dict(method=["POST"]), - ) _add_item_tags_controller(webapp, name_prefix="history_", path_prefix="/api/histories/{history_id}") _add_item_tags_controller(webapp, name_prefix="workflow_", path_prefix="/api/workflows/{workflow_id}") From c00104c7fb1378bc0714da6250c4d9dccd314bd8 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 22 Jun 2023 16:35:06 +0200 Subject: [PATCH 04/18] Regenerate the client schema --- client/src/schema/schema.ts | 178 ++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index 1f59e67a0faa..83688aefe042 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -8,6 +8,18 @@ export interface paths { /** Returns returns an API key for authenticated user based on BaseAuth headers. */ get: operations["get_api_key_api_authenticate_baseauth_get"]; }; + "/api/cloud/storage": { + /** Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. Is not yet implemented */ + get: operations["index_api_cloud_storage_get"]; + }; + "/api/cloud/storage/get": { + /** Gets given objects from a given cloud-based bucket to a Galaxy history. */ + post: operations["get_api_cloud_storage_get_post"]; + }; + "/api/cloud/storage/send": { + /** Sends given dataset(s) in a given history to a given cloud-based bucket. Each dataset is named using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`). If no dataset ID is given, this API sends all the datasets belonging to a given history to a given cloud-based bucket. */ + post: operations["send_api_cloud_storage_send_post"]; + }; "/api/configuration": { /** * Return an object containing exposable configuration settings @@ -2203,6 +2215,73 @@ export interface components { /** Item Ids */ item_ids: string[]; }; + /** + * CloudDatasets + * @description Base model definition with common configuration used by all derived models. + */ + CloudDatasets: { + /** + * Authentication ID + * @description The ID of CloudAuthz to be used for authorizing access to the resource provider. You may get a list of the defined authorizations via `/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a new authorization. + * @example 0123456789ABCDEF + */ + authz_id: string; + /** + * Bucket + * @description The name of a bucket to which data should be sent (e.g., a bucket name on AWS S3). + */ + bucket: string; + /** + * Objects + * @description A list of dataset IDs belonging to the specified history that should be sent to the given bucket. If not provided, Galaxy sends all the datasets belonging the specified history. + */ + dataset_ids?: string[]; + /** + * History ID + * @description The ID of history from which the object should be downloaded + * @example 0123456789ABCDEF + */ + history_id: string; + /** + * Spaces to tabs + * @description A boolean value. If set to 'True', and an object with same name of the dataset to be sent already exist in the bucket, Galaxy replaces the existing object with the dataset to be sent. If set to 'False', Galaxy appends datetime to the dataset name to prevent overwriting an existing object. + * @default false + */ + overwrite_existing?: boolean; + }; + /** + * CloudObjects + * @description Base model definition with common configuration used by all derived models. + */ + CloudObjects: { + /** + * Authentication ID + * @description The ID of CloudAuthz to be used for authorizing access to the resource provider. You may get a list of the defined authorizations via `/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a new authorization. + * @example 0123456789ABCDEF + */ + authz_id: string; + /** + * Bucket + * @description The name of a bucket from which data should be fetched from (e.g., a bucket name on AWS S3). + */ + bucket: string; + /** + * History ID + * @description The ID of history to which the object should be received to. + * @example 0123456789ABCDEF + */ + history_id: string; + /** + * Input arguments + * @description A summary of the input arguments, which is optional and will default to {}. + */ + input_args?: components["schemas"]["InputArguments"]; + /** + * Objects + * @description A list of the names of objects to be fetched. + */ + objects: string[]; + }; /** * CollectionElementIdentifier * @description Base model definition with common configuration used by all derived models. @@ -5368,6 +5447,36 @@ export interface components { */ uri: string; }; + /** + * InputArguments + * @description Base model definition with common configuration used by all derived models. + */ + InputArguments: { + /** + * Genome + * @description Sets the genome (e.g., `hg19`) of the objects being fetched to Galaxy. + * @default ? + */ + dbkey?: string; + /** + * File Type + * @description Sets the Galaxy datatype (e.g., `bam`) for the objects being fetched to Galaxy. See the following link for a complete list of Galaxy data types: https://galaxyproject.org/learn/datatypes/. + * @default auto + */ + file_type?: string; + /** + * Spaces to tabs + * @description A boolean value ('true' or 'false') that sets if spaces should be converted to tab in the objects being fetched to Galaxy. Applicable only if `to_posix_lines` is True + * @default false + */ + space_to_tab?: boolean; + /** + * POSIX line endings + * @description A boolean value ('true' or 'false'); if 'Yes', converts universal line endings to POSIX line endings. Set to 'False' if you upload a gzip, bz2 or zip archive containing a binary file. + * @default Yes + */ + to_posix_lines?: "Yes" | boolean; + }; /** * InstalledRepositoryToolShedStatus * @description Base model definition with common configuration used by all derived models. @@ -8860,6 +8969,75 @@ export interface operations { }; }; }; + index_api_cloud_storage_get: { + /** Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. Is not yet implemented */ + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record; + }; + }; + }; + }; + get_api_cloud_storage_get_post: { + /** Gets given objects from a given cloud-based bucket to a Galaxy history. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CloudObjects"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record[]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + send_api_cloud_storage_send_post: { + /** Sends given dataset(s) in a given history to a given cloud-based bucket. Each dataset is named using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`). If no dataset ID is given, this API sends all the datasets belonging to a given history to a given cloud-based bucket. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CloudDatasets"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": { + [key: string]: (Record[] | string) | undefined; + }; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; index_api_configuration_get: { /** * Return an object containing exposable configuration settings From 4601a323c20f6bcbdd677ad17fc02cc0b09e8197 Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 22 Jun 2023 18:44:21 +0200 Subject: [PATCH 05/18] Add pydantic models for operation to return --- lib/galaxy/schema/cloud.py | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/lib/galaxy/schema/cloud.py b/lib/galaxy/schema/cloud.py index 9eeba707ec8f..0a8e0fbc99ac 100644 --- a/lib/galaxy/schema/cloud.py +++ b/lib/galaxy/schema/cloud.py @@ -1,4 +1,5 @@ from typing import ( + Any, List, Optional, Union, @@ -10,6 +11,7 @@ ) from typing_extensions import Literal +# from galaxy.model import Dataset from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.schema import Model @@ -91,3 +93,68 @@ class CloudDatasets(Model): title="Spaces to tabs", description="A boolean value. If set to 'True', and an object with same name of the dataset to be sent already exist in the bucket, Galaxy replaces the existing object with the dataset to be sent. If set to 'False', Galaxy appends datetime to the dataset name to prevent overwriting an existing object.", ) + + +class FailedJob(Model): + object: str = Field( + default=Required, + title="Object", + description="The name of object is queued to be created", + ) + error: str = Field( + default=Required, + title="Error", + description="A descriptive error message.", + ) + + +class SuccessfulJob(Model): + object: str = Field( + default=Required, + title="Object", + description="The name of object is queued to be created", + ) + job_id: str = Field( + default=Required, + title="Job ID", + description="The ID of the queued send job.", + ) + + +class SummarySendDatasets(Model): + sent_dataset_labels: List[SuccessfulJob] = Field( + default=Required, + title="Send datasets", + description="The datasets for which Galaxy succeeded to create (and queue) send job", + ) + failed_dataset_labels: List[FailedJob] = Field( + default=Required, + title="Failed datasets", + description="The datasets for which Galaxy failed to create (and queue) send job", + ) + bucket_name: str = Field( + default=Required, + title="Bucket", + description="The name of bucket to which the listed datasets are queued to be sent", + ) + + +class SummaryGetObjects(Model): + datasets: List[Any] = Field( + default=Required, + title="Datasets", + description="A list of datasets created for the fetched files.", + ) + + +class StatusCode(Model): + detail: str = Field( + default=Required, + title="Detail", + description="The detail to expand on the status code", + ) + status: int = Field( + default=Required, + title="Code", + description="The actual status code", + ) From 6b18c92423ed6f5cc83506713f930be22439303d Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 22 Jun 2023 18:44:48 +0200 Subject: [PATCH 06/18] Refactor operations to return pydantic models --- lib/galaxy/webapps/galaxy/api/cloud.py | 33 ++++++++++++-------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 4491fa6d97f3..e2229e9b182e 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -3,16 +3,12 @@ """ import logging -from typing import ( - Any, - Dict, - List, - Union, -) +from typing import Union from fastapi import ( Body, - HTTPException, + Response, + status, ) from pydantic import Required @@ -23,6 +19,9 @@ CloudDatasets, CloudObjects, InputArguments, + StatusCode, + SummaryGetObjects, + SummarySendDatasets, ) from galaxy.webapps.galaxy.api import ( depends, @@ -46,9 +45,11 @@ class FastAPICloudController: ) def index( self, + response: Response, ): # TODO: This can be implemented leveraging PluggedMedia objects (part of the user-based object store project) - raise HTTPException(status_code=501) + response.status_code = status.HTTP_501_NOT_IMPLEMENTED + return StatusCode(detail="Not yet implemented.", status=501) @router.post( "/api/cloud/storage/get", @@ -58,7 +59,7 @@ def get( self, payload: CloudObjects = Body(default=Required), trans: ProvidesHistoryContext = DependsOnTrans, - ) -> List[Dict[Any, Any]]: + ) -> SummaryGetObjects: input_args: Union[InputArguments, dict] = payload.input_args or {} datasets = self.cloud_manager.get( trans=trans, @@ -71,17 +72,17 @@ def get( rtv = [] for dataset in datasets: rtv.append(self.datasets_serializer.serialize_to_view(dataset, view="summary")) - return rtv + return SummaryGetObjects(datasets=rtv) @router.post( "/api/cloud/storage/send", - summary="Sends given dataset(s) in a given history to a given cloud-based bucket. Each dataset is named using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`). If no dataset ID is given, this API sends all the datasets belonging to a given history to a given cloud-based bucket.", + summary="Sends given dataset(s) in a given history to a given cloud-based bucket.", ) def send( self, - payload: CloudDatasets, + payload: CloudDatasets = Body(default=Required), trans: ProvidesHistoryContext = DependsOnTrans, - ) -> Dict[str, Union[List[Any], str]]: + ) -> SummarySendDatasets: log.info( msg="Received api/send request for `{}` datasets using authnz with id `{}`, and history `{}`." "".format( @@ -99,8 +100,4 @@ def send( dataset_ids=payload.dataset_ids, overwrite_existing=payload.overwrite_existing, ) - return { - "sent_dataset_labels": sent, - "failed_dataset_labels": failed, - "bucket_name": payload.bucket, - } + return SummarySendDatasets(sent_dataset_labels=sent, failed_dataset_labels=failed, bucket_name=payload.bucket) From 211d7826c12eec5ae17713d5589bcb59099303ec Mon Sep 17 00:00:00 2001 From: Tillman Date: Thu, 22 Jun 2023 18:47:52 +0200 Subject: [PATCH 07/18] Refactor the new client schema --- client/src/schema/schema.ts | 74 ++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index 83688aefe042..99589b2cbc7a 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -17,7 +17,7 @@ export interface paths { post: operations["get_api_cloud_storage_get_post"]; }; "/api/cloud/storage/send": { - /** Sends given dataset(s) in a given history to a given cloud-based bucket. Each dataset is named using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`). If no dataset ID is given, this API sends all the datasets belonging to a given history to a given cloud-based bucket. */ + /** Sends given dataset(s) in a given history to a given cloud-based bucket. */ post: operations["send_api_cloud_storage_send_post"]; }; "/api/configuration": { @@ -3756,6 +3756,22 @@ export interface components { items_from?: string; src: components["schemas"]["Src"]; }; + /** + * FailedJob + * @description Base model definition with common configuration used by all derived models. + */ + FailedJob: { + /** + * Error + * @description A descriptive error message. + */ + error: string; + /** + * Object + * @description The name of object is queued to be created + */ + object: string; + }; /** * FetchDataPayload * @description Base model definition with common configuration used by all derived models. @@ -8012,6 +8028,22 @@ export interface components { * @enum {string} */ StoredItemOrderBy: "name-asc" | "name-dsc" | "size-asc" | "size-dsc" | "update_time-asc" | "update_time-dsc"; + /** + * SuccessfulJob + * @description Base model definition with common configuration used by all derived models. + */ + SuccessfulJob: { + /** + * Job ID + * @description The ID of the queued send job. + */ + job_id: string; + /** + * Object + * @description The name of object is queued to be created + */ + object: string; + }; /** * SuitableConverter * @description Base model definition with common configuration used by all derived models. @@ -8043,6 +8075,38 @@ export interface components { * @description Collection of converters that can be used on a particular dataset collection. */ SuitableConverters: components["schemas"]["SuitableConverter"][]; + /** + * SummaryGetObjects + * @description Base model definition with common configuration used by all derived models. + */ + SummaryGetObjects: { + /** + * Datasets + * @description A list of datasets created for the fetched files. + */ + datasets: Record[]; + }; + /** + * SummarySendDatasets + * @description Base model definition with common configuration used by all derived models. + */ + SummarySendDatasets: { + /** + * Bucket + * @description The name of bucket to which the listed datasets are queued to be sent + */ + bucket_name: string; + /** + * Failed datasets + * @description The datasets for which Galaxy failed to create (and queue) send job + */ + failed_dataset_labels: components["schemas"]["FailedJob"][]; + /** + * Send datasets + * @description The datasets for which Galaxy succeeded to create (and queue) send job + */ + sent_dataset_labels: components["schemas"]["SuccessfulJob"][]; + }; /** * TagCollection * @description The collection of tags associated with an item. @@ -8997,7 +9061,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": Record[]; + "application/json": components["schemas"]["SummaryGetObjects"]; }; }; /** @description Validation Error */ @@ -9009,7 +9073,7 @@ export interface operations { }; }; send_api_cloud_storage_send_post: { - /** Sends given dataset(s) in a given history to a given cloud-based bucket. Each dataset is named using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`). If no dataset ID is given, this API sends all the datasets belonging to a given history to a given cloud-based bucket. */ + /** Sends given dataset(s) in a given history to a given cloud-based bucket. */ parameters?: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -9025,9 +9089,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": { - [key: string]: (Record[] | string) | undefined; - }; + "application/json": components["schemas"]["SummarySendDatasets"]; }; }; /** @description Validation Error */ From 2a38d336cc0fdb72eba2c3d2c55fcea494cafbec Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 26 Jun 2023 09:34:32 +0200 Subject: [PATCH 08/18] Mark new routes as deprecated --- lib/galaxy/webapps/galaxy/api/cloud.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index e2229e9b182e..5c2a967032c6 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -42,6 +42,7 @@ class FastAPICloudController: @router.get( "/api/cloud/storage", summary="Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. Is not yet implemented", + deprecated=True, ) def index( self, @@ -54,6 +55,7 @@ def index( @router.post( "/api/cloud/storage/get", summary="Gets given objects from a given cloud-based bucket to a Galaxy history.", + deprecated=True, ) def get( self, @@ -77,6 +79,7 @@ def get( @router.post( "/api/cloud/storage/send", summary="Sends given dataset(s) in a given history to a given cloud-based bucket.", + deprecated=True, ) def send( self, From 54aecb48439a4ad3df5c93768fd544d51d5c54f9 Mon Sep 17 00:00:00 2001 From: Tillman Date: Mon, 26 Jun 2023 09:43:44 +0200 Subject: [PATCH 09/18] Regenerate the client schema --- client/src/schema/schema.ts | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index 99589b2cbc7a..23af50972827 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -9,15 +9,24 @@ export interface paths { get: operations["get_api_key_api_authenticate_baseauth_get"]; }; "/api/cloud/storage": { - /** Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. Is not yet implemented */ + /** + * Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. Is not yet implemented + * @deprecated + */ get: operations["index_api_cloud_storage_get"]; }; "/api/cloud/storage/get": { - /** Gets given objects from a given cloud-based bucket to a Galaxy history. */ + /** + * Gets given objects from a given cloud-based bucket to a Galaxy history. + * @deprecated + */ post: operations["get_api_cloud_storage_get_post"]; }; "/api/cloud/storage/send": { - /** Sends given dataset(s) in a given history to a given cloud-based bucket. */ + /** + * Sends given dataset(s) in a given history to a given cloud-based bucket. + * @deprecated + */ post: operations["send_api_cloud_storage_send_post"]; }; "/api/configuration": { @@ -9034,7 +9043,10 @@ export interface operations { }; }; index_api_cloud_storage_get: { - /** Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. Is not yet implemented */ + /** + * Lists cloud-based buckets (e.g., S3 bucket, Azure blob) user has defined. Is not yet implemented + * @deprecated + */ responses: { /** @description Successful Response */ 200: { @@ -9045,7 +9057,10 @@ export interface operations { }; }; get_api_cloud_storage_get_post: { - /** Gets given objects from a given cloud-based bucket to a Galaxy history. */ + /** + * Gets given objects from a given cloud-based bucket to a Galaxy history. + * @deprecated + */ parameters?: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -9073,7 +9088,10 @@ export interface operations { }; }; send_api_cloud_storage_send_post: { - /** Sends given dataset(s) in a given history to a given cloud-based bucket. */ + /** + * Sends given dataset(s) in a given history to a given cloud-based bucket. + * @deprecated + */ parameters?: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { From df14342640c931ed9a7edae823a6ee1e32e44a5b Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 11 Jul 2023 13:21:51 +0200 Subject: [PATCH 10/18] Refactor InputArguments and remove unused import --- lib/galaxy/schema/cloud.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/schema/cloud.py b/lib/galaxy/schema/cloud.py index 0a8e0fbc99ac..e41d669e281c 100644 --- a/lib/galaxy/schema/cloud.py +++ b/lib/galaxy/schema/cloud.py @@ -11,7 +11,6 @@ ) from typing_extensions import Literal -# from galaxy.model import Dataset from galaxy.schema.fields import DecodedDatabaseIdField from galaxy.schema.schema import Model @@ -19,8 +18,8 @@ class InputArguments(Model): dbkey: Optional[str] = Field( default="?", - title="Genome", - description="Sets the genome (e.g., `hg19`) of the objects being fetched to Galaxy.", + title="Database Key", + description="Sets the database key of the objects being fetched to Galaxy.", ) file_type: Optional[str] = Field( default="auto", From 361b03139d3012f29d4d07f655342ef266f46854 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 11 Jul 2023 14:14:22 +0200 Subject: [PATCH 11/18] Refactor get operation --- lib/galaxy/webapps/galaxy/api/cloud.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 5c2a967032c6..1e63a83f6ef5 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -3,7 +3,7 @@ """ import logging -from typing import Union +from typing import List from fastapi import ( Body, @@ -18,9 +18,7 @@ from galaxy.schema.cloud import ( CloudDatasets, CloudObjects, - InputArguments, StatusCode, - SummaryGetObjects, SummarySendDatasets, ) from galaxy.webapps.galaxy.api import ( @@ -61,20 +59,19 @@ def get( self, payload: CloudObjects = Body(default=Required), trans: ProvidesHistoryContext = DependsOnTrans, - ) -> SummaryGetObjects: - input_args: Union[InputArguments, dict] = payload.input_args or {} + ) -> List[dict]: datasets = self.cloud_manager.get( trans=trans, history_id=payload.history_id, bucket_name=payload.bucket, objects=payload.objects, authz_id=payload.authz_id, - input_args=input_args, + input_args=payload.input_args, ) rtv = [] for dataset in datasets: rtv.append(self.datasets_serializer.serialize_to_view(dataset, view="summary")) - return SummaryGetObjects(datasets=rtv) + return rtv @router.post( "/api/cloud/storage/send", From ca57d74bdfd4603b7eae347b0f6aadc54c4161b6 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 11 Jul 2023 14:19:49 +0200 Subject: [PATCH 12/18] Refactor SummarySendDatasets class and remove unused classes --- lib/galaxy/schema/cloud.py | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/schema/cloud.py b/lib/galaxy/schema/cloud.py index e41d669e281c..157c59a5eb2e 100644 --- a/lib/galaxy/schema/cloud.py +++ b/lib/galaxy/schema/cloud.py @@ -94,39 +94,13 @@ class CloudDatasets(Model): ) -class FailedJob(Model): - object: str = Field( - default=Required, - title="Object", - description="The name of object is queued to be created", - ) - error: str = Field( - default=Required, - title="Error", - description="A descriptive error message.", - ) - - -class SuccessfulJob(Model): - object: str = Field( - default=Required, - title="Object", - description="The name of object is queued to be created", - ) - job_id: str = Field( - default=Required, - title="Job ID", - description="The ID of the queued send job.", - ) - - class SummarySendDatasets(Model): - sent_dataset_labels: List[SuccessfulJob] = Field( + sent_dataset_labels: List[str] = Field( default=Required, title="Send datasets", description="The datasets for which Galaxy succeeded to create (and queue) send job", ) - failed_dataset_labels: List[FailedJob] = Field( + failed_dataset_labels: List[str] = Field( default=Required, title="Failed datasets", description="The datasets for which Galaxy failed to create (and queue) send job", From c2eaa2be5a0b314e6ebb28a9810747ef5a84a810 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 11 Jul 2023 17:44:24 +0200 Subject: [PATCH 13/18] Refactor get operation --- lib/galaxy/webapps/galaxy/api/cloud.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 1e63a83f6ef5..2a02a5a516b0 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -18,6 +18,7 @@ from galaxy.schema.cloud import ( CloudDatasets, CloudObjects, + Dataset, StatusCode, SummarySendDatasets, ) @@ -59,7 +60,7 @@ def get( self, payload: CloudObjects = Body(default=Required), trans: ProvidesHistoryContext = DependsOnTrans, - ) -> List[dict]: + ) -> List[Dataset]: datasets = self.cloud_manager.get( trans=trans, history_id=payload.history_id, @@ -70,7 +71,7 @@ def get( ) rtv = [] for dataset in datasets: - rtv.append(self.datasets_serializer.serialize_to_view(dataset, view="summary")) + rtv.append(Dataset(**self.datasets_serializer.serialize_to_view(dataset, view="summary"))) return rtv @router.post( From 858daf991a2a5b2740933a93d35f8194f82728e1 Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 11 Jul 2023 17:46:54 +0200 Subject: [PATCH 14/18] Refactor pydantic model for get operation --- lib/galaxy/schema/cloud.py | 56 ++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/schema/cloud.py b/lib/galaxy/schema/cloud.py index 157c59a5eb2e..4354509e47b1 100644 --- a/lib/galaxy/schema/cloud.py +++ b/lib/galaxy/schema/cloud.py @@ -112,11 +112,57 @@ class SummarySendDatasets(Model): ) -class SummaryGetObjects(Model): - datasets: List[Any] = Field( - default=Required, - title="Datasets", - description="A list of datasets created for the fetched files.", +class Dataset(Model): + # TODO Add descriptions for fields + id: Optional[Any] = Field( + default=None, + title="ID", + description="", + ) + create_time: Optional[Any] = Field( + default=None, + title="Create time", + description="", + ) + update_time: Optional[Any] = Field( + default=None, + title="Update time", + description="", + ) + state: Optional[Any] = Field( + default=None, + title="State", + description="", + ) + deleted: Optional[Any] = Field( + default=None, + title="Deleted", + description="", + ) + purged: Optional[Any] = Field( + default=None, + title="Purged", + description="", + ) + purgable: Optional[Any] = Field( + default=None, + title="Purgable", + description="", + ) + file_size: Optional[Any] = Field( + default=None, + title="File size", + description="", + ) + total_size: Optional[Any] = Field( + default=None, + title="Total size", + description="", + ) + uuid: Optional[Any] = Field( + default=None, + title="UUID", + description="", ) From c359213a2e5a22ee65dcd3718efbb1569ba66d0d Mon Sep 17 00:00:00 2001 From: Tillman Date: Tue, 11 Jul 2023 17:55:15 +0200 Subject: [PATCH 15/18] Regenerate the client schema --- client/src/schema/schema.ts | 79 +++++++++++++++---------------------- 1 file changed, 31 insertions(+), 48 deletions(-) diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index 23af50972827..7a8b54651a20 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -3048,6 +3048,32 @@ export interface components { | components["schemas"]["NestedElement"] )[]; }; + /** + * Dataset + * @description Base model definition with common configuration used by all derived models. + */ + Dataset: { + /** Create time */ + create_time?: Record; + /** Deleted */ + deleted?: Record; + /** File size */ + file_size?: Record; + /** ID */ + id?: Record; + /** Purgable */ + purgable?: Record; + /** Purged */ + purged?: Record; + /** State */ + state?: Record; + /** Total size */ + total_size?: Record; + /** Update time */ + update_time?: Record; + /** UUID */ + uuid?: Record; + }; /** * DatasetAssociationRoles * @description Base model definition with common configuration used by all derived models. @@ -3765,22 +3791,6 @@ export interface components { items_from?: string; src: components["schemas"]["Src"]; }; - /** - * FailedJob - * @description Base model definition with common configuration used by all derived models. - */ - FailedJob: { - /** - * Error - * @description A descriptive error message. - */ - error: string; - /** - * Object - * @description The name of object is queued to be created - */ - object: string; - }; /** * FetchDataPayload * @description Base model definition with common configuration used by all derived models. @@ -5478,8 +5488,8 @@ export interface components { */ InputArguments: { /** - * Genome - * @description Sets the genome (e.g., `hg19`) of the objects being fetched to Galaxy. + * Database Key + * @description Sets the database key of the objects being fetched to Galaxy. * @default ? */ dbkey?: string; @@ -8037,22 +8047,6 @@ export interface components { * @enum {string} */ StoredItemOrderBy: "name-asc" | "name-dsc" | "size-asc" | "size-dsc" | "update_time-asc" | "update_time-dsc"; - /** - * SuccessfulJob - * @description Base model definition with common configuration used by all derived models. - */ - SuccessfulJob: { - /** - * Job ID - * @description The ID of the queued send job. - */ - job_id: string; - /** - * Object - * @description The name of object is queued to be created - */ - object: string; - }; /** * SuitableConverter * @description Base model definition with common configuration used by all derived models. @@ -8084,17 +8078,6 @@ export interface components { * @description Collection of converters that can be used on a particular dataset collection. */ SuitableConverters: components["schemas"]["SuitableConverter"][]; - /** - * SummaryGetObjects - * @description Base model definition with common configuration used by all derived models. - */ - SummaryGetObjects: { - /** - * Datasets - * @description A list of datasets created for the fetched files. - */ - datasets: Record[]; - }; /** * SummarySendDatasets * @description Base model definition with common configuration used by all derived models. @@ -8109,12 +8092,12 @@ export interface components { * Failed datasets * @description The datasets for which Galaxy failed to create (and queue) send job */ - failed_dataset_labels: components["schemas"]["FailedJob"][]; + failed_dataset_labels: string[]; /** * Send datasets * @description The datasets for which Galaxy succeeded to create (and queue) send job */ - sent_dataset_labels: components["schemas"]["SuccessfulJob"][]; + sent_dataset_labels: string[]; }; /** * TagCollection @@ -9076,7 +9059,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["SummaryGetObjects"]; + "application/json": components["schemas"]["Dataset"][]; }; }; /** @description Validation Error */ From 742e936858c6c370a311c82bd7b2005d3c65a710 Mon Sep 17 00:00:00 2001 From: Tillman Date: Wed, 12 Jul 2023 12:45:13 +0200 Subject: [PATCH 16/18] Refactor and add pydantic models for CloudAPI --- lib/galaxy/schema/cloud.py | 66 +++++-------------------------------- lib/galaxy/schema/schema.py | 13 ++++++++ 2 files changed, 22 insertions(+), 57 deletions(-) diff --git a/lib/galaxy/schema/cloud.py b/lib/galaxy/schema/cloud.py index 4354509e47b1..8b462def4a94 100644 --- a/lib/galaxy/schema/cloud.py +++ b/lib/galaxy/schema/cloud.py @@ -1,5 +1,4 @@ from typing import ( - Any, List, Optional, Union, @@ -12,7 +11,10 @@ from typing_extensions import Literal from galaxy.schema.fields import DecodedDatabaseIdField -from galaxy.schema.schema import Model +from galaxy.schema.schema import ( + DatasetSummary, + Model, +) class InputArguments(Model): @@ -94,7 +96,7 @@ class CloudDatasets(Model): ) -class SummarySendDatasets(Model): +class CloudDatasetsResponse(Model): sent_dataset_labels: List[str] = Field( default=Required, title="Send datasets", @@ -112,60 +114,6 @@ class SummarySendDatasets(Model): ) -class Dataset(Model): - # TODO Add descriptions for fields - id: Optional[Any] = Field( - default=None, - title="ID", - description="", - ) - create_time: Optional[Any] = Field( - default=None, - title="Create time", - description="", - ) - update_time: Optional[Any] = Field( - default=None, - title="Update time", - description="", - ) - state: Optional[Any] = Field( - default=None, - title="State", - description="", - ) - deleted: Optional[Any] = Field( - default=None, - title="Deleted", - description="", - ) - purged: Optional[Any] = Field( - default=None, - title="Purged", - description="", - ) - purgable: Optional[Any] = Field( - default=None, - title="Purgable", - description="", - ) - file_size: Optional[Any] = Field( - default=None, - title="File size", - description="", - ) - total_size: Optional[Any] = Field( - default=None, - title="Total size", - description="", - ) - uuid: Optional[Any] = Field( - default=None, - title="UUID", - description="", - ) - - class StatusCode(Model): detail: str = Field( default=Required, @@ -177,3 +125,7 @@ class StatusCode(Model): title="Code", description="The actual status code", ) + + +class DatasetSummaryList(Model): + __root__: List[DatasetSummary] diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 863f4543f149..5410858d531d 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -3463,3 +3463,16 @@ class PageSummaryList(Model): default=[], title="List with summary information of Pages.", ) + + +class DatasetSummary(Model): + id: EncodedDatabaseIdField = EntityIdField + create_time: Optional[datetime] = CreateTimeField + update_time: Optional[datetime] = UpdateTimeField + state: DatasetState = DatasetStateField + deleted: bool + purged: bool + purgable: bool + file_size: int + total_size: int + uuid: UUID4 = UuidField From 8bcd05c2a039e9e4afd8e5db178e4cb4e6e6f4e9 Mon Sep 17 00:00:00 2001 From: Tillman Date: Wed, 12 Jul 2023 12:46:37 +0200 Subject: [PATCH 17/18] Refactor get and send operation --- lib/galaxy/webapps/galaxy/api/cloud.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 2a02a5a516b0..7bb61c2ec7b9 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -3,7 +3,6 @@ """ import logging -from typing import List from fastapi import ( Body, @@ -17,10 +16,10 @@ from galaxy.managers.datasets import DatasetSerializer from galaxy.schema.cloud import ( CloudDatasets, + CloudDatasetsResponse, CloudObjects, - Dataset, + DatasetSummaryList, StatusCode, - SummarySendDatasets, ) from galaxy.webapps.galaxy.api import ( depends, @@ -60,7 +59,7 @@ def get( self, payload: CloudObjects = Body(default=Required), trans: ProvidesHistoryContext = DependsOnTrans, - ) -> List[Dataset]: + ) -> DatasetSummaryList: datasets = self.cloud_manager.get( trans=trans, history_id=payload.history_id, @@ -71,8 +70,8 @@ def get( ) rtv = [] for dataset in datasets: - rtv.append(Dataset(**self.datasets_serializer.serialize_to_view(dataset, view="summary"))) - return rtv + rtv.append(self.datasets_serializer.serialize_to_view(dataset, view="summary")) + return DatasetSummaryList.construct(__root__=rtv) @router.post( "/api/cloud/storage/send", @@ -83,7 +82,7 @@ def send( self, payload: CloudDatasets = Body(default=Required), trans: ProvidesHistoryContext = DependsOnTrans, - ) -> SummarySendDatasets: + ) -> CloudDatasetsResponse: log.info( msg="Received api/send request for `{}` datasets using authnz with id `{}`, and history `{}`." "".format( @@ -101,4 +100,4 @@ def send( dataset_ids=payload.dataset_ids, overwrite_existing=payload.overwrite_existing, ) - return SummarySendDatasets(sent_dataset_labels=sent, failed_dataset_labels=failed, bucket_name=payload.bucket) + return CloudDatasetsResponse(sent_dataset_labels=sent, failed_dataset_labels=failed, bucket_name=payload.bucket) From fe4f9904e79f6979023c47568ac593f8e55df7f6 Mon Sep 17 00:00:00 2001 From: Tillman Date: Wed, 12 Jul 2023 12:54:15 +0200 Subject: [PATCH 18/18] Regenerate the client schema --- client/src/schema/schema.ts | 122 +++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 49 deletions(-) diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index 7a8b54651a20..1ea7124d5327 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -2258,6 +2258,27 @@ export interface components { */ overwrite_existing?: boolean; }; + /** + * CloudDatasetsResponse + * @description Base model definition with common configuration used by all derived models. + */ + CloudDatasetsResponse: { + /** + * Bucket + * @description The name of bucket to which the listed datasets are queued to be sent + */ + bucket_name: string; + /** + * Failed datasets + * @description The datasets for which Galaxy failed to create (and queue) send job + */ + failed_dataset_labels: string[]; + /** + * Send datasets + * @description The datasets for which Galaxy succeeded to create (and queue) send job + */ + sent_dataset_labels: string[]; + }; /** * CloudObjects * @description Base model definition with common configuration used by all derived models. @@ -3048,32 +3069,6 @@ export interface components { | components["schemas"]["NestedElement"] )[]; }; - /** - * Dataset - * @description Base model definition with common configuration used by all derived models. - */ - Dataset: { - /** Create time */ - create_time?: Record; - /** Deleted */ - deleted?: Record; - /** File size */ - file_size?: Record; - /** ID */ - id?: Record; - /** Purgable */ - purgable?: Record; - /** Purged */ - purged?: Record; - /** State */ - state?: Record; - /** Total size */ - total_size?: Record; - /** Update time */ - update_time?: Record; - /** UUID */ - uuid?: Record; - }; /** * DatasetAssociationRoles * @description Base model definition with common configuration used by all derived models. @@ -3297,6 +3292,56 @@ export interface components { */ sources: Record[]; }; + /** + * DatasetSummary + * @description Base model definition with common configuration used by all derived models. + */ + DatasetSummary: { + /** + * Create Time + * Format: date-time + * @description The time and date this item was created. + */ + create_time?: string; + /** Deleted */ + deleted: boolean; + /** File Size */ + file_size: number; + /** + * ID + * @description The encoded ID of this entity. + * @example 0123456789ABCDEF + */ + id: string; + /** Purgable */ + purgable: boolean; + /** Purged */ + purged: boolean; + /** + * State + * @description The current state of this dataset. + */ + state: components["schemas"]["DatasetState"]; + /** Total Size */ + total_size: number; + /** + * Update Time + * Format: date-time + * @description The last time and date this item was updated. + */ + update_time?: string; + /** + * UUID + * Format: uuid4 + * @description Universal unique identifier for this dataset. + */ + uuid: string; + }; + /** + * DatasetSummaryList + * @description Base model definition with common configuration used by all derived models. + */ + DatasetSummaryList: components["schemas"]["DatasetSummary"][]; /** * DatasetTextContentDetails * @description Base model definition with common configuration used by all derived models. @@ -8078,27 +8123,6 @@ export interface components { * @description Collection of converters that can be used on a particular dataset collection. */ SuitableConverters: components["schemas"]["SuitableConverter"][]; - /** - * SummarySendDatasets - * @description Base model definition with common configuration used by all derived models. - */ - SummarySendDatasets: { - /** - * Bucket - * @description The name of bucket to which the listed datasets are queued to be sent - */ - bucket_name: string; - /** - * Failed datasets - * @description The datasets for which Galaxy failed to create (and queue) send job - */ - failed_dataset_labels: string[]; - /** - * Send datasets - * @description The datasets for which Galaxy succeeded to create (and queue) send job - */ - sent_dataset_labels: string[]; - }; /** * TagCollection * @description The collection of tags associated with an item. @@ -9059,7 +9083,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["Dataset"][]; + "application/json": components["schemas"]["DatasetSummaryList"]; }; }; /** @description Validation Error */ @@ -9090,7 +9114,7 @@ export interface operations { /** @description Successful Response */ 200: { content: { - "application/json": components["schemas"]["SummarySendDatasets"]; + "application/json": components["schemas"]["CloudDatasetsResponse"]; }; }; /** @description Validation Error */