From 1e0a1c67f3b6bed8aad6e8a7f1137ceca34b8848 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 16 Jun 2023 15:35:29 -0400 Subject: [PATCH 1/6] Fix select([foo]) SA2.0 warnings --- lib/galaxy/jobs/handler.py | 6 +++--- lib/galaxy/managers/hdas.py | 2 +- lib/galaxy/managers/histories.py | 2 +- lib/galaxy/managers/history_contents.py | 2 +- lib/galaxy/managers/jobs.py | 4 ++-- lib/galaxy/managers/notification.py | 2 +- lib/galaxy/model/__init__.py | 4 ++-- lib/galaxy/webapps/reports/controllers/jobs.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index daec2a0e0193..1b25b29a1d93 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -123,7 +123,7 @@ def __init__( def setup_query(self): subq = ( - select([self.grab_this.id]) + select(self.grab_this.id) .where( and_( self.grab_this.table.c.handler.in_(self.self_handler_tags), @@ -817,7 +817,7 @@ def get_user_job_count(self, user_id): rval = self.user_job_count.get(user_id, 0) if not self.app.config.cache_user_job_count: result = self.sa_session.execute( - select([func.count(model.Job.table.c.id)]).where( + select(func.count(model.Job.table.c.id)).where( and_( model.Job.table.c.state.in_( (model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.RESUBMITTED) @@ -836,7 +836,7 @@ def __cache_user_job_count(self): if self.user_job_count is None and self.app.config.cache_user_job_count: self.user_job_count = {} query = self.sa_session.execute( - select([model.Job.table.c.user_id, func.count(model.Job.table.c.user_id)]) + select(model.Job.table.c.user_id, func.count(model.Job.table.c.user_id)) .where( and_( model.Job.table.c.state.in_( diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index 82aeb0709948..f2d99d887832 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -360,7 +360,7 @@ def __init__(self, hda_manager: HDAManager, dataset_manager: datasets.DatasetMan def get_discarded_summary(self, user: model.User) -> CleanableItemsSummary: stmt = ( - select([func.sum(model.Dataset.total_size), func.count(model.HistoryDatasetAssociation.id)]) + select(func.sum(model.Dataset.total_size), func.count(model.HistoryDatasetAssociation.id)) .select_from(model.HistoryDatasetAssociation) .join(model.Dataset, model.HistoryDatasetAssociation.table.c.dataset_id == model.Dataset.id) .join(model.History, model.HistoryDatasetAssociation.table.c.history_id == model.History.id) diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index af575c86459a..ffc1073f6a68 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -417,7 +417,7 @@ def __init__(self, history_manager: HistoryManager): } def get_discarded_summary(self, user: model.User) -> CleanableItemsSummary: - stmt = select([func.sum(model.History.disk_size), func.count(model.History.id)]).where( + stmt = select(func.sum(model.History.disk_size), func.count(model.History.id)).where( model.History.user_id == user.id, model.History.deleted == true(), model.History.purged == false(), diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 12c19af28ac5..cbb0f9fa3051 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -180,7 +180,7 @@ def state_counts(self, history): ] contents_subquery = self._union_of_contents_query(history, filters=filters).subquery() statement = ( - sql.select([sql.column("state"), func.count("*")]) + sql.select(sql.column("state"), func.count("*")) .select_from(contents_subquery) .group_by(sql.column("state")) ) diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 54c01bcea4aa..bfb648ca5067 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -416,7 +416,7 @@ def replace_dataset_ids(path, key, value): c = aliased(model.HistoryDatasetAssociation) d = aliased(model.JobParameter) e = aliased(model.HistoryDatasetAssociationHistory) - stmt = select([model.HistoryDatasetAssociation.id]).where( + stmt = select(model.HistoryDatasetAssociation.id).where( model.HistoryDatasetAssociation.id == e.history_dataset_association_id ) name_condition = [] @@ -789,7 +789,7 @@ def summarize_jobs_to_dict(sa_session, jobs_source): model.ImplicitCollectionJobsJobAssociation.table.join(model.Job) ) statement = ( - select([model.Job.state, func.count("*")]) + select(model.Job.state, func.count("*")) .select_from(join) .where(model.ImplicitCollectionJobs.id == jobs_source.id) .group_by(model.Job.state) diff --git a/lib/galaxy/managers/notification.py b/lib/galaxy/managers/notification.py index 87b75db17139..3e12fada5dd5 100644 --- a/lib/galaxy/managers/notification.py +++ b/lib/galaxy/managers/notification.py @@ -184,7 +184,7 @@ def get_user_total_unread_notification_count(self, user: User) -> int: Only published and not expired notifications are accounted. """ stmt = ( - select([func.count(UserNotificationAssociation.id)]) + select(func.count(UserNotificationAssociation.id)) .select_from(UserNotificationAssociation) .join( Notification, diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 372a39e94819..4ba24cc4f205 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3313,7 +3313,7 @@ def disk_size(cls): distinct_datasets_alias = aliased(distinct_datasets.subquery(), name="datasets") # then, bind as property of history using the cls.id size_query = ( - select([func.coalesce(func.sum(distinct_datasets_alias.c.dataset_size), 0)]) + select(func.coalesce(func.sum(distinct_datasets_alias.c.dataset_size), 0)) .select_from(distinct_datasets_alias) .where(distinct_datasets_alias.c.history_id == cls.id) ) @@ -8595,7 +8595,7 @@ class WorkflowInvocationStep(Base, Dictifiable, Serializable): viewonly=True, ) order_index = column_property( - select([WorkflowStep.order_index]).where(WorkflowStep.id == workflow_step_id).scalar_subquery() + select(WorkflowStep.order_index).where(WorkflowStep.id == workflow_step_id).scalar_subquery() ) subworkflow_invocation_id: column_property diff --git a/lib/galaxy/webapps/reports/controllers/jobs.py b/lib/galaxy/webapps/reports/controllers/jobs.py index 333d3c6da8fe..f324b19bc010 100644 --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -857,7 +857,7 @@ def per_user(self, trans, **kwd): q_time.stop() query1time = q_time.time_elapsed() - users = sa.select([model.User.table.c.email], from_obj=[model.User.table]) + users = sa.select(model.User.table.c.email).select_from(model.User.table) all_jobs_per_user = sa.select( ( From b4e843cecb15d1be2bf7c1ab603e5eb9a59472ea Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 21 Jun 2023 17:24:50 -0400 Subject: [PATCH 2/6] Fix select([foo]) in lib/tool_shed --- lib/galaxy/jobs/handler.py | 12 +++---- lib/galaxy/managers/hdas.py | 10 +++--- lib/galaxy/managers/job_connections.py | 48 +++++++++----------------- lib/galaxy/managers/jobs.py | 8 ++--- lib/galaxy/model/__init__.py | 10 +++--- lib/tool_shed/util/repository_util.py | 2 +- 6 files changed, 33 insertions(+), 57 deletions(-) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 1b25b29a1d93..10e915319381 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -866,7 +866,7 @@ def get_user_job_count_per_destination(self, user_id): rval.update(cached) result = self.sa_session.execute( select( - [model.Job.table.c.destination_id, func.count(model.Job.table.c.destination_id).label("job_count")] + model.Job.table.c.destination_id, func.count(model.Job.table.c.destination_id).label("job_count") ) .where( and_( @@ -887,11 +887,9 @@ def __cache_user_job_count_per_destination(self): self.user_job_count_per_destination = {} result = self.sa_session.execute( select( - [ - model.Job.table.c.user_id, - model.Job.table.c.destination_id, - func.count(model.Job.table.c.user_id).label("job_count"), - ] + model.Job.table.c.user_id, + model.Job.table.c.destination_id, + func.count(model.Job.table.c.user_id).label("job_count"), ) .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, model.Job.states.RUNNING)))) .group_by(model.Job.table.c.user_id, model.Job.table.c.destination_id) @@ -987,7 +985,7 @@ def __cache_total_job_count_per_destination(self): self.total_job_count_per_destination = {} result = self.sa_session.execute( select( - [model.Job.table.c.destination_id, func.count(model.Job.table.c.destination_id).label("job_count")] + model.Job.table.c.destination_id, func.count(model.Job.table.c.destination_id).label("job_count") ) .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, model.Job.states.RUNNING)))) .group_by(model.Job.table.c.destination_id) diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index f2d99d887832..5b8aee4aec7e 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -385,12 +385,10 @@ def get_discarded( ) -> List[StoredItem]: stmt = ( select( - [ - model.HistoryDatasetAssociation.id, - model.HistoryDatasetAssociation.name, - model.HistoryDatasetAssociation.update_time, - model.Dataset.total_size, - ] + model.HistoryDatasetAssociation.id, + model.HistoryDatasetAssociation.name, + model.HistoryDatasetAssociation.update_time, + model.Dataset.total_size, ) .select_from(model.HistoryDatasetAssociation) .join(model.Dataset, model.HistoryDatasetAssociation.table.c.dataset_id == model.Dataset.id) diff --git a/lib/galaxy/managers/job_connections.py b/lib/galaxy/managers/job_connections.py index e387f36527a5..c9000869cce3 100644 --- a/lib/galaxy/managers/job_connections.py +++ b/lib/galaxy/managers/job_connections.py @@ -66,10 +66,8 @@ def _get_union_results(self, *selects): def outputs_derived_from_input_hda(self, input_hda_id: int): hda_select = ( select( - [ - literal("HistoryDatasetAssociation").label("src"), - model.JobToOutputDatasetAssociation.dataset_id.label("id"), - ] + literal("HistoryDatasetAssociation").label("src"), + model.JobToOutputDatasetAssociation.dataset_id.label("id"), ) .join( model.JobToInputDatasetAssociation, @@ -80,10 +78,8 @@ def outputs_derived_from_input_hda(self, input_hda_id: int): ) hdca_select = ( select( - [ - literal("HistoryDatasetCollectionAssociation").label("src"), - model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), - ] + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), ) .join( model.JobToInputDatasetAssociation, @@ -97,10 +93,8 @@ def outputs_derived_from_input_hda(self, input_hda_id: int): def outputs_derived_from_input_hdca(self, input_hdca_id: int): hda_select = ( select( - [ - literal("HistoryDatasetAssociation").label("src"), - model.JobToOutputDatasetAssociation.dataset_id.label("id"), - ] + literal("HistoryDatasetAssociation").label("src"), + model.JobToOutputDatasetAssociation.dataset_id.label("id"), ) .join( model.JobToInputDatasetCollectionAssociation, @@ -111,10 +105,8 @@ def outputs_derived_from_input_hdca(self, input_hdca_id: int): ) hdca_select = ( select( - [ - literal("HistoryDatasetCollectionAssociation").label("src"), - model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), - ] + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToOutputDatasetCollectionAssociation.dataset_collection_id.label("id"), ) .join( model.JobToInputDatasetCollectionAssociation, @@ -129,10 +121,8 @@ def outputs_derived_from_input_hdca(self, input_hdca_id: int): def inputs_for_hda(self, input_hda_id: int): input_hdas = ( select( - [ - literal("HistoryDatasetAssociation").label("src"), - model.JobToInputDatasetAssociation.dataset_id.label("id"), - ] + literal("HistoryDatasetAssociation").label("src"), + model.JobToInputDatasetAssociation.dataset_id.label("id"), ) .join( model.JobToOutputDatasetAssociation, @@ -143,10 +133,8 @@ def inputs_for_hda(self, input_hda_id: int): ) input_hdcas = ( select( - [ - literal("HistoryDatasetCollectionAssociation").label("src"), - model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), - ] + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), ) .join( model.JobToOutputDatasetAssociation, @@ -160,10 +148,8 @@ def inputs_for_hda(self, input_hda_id: int): def inputs_for_hdca(self, input_hdca_id: int): input_hdas = ( select( - [ - literal("HistoryDatasetAssociation").label("src"), - model.JobToInputDatasetAssociation.dataset_id.label("id"), - ] + literal("HistoryDatasetAssociation").label("src"), + model.JobToInputDatasetAssociation.dataset_id.label("id"), ) .join( model.JobToOutputDatasetCollectionAssociation, @@ -174,10 +160,8 @@ def inputs_for_hdca(self, input_hdca_id: int): ) input_hdcas = ( select( - [ - literal("HistoryDatasetCollectionAssociation").label("src"), - model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), - ] + literal("HistoryDatasetCollectionAssociation").label("src"), + model.JobToInputDatasetCollectionAssociation.dataset_collection_id.label("id"), ) .join( model.JobToOutputDatasetCollectionAssociation, diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index bfb648ca5067..38eb5bab4d54 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -614,11 +614,9 @@ def invocation_job_source_iter(sa_session, invocation_id): join = model.WorkflowInvocationStep.table.join(model.WorkflowInvocation) statement = ( select( - [ - model.WorkflowInvocationStep.job_id, - model.WorkflowInvocationStep.implicit_collection_jobs_id, - model.WorkflowInvocationStep.state, - ] + model.WorkflowInvocationStep.job_id, + model.WorkflowInvocationStep.implicit_collection_jobs_id, + model.WorkflowInvocationStep.state, ) .select_from(join) .where(model.WorkflowInvocation.id == invocation_id) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4ba24cc4f205..d2f13613b1e3 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3296,12 +3296,10 @@ def disk_size(cls): ) distinct_datasets = ( select( - [ - # use labels here to better access from the query above - HistoryDatasetAssociation.table.c.history_id.label("history_id"), - Dataset.total_size.label("dataset_size"), - Dataset.id.label("dataset_id"), - ] + # use labels here to better access from the query above + HistoryDatasetAssociation.table.c.history_id.label("history_id"), + Dataset.total_size.label("dataset_size"), + Dataset.id.label("dataset_id"), ) .where(HistoryDatasetAssociation.table.c.purged != true()) .where(Dataset.table.c.purged != true()) diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 9d9e805ae669..afd773f05242 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -338,7 +338,7 @@ def get_repositories_by_category( .filter(app.model.RepositoryCategoryAssociation.category_id == category_id) ) if installable: - subquery = select([app.model.RepositoryMetadata.table.c.repository_id]) + subquery = select(app.model.RepositoryMetadata.table.c.repository_id) query = query.filter(app.model.Repository.id.in_(subquery)) if sort_key == "owner": query = ( From fd11cda073fedcb1585e5a8931a6f79c6e86ece8 Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 21 Jun 2023 17:43:23 -0400 Subject: [PATCH 3/6] Fix select(foo=bar, baz=bam) SA20 warnings - func.count() does not need "*" argument - select_from is not needed if it can be derived from the columns --- lib/galaxy/model/tags.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 6b0c1b9f9923..ae6a6c7dd2b1 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -95,19 +95,17 @@ def get_community_tags(self, item=None, limit=None): if not item_tag_assoc_class: return [] # Build select statement. - cols_to_select = [item_tag_assoc_class.table.c.tag_id, func.count("*")] from_obj = item_tag_assoc_class.table.join(item_class.table).join(galaxy.model.Tag.table) where_clause = self.get_id_col_in_item_tag_assoc_table(item_class) == item.id - order_by = [func.count("*").desc()] group_by = item_tag_assoc_class.table.c.tag_id # Do query and get result set. - query = select( - columns=cols_to_select, - from_obj=from_obj, - whereclause=where_clause, - group_by=group_by, - order_by=order_by, - limit=limit, + query = ( + select(item_tag_assoc_class.table.c.tag_id, func.count()) + .select_from(from_obj) + .where(where_clause) + .group_by(group_by) + .order_by(func.count().desc()) + .limit(limit) ) result_set = self.sa_session.execute(query) # Return community tags. @@ -118,9 +116,7 @@ def get_community_tags(self, item=None, limit=None): return community_tags def get_tool_tags(self): - query = select( - columns=[galaxy.model.ToolTagAssociation.table.c.tag_id], from_obj=galaxy.model.ToolTagAssociation.table - ).distinct() + query = select(galaxy.model.ToolTagAssociation.table.c.tag_id).distinct() result_set = self.sa_session.execute(query) tags = [] From 09fe7b9a544880f7b42a0c860e5b52d1eb50e493 Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 22 Jun 2023 13:37:35 -0400 Subject: [PATCH 4/6] Fix select stmts in scripts --- .../admin_cleanup_datasets.py | 35 ++++++++----- scripts/cleanup_datasets/cleanup_datasets.py | 52 ++++++++++--------- ...deprecate_repositories_without_metadata.py | 24 ++++----- 3 files changed, 58 insertions(+), 53 deletions(-) diff --git a/scripts/cleanup_datasets/admin_cleanup_datasets.py b/scripts/cleanup_datasets/admin_cleanup_datasets.py index 57003c6f11da..cc24664eb011 100755 --- a/scripts/cleanup_datasets/admin_cleanup_datasets.py +++ b/scripts/cleanup_datasets/admin_cleanup_datasets.py @@ -202,14 +202,18 @@ def administrative_delete_datasets( # Get HDAs older than cutoff time (ignore tool_id at this point) # We really only need the id column here, but sqlalchemy barfs when # trying to select only 1 column - hda_ids_query = sa.select( - (app.model.HistoryDatasetAssociation.__table__.c.id, app.model.HistoryDatasetAssociation.__table__.c.deleted), - whereclause=and_( - app.model.Dataset.__table__.c.deleted == false(), - app.model.HistoryDatasetAssociation.__table__.c.update_time < cutoff_time, - app.model.HistoryDatasetAssociation.__table__.c.deleted == false(), - ), - from_obj=[sa.outerjoin(app.model.Dataset.__table__, app.model.HistoryDatasetAssociation.__table__)], + hda_ids_query = ( + sa.select( + app.model.HistoryDatasetAssociation.__table__.c.id, app.model.HistoryDatasetAssociation.__table__.c.deleted + ) + .where( + and_( + app.model.Dataset.__table__.c.deleted == false(), + app.model.HistoryDatasetAssociation.__table__.c.update_time < cutoff_time, + app.model.HistoryDatasetAssociation.__table__.c.deleted == false(), + ) + ) + .select_from(sa.outerjoin(app.model.Dataset.__table__, app.model.HistoryDatasetAssociation.__table__)) ) # Add all datasets associated with Histories to our list @@ -230,16 +234,19 @@ def administrative_delete_datasets( # Process each of the Dataset objects for hda_id in hda_ids: - user_query = sa.select( - [app.model.HistoryDatasetAssociation.__table__, app.model.History.__table__, app.model.User.__table__], - whereclause=and_(app.model.HistoryDatasetAssociation.__table__.c.id == hda_id), - from_obj=[ + user_query = ( + sa.select( + app.model.HistoryDatasetAssociation.__table__, app.model.History.__table__, app.model.User.__table__ + ) + .where(and_(app.model.HistoryDatasetAssociation.__table__.c.id == hda_id)) + .select_from( sa.join(app.model.User.__table__, app.model.History.__table__).join( app.model.HistoryDatasetAssociation.__table__ ) - ], - use_labels=True, + ) + .set_label_style() ) + for result in app.sa_session.execute(user_query): user_notifications[result[app.model.User.__table__.c.email]].append( ( diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index 0138d26952fa..b4df76327123 100755 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -376,35 +376,39 @@ def delete_datasets(app, cutoff_time, remove_from_disk, info_only=False, force_r # Marks datasets as deleted if associated items are all deleted. start = time.time() if force_retry: - history_dataset_ids_query = sa.select( - (app.model.Dataset.__table__.c.id, app.model.Dataset.__table__.c.state), - whereclause=app.model.HistoryDatasetAssociation.__table__.c.update_time < cutoff_time, - from_obj=[sa.outerjoin(app.model.Dataset.__table__, app.model.HistoryDatasetAssociation.__table__)], + history_dataset_ids_query = ( + sa.select(app.model.Dataset.__table__.c.id, app.model.Dataset.__table__.c.state) + .where(app.model.HistoryDatasetAssociation.__table__.c.update_time < cutoff_time) + .select_from(sa.outerjoin(app.model.Dataset.__table__, app.model.HistoryDatasetAssociation.__table__)) ) - library_dataset_ids_query = sa.select( - (app.model.LibraryDataset.__table__.c.id, app.model.LibraryDataset.__table__.c.deleted), - whereclause=app.model.LibraryDataset.__table__.c.update_time < cutoff_time, - from_obj=[app.model.LibraryDataset.__table__], + library_dataset_ids_query = ( + sa.select(app.model.LibraryDataset.__table__.c.id, app.model.LibraryDataset.__table__.c.deleted) + .where(app.model.LibraryDataset.__table__.c.update_time < cutoff_time) + .select_from(app.model.LibraryDataset.__table__) ) else: # We really only need the id column here, but sqlalchemy barfs when trying to select only 1 column - history_dataset_ids_query = sa.select( - (app.model.Dataset.__table__.c.id, app.model.Dataset.__table__.c.state), - whereclause=and_( - app.model.Dataset.__table__.c.deleted == false(), - app.model.HistoryDatasetAssociation.__table__.c.update_time < cutoff_time, - app.model.HistoryDatasetAssociation.__table__.c.deleted == true(), - ), - from_obj=[sa.outerjoin(app.model.Dataset.__table__, app.model.HistoryDatasetAssociation.__table__)], + history_dataset_ids_query = ( + sa.select(app.model.Dataset.__table__.c.id, app.model.Dataset.__table__.c.state) + .where( + and_( + app.model.Dataset.__table__.c.deleted == false(), + app.model.HistoryDatasetAssociation.__table__.c.update_time < cutoff_time, + app.model.HistoryDatasetAssociation.__table__.c.deleted == true(), + ) + ) + .select_from(sa.outerjoin(app.model.Dataset.__table__, app.model.HistoryDatasetAssociation.__table__)) ) - library_dataset_ids_query = sa.select( - (app.model.LibraryDataset.__table__.c.id, app.model.LibraryDataset.__table__.c.deleted), - whereclause=and_( - app.model.LibraryDataset.__table__.c.deleted == true(), - app.model.LibraryDataset.__table__.c.purged == false(), - app.model.LibraryDataset.__table__.c.update_time < cutoff_time, - ), - from_obj=[app.model.LibraryDataset.__table__], + library_dataset_ids_query = ( + sa.select(app.model.LibraryDataset.__table__.c.id, app.model.LibraryDataset.__table__.c.deleted) + .where( + and_( + app.model.LibraryDataset.__table__.c.deleted == true(), + app.model.LibraryDataset.__table__.c.purged == false(), + app.model.LibraryDataset.__table__.c.update_time < cutoff_time, + ) + ) + .select_from(app.model.LibraryDataset.__table__) ) deleted_dataset_count = 0 deleted_instance_count = 0 diff --git a/scripts/tool_shed/deprecate_repositories_without_metadata.py b/scripts/tool_shed/deprecate_repositories_without_metadata.py index b3cf0d15a54f..beddab51dbd0 100644 --- a/scripts/tool_shed/deprecate_repositories_without_metadata.py +++ b/scripts/tool_shed/deprecate_repositories_without_metadata.py @@ -135,27 +135,21 @@ def deprecate_repositories(app, cutoff_time, days=14, info_only=False, verbose=F repository_ids_to_not_check = [] # Get a unique list of repository ids from the repository_metadata table. Any repository ID found in this table is not # empty, and will not be checked. - metadata_records = app.sa_session.execute( - sa.select( - [distinct(app.model.RepositoryMetadata.table.c.repository_id)], from_obj=app.model.RepositoryMetadata.table - ) - ) + stmt = sa.select(distinct(app.model.RepositoryMetadata.table.c.repository_id)) + metadata_records = app.sa_session.execute(stmt) for metadata_record in metadata_records: repository_ids_to_not_check.append(metadata_record.repository_id) # Get the repositories that are A) not present in the above list, and b) older than the specified time. # This will yield a list of repositories that have been created more than n days ago, but never populated. - query_result = app.sa_session.execute( - sa.select( - [app.model.Repository.table.c.id], - whereclause=and_( - app.model.Repository.table.c.create_time < cutoff_time, - app.model.Repository.table.c.deprecated == false(), - app.model.Repository.table.c.deleted == false(), - not_(app.model.Repository.table.c.id.in_(repository_ids_to_not_check)), - ), - from_obj=[app.model.Repository.table], + stmt = sa.select(app.model.Repository.table.c.id).where( + and_( + app.model.Repository.table.c.create_time < cutoff_time, + app.model.Repository.table.c.deprecated == false(), + app.model.Repository.table.c.deleted == false(), + not_(app.model.Repository.table.c.id.in_(repository_ids_to_not_check)), ) ) + query_result = app.sa_session.execute(stmt) repositories = [] repositories_by_owner = {} repository_ids = [row.id for row in query_result] From d2837c62e578e6ab7d391d26158578143a8264cc Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 22 Jun 2023 13:57:35 -0400 Subject: [PATCH 5/6] Fix select stmt in webapps/galaxy --- lib/galaxy/webapps/galaxy/controllers/tag.py | 44 +++++++++----------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/tag.py b/lib/galaxy/webapps/galaxy/controllers/tag.py index 32a8fd4c4ffa..2271945eae79 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tag.py +++ b/lib/galaxy/webapps/galaxy/controllers/tag.py @@ -84,21 +84,18 @@ def _get_tag_autocomplete_names(self, trans, q, limit, timestamp, user=None, ite item_class = item.__class__ item_tag_assoc_class = self.get_tag_handler(trans).get_tag_assoc_class(item_class) # Build select statement. - cols_to_select = [item_tag_assoc_class.table.c.tag_id, func.count("*")] from_obj = item_tag_assoc_class.table.join(item_class.table).join(trans.app.model.Tag.table) where_clause = and_( trans.app.model.Tag.table.c.name.like(f"{q}%"), item_tag_assoc_class.table.c.user_id == user.id ) - order_by = [func.count("*").desc()] - group_by = item_tag_assoc_class.table.c.tag_id # Do query and get result set. - query = select( - columns=cols_to_select, - from_obj=from_obj, - whereclause=where_clause, - group_by=group_by, - order_by=order_by, - limit=limit, + query = ( + select(item_tag_assoc_class.table.c.tag_id, func.count()) + .select_from(from_obj) + .where(where_clause) + .group_by(item_tag_assoc_class.table.c.tag_id) + .order_by(func.count().desc()) + .limit(limit) ) result_set = trans.sa_session.execute(query) # Create and return autocomplete data. @@ -133,23 +130,20 @@ def _get_tag_autocomplete_values(self, trans, q, limit, timestamp, user=None, it item_class = item.__class__ item_tag_assoc_class = self.get_tag_handler(trans).get_tag_assoc_class(item_class) # Build select statement. - cols_to_select = [item_tag_assoc_class.table.c.value, func.count("*")] from_obj = item_tag_assoc_class.table.join(item_class.table).join(trans.app.model.Tag.table) where_clause = and_( item_tag_assoc_class.table.c.user_id == user.id, trans.app.model.Tag.table.c.id == tag.id, item_tag_assoc_class.table.c.value.like(f"{tag_value}%"), ) - order_by = [func.count("*").desc(), item_tag_assoc_class.table.c.value] - group_by = item_tag_assoc_class.table.c.value # Do query and get result set. - query = select( - columns=cols_to_select, - from_obj=from_obj, - whereclause=where_clause, - group_by=group_by, - order_by=order_by, - limit=limit, + query = ( + select(item_tag_assoc_class.table.c.value, func.count()) + .select_from_obj(from_obj) + .where(where_clause) + .group_by(item_tag_assoc_class.table.c.value) + .order_by(func.count().desc(), item_tag_assoc_class.table.c.value) + .limit(limit) ) result_set = trans.sa_session.execute(query) # Create and return autocomplete data. @@ -165,14 +159,16 @@ def _get_usernames_for_tag(self, trans, user, tag, item_class, item_tag_assoc_cl most popular to least popular name. """ # Build select stmt. - cols_to_select = [item_tag_assoc_class.table.c.user_tname, func.count("*")] where_clause = and_( item_tag_assoc_class.table.c.user_id == user.id, item_tag_assoc_class.table.c.tag_id == tag.id ) - group_by = item_tag_assoc_class.table.c.user_tname - order_by = [func.count("*").desc()] # Do query and get result set. - query = select(columns=cols_to_select, whereclause=where_clause, group_by=group_by, order_by=order_by) + query = ( + select(item_tag_assoc_class.table.c.user_tname, func.count()) + .where(where_clause) + .group_by(item_tag_assoc_class.table.c.user_tname) + .order_by(func.count().desc()) + ) result_set = trans.sa_session.execute(query) user_tag_names = list() for row in result_set: From 00a600e095eec2a2f0bdf1f3199a31fc789608fc Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 22 Jun 2023 16:14:04 -0400 Subject: [PATCH 6/6] Fix select stmt in webapps/reports --- .../webapps/reports/controllers/history.py | 60 ++-- .../webapps/reports/controllers/home.py | 2 +- .../webapps/reports/controllers/jobs.py | 293 +++++++++--------- .../webapps/reports/controllers/tools.py | 78 ++--- .../webapps/reports/controllers/users.py | 66 ++-- .../webapps/reports/controllers/workflows.py | 92 +++--- 6 files changed, 287 insertions(+), 304 deletions(-) diff --git a/lib/galaxy/webapps/reports/controllers/history.py b/lib/galaxy/webapps/reports/controllers/history.py index c183a82dcd1e..4e36891fc0bb 100644 --- a/lib/galaxy/webapps/reports/controllers/history.py +++ b/lib/galaxy/webapps/reports/controllers/history.py @@ -63,43 +63,47 @@ def history_and_dataset_per_user(self, trans, **kwd): # from history h, galaxy_user u # where h.user_id = u.id and h.deleted='f' # group by email order by email desc - histories = sa.select( - ( + histories = ( + sa.select( sa.func.count(galaxy.model.History.table.c.id).label("history"), galaxy.model.User.table.c.email.label("email"), - ), - from_obj=[sa.outerjoin(galaxy.model.History.table, galaxy.model.User.table)], - whereclause=and_( - galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id, - galaxy.model.History.table.c.deleted == "f", - ), - group_by=["email"], - order_by=[sa.desc("email"), "history"], + ) + .select_from(sa.outerjoin(galaxy.model.History.table, galaxy.model.User.table)) + .where( + and_( + galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id, + galaxy.model.History.table.c.deleted == "f", + ) + ) + .group_by("email") + .order_by(sa.desc("email"), "history") ) # select u.email, count(d.id) # from galaxy_user u, dataset d, history_dataset_association hd,history h # where d.id=hd.dataset_id and h.id=hd.history_id and u.id = h.user_id and h.deleted='f' # group by u.email; - datasets = sa.select( - ( + datasets = ( + sa.select( sa.func.count(galaxy.model.Dataset.table.c.id).label("dataset"), sa.func.sum(galaxy.model.Dataset.table.c.total_size).label("size"), galaxy.model.User.table.c.email.label("email"), - ), - from_obj=[ + ) + .select_from( galaxy.model.User.table, galaxy.model.Dataset.table, galaxy.model.HistoryDatasetAssociation.table, galaxy.model.History.table, - ], - whereclause=and_( - galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id, - galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id, - galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id, - galaxy.model.History.table.c.deleted == "f", - ), - group_by=["email"], + ) + .where( + and_( + galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id, + galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id, + galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id, + galaxy.model.History.table.c.deleted == "f", + ) + ) + .group_by("email") ) # execute requests, replace None fields by "Unknown" @@ -178,11 +182,13 @@ def history_and_dataset_type(self, trans, **kwd): galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id, galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id, ) - histories = sa.select( - (galaxy.model.Dataset.table.c.state.label("state"), galaxy.model.History.table.c.name.label("name")), - from_obj=from_obj, - whereclause=whereclause, - order_by=["name"], + histories = ( + sa.select( + galaxy.model.Dataset.table.c.state.label("state"), galaxy.model.History.table.c.name.label("name") + ) + .select_from(from_obj) + .where(whereclause) + .order_by("name") ) # execute requests, replace None fields by "Unknown" diff --git a/lib/galaxy/webapps/reports/controllers/home.py b/lib/galaxy/webapps/reports/controllers/home.py index b8ca77697ff4..390b786da9f8 100644 --- a/lib/galaxy/webapps/reports/controllers/home.py +++ b/lib/galaxy/webapps/reports/controllers/home.py @@ -35,7 +35,7 @@ def run_stats(self, trans, **kwd): et_dy_data = [] recent_jobs = sa.select( - ((model.Job.id), (model.Job.create_time).label("create_time"), (model.Job.update_time).label("update_time")) + model.Job.id, model.Job.create_time.label("create_time"), model.Job.update_time.label("update_time") ) for job in trans.sa_session.execute(recent_jobs): diff --git a/lib/galaxy/webapps/reports/controllers/jobs.py b/lib/galaxy/webapps/reports/controllers/jobs.py index f324b19bc010..ede8c033c843 100644 --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -387,7 +387,6 @@ def specified_month_all(self, trans, **kwd): sort_id = specs.sort_id order = specs.order arrow = specs.arrow - _order = specs.exc_order if "entries" in kwd: entries = int(kwd.get("entries")) @@ -419,31 +418,31 @@ def specified_month_all(self, trans, **kwd): year_label = start_date.strftime("%Y") # Use to make the page table - month_jobs = sa.select( - ( + month_jobs = ( + sa.select( sa.func.date(model.Job.table.c.create_time).label("date"), sa.func.count(model.Job.table.c.id).label("total_jobs"), - ), - whereclause=sa.and_( - model.Job.table.c.user_id != monitor_user_id, - model.Job.table.c.create_time >= start_date, - model.Job.table.c.create_time < end_date, - ), - from_obj=[model.Job.table], - group_by=["date"], - order_by=[_order], - offset=offset, - limit=limit, + ) + .where( + sa.and_( + model.Job.table.c.user_id != monitor_user_id, + model.Job.table.c.create_time >= start_date, + model.Job.table.c.create_time < end_date, + ) + ) + .group_by("date") + .order_by(specs.exc_order) + .offset(offset) + .limit(limit) ) # Use to make trendline - all_jobs = sa.select( - (model.Job.table.c.create_time.label("date"), model.Job.table.c.id.label("id")), - whereclause=sa.and_( + all_jobs = sa.select(model.Job.table.c.create_time.label("date"), model.Job.table.c.id.label("id")).where( + sa.and_( model.Job.table.c.user_id != monitor_user_id, model.Job.table.c.create_time >= start_date, model.Job.table.c.create_time < end_date, - ), + ) ) trends = dict() @@ -496,7 +495,6 @@ def specified_month_in_error(self, trans, **kwd): sort_id = specs.sort_id order = specs.order arrow = specs.arrow - _order = specs.exc_order offset = 0 limit = 10 @@ -528,33 +526,35 @@ def specified_month_in_error(self, trans, **kwd): month_label = start_date.strftime("%B") year_label = start_date.strftime("%Y") - month_jobs_in_error = sa.select( - ( + month_jobs_in_error = ( + sa.select( sa.func.date(model.Job.table.c.create_time).label("date"), sa.func.count(model.Job.table.c.id).label("total_jobs"), - ), - whereclause=sa.and_( - model.Job.table.c.user_id != monitor_user_id, - model.Job.table.c.state == "error", - model.Job.table.c.create_time >= start_date, - model.Job.table.c.create_time < end_date, - ), - from_obj=[model.Job.table], - group_by=["date"], - order_by=[_order], - offset=offset, - limit=limit, + ) + .where( + sa.and_( + model.Job.table.c.user_id != monitor_user_id, + model.Job.table.c.state == "error", + model.Job.table.c.create_time >= start_date, + model.Job.table.c.create_time < end_date, + ) + ) + .group_by("date") + .order_by(specs.exc_order) + .offset(offset) + .limit(limit) ) # Use to make trendline all_jobs_in_error = sa.select( - (model.Job.table.c.create_time.label("date"), model.Job.table.c.id.label("id")), - whereclause=sa.and_( + model.Job.table.c.create_time.label("date"), model.Job.table.c.id.label("id") + ).where( + sa.and_( model.Job.table.c.user_id != monitor_user_id, model.Job.table.c.state == "error", model.Job.table.c.create_time >= start_date, model.Job.table.c.create_time < end_date, - ), + ) ) trends = dict() @@ -635,38 +635,37 @@ def per_month_all(self, trans, **kwd): # Use to make the page table if by_destination == "true": - jobs_by_month = sa.select( - ( + jobs_by_month = ( + sa.select( self.select_month(model.Job.table.c.create_time).label("date"), model.Job.table.c.destination_id.label("destination_id"), sa.func.sum(model.Job.table.c.update_time - model.Job.table.c.create_time).label("execute_time"), sa.func.count(model.Job.table.c.id).label("total_jobs"), model.User.table.c.email.label("user_email"), - ), - whereclause=model.Job.table.c.user_id != monitor_user_id, - from_obj=[sa.join(model.Job.table, model.User.table)], - group_by=["user_email", "date", "destination_id"], - order_by=[_order], - offset=offset, - limit=limit, + ) + .where(model.Job.table.c.user_id != monitor_user_id) + .select_from(sa.join(model.Job.table, model.User.table)) + .group_by("user_email", "date", "destination_id") + .order_by(_order) + .offset(offset) + .limit(limit) ) else: - jobs_by_month = sa.select( - ( + jobs_by_month = ( + sa.select( self.select_month(model.Job.table.c.create_time).label("date"), sa.func.count(model.Job.table.c.id).label("total_jobs"), - ), - whereclause=model.Job.table.c.user_id != monitor_user_id, - from_obj=[model.Job.table], - group_by=self.group_by_month(model.Job.table.c.create_time), - order_by=[_order], - offset=offset, - limit=limit, + ) + .where(model.Job.table.c.user_id != monitor_user_id) + .group_by(self.group_by_month(model.Job.table.c.create_time)) + .order_by(_order) + .offset(offset) + .limit(limit) ) # Use to make sparkline all_jobs = sa.select( - (self.select_day(model.Job.table.c.create_time).label("date"), model.Job.table.c.id.label("id")) + self.select_day(model.Job.table.c.create_time).label("date"), model.Job.table.c.id.label("id") ) trends = self._calculate_trends_for_jobs(trans.sa_session, all_jobs) @@ -730,24 +729,22 @@ def per_month_in_error(self, trans, **kwd): monitor_user_id = get_monitor_id(trans, monitor_email) # Use to make the page table - jobs_in_error_by_month = sa.select( - ( + jobs_in_error_by_month = ( + sa.select( self.select_month(model.Job.table.c.create_time).label("date"), sa.func.count(model.Job.table.c.id).label("total_jobs"), - ), - whereclause=sa.and_(model.Job.table.c.state == "error", model.Job.table.c.user_id != monitor_user_id), - from_obj=[model.Job.table], - group_by=self.group_by_month(model.Job.table.c.create_time), - order_by=[_order], - offset=offset, - limit=limit, + ) + .where(sa.and_(model.Job.table.c.state == "error", model.Job.table.c.user_id != monitor_user_id)) + .group_by(self.group_by_month(model.Job.table.c.create_time)) + .order_by(_order) + .offset(offset) + .limit(limit) ) # Use to make trendline all_jobs = sa.select( - (self.select_day(model.Job.table.c.create_time).label("date"), model.Job.table.c.id.label("id")), - whereclause=sa.and_(model.Job.table.c.state == "error", model.Job.table.c.user_id != monitor_user_id), - ) + self.select_day(model.Job.table.c.create_time).label("date"), model.Job.table.c.id.label("id") + ).where(sa.and_(model.Job.table.c.state == "error", model.Job.table.c.user_id != monitor_user_id)) trends = self._calculate_trends_for_jobs(trans.sa_session, all_jobs) jobs = self._calculate_job_table(trans.sa_session, jobs_in_error_by_month) @@ -814,27 +811,30 @@ def per_user(self, trans, **kwd): jobs = [] if by_destination == "true": - jobs_per_user = sa.select( - ( + jobs_per_user = ( + sa.select( model.User.table.c.email.label("user_email"), sa.func.count(model.Job.table.c.id).label("total_jobs"), model.Job.table.c.destination_id.label("destination_id"), - ), - from_obj=[sa.outerjoin(model.Job.table, model.User.table)], - group_by=["user_email", "destination_id"], - order_by=[_order], - offset=offset, - limit=limit, + ) + .select_from(sa.outerjoin(model.Job.table, model.User.table)) + .group_by("user_email", "destination_id") + .order_by(_order) + .offset(offset) + .limit(limit) ) else: - jobs_per_user = sa.select( - (model.User.table.c.email.label("user_email"), sa.func.count(model.Job.table.c.id).label("total_jobs")), - from_obj=[sa.outerjoin(model.Job.table, model.User.table)], - group_by=["user_email"], - order_by=[_order], - offset=offset, - limit=limit, + jobs_per_user = ( + sa.select( + model.User.table.c.email.label("user_email"), + sa.func.count(model.Job.table.c.id).label("total_jobs"), + ) + .select_from(sa.outerjoin(model.Job.table, model.User.table)) + .group_by("user_email") + .order_by(_order) + .offset(offset) + .limit(limit) ) q_time.start() @@ -859,14 +859,14 @@ def per_user(self, trans, **kwd): users = sa.select(model.User.table.c.email).select_from(model.User.table) - all_jobs_per_user = sa.select( - ( + all_jobs_per_user = ( + sa.select( model.Job.table.c.id.label("id"), model.Job.table.c.create_time.label("date"), model.User.table.c.email.label("user_email"), - ), - from_obj=[sa.outerjoin(model.Job.table, model.User.table)], - whereclause=model.User.table.c.email.in_(users), + ) + .select_from(sa.outerjoin(model.Job.table, model.User.table)) + .where(model.User.table.c.email.in_(users)) ) currday = datetime.today() @@ -939,34 +939,34 @@ def user_per_month(self, trans, **kwd): _order = specs.exc_order if by_destination == "true": - q = sa.select( - ( + q = ( + sa.select( self.select_month(model.Job.table.c.create_time).label("date"), model.Job.table.c.destination_id.label("destination_id"), sa.func.sum(model.Job.table.c.update_time - model.Job.table.c.create_time).label("execute_time"), sa.func.count(model.Job.table.c.id).label("total_jobs"), - ), - whereclause=model.User.table.c.email == email, - from_obj=[sa.join(model.Job.table, model.User.table)], - group_by=["date", "destination_id"], - order_by=[_order], + ) + .where(model.User.table.c.email == email) + .select_from(sa.join(model.Job.table, model.User.table)) + .group_by("date", "destination_id") + .order_by(_order) ) else: - q = sa.select( - ( + q = ( + sa.select( self.select_month(model.Job.table.c.create_time).label("date"), sa.func.count(model.Job.table.c.id).label("total_jobs"), - ), - whereclause=model.User.table.c.email == email, - from_obj=[sa.join(model.Job.table, model.User.table)], - group_by=self.group_by_month(model.Job.table.c.create_time), - order_by=[_order], + ) + .where(model.User.table.c.email == email) + .select_from(sa.join(model.Job.table, model.User.table)) + .group_by(self.group_by_month(model.Job.table.c.create_time)) + .order_by(_order) ) - all_jobs_per_user = sa.select( - (model.Job.table.c.create_time.label("date"), model.Job.table.c.id.label("job_id")), - whereclause=sa.and_(model.User.table.c.email == email), - from_obj=[sa.join(model.Job.table, model.User.table)], + all_jobs_per_user = ( + sa.select(model.Job.table.c.create_time.label("date"), model.Job.table.c.id.label("job_id")) + .where(sa.and_(model.User.table.c.email == email)) + .select_from(sa.join(model.Job.table, model.User.table)) ) trends = dict() @@ -1065,25 +1065,22 @@ def per_tool(self, trans, **kwd): monitor_user_id = get_monitor_id(trans, monitor_email) jobs = [] - q = sa.select( - (model.Job.table.c.tool_id.label("tool_id"), sa.func.count(model.Job.table.c.id).label("total_jobs")), - whereclause=model.Job.table.c.user_id != monitor_user_id, - from_obj=[model.Job.table], - group_by=["tool_id"], - order_by=[_order], - offset=offset, - limit=limit, + q = ( + sa.select( + model.Job.table.c.tool_id.label("tool_id"), sa.func.count(model.Job.table.c.id).label("total_jobs") + ) + .where(model.Job.table.c.user_id != monitor_user_id) + .group_by("tool_id") + .order_by(_order) + .offset(offset) + .limit(limit) ) all_jobs_per_tool = sa.select( - ( - model.Job.table.c.tool_id.label("tool_id"), - model.Job.table.c.id.label("id"), - self.select_day(model.Job.table.c.create_time).label("date"), - ), - whereclause=model.Job.table.c.user_id != monitor_user_id, - from_obj=[model.Job.table], - ) + model.Job.table.c.tool_id.label("tool_id"), + model.Job.table.c.id.label("id"), + self.select_day(model.Job.table.c.create_time).label("date"), + ).where(model.Job.table.c.user_id != monitor_user_id) currday = date.today() trends = dict() @@ -1166,25 +1163,22 @@ def errors_per_tool(self, trans, **kwd): # In case we don't know which is the monitor user we will query for all jobs monitor_user_id = get_monitor_id(trans, monitor_email) - jobs_in_error_per_tool = sa.select( - (model.Job.table.c.tool_id.label("tool_id"), sa.func.count(model.Job.table.c.id).label("total_jobs")), - whereclause=sa.and_(model.Job.table.c.state == "error", model.Job.table.c.user_id != monitor_user_id), - from_obj=[model.Job.table], - group_by=["tool_id"], - order_by=[_order], - offset=offset, - limit=limit, + jobs_in_error_per_tool = ( + sa.select( + model.Job.table.c.tool_id.label("tool_id"), sa.func.count(model.Job.table.c.id).label("total_jobs") + ) + .where(sa.and_(model.Job.table.c.state == "error", model.Job.table.c.user_id != monitor_user_id)) + .group_by("tool_id") + .order_by(_order) + .offset(offset) + .limit(limit) ) all_jobs_per_tool_errors = sa.select( - ( - self.select_day(model.Job.table.c.create_time).label("date"), - model.Job.table.c.id.label("id"), - model.Job.table.c.tool_id.label("tool_id"), - ), - whereclause=sa.and_(model.Job.table.c.state == "error", model.Job.table.c.user_id != monitor_user_id), - from_obj=[model.Job.table], - ) + self.select_day(model.Job.table.c.create_time).label("date"), + model.Job.table.c.id.label("id"), + model.Job.table.c.tool_id.label("tool_id"), + ).where(sa.and_(model.Job.table.c.state == "error", model.Job.table.c.user_id != monitor_user_id)) currday = date.today() trends = dict() @@ -1243,27 +1237,22 @@ def tool_per_month(self, trans, **kwd): tool_id = params.get("tool_id", "Add a column1") specified_date = params.get("specified_date", datetime.utcnow().strftime("%Y-%m-%d")) - q = sa.select( - ( + q = ( + sa.select( self.select_month(model.Job.table.c.create_time).label("date"), sa.func.count(model.Job.table.c.id).label("total_jobs"), - ), - whereclause=sa.and_(model.Job.table.c.tool_id == tool_id, model.Job.table.c.user_id != monitor_user_id), - from_obj=[model.Job.table], - group_by=self.group_by_month(model.Job.table.c.create_time), - order_by=[_order], + ) + .where(sa.and_(model.Job.table.c.tool_id == tool_id, model.Job.table.c.user_id != monitor_user_id)) + .group_by(self.group_by_month(model.Job.table.c.create_time)) + .order_by(_order) ) # Use to make sparkline all_jobs_for_tool = sa.select( - ( - self.select_month(model.Job.table.c.create_time).label("month"), - self.select_day(model.Job.table.c.create_time).label("day"), - model.Job.table.c.id.label("id"), - ), - whereclause=sa.and_(model.Job.table.c.tool_id == tool_id, model.Job.table.c.user_id != monitor_user_id), - from_obj=[model.Job.table], - ) + self.select_month(model.Job.table.c.create_time).label("month"), + self.select_day(model.Job.table.c.create_time).label("day"), + model.Job.table.c.id.label("id"), + ).where(sa.and_(model.Job.table.c.tool_id == tool_id, model.Job.table.c.user_id != monitor_user_id)) trends = dict() for job in trans.sa_session.execute(all_jobs_for_tool): job_day = int(job.day.strftime("%-d")) - 1 diff --git a/lib/galaxy/webapps/reports/controllers/tools.py b/lib/galaxy/webapps/reports/controllers/tools.py index 38f7738a78fd..0979f522b3e7 100644 --- a/lib/galaxy/webapps/reports/controllers/tools.py +++ b/lib/galaxy/webapps/reports/controllers/tools.py @@ -90,19 +90,21 @@ def tools_and_job_state(self, trans, **kwd): data = {} # select count(id), tool_id from job where state='ok' group by tool_id; - tools_and_jobs_ok = sa.select( - (galaxy.model.Job.table.c.tool_id.label("tool"), sa.func.count(galaxy.model.Job.table.c.id).label("job")), - from_obj=[galaxy.model.Job.table], - whereclause=(galaxy.model.Job.table.c.state == "ok"), - group_by=["tool"], + tools_and_jobs_ok = ( + sa.select( + galaxy.model.Job.table.c.tool_id.label("tool"), sa.func.count(galaxy.model.Job.table.c.id).label("job") + ) + .where(galaxy.model.Job.table.c.state == "ok") + .group_by("tool") ) # select count(id), tool_id from job where state='error' group by tool_id; - tools_and_jobs_error = sa.select( - (galaxy.model.Job.table.c.tool_id.label("tool"), sa.func.count(galaxy.model.Job.table.c.id).label("job")), - from_obj=[galaxy.model.Job.table], - whereclause=(galaxy.model.Job.table.c.state == "error"), - group_by=["tool"], + tools_and_jobs_error = ( + sa.select( + galaxy.model.Job.table.c.tool_id.label("tool"), sa.func.count(galaxy.model.Job.table.c.id).label("job") + ) + .where(galaxy.model.Job.table.c.state == "error") + .group_by("tool") ) tools_and_jobs_ok = dict(list(trans.sa_session.execute(tools_and_jobs_ok))) @@ -148,25 +150,23 @@ def tools_and_job_state_per_month(self, trans, **kwd): data = {} # select count(id), create_time from job where state='ok' and tool_id=$tool group by date; - date_and_jobs_ok = sa.select( - ( + date_and_jobs_ok = ( + sa.select( sa.func.date(galaxy.model.Job.table.c.create_time).label("date"), sa.func.count(galaxy.model.Job.table.c.id).label("job"), - ), - from_obj=[galaxy.model.Job.table], - whereclause=and_(galaxy.model.Job.table.c.state == "ok", galaxy.model.Job.table.c.tool_id == tool), - group_by=["date"], + ) + .where(and_(galaxy.model.Job.table.c.state == "ok", galaxy.model.Job.table.c.tool_id == tool)) + .group_by("date") ) # select count(id), create_time from job where state='error' and tool_id=$tool group by date; - date_and_jobs_error = sa.select( - ( + date_and_jobs_error = ( + sa.select( sa.func.date(galaxy.model.Job.table.c.create_time).label("date"), sa.func.count(galaxy.model.Job.table.c.id).label("job"), - ), - from_obj=[galaxy.model.Job.table], - whereclause=and_(galaxy.model.Job.table.c.state == "error", galaxy.model.Job.table.c.tool_id == tool), - group_by=["date"], + ) + .where(and_(galaxy.model.Job.table.c.state == "error", galaxy.model.Job.table.c.tool_id == tool)) + .group_by("date") ) date_and_jobs_ok = dict(list(trans.sa_session.execute(date_and_jobs_ok))) @@ -215,13 +215,10 @@ def tool_execution_time(self, trans, **kwd): sort_keys = (lambda v: v.lower(), lambda v: data[v]["avg"], lambda v: data[v]["min"], lambda v: data[v]["max"]) jobs_times = sa.select( - ( - galaxy.model.Job.table.c.tool_id.label("name"), - galaxy.model.Job.table.c.create_time.label("create_time"), - galaxy.model.Job.table.c.update_time.label("update_time"), - galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time, - ), - from_obj=[galaxy.model.Job.table], + galaxy.model.Job.table.c.tool_id.label("name"), + galaxy.model.Job.table.c.create_time.label("create_time"), + galaxy.model.Job.table.c.update_time.label("update_time"), + galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time, ) jobs_times = [ @@ -285,16 +282,15 @@ def tool_execution_time_per_month(self, trans, **kwd): ordered_data = {} sort_keys = [(lambda v, i=i: v[i]) for i in range(4)] - jobs_times = sa.select( - ( + jobs_times = ( + sa.select( sa.func.date_trunc("month", galaxy.model.Job.table.c.create_time).label("date"), sa.func.max(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), sa.func.avg(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), sa.func.min(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), - ), - from_obj=[galaxy.model.Job.table], - whereclause=galaxy.model.Job.table.c.tool_id == tool, - group_by=["date"], + ) + .where(galaxy.model.Job.table.c.tool_id == tool) + .group_by("date") ) months = list(trans.sa_session.execute(jobs_times.execute)) @@ -332,16 +328,8 @@ def tool_error_messages(self, trans, **kwd): tool_errors = [ [unicodify(a), b] for a, b in trans.sa_session.execute( - sa.select( - ( - galaxy.model.Job.table.c.tool_stderr, - galaxy.model.Job.table.c.create_time, - ), - from_obj=[galaxy.model.Job.table], - whereclause=and_( - galaxy.model.Job.table.c.tool_id == tool_name, - galaxy.model.Job.table.c.state == "error", - ), + sa.select(galaxy.model.Job.table.c.tool_stderr, galaxy.model.Job.table.c.create_time).where( + and_(galaxy.model.Job.table.c.tool_id == tool_name, galaxy.model.Job.table.c.state == "error") ) ) ] diff --git a/lib/galaxy/webapps/reports/controllers/users.py b/lib/galaxy/webapps/reports/controllers/users.py index 773f2dcd691c..543edb7979ac 100644 --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -37,16 +37,14 @@ def registered_users_per_month(self, trans, **kwd): sort_id = specs.sort_id order = specs.order arrow = specs.arrow - _order = specs.exc_order - q = sa.select( - ( + q = ( + sa.select( self.select_month(galaxy.model.User.table.c.create_time).label("date"), sa.func.count(galaxy.model.User.table.c.id).label("num_users"), - ), - from_obj=[galaxy.model.User.table], - group_by=self.group_by_month(galaxy.model.User.table.c.create_time), - order_by=[_order], + ) + .group_by(self.group_by_month(galaxy.model.User.table.c.create_time)) + .order_by(specs.exc_order) ) users = [] for row in trans.sa_session.execute(q): @@ -71,17 +69,20 @@ def specified_month(self, trans, **kwd): end_date = start_date + timedelta(days=calendar.monthrange(year, month)[1]) month_label = start_date.strftime("%B") year_label = start_date.strftime("%Y") - q = sa.select( - ( + q = ( + sa.select( self.select_day(galaxy.model.User.table.c.create_time).label("date"), sa.func.count(galaxy.model.User.table.c.id).label("num_users"), - ), - whereclause=sa.and_( - galaxy.model.User.table.c.create_time >= start_date, galaxy.model.User.table.c.create_time < end_date - ), - from_obj=[galaxy.model.User.table], - group_by=self.group_by_day(galaxy.model.User.table.c.create_time), - order_by=[sa.desc("date")], + ) + .where( + sa.and_( + galaxy.model.User.table.c.create_time >= start_date, + galaxy.model.User.table.c.create_time < end_date, + ) + ) + .select_from(galaxy.model.User.table) + .group_by(self.group_by_day(galaxy.model.User.table.c.create_time)) + .order_by(sa.desc("date")) ) users = [] for row in trans.sa_session.execute(q): @@ -109,13 +110,18 @@ def specified_date(self, trans, **kwd): day_label = start_date.strftime("%A") month_label = start_date.strftime("%B") year_label = start_date.strftime("%Y") - q = sa.select( - (self.select_day(galaxy.model.User.table.c.create_time).label("date"), galaxy.model.User.table.c.email), - whereclause=sa.and_( - galaxy.model.User.table.c.create_time >= start_date, galaxy.model.User.table.c.create_time < end_date - ), - from_obj=[galaxy.model.User.table], - order_by=[galaxy.model.User.table.c.email], + q = ( + sa.select( + self.select_day(galaxy.model.User.table.c.create_time).label("date"), galaxy.model.User.table.c.email + ) + .where( + sa.and_( + galaxy.model.User.table.c.create_time >= start_date, + galaxy.model.User.table.c.create_time < end_date, + ) + ) + .select_from(galaxy.model.User.table) + .order_by(galaxy.model.User.table.c.email) ) users = [] for row in trans.sa_session.execute(q): @@ -221,15 +227,15 @@ def history_per_user(self, trans, **kwd): reverse = descending == 1 sort_keys = (lambda v: v[0].lower(), lambda v: v[1]) - req = sa.select( - ( + req = ( + sa.select( sa.func.count(galaxy.model.History.table.c.id).label("history"), galaxy.model.User.table.c.username.label("username"), - ), - from_obj=[sa.outerjoin(galaxy.model.History.table, galaxy.model.User.table)], - whereclause=galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id, - group_by=["username"], - order_by=[sa.desc("username"), "history"], + ) + .select_from(sa.outerjoin(galaxy.model.History.table, galaxy.model.User.table)) + .where(galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id) + .group_by("username") + .order_by(sa.desc("username"), "history") ) histories = [ diff --git a/lib/galaxy/webapps/reports/controllers/workflows.py b/lib/galaxy/webapps/reports/controllers/workflows.py index d458b56c353c..999a25fe9897 100644 --- a/lib/galaxy/webapps/reports/controllers/workflows.py +++ b/lib/galaxy/webapps/reports/controllers/workflows.py @@ -188,20 +188,20 @@ def per_month_all(self, trans, **kwd): else: page = 1 - q = sa.select( - ( + q = ( + sa.select( self.select_month(model.StoredWorkflow.table.c.create_time).label("date"), sa.func.count(model.StoredWorkflow.table.c.id).label("total_workflows"), - ), - from_obj=[sa.outerjoin(model.StoredWorkflow.table, model.User.table)], - group_by=self.group_by_month(model.StoredWorkflow.table.c.create_time), - order_by=[_order], - offset=offset, - limit=limit, + ) + .select_from(sa.outerjoin(model.StoredWorkflow.table, model.User.table)) + .group_by(self.group_by_month(model.StoredWorkflow.table.c.create_time)) + .order_by(_order) + .offset(offset) + .limit(limit) ) all_workflows = sa.select( - (self.select_day(model.StoredWorkflow.table.c.create_time).label("date"), model.StoredWorkflow.table.c.id) + self.select_day(model.StoredWorkflow.table.c.create_time).label("date"), model.StoredWorkflow.table.c.id ) trends = dict() @@ -274,26 +274,23 @@ def per_user(self, trans, **kwd): page = 1 workflows = [] - q = sa.select( - ( + q = ( + sa.select( model.User.table.c.email.label("user_email"), sa.func.count(model.StoredWorkflow.table.c.id).label("total_workflows"), - ), - from_obj=[sa.outerjoin(model.StoredWorkflow.table, model.User.table)], - group_by=["user_email"], - order_by=[_order], - offset=offset, - limit=limit, + ) + .select_from(sa.outerjoin(model.StoredWorkflow.table, model.User.table)) + .group_by("user_email") + .order_by(_order) + .offset(offset) + .limit(limit) ) all_workflows_per_user = sa.select( - ( - model.User.table.c.email.label("user_email"), - self.select_day(model.StoredWorkflow.table.c.create_time).label("date"), - model.StoredWorkflow.table.c.id, - ), - from_obj=[sa.outerjoin(model.StoredWorkflow.table, model.User.table)], - ) + model.User.table.c.email.label("user_email"), + self.select_day(model.StoredWorkflow.table.c.create_time).label("date"), + model.StoredWorkflow.table.c.id, + ).select_from(sa.outerjoin(model.StoredWorkflow.table, model.User.table)) currday = datetime.today() trends = dict() for workflow in trans.sa_session.execute(all_workflows_per_user): @@ -345,22 +342,19 @@ def user_per_month(self, trans, **kwd): email = util.restore_text(params.get("email", "")) user_id = trans.security.decode_id(params.get("id", "")) - q = sa.select( - ( + q = ( + sa.select( self.select_month(model.StoredWorkflow.table.c.create_time).label("date"), sa.func.count(model.StoredWorkflow.table.c.id).label("total_workflows"), - ), - whereclause=model.StoredWorkflow.table.c.user_id == user_id, - from_obj=[model.StoredWorkflow.table], - group_by=self.group_by_month(model.StoredWorkflow.table.c.create_time), - order_by=[_order], + ) + .where(model.StoredWorkflow.table.c.user_id == user_id) + .group_by(self.group_by_month(model.StoredWorkflow.table.c.create_time)) + .order_by(_order) ) all_workflows_user_month = sa.select( - (self.select_day(model.StoredWorkflow.table.c.create_time).label("date"), model.StoredWorkflow.table.c.id), - whereclause=model.StoredWorkflow.table.c.user_id == user_id, - from_obj=[model.StoredWorkflow.table], - ) + self.select_day(model.StoredWorkflow.table.c.create_time).label("date"), model.StoredWorkflow.table.c.id + ).where(model.StoredWorkflow.table.c.user_id == user_id) trends = dict() for workflow in trans.sa_session.execute(all_workflows_user_month): @@ -428,28 +422,28 @@ def per_workflow(self, trans, **kwd): # In case we don't know which is the monitor user we will query for all jobs - q = sa.select( - ( + q = ( + sa.select( model.Workflow.table.c.id.label("workflow_id"), sa.func.min(model.Workflow.table.c.name).label("workflow_name"), sa.func.count(model.WorkflowInvocation.table.c.id).label("total_runs"), - ), - from_obj=[model.Workflow.table, model.WorkflowInvocation.table], - whereclause=sa.and_(model.WorkflowInvocation.table.c.workflow_id == model.Workflow.table.c.id), - group_by=[model.Workflow.table.c.id], - order_by=[_order], - offset=offset, - limit=limit, + ) + .select_from(model.Workflow.table, model.WorkflowInvocation.table) + .where(sa.and_(model.WorkflowInvocation.table.c.workflow_id == model.Workflow.table.c.id)) + .group_by(model.Workflow.table.c.id) + .order_by(_order) + .offset(offset) + .limit(limit) ) - all_runs_per_workflow = sa.select( - ( + all_runs_per_workflow = ( + sa.select( model.Workflow.table.c.id.label("workflow_id"), model.Workflow.table.c.name.label("workflow_name"), self.select_day(model.WorkflowInvocation.table.c.create_time).label("date"), - ), - from_obj=[model.Workflow.table, model.WorkflowInvocation.table], - whereclause=sa.and_(model.WorkflowInvocation.table.c.workflow_id == model.Workflow.table.c.id), + ) + .select_from(model.Workflow.table, model.WorkflowInvocation.table) + .where(sa.and_(model.WorkflowInvocation.table.c.workflow_id == model.Workflow.table.c.id)) ) currday = date.today()