From d6363c6e9d1514cd9e0173699d0c395bff80e6fb Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Fri, 11 Oct 2024 10:35:52 -0700 Subject: [PATCH] Added support for temporary table upload Co-authored-by: Adam Ginsburg Co-authored-by: Adam Ginsburg --- CHANGES.rst | 2 + astroquery/alma/core.py | 20 +++++++-- astroquery/alma/tests/test_alma.py | 19 +++++--- astroquery/alma/tests/test_alma_remote.py | 54 ++++++++++++++++++----- docs/alma/alma.rst | 29 ++++++++++++ 5 files changed, 102 insertions(+), 22 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 79a1cb271d..48b88106cb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -22,6 +22,8 @@ alma - Added support for frequency_resolution in KHz [#3035] +- Added support for temporary upload tables in query_tap [#3118] + - Changed the way galactic ranges are used in queries [#3105] ehst diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index ce9509cee8..ee92030e22 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -138,7 +138,7 @@ 'Project': { 'Project code': ['project_code', 'proposal_id', _gen_str_sql], 'Project title': ['project_title', 'obs_title', _gen_str_sql], - 'PI name': ['pi_name', 'obs_creator_name', _gen_str_sql], + 'PI name': ['pi_name', 'pi_name', _gen_str_sql], 'Proposal authors': ['proposal_authors', 'proposal_authors', _gen_str_sql], 'Project abstract': ['project_abstract', 'proposal_abstract', _gen_str_sql], 'Publication count': ['publication_count', 'NA', _gen_str_sql], @@ -678,19 +678,33 @@ def query_sia(self, *, pos=None, band=None, time=None, pol=None, query_sia.__doc__ = query_sia.__doc__.replace('_SIA2_PARAMETERS', SIA2_PARAMETERS_DESC) - def query_tap(self, query, maxrec=None): + def query_tap(self, query, *, maxrec=None, uploads=None): """ Send query to the ALMA TAP. Results in pyvo.dal.TapResult format. result.table in Astropy table format Parameters ---------- + query : str + ADQL query to execute maxrec : int maximum number of records to return + uploads : dict + a mapping from temporary table names to objects containing a votable. These + temporary tables can be referred to in queries. The keys in the dictionary are + the names of temporary tables which need to be prefixed with the TAP_UPLOAD + schema in the actual query. The values are either astropy.table.Table instances + or file names or file like handles such as io.StringIO to table definition in + IVOA VOTable format. + + Examples + ---------- + >>> uploads = {'tmptable': '/tmp/tmptable_def.xml'} + >>> rslt = query_tap(self, query, maxrec=None, uploads=uploads) """ log.debug('TAP query: {}'.format(query)) - return self.tap.search(query, language='ADQL', maxrec=maxrec) + return self.tap.search(query, language='ADQL', maxrec=maxrec, uploads=uploads) def help_tap(self): print('Table to query is "voa.ObsCore".') diff --git a/astroquery/alma/tests/test_alma.py b/astroquery/alma/tests/test_alma.py index a4a1110670..6bbdc50911 100644 --- a/astroquery/alma/tests/test_alma.py +++ b/astroquery/alma/tests/test_alma.py @@ -269,7 +269,7 @@ def test_query(): "select * from ivoa.obscore WHERE " "(INTERSECTS(CIRCLE('ICRS',1.0,2.0,1.0), s_region) = 1) " "AND science_observation='T' AND data_rights='Public'", - language='ADQL', maxrec=None) + language='ADQL', maxrec=None, uploads=None) # one row result tap_mock = Mock() @@ -291,7 +291,7 @@ def test_query(): "(INTERSECTS(CIRCLE('ICRS',1.0,2.0,0.16666666666666666), s_region) = 1) " "AND band_list LIKE '%3%' AND science_observation='T' AND " "data_rights='Proprietary'", - language='ADQL', maxrec=None) + language='ADQL', maxrec=None, uploads=None) # repeat for legacy columns mock_result = Mock() @@ -313,7 +313,7 @@ def test_query(): "(INTERSECTS(CIRCLE('ICRS',1.0,2.0,0.16666666666666666), s_region) = 1) " "AND band_list LIKE '%3%' AND science_observation='T' AND " "data_rights='Proprietary'", - language='ADQL', maxrec=None) + language='ADQL', maxrec=None, uploads=None) row_legacy = result_legacy[0] row = result[0] for item in _OBSCORE_TO_ALMARESULT.items(): @@ -347,7 +347,7 @@ def test_query(): "(band_list LIKE '%1%' OR band_list LIKE '%3%') AND " "t_min=55197.0 AND pol_states='/XX/YY/' AND s_fov=0.012313 AND " "t_exptime=25 AND science_observation='F'", - language='ADQL', maxrec=None + language='ADQL', maxrec=None, uploads=None ) tap_mock.reset() @@ -361,7 +361,7 @@ def test_query(): "AND spectral_resolution=2000000 " "AND (INTERSECTS(CIRCLE('ICRS',1.0,2.0,1.0), " "s_region) = 1) AND science_observation='T' AND data_rights='Public'", - language='ADQL', maxrec=None) + language='ADQL', maxrec=None, uploads=None) @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyUserWarning") @@ -499,9 +499,14 @@ def test_tap(): alma._tap = tap_mock result = alma.query_tap('select * from ivoa.ObsCore') assert len(result.table) == 0 - tap_mock.search.assert_called_once_with('select * from ivoa.ObsCore', - language='ADQL', maxrec=None) + language='ADQL', maxrec=None, uploads=None) + + tap_mock.search.reset_mock() + result = alma.query_tap('select * from ivoa.ObsCore', maxrec=10, uploads={'tmptable': 'votable_file.xml'}) + assert len(result.table) == 0 + tap_mock.search.assert_called_once_with( + 'select * from ivoa.ObsCore', language='ADQL', maxrec=10, uploads={'tmptable': 'votable_file.xml'}) @pytest.mark.parametrize('data_archive_url', diff --git a/astroquery/alma/tests/test_alma_remote.py b/astroquery/alma/tests/test_alma_remote.py index 46ce6f91a2..32131ca08e 100644 --- a/astroquery/alma/tests/test_alma_remote.py +++ b/astroquery/alma/tests/test_alma_remote.py @@ -1,6 +1,7 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst from datetime import datetime, timezone import os +from io import StringIO from pathlib import Path from urllib.parse import urlparse import re @@ -23,7 +24,6 @@ except ImportError: HAS_REGIONS = False -# ALMA tests involving staging take too long, leading to travis timeouts # TODO: make this a configuration item SKIP_SLOW = True @@ -91,6 +91,7 @@ def test_SgrAstar(self, tmp_path, alma): assert '2013.1.00857.S' in result_s['Project code'] + @pytest.mark.skipif("SKIP_SLOW") def test_freq(self, alma): payload = {'frequency': '85..86'} result = alma.query(payload) @@ -98,7 +99,7 @@ def test_freq(self, alma): for row in result: # returned em_min and em_max are in m assert row['frequency'] >= 85 - assert row['frequency'] <= 100 + assert row['frequency'] <= 86 assert '3' in row['band_list'] def test_bands(self, alma): @@ -216,10 +217,7 @@ def test_data_info(self, tmp_path, alma): trimmed_access_url_list = [e for e in data_info_tar['access_url'].data if len(e) > 0] trimmed_access_urls = (trimmed_access_url_list,) mock_calls = download_files_mock.mock_calls[0][1] - print(f"\n\nComparing {mock_calls} to {trimmed_access_urls}\n\n") - # comparison = download_files_mock.mock_calls[0][1] == data_info_tar['access_url'] assert mock_calls == trimmed_access_urls - # assert comparison.all() def test_download_data(self, tmp_path, alma): # test only fits files from a program @@ -239,9 +237,8 @@ def test_download_data(self, tmp_path, alma): alma._download_file.call_count == len(results) assert len(results) == len(urls) + @pytest.mark.skipif("SKIP_SLOW") def test_download_and_extract(self, tmp_path, alma): - # TODO: slowish, runs for ~90s - alma.cache_location = tmp_path alma._cycle0_tarfile_content_table = {'ID': ''} @@ -345,16 +342,18 @@ def test_misc(self, alma): result = alma.query_object('M83', public=True, science=True) assert len(result) > 0 - result = alma.query(payload={'pi_name': '*Bally*'}, public=False, - maxrec=10) + with pytest.warns(expected_warning=DALOverflowWarning, + match="Partial result set. Potential causes MAXREC, async storage space, etc."): + result = alma.query(payload={'pi_name': 'Bally*'}, public=True, + maxrec=10) assert result # Add overwrite=True in case the test previously died unexpectedly # and left the temp file. result.write('/tmp/alma-onerow.txt', format='ascii', overwrite=True) for row in result: - assert 'Bally' in row['obs_creator_name'] + assert 'Bally' in row['pi_name'] result = alma.query(payload=dict(project_code='2016.1.00165.S'), - public=False) + public=True) assert result for row in result: assert '2016.1.00165.S' == row['proposal_id'] @@ -398,8 +397,9 @@ def test_misc(self, alma): assert result for row in result: assert '6' == row['band_list'] - assert 'ginsburg' in row['obs_creator_name'].lower() + assert 'ginsburg' in row['pi_name'].lower() + @pytest.mark.skip("Not sure what this is supposed to do") def test_user(self, alma): # miscellaneous set of tests from current users rslt = alma.query({'band_list': [6], 'project_code': '2012.1.*'}, @@ -561,6 +561,36 @@ def test_big_download_regression(alma): alma.download_files([files['access_url'][3]]) +@pytest.mark.remote_data +def test_tap_upload(): + tmp_table = StringIO(''' + + + + + external URI for the physical artifact + + + + + + + + +
2013.1.01365.S
+ + ''') + + alma = Alma() + res = alma.query_tap( + 'select top 3 proposal_id from ivoa.ObsCore oc join TAP_UPLOAD.proj_codes pc on oc.proposal_id=pc.prop_id', + uploads={'proj_codes': tmp_table}) + assert len(res) == 3 + for row in res: + assert row['proposal_id'] == '2013.1.01365.S' + + @pytest.mark.remote_data def test_download_html_file(alma, tmp_path): alma.cache_location = tmp_path diff --git a/docs/alma/alma.rst b/docs/alma/alma.rst index 5dd9a86e91..a2d39e3b79 100644 --- a/docs/alma/alma.rst +++ b/docs/alma/alma.rst @@ -210,6 +210,35 @@ One can also query by keyword, spatial resolution, etc: ... "in ('Disks around high-mass stars', 'Asymptotic Giant Branch (AGB) stars') " ... "AND science_observation='T'") # doctest: +IGNORE_OUTPUT +''query_tap'' also supports uploading temporary tables that can be used to join to in queries. +These temporary tables can be defined as ''astropy.table.Table'' instances or references to file names or +file like handles (''io.StringIO'' instances for example) of table definitions in IVOA VOTable format. +Below is a very simple example of using ''astropy.table.Table'' temporary table with the "proj_codes" name. +Note that the table name must always be prefixed with the "TAP_UPLOAD" schema when referenced in queries. + +.. doctest-remote-data:: + + >>> from astropy.table import Table + >>> tmp_table = Table([['2013.1.01365.S', '2013.A.00014.S']], names=['prop_id'], dtype=['S']) + >>> Alma.query_tap('select distinct target_name from ivoa.ObsCore oc join TAP_UPLOAD.proj_codes pc on oc.proposal_id=pc.prop_id order by target_name', + ... uploads={'proj_codes': tmp_table}) + + target_name + str256 + ------------ + Ceres + J0042-4030 + J0334-4008 + J1733-130 + J1751+0939 + J1751+096 + J1851+0035 + J1924-2914 + Neptune + SGP-UR-54092 + Titan + Uranus + W43-MM1 Use the ``help_tap`` method to learn about the ALMA 'ObsCore' keywords and their types.