diff --git a/CHANGES.rst b/CHANGES.rst index 6710e9e9cc..0b6263f7ae 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,11 @@ New Tools and Services Service fixes and enhancements ------------------------------ +heasarc +^^^^^^^ + +- Refactor heasarc to use the VO backend. The old Heasarc class is now HeasarcBrowser [#2997] + mpc ^^^ diff --git a/astroquery/heasarc/__init__.py b/astroquery/heasarc/__init__.py index f304997cf1..7db561b455 100644 --- a/astroquery/heasarc/__init__.py +++ b/astroquery/heasarc/__init__.py @@ -29,7 +29,7 @@ class Conf(_config.ConfigNamespace): timeout = _config.ConfigItem( 30, 'Time limit for connecting to HEASARC server.') - + VO_URL = _config.ConfigItem( 'https://heasarc.gsfc.nasa.gov/xamin/vo', 'Base Url for VO services') diff --git a/astroquery/heasarc/core.py b/astroquery/heasarc/core.py index b09e7e3492..bf68da48e1 100644 --- a/astroquery/heasarc/core.py +++ b/astroquery/heasarc/core.py @@ -21,6 +21,7 @@ from .heasarc_browse import HeasarcBrowseClass + @async_to_sync class HeasarcClass(BaseQuery): """Class for accessing HEASARC data using XAMIN. @@ -48,14 +49,16 @@ def __init__(self): def tap(self): """TAP service""" if self._tap is None: - self._tap = pyvo.dal.TAPService(f'{self.VO_URL}/tap', session=self._session) + self._tap = pyvo.dal.TAPService( + f'{self.VO_URL}/tap', session=self._session + ) self._session = self._tap._session return self._tap - + @property def _meta(self): """Queries and holds meta-information about the tables. - + This is a table that holds useful information such as the list of default columns per table, the reasonable default search radius per table that is appropriate for a mission etc. @@ -65,20 +68,21 @@ def _meta(self): These are not meant to be used directly by the user. """ if self._meta_info is None: - query = ("SELECT split_part(name, '.', 1) AS table, " - "split_part(name, '.', 2) AS par, " - "CAST(value AS DECIMAL) AS value " - "FROM metainfo " - "WHERE (type = 'parameter' and relation = 'order') " - "OR relation LIKE 'defaultSearchRadius' " - "ORDER BY value" - ) + query = ( + "SELECT split_part(name, '.', 1) AS table, " + "split_part(name, '.', 2) AS par, " + "CAST(value AS DECIMAL) AS value " + "FROM metainfo " + "WHERE (type = 'parameter' and relation = 'order') " + "OR relation LIKE 'defaultSearchRadius' " + "ORDER BY value" + ) self._meta_info = self.query_tap(query).to_table() - self._meta_info['value'] = np.array(self._meta_info['value'], np.float32) + self._meta_info['value'] = np.array( + self._meta_info['value'], np.float32) self._meta_info = self._meta_info[self._meta_info['value'] > 0] return self._meta_info - def _get_default_cols(self, table_name): """Get a list of default columns for a table @@ -93,13 +97,13 @@ def _get_default_cols(self, table_name): """ meta = self._meta[ - (self._meta['table'] == table_name) & - (self._meta['par'] != '') - ] + (self._meta['table'] == table_name) + and (self._meta['par'] != '') + ] meta.sort('value') defaults = meta['par'] return defaults - + def get_default_radius(self, table_name): """Get a mission-appropriate default radius for a table @@ -108,15 +112,15 @@ def get_default_radius(self, table_name): table_name: str The name of table as a str - Return - ------ + Returns + ------- The radius as ~astropy.units """ meta = self._meta[ - (self._meta['table'] == table_name) & - (self._meta['par'] == '') - ] + (self._meta['table'] == table_name) + and (self._meta['par'] == '') + ] radius = np.double(meta['value'][0]) * u.arcmin return radius @@ -134,7 +138,6 @@ def set_session(self, session): self._session = session - def tables(self, *, master=False, keywords=None): """Return a dictionay of all available table in the form {name: description} @@ -148,8 +151,8 @@ def tables(self, *, master=False, keywords=None): terms for tables. Words with a str separated by a space are AND'ed, while words in a list are OR'ed - Return - ------ + Returns + ------- `~astropy.table.Table` with columns: name, description """ @@ -164,31 +167,34 @@ def tables(self, *, master=False, keywords=None): if 'TAP' in lab or (master and 'mast' not in lab): continue if keywords is not None: - matched = any([all([wrd.lower() in f'{lab} {tab.description}'.lower() - for wrd in wrds.split()]) - for wrds in keywords]) + matched = any( + [ + all([wrd.lower() in f'{lab} {tab.description}'.lower() + for wrd in wrds.split()]) + for wrds in keywords + ] + ) if not matched: continue names.append(lab) desc.append(tab.description) - return Table({'name': names, 'description':desc}) - + return Table({'name': names, 'description': desc}) @deprecated( since='TBD', - message='Heasarc.query_mission_list is deprecated. Use ~Heasarc.tables instead', + message=('Heasarc.query_mission_list is deprecated. ' + 'Use ~Heasarc.tables instead'), ) def query_mission_list(self, *, cache=True, get_query_payload=False): """Returns a list of all available mission tables with descriptions. - NOTE: This method is deprecated, and is included only for limited backward - compatibility with the old astroquery.Heasarc that uses the Browse interface. - Please use ~Heasarc.tables instead. + This method is deprecated, and is included only for limited + backward compatibility with the old astroquery.Heasarc that uses + the Browse interface. Please use ~Heasarc.tables instead. """ return self.tables(master=False) - def columns(self, table_name, full=False): """Return a dictionay of the columns available in table_name @@ -208,10 +214,11 @@ def columns(self, table_name, full=False): """ tables = self.tap.tables if table_name not in tables.keys(): - msg = f'{table_name} is not available as a public table. ' - msg += 'Try passing keywords to ~Heasarc.tables() to find the table name' + msg = (f'{table_name} is not available as a public table. ' + 'Try passing keywords to ~Heasarc.tables() to find ' + 'the table name') raise ValueError(msg) - + default_cols = self._get_default_cols(table_name) names, desc, unit = [], [], [] @@ -220,21 +227,22 @@ def columns(self, table_name, full=False): names.append(col.name) desc.append(col.description) unit.append(col.unit or '') - cols = Table({'name': names, 'description':desc, 'unit': unit}) + cols = Table({'name': names, 'description': desc, 'unit': unit}) return cols @deprecated( since='TBD', - message='Heasarc.query_mission_cols is deprecated. Use ~Heasarc.columns instead', + message=('Heasarc.query_mission_cols is deprecated. ' + 'Use ~Heasarc.columns instead'), ) - def query_mission_cols(self, mission, *, cache=True, get_query_payload=False, - **kwargs): + def query_mission_cols(self, mission, *, cache=True, + get_query_payload=False, **kwargs): """Query around a specific object within a given mission catalog - NOTE: This method is deprecated, and is included only for limited backward - compatibility with the old astroquery.Heasarc that uses the Browse interface. - Please use ~Heasarc.columns instead. + NOTE: This method is deprecated, and is included only for limited + backward compatibility with the old astroquery.Heasarc that uses + the Browse interface. Please use ~Heasarc.columns instead. Parameters ---------- @@ -256,8 +264,6 @@ def query_mission_cols(self, mission, *, cache=True, get_query_payload=False, cols = self.columns(mission, full=full) cols = [col.upper() for col in cols['name'] if '__' not in col] return cols - - def query_tap(self, query, *, maxrec=None): """ @@ -288,38 +294,44 @@ def query_tap(self, query, *, maxrec=None): @deprecated_renamed_argument( ('mission', 'fields', 'resultmax', 'entry', 'coordsys', 'equinox', - 'displaymode', 'action', 'sortvar', 'cache'), - ('table', 'columns', 'maxrec', None, None, None, None, None, None, None), - since=('TBD', 'TBD', 'TBD', 'TBD', 'TBD', 'TBD', 'TBD', 'TBD', 'TBD', 'TBD'), - arg_in_kwargs=(False, True, True, True, True, True, True, True, True, False) + 'displaymode', 'action', 'sortvar', 'cache'), + ('table', 'columns', 'maxrec', None, None, None, + None, None, None, None), + since=('TBD', 'TBD', 'TBD', 'TBD', 'TBD', 'TBD', + 'TBD', 'TBD', 'TBD', 'TBD'), + arg_in_kwargs=(False, True, True, True, True, True, + True, True, True, False) ) - def query_region(self, position=None, table=None, radius=None, * , + def query_region(self, position=None, table=None, radius=None, *, spatial='cone', width=None, polygon=None, get_query_payload=False, columns=None, cache=False, verbose=False, maxrec=None, **kwargs): - """ - Queries the HEASARC TAP server around a coordinate and returns a `~astropy.table.Table` object. + """Queries the HEASARC TAP server around a coordinate and returns a + `~astropy.table.Table` object. Parameters ---------- position : str, `astropy.coordinates` object - Gives the position of the center of the cone or box if performing a cone or box search. - Required if spatial is ``'cone'`` or ``'box'``. Ignored if spatial is ``'polygon'`` or - ``'all-sky'``. + Gives the position of the center of the cone or box if performing + a cone or box search. Required if spatial is ``'cone'`` or + ``'box'``. Ignored if spatial is ``'polygon'`` or ``'all-sky'``. table : str The table to query. To list the available tables, use :meth:`~astroquery.heasarc.HeasarcClass.tables`. spatial : str Type of spatial query: ``'cone'``, ``'box'``, ``'polygon'``, and ``'all-sky'``. Defaults to ``'cone'``. - radius : str or `~astropy.units.Quantity` object, [optional for spatial == ``'cone'``] + radius : str or `~astropy.units.Quantity` object, [optional for + spatial == ``'cone'``]. The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from - `astropy.units` may also be used. If None, a default value appropriate for the - selected table is used. To see the default radius for the table, see + `astropy.units` may also be used. If None, a default value + appropriate for the selected table is used. To see the default + radius for the table, see ~astroquery.heasarc.Heasarc.get_default_radius. - width : str, `~astropy.units.Quantity` object [Required for spatial == ``'box'``.] + width : str, `~astropy.units.Quantity` object [Required for + spatial == ``'box'``.] The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `astropy.units` may also be used. @@ -358,21 +370,27 @@ def query_region(self, position=None, table=None, radius=None, * , if '__row' not in columns: columns += ',__row' - if spatial.lower() == 'all-sky': where = '' elif spatial.lower() == 'polygon': try: - coords_list = [parse_coordinates(coord).icrs for coord in polygon] + coords_list = [parse_coordinates(coord).icrs + for coord in polygon] except TypeError: try: - coords_list = [coordinates.SkyCoord(*coord).icrs for coord in polygon] + coords_list = [coordinates.SkyCoord(*coord).icrs + for coord in polygon] except u.UnitTypeError: warnings.warn("Polygon endpoints are being interpreted as " - "RA/Dec pairs specified in decimal degree units.") - coords_list = [coordinates.SkyCoord(*coord, unit='deg').icrs for coord in polygon] - - coords_str = [f'{coord.ra.deg},{coord.dec.deg}' for coord in coords_list] + "RA/Dec pairs specified in decimal degree " + "units.") + coords_list = [ + coordinates.SkyCoord(*coord, unit='deg').icrs + for coord in polygon + ] + + coords_str = [f'{coord.ra.deg},{coord.dec.deg}' + for coord in coords_list] where = (" WHERE CONTAINS(POINT('ICRS',ra,dec)," f"POLYGON('ICRS',{','.join(coords_str)}))=1") else: @@ -384,8 +402,8 @@ def query_region(self, position=None, table=None, radius=None, * , radius = self.get_default_radius(table) elif isinstance(radius, str): radius = coordinates.Angle(radius) - where = (" WHERE CONTAINS(POINT('ICRS',ra,dec)," - f"CIRCLE('ICRS',{ra},{dec},{radius.to(u.deg).value}))=1") + where = (" WHERE CONTAINS(POINT('ICRS',ra,dec),CIRCLE(" + f"'ICRS',{ra},{dec},{radius.to(u.deg).value}))=1") # add search_offset_ for the case of cone columns += (",DISTANCE(POINT('ICRS',ra,dec), " f"POINT('ICRS',{ra},{dec})) as search_offset_") @@ -393,10 +411,11 @@ def query_region(self, position=None, table=None, radius=None, * , if isinstance(width, str): width = coordinates.Angle(width) where = (" WHERE CONTAINS(POINT('ICRS',ra,dec)," - f"BOX('ICRS',{ra},{dec},{width.to(u.deg).value},{width.to(u.deg).value}))=1") + f"BOX('ICRS',{ra},{dec},{width.to(u.deg).value}," + f"{width.to(u.deg).value}))=1") else: - raise ValueError("Unrecognized spatial query type. Must be one of " - "'cone', 'box', 'polygon', or 'all-sky'.") + raise ValueError("Unrecognized spatial query type. Must be one" + " of 'cone', 'box', 'polygon', or 'all-sky'.") adql = f'SELECT {columns} FROM {table}{where}' @@ -412,10 +431,15 @@ def query_region(self, position=None, table=None, radius=None, * , if 'search_offset_' in table.colnames: table['search_offset_'].unit = u.arcmin if len(table) == 0: - warnings.warn(NoResultsWarning("No matching rows were found in the query.")) + warnings.warn( + NoResultsWarning("No matching rows were found in the query.") + ) return table - @deprecated(since='TBD', message='query_object is being deprecated. Use query_region instead') + @deprecated( + since='TBD', + message='query_object is being deprecated. Use query_region instead' + ) def query_object(self, object_name, mission, *, cache=True, get_query_payload=False, **kwargs): @@ -440,7 +464,6 @@ def query_object(self, object_name, mission, *, return self.query_region(pos, table=mission, spatial='cone', get_query_payload=get_query_payload) - def get_links(self, query_result=None, tablename=None): """Get links to data products Use vo/datalinks to query the data products for some query_results. @@ -460,7 +483,10 @@ def get_links(self, query_result=None, tablename=None): table : A `~astropy.table.Table` object. """ if query_result is None: - if not hasattr(self, '_query_result') or self._query_result is None: + if ( + not hasattr(self, '_query_result') + or self._query_result is None + ): raise ValueError('query_result is None, and none ' 'found from a previous search') else: @@ -472,11 +498,15 @@ def get_links(self, query_result=None, tablename=None): # make sure we have a column __row if '__row' not in query_result.colnames: raise ValueError('No __row column found in query_result. ' - 'query_result needs to be the output of query_region or a subset.') + 'query_result needs to be the output of ' + 'query_region or a subset.') if tablename is None: tablename = self._tablename - if not (isinstance(tablename, str) and tablename in self.tap.tables.keys()): + if ( + not isinstance(tablename, str) + and tablename in self.tap.tables.keys() + ): raise ValueError(f'Unknown table name: {tablename}') # datalink url @@ -492,11 +522,14 @@ def get_links(self, query_result=None, tablename=None): dl_result = dl_result[['ID', 'access_url', 'content_length']] # add sciserver and s3 columns - newcol = [f"/FTP/{row.split('FTP/')[1]}".replace('//', '/') if 'FTP' in row else '' - for row in dl_result['access_url']] + newcol = [ + f"/FTP/{row.split('FTP/')[1]}".replace('//', '/') + if 'FTP' in row else '' + for row in dl_result['access_url'] + ] dl_result.add_column(newcol, name='sciserver', index=2) newcol = [f"s3://{self.S3_BUCKET}/{row[5:]}" if row != '' else '' - for row in dl_result['sciserver']] + for row in dl_result['sciserver']] dl_result.add_column(newcol, name='aws', index=3) return dl_result @@ -509,20 +542,26 @@ def enable_cloud(self, provider='aws', profile=None): Parameters ---------- provider : str - Which cloud data provider to use. Currently, only 'aws' is supported + Which cloud data provider to use. Currently, only 'aws' is + supported. profile : str - Profile to use to identify yourself to the cloud provider (usually in ~/.aws/config). + Profile to use to identify yourself to the cloud provider + (usually in ~/.aws/config). """ try: import boto3 import botocore except ImportError: - raise ImportError('The cloud feature requires the boto3 package. Install it first.') + raise ImportError( + 'The cloud feature requires the boto3 package. ' + 'Install it first.' + ) if profile is None: log.info('Enabling annonymous cloud data access ...') - config = botocore.client.Config(signature_version=botocore.UNSIGNED) + config = botocore.client.Config( + signature_version=botocore.UNSIGNED) self.s3_resource = boto3.resource('s3', config=config) elif isinstance(profile, bool) and not profile: @@ -536,7 +575,6 @@ def enable_cloud(self, provider='aws', profile=None): self.s3_client = self.s3_resource.meta.client - def download_data(self, links, host='heasarc', location='.'): """Download data products in links with a choice of getting the data from either the heasarc server, sciserver, or the cloud in AWS. @@ -548,16 +586,18 @@ def download_data(self, links, host='heasarc', location='.'): The result from get_links host : str The data host. The options are: heasarc (defaul), sciserver, aws. - If host == 'sciserver', data is copied from the local mounted data drive. - If host == 'aws', data is downloaded from Amazon S3 Open Data Repository. + If host == 'sciserver', data is copied from the local mounted + data drive. + If host == 'aws', data is downloaded from Amazon S3 Open + Data Repository. location : str local folder where the downloaded file will be saved. Default is current working directory - Note - ---- - Downloading more than ~10 observations from the HEASARC will likely fail. - If you have more than 10 links, group them and make several requests. + Note that ff you are downloading large datasets (more 10 10GB), + from the main heasarc server, it is recommended that you split + it up, so that if the downloaded is intrrupted, you do not need + to start again. """ if len(links) == 0: @@ -569,7 +609,8 @@ def download_data(self, links, host='heasarc', location='.'): host_column = 'access_url' if host == 'heasarc' else host if host_column not in links.colnames: raise ValueError( - f'No {host_column} column found in the table. Call ~get_links first' + f'No {host_column} column found in the table. Call ' + '~get_links first' ) if host == 'heasarc': @@ -587,7 +628,6 @@ def download_data(self, links, host='heasarc', location='.'): log.info('Downloading data AWS S3 ...') self._download_s3(links, location) - def _download_heasarc(self, links, location='.'): """Download data from the heasarc main server using xamin's tar servlet @@ -607,9 +647,11 @@ def _download_heasarc(self, links, location='.'): if 'content_length' in links.columns: size = links['content_length'].sum() / 2**30 if size > 10: - warnings.warn(f"The size of the requested file is large {size:.3f} GB. " - "If the download is interrupted, you may need to start again. " - "Consider downloading the data in chunks") + warnings.warn( + f"The size of the requested file is large {size:.3f} GB. " + "If the download is interrupted, you may need to start " + "again. Consider downloading the data in chunks." + ) file_list = [f"/FTP/{link.split('FTP/')[1]}" for link in links['access_url']] @@ -639,8 +681,9 @@ def _download_heasarc(self, links, location='.'): tfile.close() os.remove(local_filepath) else: - raise ValueError('An error ocurred when downloading the data. Retry again.') - + raise ValueError( + 'An error ocurred when downloading the data. Retry again.' + ) def _copy_sciserver(self, links, location='.'): """Copy data from the local archive on sciserver @@ -664,14 +707,14 @@ def _copy_sciserver(self, links, location='.'): raise ValueError( f'No data found in {link}. ' 'Make sure you are running this on Sciserver. ' - 'If you think data is missing, please contact the Heasarc Help desk' + 'If you think data is missing, please contact the ' + 'Heasarc Help desk' ) if os.path.isdir(link): shutil.copytree(link, location) else: shutil.copy(link, location) - def _download_s3(self, links, location='.'): """Download data from AWS S3 Assuming open access. @@ -702,5 +745,6 @@ def _s3_tree_download(client, bucket_name, path, local): path = key.replace(f's3://{self.S3_BUCKET}/', '') _s3_tree_download(self.s3_client, self.S3_BUCKET, path, location) + Heasarc = HeasarcClass() -HeasarcBrowse = HeasarcBrowseClass() \ No newline at end of file +HeasarcBrowse = HeasarcBrowseClass() diff --git a/astroquery/heasarc/tests/test_heasarc.py b/astroquery/heasarc/tests/test_heasarc.py index e21911f6b1..df882bf914 100644 --- a/astroquery/heasarc/tests/test_heasarc.py +++ b/astroquery/heasarc/tests/test_heasarc.py @@ -11,155 +11,233 @@ from astroquery.exceptions import InvalidQueryError try: - # Both boto3, botocore and moto are optional dependencies, but the former 2 are - # dependencies of the latter, so it's enough to handle them with one variable + # Both boto3, botocore and moto are optional dependencies, + # but the former 2 are dependencies of the latter, so it's enough + # to handle them with one variable import boto3 from moto import mock_aws + DO_AWS_S3 = True except ImportError: DO_AWS_S3 = False -OBJ_LIST = ["182d38m08.64s 39d24m21.06s", - SkyCoord(l=155.0771, b=75.0631, unit=(u.deg, u.deg), frame='galactic')] +OBJ_LIST = [ + "182d38m08.64s 39d24m21.06s", + SkyCoord(l=155.0771, b=75.0631, unit=(u.deg, u.deg), frame="galactic"), +] + +SIZE_LIST = [2 * u.arcmin, "0d2m0s"] -SIZE_LIST = [2 * u.arcmin, '0d2m0s'] @pytest.mark.parametrize("coordinates", OBJ_LIST) @pytest.mark.parametrize("radius", SIZE_LIST) def test_query_region_cone(coordinates, radius): # use columns='*' to avoid remote call to obtain the default columns - query = Heasarc.query_region(coordinates, table='suzamaster', spatial='cone', radius=radius, - columns='*', get_query_payload=True) - - # We don't fully float compare in this string, there are slight differences due to the name-coordinate - # resolution and conversions - assert "SELECT *,DISTANCE(POINT('ICRS',ra,dec), POINT('ICRS',182.63" in query - assert "FROM suzamaster WHERE CONTAINS(POINT('ICRS',ra,dec),CIRCLE('ICRS',182.63" in query + query = Heasarc.query_region( + coordinates, + table="suzamaster", + spatial="cone", + radius=radius, + columns="*", + get_query_payload=True, + ) + + # We don't fully float compare in this string, there are slight + # differences due to the name-coordinate resolution and conversions + assert ("SELECT *,DISTANCE(POINT('ICRS',ra,dec), " + "POINT('ICRS',182.63") in query + assert ( + "FROM suzamaster WHERE CONTAINS(POINT('ICRS',ra,dec)," + "CIRCLE('ICRS',182.63" in query + ) assert ",39.40" in query assert ",0.0333" in query + @pytest.mark.parametrize("coordinates", OBJ_LIST) @pytest.mark.parametrize("width", SIZE_LIST) def test_query_region_box(coordinates, width): - query = Heasarc.query_region(coordinates, table='suzamaster', spatial='box', width=2 * u.arcmin, - columns='*', get_query_payload=True) - - assert "SELECT * FROM suzamaster WHERE CONTAINS(POINT('ICRS',ra,dec),BOX('ICRS',182.63" in query + query = Heasarc.query_region( + coordinates, + table="suzamaster", + spatial="box", + width=2 * u.arcmin, + columns="*", + get_query_payload=True, + ) + + assert ( + "SELECT * FROM suzamaster WHERE CONTAINS(POINT('ICRS',ra,dec)," + "BOX('ICRS',182.63" in query + ) assert ",39.40" in query assert ",0.0333" in query -poly1 = [SkyCoord(ra=10.1 * u.deg, dec=10.1 * u.deg), - SkyCoord(ra=10.0 * u.deg, dec=10.1 * u.deg), - SkyCoord(ra=10.0 * u.deg, dec=10.0 * u.deg)] -poly2 = [(10.1 * u.deg, 10.1 * u.deg), (10.0 * u.deg, 10.1 * u.deg), - (10.0 * u.deg, 10.0 * u.deg)] +poly1 = [ + SkyCoord(ra=10.1 * u.deg, dec=10.1 * u.deg), + SkyCoord(ra=10.0 * u.deg, dec=10.1 * u.deg), + SkyCoord(ra=10.0 * u.deg, dec=10.0 * u.deg), +] +poly2 = [ + (10.1 * u.deg, 10.1 * u.deg), + (10.0 * u.deg, 10.1 * u.deg), + (10.0 * u.deg, 10.0 * u.deg), +] + @pytest.mark.parametrize("polygon", [poly1, poly2]) def test_query_region_polygon(polygon): # position is not used for polygon - query1 = Heasarc.query_region(table="suzamaster", spatial="polygon", polygon=polygon, - columns='*', get_query_payload=True) - query2 = Heasarc.query_region("ngc4151", table="suzamaster", spatial="polygon", polygon=polygon, - columns='*', get_query_payload=True) + query1 = Heasarc.query_region( + table="suzamaster", + spatial="polygon", + polygon=polygon, + columns="*", + get_query_payload=True, + ) + query2 = Heasarc.query_region( + "ngc4151", + table="suzamaster", + spatial="polygon", + polygon=polygon, + columns="*", + get_query_payload=True, + ) assert query1 == query2 - assert query1 == ("SELECT * FROM suzamaster " - "WHERE CONTAINS(POINT('ICRS',ra,dec),POLYGON('ICRS',10.1,10.1,10.0,10.1,10.0,10.0))=1") + assert query1 == ( + "SELECT * FROM suzamaster " + "WHERE CONTAINS(POINT('ICRS',ra,dec),POLYGON('ICRS'," + "10.1,10.1,10.0,10.1,10.0,10.0))=1" + ) + def test_query_allsky(): - query1 = Heasarc.query_region(table="suzamaster", spatial="all-sky", - columns='*', get_query_payload=True) - query2 = Heasarc.query_region("m31", table="suzamaster", spatial="all-sky", - columns='*', get_query_payload=True) + query1 = Heasarc.query_region( + table="suzamaster", spatial="all-sky", columns="*", + get_query_payload=True + ) + query2 = Heasarc.query_region( + "m31", + table="suzamaster", + spatial="all-sky", + columns="*", + get_query_payload=True, + ) assert query1 == query2 == "SELECT * FROM suzamaster" -@pytest.mark.parametrize('spatial', ['space', 'invalid']) + +@pytest.mark.parametrize("spatial", ["space", "invalid"]) def test_spatial_invalid(spatial): with pytest.raises(ValueError): - Heasarc.query_region(OBJ_LIST[0], table='invalid_spatial', columns='*', spatial=spatial) + Heasarc.query_region( + OBJ_LIST[0], table="invalid_spatial", columns="*", spatial=spatial + ) + def test_no_table(): with pytest.raises(InvalidQueryError): - Heasarc.query_region("m31", spatial='cone', columns='*') + Heasarc.query_region("m31", spatial="cone", columns="*") def test_get_link(): - with pytest.raises(ValueError, match='query_result is None'): + with pytest.raises(ValueError, match="query_result is None"): Heasarc.get_links() - with pytest.raises(TypeError, match='query_result need to be an astropy.table.Table'): - Heasarc.get_links([1,2]) + with pytest.raises( + TypeError, match="query_result need to be an astropy.table.Table" + ): + Heasarc.get_links([1, 2]) - with pytest.raises(ValueError, match='Unknown table name:'): - Heasarc.get_links(Table({'__row':[1,2,3.]}), tablename=1) + with pytest.raises(ValueError, match="No __row column found"): + Heasarc.get_links(Table({"id": [1, 2, 3.0]}), tablename="xray") - with pytest.raises(ValueError, match='No __row column found'): - Heasarc.get_links(Table({'id':[1,2,3.]}), tablename='xray') def test_download_data__empty(): - with pytest.raises(ValueError, match='Input links table is empty'): + with pytest.raises(ValueError, match="Input links table is empty"): Heasarc.download_data(Table()) + def test_download_data__wronghost(): - with pytest.raises(ValueError, match='host has to be one of heasarc, sciserver, aws'): - Heasarc.download_data(Table({'id':[1]}), host='nohost') + with pytest.raises( + ValueError, match="host has to be one of heasarc, sciserver, aws" + ): + Heasarc.download_data(Table({"id": [1]}), host="nohost") -@pytest.mark.parametrize('host', ['heasarc', 'sciserver', 'aws']) + +@pytest.mark.parametrize("host", ["heasarc", "sciserver", "aws"]) def test_download_data__missingcolumn(host): - host_col = 'access_url' if host == 'heasarc' else host - with pytest.raises(ValueError, match=f'No {host_col} column found in the table. Call ~get_links first'): - Heasarc.download_data(Table({'id':[1]}), host=host) + host_col = "access_url" if host == "heasarc" else host + with pytest.raises( + ValueError, + match=f"No {host_col} column found in the table. Call ~get_links first" + ): + Heasarc.download_data(Table({"id": [1]}), host=host) + def test_download_data__outside_sciserver(): - with pytest.raises(FileNotFoundError, match='No data archive found. This should be run on Sciserver'): - Heasarc.download_data(Table({'sciserver':['some-link']}), host='sciserver') + with pytest.raises( + FileNotFoundError, + match="No data archive found. This should be run on Sciserver", + ): + Heasarc.download_data( + Table({"sciserver": ["some-link"]}), host="sciserver" + ) -## S3 mock tests ## -s3_bucket = 'nasa-heasarc' -s3_key1 = 'some/location/file1.txt' -s3_key2 = 'some/location/sub/file2.txt' -s3_key3 = 'some/location/sub/sub2/file3.txt' -s3_dir = 'some/location' +# S3 mock tests +s3_bucket = "nasa-heasarc" +s3_key1 = "some/location/file1.txt" +s3_key2 = "some/location/sub/file2.txt" +s3_key3 = "some/location/sub/sub2/file3.txt" +s3_dir = "some/location" -@pytest.fixture(name='s3_mock') +@pytest.fixture(name="s3_mock") def _s3_mock(mocker): with mock_aws(): - conn = boto3.resource('s3', region_name='us-east-1') + conn = boto3.resource("s3", region_name="us-east-1") conn.create_bucket(Bucket=s3_bucket) s3_client = conn.meta.client - s3_client.put_object(Bucket=s3_bucket, Key=s3_key1, Body='my content') - s3_client.put_object(Bucket=s3_bucket, Key=s3_key2, Body='my other content') - s3_client.put_object(Bucket=s3_bucket, Key=s3_key3, Body='my other content') + s3_client.put_object(Bucket=s3_bucket, Key=s3_key1, + Body="my content") + s3_client.put_object(Bucket=s3_bucket, Key=s3_key2, + Body="my other content") + s3_client.put_object(Bucket=s3_bucket, Key=s3_key3, + Body="my other content") yield conn -@pytest.mark.skipif('not DO_AWS_S3') + +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +@pytest.mark.skipif("not DO_AWS_S3") def test_s3_mock_basic(s3_mock): - body = s3_mock.Object(s3_bucket, s3_key1).get()['Body'] - content = body.read().decode('utf-8') - assert content == 'my content' + body = s3_mock.Object(s3_bucket, s3_key1).get()["Body"] + content = body.read().decode("utf-8") + assert content == "my content" + -@pytest.mark.skipif('not DO_AWS_S3') +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +@pytest.mark.skipif("not DO_AWS_S3") def test_s3_mock_file(s3_mock): - links = Table({'aws':[f's3://{s3_bucket}/{s3_key1}']}) + links = Table({"aws": [f"s3://{s3_bucket}/{s3_key1}"]}) Heasarc.enable_cloud(profile=False) - Heasarc.download_data(links, host='aws', location='.') - file = s3_key1.split('/')[-1] - assert(os.path.exists(file)) + Heasarc.download_data(links, host="aws", location=".") + file = s3_key1.split("/")[-1] + assert os.path.exists(file) os.remove(file) -@pytest.mark.skipif('not DO_AWS_S3') + +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +@pytest.mark.skipif("not DO_AWS_S3") def test_s3_mock_directory(s3_mock): - links = Table({'aws':[f's3://{s3_bucket}/{s3_dir}']}) + links = Table({"aws": [f"s3://{s3_bucket}/{s3_dir}"]}) Heasarc.enable_cloud(profile=False) - Heasarc.download_data(links, host='aws', location='.') - assert(os.path.exists('location')) - assert(os.path.exists('location/file1.txt')) - assert(os.path.exists('location/sub/file2.txt')) - assert(os.path.exists('location/sub/sub2/file3.txt')) - shutil.rmtree('location') - \ No newline at end of file + Heasarc.download_data(links, host="aws", location=".") + assert os.path.exists("location") + assert os.path.exists("location/file1.txt") + assert os.path.exists("location/sub/file2.txt") + assert os.path.exists("location/sub/sub2/file3.txt") + shutil.rmtree("location") diff --git a/astroquery/heasarc/tests/test_heasarc_remote.py b/astroquery/heasarc/tests/test_heasarc_remote.py index c7263362e1..ed00e90774 100644 --- a/astroquery/heasarc/tests/test_heasarc_remote.py +++ b/astroquery/heasarc/tests/test_heasarc_remote.py @@ -6,10 +6,11 @@ import astropy.units as u from astropy.table import Table from astropy.coordinates import SkyCoord +from packaging.version import Version -from pyvo.dal.exceptions import DALOverflowWarning from astropy.utils.exceptions import AstropyDeprecationWarning from astroquery.exceptions import NoResultsWarning +import pyvo from astroquery.heasarc import Heasarc @@ -21,24 +22,43 @@ ] DEFAULT_COLS = [ - ["nicermastr", - [ - "name", "ra", "dec", "time", "obsid", "exposure", - "processing_status", "processing_date", "public_date", "obs_type", + [ + "nicermastr", + [ + "name", + "ra", + "dec", + "time", + "obsid", + "exposure", + "processing_status", + "processing_date", + "public_date", + "obs_type", ], ], - ["numaster", [ - "name", "ra", "dec", "time", "obsid", "status", "exposure_a", - "observation_mode", "obs_type", "processing_date", "public_date","issue_flag", + "numaster", + [ + "name", + "ra", + "dec", + "time", + "obsid", + "status", + "exposure_a", + "observation_mode", + "obs_type", + "processing_date", + "public_date", + "issue_flag", + ], ], - ] ] @pytest.mark.remote_data class TestHeasarc: - @pytest.mark.parametrize("coordinates", OBJ_LIST) def test_query_region_cone(self, coordinates): """ @@ -61,7 +81,8 @@ def test_query_columns_radius(self): Test selection of only a few columns, and using a bigger radius """ result = Heasarc.query_region( - "NGC 4151", table="suzamaster", columns="ra,dec,obsid", radius=10 * u.arcmin + "NGC 4151", table="suzamaster", columns="ra,dec,obsid", + radius=10 * u.arcmin ) assert len(result) == 4 # assert only selected columns are returned @@ -84,42 +105,50 @@ def test_query_region_polygon(self): result = Heasarc.query_region( table="suzamaster", spatial="polygon", polygon=polygon ) - assert(warnings[0].category == UserWarning) - assert("Polygon endpoints are being interpreted" in warnings[0].message.args[0]) - assert(warnings[1].category == NoResultsWarning) + assert warnings[0].category == UserWarning + assert ("Polygon endpoints are being interpreted" in + warnings[0].message.args[0]) + assert warnings[1].category == NoResultsWarning assert isinstance(result, Table) def test_list_tables(self): tables = Heasarc.tables() - # Number of available tables may change over time, test only for significant drop. - # (at the time of writing there are 1020 tables in the list). + # Number of available tables may change over time, test only for + # significant drop. (at the time of writing there are 1020 tables + # in the list). assert len(tables) > 1000 - + def test_list_tables__master(self): - tables = list(Heasarc.tables(master=True)['name']) - assert 'numaster' in tables - assert 'nicermastr' in tables - assert 'xmmmaster' in tables - assert 'swiftmastr' in tables - - def test_list_tables__keywords(self): - tables = list(Heasarc.tables(keywords='nustar', master=True)['name']) - assert len(tables) == 1 and 'numaster' in tables - - tables = list(Heasarc.tables(keywords='xmm', master=True)['name']) - assert len(tables) == 1 and 'xmmmaster' in tables - - tables = list(Heasarc.tables(keywords=['swift', 'rosat'], master=True)['name']) - assert 'swiftmastr' in tables - assert 'rosmaster' in tables - assert 'rassmaster' in tables - + tables = list(Heasarc.tables(master=True)["name"]) + assert "numaster" in tables + assert "nicermastr" in tables + assert "xmmmaster" in tables + assert "swiftmastr" in tables + def test_list_tables__keywords(self): + tables = list(Heasarc.tables(keywords="nustar", master=True)["name"]) + assert len(tables) == 1 and "numaster" in tables + + tables = list(Heasarc.tables(keywords="xmm", master=True)["name"]) + assert len(tables) == 1 and "xmmmaster" in tables + + tables = list(Heasarc.tables(keywords=["swift", "rosat"], + master=True)["name"]) + assert "swiftmastr" in tables + assert "rosmaster" in tables + assert "rassmaster" in tables + + @pytest.mark.skipif( + Version(pyvo.__version__) < Version('1.4'), + reason="DALOverflowWarning is available only in pyvo>=1.4" + ) def test_tap__maxrec(self): + from pyvo.dal.exceptions import DALOverflowWarning query = "SELECT TOP 10 ra,dec FROM xray" with pytest.warns( expected_warning=DALOverflowWarning, - match="Partial result set. Potential causes MAXREC, async storage space, etc.", + match=("Partial result set. Potential causes MAXREC, " + "async storage space, etc."), ): result = Heasarc.query_tap(query=query, maxrec=5) assert len(result) == 5 @@ -133,10 +162,12 @@ def test__get_default_cols(self, tdefault): def test_get_links__wrongtable(self): with pytest.raises(ValueError, match="Unknown table name:"): - Heasarc.get_links(Table({"__row": [1, 2, 3.0]}), tablename="wrongtable") + Heasarc.get_links(Table({"__row": [1, 2, 3.0]}), + tablename="wrongtable") def test_get_links__xmmmaster(self): - links = Heasarc.get_links(Table({"__row": [4154, 4155]}), tablename="xmmmaster") + links = Heasarc.get_links(Table({"__row": [4154, 4155]}), + tablename="xmmmaster") assert len(links) == 2 assert "access_url" in links.colnames assert "sciserver" in links.colnames @@ -144,25 +175,23 @@ def test_get_links__xmmmaster(self): def test_download_data__heasarc_file(self): filename = "00README" - tab = Table( - { - "access_url": [ - f"https://heasarc.gsfc.nasa.gov/FTP/rxte/data/archive/{filename}" - ] - } - ) + tab = Table({ + "access_url": [ + ("https://heasarc.gsfc.nasa.gov/FTP/rxte/" + f"data/archive/{filename}") + ] + }) Heasarc.download_data(tab, host="heasarc", location=".") assert os.path.exists(filename) os.remove(filename) def test_download_data__heasarc_folder(self): - tab = Table( - { - "access_url": [ - "https://heasarc.gsfc.nasa.gov/FTP/rxte/data/archive/AO10/P91129/91129-01-68-00A/stdprod" - ] - } - ) + tab = Table({ + "access_url": [ + ("https://heasarc.gsfc.nasa.gov/FTP/rxte/data/archive/" + "AO10/P91129/91129-01-68-00A/stdprod") + ] + }) Heasarc.download_data(tab, host="heasarc", location=".") assert os.path.exists("stdprod") assert os.path.exists("stdprod/FHed_1791a7b9-1791a931.gz") @@ -172,7 +201,9 @@ def test_download_data__heasarc_folder(self): def test_download_data__s3_file(self): filename = "00README" - tab = Table({"aws": [f"s3://nasa-heasarc/rxte/data/archive/{filename}"]}) + tab = Table( + {"aws": [f"s3://nasa-heasarc/rxte/data/archive/{filename}"]} + ) Heasarc.download_data(tab, host="aws", location=".") assert os.path.exists(filename) os.remove(filename) @@ -182,7 +213,8 @@ def test_download_data__s3_folder(self, slash): tab = Table( { "aws": [ - f"s3://nasa-heasarc/rxte/data/archive/AO10/P91129/91129-01-68-00A/stdprod{slash}" + ("s3://nasa-heasarc/rxte/data/archive/AO10/" + "P91129/91129-01-68-00A/stdprod{slash}") ] } ) @@ -195,51 +227,65 @@ def test_download_data__s3_folder(self, slash): def test_query_mission_cols(self): with pytest.warns(AstropyDeprecationWarning): - Heasarc.query_mission_cols(mission='xmmmaster') - + Heasarc.query_mission_cols(mission="xmmmaster") + def test_query_mission_list(self): with pytest.warns(AstropyDeprecationWarning): Heasarc.query_mission_list() - - @pytest.mark.parametrize('pars', [['mission', 'xmmmaster'], ['fields', '*'], - ['resultmax', 10000], ['entry', None], ['coordsys', None], ['equinox', None], - ['displaymode', None], ['action', None], ['sortvar', None], ['cache', None]]) + + @pytest.mark.parametrize( + "pars", + [ + ["mission", "xmmmaster"], + ["fields", "*"], + ["resultmax", 10000], + ["entry", None], + ["coordsys", None], + ["equinox", None], + ["displaymode", None], + ["action", None], + ["sortvar", None], + ["cache", None], + ], + ) def test_deprecated_pars(self, pars): """Test deprecated keywords. - - resultmax needs to be large so we don't get the pyvo DALOverflowWarning """ keyword, value = pars pos = OBJ_LIST[2] with pytest.warns(AstropyDeprecationWarning): - if keyword == 'mission': + if keyword == "mission": Heasarc.query_region(pos, mission=value) else: - Heasarc.query_region(pos, mission='xmmmaster', **{keyword:value}) + Heasarc.query_region(pos, mission="xmmmaster", + **{keyword: value}) + @pytest.mark.remote_data class TestHeasarcBrowse: """Tests for backward compatibility with the old astroquery.heasarc""" - + def test_custom_args(self): - object_name = 'Crab' - mission = 'intscw' + object_name = "Crab" + mission = "intscw" heasarc = Heasarc with pytest.warns(AstropyDeprecationWarning): - table = heasarc.query_object(object_name, - mission=mission, - radius='1 degree', - time="2020-09-01 .. 2020-12-01", - resultmax=10, - good_isgri=">1000", - cache=False - ) + table = heasarc.query_object( + object_name, + mission=mission, + radius="1 degree", + time="2020-09-01 .. 2020-12-01", + resultmax=10, + good_isgri=">1000", + cache=False, + ) assert len(table) > 0 - + def test_basic_example(self): - mission = 'rosmaster' - object_name = '3c273' + mission = "rosmaster" + object_name = "3c273" heasarc = Heasarc with pytest.warns(AstropyDeprecationWarning): @@ -253,12 +299,13 @@ def test_mission_list(self): missions = heasarc.query_mission_list() # Assert that there are indeed a large number of tables - # Number of tables could change, but should be > 900 (currently 956) + # Number of tables could change, but should be > 900 + # (currently 956) assert len(missions) > 900 - + def test_mission_cols(self): heasarc = Heasarc - mission = 'rosmaster' + mission = "rosmaster" with pytest.warns(AstropyDeprecationWarning): cols = heasarc.query_mission_cols(mission=mission) @@ -266,22 +313,22 @@ def test_mission_cols(self): assert len(cols) == 28 # Test that the cols list contains known names - assert 'EXPOSURE' in cols - assert 'RA' in cols - assert 'DEC' in cols + assert "EXPOSURE" in cols + assert "RA" in cols + assert "DEC" in cols def test_query_region(self): heasarc = Heasarc - mission = 'rosmaster' + mission = "rosmaster" skycoord_3C_273 = SkyCoord("12h29m06.70s +02d03m08.7s", frame="icrs") - with pytest.warns(AstropyDeprecationWarning): table = heasarc.query_region( - skycoord_3C_273, mission=mission, radius="1 degree") + skycoord_3C_273, mission=mission, radius="1 degree" + ) assert len(table) == 63 - + def test_query_region_nohits(self): """ Regression test for #2560: HEASARC returns a FITS file as a null result @@ -292,8 +339,11 @@ def test_query_region_nohits(self): # This was an example coordinate that returned nothing # Since Fermi is still active, it is possible that sometime in the # future an event will occur here. - table = heasarc.query_region(SkyCoord(0.28136*u.deg, -0.09789*u.deg, frame='fk5'), - mission='hitomaster', radius=0.1*u.deg) - assert(warnings[0].category == AstropyDeprecationWarning) - assert(warnings[1].category == NoResultsWarning) - assert(len(table) == 0) + table = heasarc.query_region( + SkyCoord(0.28136 * u.deg, -0.09789 * u.deg, frame="fk5"), + mission="hitomaster", + radius=0.1 * u.deg, + ) + assert warnings[0].category == AstropyDeprecationWarning + assert warnings[1].category == NoResultsWarning + assert len(table) == 0 diff --git a/docs/heasarc/heasarc.rst b/docs/heasarc/heasarc.rst index 1d3eb7c552..a261dfc71a 100644 --- a/docs/heasarc/heasarc.rst +++ b/docs/heasarc/heasarc.rst @@ -15,7 +15,7 @@ There main interface for the Heasarc services``heasarc.Heasac`` now uses Virtual Observatory protocols with the Xamin interface, which offers more powerful search options than the old Browse interface. -- :ref:`Heasarc Main (Xamin) Interface`. +- :ref:`Heasarc Main Interface`. - :ref:`Old Browse Interface`. .. _Heasarc Main Interface: @@ -284,12 +284,10 @@ in the XMM master table ``xmmmaster``: Old Browse Interface ==================== -:::{admonition} Limited Support -:class: warning -The old Browse interface has only limited support from the Heasarc, -please consider using the main `~astroquery.heasarc.HeasarcClas` interface. -::: +.. warning:: + The old Browse interface has limited support from the Heasarc, + please consider using the main `~astroquery.heasarc.HeasarcClass` interface. Getting lists of available datasets -----------------------------------