# HG changeset patch # User astroteam # Date 1698129502 0 # Node ID 667fc28d803ce6c69fe26b3c0b125886fe9ad182 # Parent 0ddfc343f9f9ac711d2ee3377aed21168a372321 planemo upload for repository https://github.com/esg-epfl-apc/tools-astro/tree/main/tools/ commit f9ba105adfaad1b2a16dd570652aa27c508d3c4d diff -r 0ddfc343f9f9 -r 667fc28d803c astronomical_archives.py --- a/astronomical_archives.py Mon Sep 04 14:20:34 2023 +0000 +++ b/astronomical_archives.py Tue Oct 24 06:38:22 2023 +0000 @@ -1,17 +1,47 @@ +import errno +import functools import json import os +import signal import sys import urllib from urllib import request +from astropy.coordinates import SkyCoord + import pyvo from pyvo import DALAccessError, DALQueryError, DALServiceError from pyvo import registry + MAX_ALLOWED_ENTRIES = 100 MAX_REGISTRIES_TO_SEARCH = 100 +class TimeoutException(Exception): + pass + + +def timeout(seconds=10, error_message=os.strerror(errno.ETIME)): + def decorator(func): + def _handle_timeout(signum, frame): + raise TimeoutException(error_message) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + signal.signal(signal.SIGALRM, _handle_timeout) + signal.alarm(seconds) + try: + result = func(*args, **kwargs) + finally: + signal.alarm(0) + return result + + return wrapper + + return decorator + + class Service: # https://pyvo.readthedocs.io/en/latest/api/pyvo.registry.Servicetype.html @@ -92,6 +122,7 @@ self.archive_service = None self.tables = None + @timeout(10) def get_resources(self, query, number_of_results, @@ -258,6 +289,26 @@ return name +class ConeService(TapArchive): + + def _get_service(self): + if self.access_url: + self.archive_service = pyvo.dal.SCSService(self.access_url) + + def get_resources_from_service_list(self, service_list, target, radius): + + resource_list_hydrated = [] + + for service in service_list: + resources = service.search(target, radius) + for i in range(resources.__len__()): + resource_url = resources.getrecord(i).getdataurl() + if resource_url: + resource_list_hydrated.append(resource_url) + + return resource_list_hydrated + + class RegistrySearchParameters: def __init__(self, keyword=None, waveband=None, service_type=None): @@ -338,6 +389,24 @@ return archive_list +class ConeServiceRegistry: + + def __init__(self): + pass + + @staticmethod + def search_services(keyword, number_of_registries): + + service_list = [] + + service_list = registry.search(servicetype="scs", keywords=keyword) + + if service_list: + service_list = service_list[:number_of_registries] + + return service_list + + class TapQuery: def __init__(self, query): @@ -410,6 +479,7 @@ self._output_error = output_error self._set_run_main_parameters() + self._is_initialised, error_message = self._set_archive() if self._is_initialised and error_message is None: @@ -479,10 +549,41 @@ else: return False, error_message + def _set_cone_service(self): + + qs = 'query_section' + qsl = 'query_selection' + csts = 'cone_search_target_selection' + + error_message = None + is_service_initialised = True + + keyword = self._json_parameters[qs][qsl][csts]['keyword'] + + service_list = ConeServiceRegistry.search_services( + keyword, + MAX_REGISTRIES_TO_SEARCH) + + if len(service_list) >= 1: + self._services = service_list + else: + is_service_initialised = False + error_message = "no services matching search parameters" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_ARCHIVE_CONNECTION, + error_message) + + return is_service_initialised, error_message + def _set_query(self): qs = 'query_section' qsl = 'query_selection' + csts = 'cone_search_target_selection' + cs = 'cone_section' + ts = 'target_selection' + con = 'cone_object_name' if self._query_type == 'obscore_query': @@ -517,6 +618,34 @@ order_by = \ self._json_parameters[qs][qsl]['order_by'] + if self._json_parameters[qs][qsl][cs][csts][ts] == 'coordinates': + ra = self._json_parameters[qs][qsl][cs][csts]['ra'] + dec = self._json_parameters[qs][qsl][cs][csts]['dec'] + else: + obs_target = self._json_parameters[qs][qsl][cs][csts][con] + + if obs_target != 'none' and obs_target is not None: + target = CelestialObject(obs_target) + target_coordinates = target.get_coordinates_in_degrees() + + ra = target_coordinates['ra'] + dec = target_coordinates['dec'] + else: + ra = None + dec = None + + radius = self._json_parameters[qs][qsl][cs]['radius'] + + if (ra != '' and ra is not None)\ + and (dec != '' and dec is not None)\ + and (radius != '' and radius is not None): + cone_condition = \ + ADQLConeSearchQuery.get_search_circle_condition(ra, + dec, + radius) + else: + cone_condition = None + obscore_query_object = ADQLObscoreQuery(dataproduct_type, obs_collection, obs_title, @@ -531,6 +660,7 @@ calibration_level, t_min, t_max, + cone_condition, order_by) self._adql_query = obscore_query_object.get_query() @@ -558,6 +688,33 @@ else: self._adql_query = ADQLObscoreQuery.base_query + def _set_cone_query(self): + + qs = 'query_section' + qsl = 'query_selection' + csts = 'cone_search_target_selection' + ts = 'target_selection' + con = 'cone_object_name' + + search_radius = self._json_parameters[qs][qsl]['radius'] + time = None + + if self._json_parameters[qs][qsl][csts][ts] == 'coordinates': + ra = self._json_parameters[qs][qsl][csts]['ra'] + dec = self._json_parameters[qs][qsl][csts]['dec'] + time = self._json_parameters[qs][qsl][csts]['time'] + else: + target = CelestialObject(self._json_parameters[qs][qsl][csts][con]) + + target_coordinates = target.get_coordinates_in_degrees() + + ra = target_coordinates['ra'] + dec = target_coordinates['dec'] + + cone_query_object = ADQLConeSearchQuery(ra, dec, search_radius, time) + + self._adql_query = cone_query_object.get_query() + def _set_output(self): self._number_of_files = \ int( @@ -594,12 +751,20 @@ self._archive_type) for archive in self._archives: - _file_url, error_message = archive.get_resources( - self._adql_query, - self._number_of_files, - self._url_field) + try: + _file_url, error_message = archive.get_resources( + self._adql_query, + self._number_of_files, + self._url_field) - file_url.extend(_file_url) + file_url.extend(_file_url) + except TimeoutException: + error_message = \ + "Archive is taking too long to respond (timeout)" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_ARCHIVE_CONNECTION, + error_message) if len(file_url) >= int(self._number_of_files): file_url = file_url[:int(self._number_of_files)] @@ -708,12 +873,11 @@ FileHandler.write_file_to_output(summary_file, self._output_error) - else: summary_file = Logger.create_log_file("Archive", self._adql_query) - summary_file += "Unable to initialize archive" + summary_file += "Unable to initialize archives" FileHandler.write_file_to_output(summary_file, self._output_error) @@ -745,6 +909,7 @@ calibration_level, t_min, t_max, + cone_condition, order_by): super().__init__() @@ -770,6 +935,11 @@ if dataproduct_type == 'none' or dataproduct_type is None: dataproduct_type = '' + if cone_condition is not None: + self.cone_condition = cone_condition + else: + self.cone_condition = None + self.parameters = { 'dataproduct_type': dataproduct_type, 'obs_collection': obs_collection, @@ -782,9 +952,9 @@ 'target_name': target_name, 'obs_publisher_id': obs_publisher_id, 's_fov': s_fov, - 'calibration_level': calibration_level, + 'calib_level': calibration_level, 't_min': t_min, - 't_max': t_max + 't_max': t_max, } self.order_by = order_by @@ -807,11 +977,21 @@ return super()._get_order_by_clause(obscore_order_type) def get_where_statement(self): - return self._get_where_clause(self.parameters) + where_clause = self._get_where_clause(self.parameters) + + if where_clause == '' and self.cone_condition is not None: + where_clause = 'WHERE ' + self.get_cone_condition() + elif where_clause != '' and self.cone_condition is not None: + where_clause += 'AND ' + self.get_cone_condition() + + return where_clause def _get_where_clause(self, parameters): return super()._get_where_clause(parameters) + def get_cone_condition(self): + return self.cone_condition + class ADQLTapQuery(BaseADQLQuery): base_query = 'SELECT TOP '+str(MAX_ALLOWED_ENTRIES)+' * FROM ' @@ -833,6 +1013,69 @@ return ADQLTapQuery.base_query + str(table) +class ADQLConeSearchQuery: + + base_query = "SELECT TOP 100 * FROM ivoa.obscore" + + def __init__(self, ra, dec, radius, time=None): + + self.ra = ra + self.dec = dec + self.radius = radius + self.time = time + + self._query = ADQLObscoreQuery.base_query + + if self.ra and self.dec and self.radius: + self._query += " WHERE " + self._query += self._get_search_circle(ra, dec, radius) + + if self.time: + self._query += self._get_search_time() + + def _get_search_circle(self, ra, dec, radius): + return "(CONTAINS" \ + "(POINT('ICRS', s_ra, s_dec), " \ + "CIRCLE('ICRS', "+str(ra)+", "+str(dec)+", "+str(radius)+")" \ + ") = 1)" + + def _get_search_time(self): + return " AND t_min <= "+self.time+" AND t_max >= "+self.time + + def get_query(self): + return self._query + + @staticmethod + def get_search_circle_condition(ra, dec, radius): + return "(CONTAINS" \ + "(POINT('ICRS', s_ra, s_dec)," \ + "CIRCLE('ICRS', "+str(ra)+", "+str(dec)+", "+str(radius)+")" \ + ") = 1) " + + +class CelestialObject: + + def __init__(self, name): + self.name = name + self.coordinates = None + + self.coordinates = SkyCoord.from_name(self.name) + + def get_coordinates_in_degrees(self): + + coordinates = { + 'ra': '', + 'dec': '' + } + + ra_dec = self.coordinates.ravel() + + coordinates['ra'] = ra_dec.ra.degree[0] + coordinates['dec'] = ra_dec.dec.degree[0] + + return coordinates + + class HTMLReport: _html_report_base_header = '' _html_report_base_body = '' diff -r 0ddfc343f9f9 -r 667fc28d803c astronomical_archives.xml --- a/astronomical_archives.xml Mon Sep 04 14:20:34 2023 +0000 +++ b/astronomical_archives.xml Tue Oct 24 06:38:22 2023 +0000 @@ -1,4 +1,4 @@ - + queries astronomical archives through Virtual Observatory protocols operation_0224 @@ -23,15 +23,12 @@ - + - - - @@ -58,11 +55,26 @@ - +
+ + + + + + + + + + + + + + +
@@ -93,12 +105,13 @@ - - + + - + + @@ -132,7 +145,6 @@ - @@ -671,7 +683,9 @@ ----- -**ChiVO TAP** https://vo.chivo.cl/tap ChiVO TAP service +.. workaround for temporary unavailable urls + +**ChiVO TAP** https vo.chivo.cl/tap ChiVO TAP service ----- @@ -925,7 +939,9 @@ ----- -**FAI NVO DC TAP** http://vo.fai.kz/tap FAI archives TAP service +.. Workaround for temporary unavailable urls + +*FAI NVO DC TAP** http vo.fai.kz/tap FAI archives TAP service ----- diff -r 0ddfc343f9f9 -r 667fc28d803c tool-data/astronomical_archives_gen.loc.sample --- a/tool-data/astronomical_archives_gen.loc.sample Mon Sep 04 14:20:34 2023 +0000 +++ b/tool-data/astronomical_archives_gen.loc.sample Tue Oct 24 06:38:22 2023 +0000 @@ -137,4 +137,4 @@ 134 APPLAUSE - Archives of Photographic PLates for Astronomical USE TAP Service https://www.plate-archive.org/tap 135 TAP interface of the 3XMM-DR5 XMM-Newton Catalogue http://xcatdb.unistra.fr/3xmmdr5/tap 136 TAP interface of the 3XMM-dr7 XMM-Newton Catalogue http://xcatdb.unistra.fr/3xmmdr7/tap -137 TAP interface of the 4XMM dr13 XMM-Newton Catalogue https://xcatdb.unistra.fr/xtapdb +137 TAP interface of the 4XMM dr13 XMM-Newton Catalogue https://xcatdb.unistra.fr/xtapdb \ No newline at end of file