Mercurial > repos > astroteam > astronomical_archives

changeset 1:667fc28d803c draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/esg-epfl-apc/tools-astro/tree/main/tools/ commit f9ba105adfaad1b2a16dd570652aa27c508d3c4d
author astroteam
date Tue, 24 Oct 2023 06:38:22 +0000
parents 0ddfc343f9f9
children 7398ea3d9ac4
files astronomical_archives.py astronomical_archives.xml tool-data/astronomical_archives_gen.loc.sample
diffstat 3 files changed, 282 insertions(+), 23 deletions(-) [+]
line wrap: on
line diff
--- a/astronomical_archives.py	Mon Sep 04 14:20:34 2023 +0000
+++ b/astronomical_archives.py	Tue Oct 24 06:38:22 2023 +0000
@@ -1,17 +1,47 @@
+import errno
+import functools
 import json
 import os
+import signal
 import sys
 import urllib
 from urllib import request
 
+from astropy.coordinates import SkyCoord
+
 import pyvo
 from pyvo import DALAccessError, DALQueryError, DALServiceError
 from pyvo import registry
 
+
 MAX_ALLOWED_ENTRIES = 100
 MAX_REGISTRIES_TO_SEARCH = 100
 
 
+class TimeoutException(Exception):
+    pass
+
+
+def timeout(seconds=10, error_message=os.strerror(errno.ETIME)):
+    def decorator(func):
+        def _handle_timeout(signum, frame):
+            raise TimeoutException(error_message)
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            signal.signal(signal.SIGALRM, _handle_timeout)
+            signal.alarm(seconds)
+            try:
+                result = func(*args, **kwargs)
+            finally:
+                signal.alarm(0)
+            return result
+
+        return wrapper
+
+    return decorator
+
+
 class Service:
     # https://pyvo.readthedocs.io/en/latest/api/pyvo.registry.Servicetype.html
 
@@ -92,6 +122,7 @@
         self.archive_service = None
         self.tables = None
 
+    @timeout(10)
     def get_resources(self,
                       query,
                       number_of_results,
@@ -258,6 +289,26 @@
         return name
 
 
+class ConeService(TapArchive):
+
+    def _get_service(self):
+        if self.access_url:
+            self.archive_service = pyvo.dal.SCSService(self.access_url)
+
+    def get_resources_from_service_list(self, service_list, target, radius):
+
+        resource_list_hydrated = []
+
+        for service in service_list:
+            resources = service.search(target, radius)
+            for i in range(resources.__len__()):
+                resource_url = resources.getrecord(i).getdataurl()
+                if resource_url:
+                    resource_list_hydrated.append(resource_url)
+
+        return resource_list_hydrated
+
+
 class RegistrySearchParameters:
 
     def __init__(self, keyword=None, waveband=None, service_type=None):
@@ -338,6 +389,24 @@
         return archive_list
 
 
+class ConeServiceRegistry:
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def search_services(keyword, number_of_registries):
+
+        service_list = []
+
+        service_list = registry.search(servicetype="scs", keywords=keyword)
+
+        if service_list:
+            service_list = service_list[:number_of_registries]
+
+        return service_list
+
+
 class TapQuery:
 
     def __init__(self, query):
@@ -410,6 +479,7 @@
         self._output_error = output_error
 
         self._set_run_main_parameters()
+
         self._is_initialised, error_message = self._set_archive()
 
         if self._is_initialised and error_message is None:
@@ -479,10 +549,41 @@
         else:
             return False, error_message
 
+    def _set_cone_service(self):
+
+        qs = 'query_section'
+        qsl = 'query_selection'
+        csts = 'cone_search_target_selection'
+
+        error_message = None
+        is_service_initialised = True
+
+        keyword = self._json_parameters[qs][qsl][csts]['keyword']
+
+        service_list = ConeServiceRegistry.search_services(
+            keyword,
+            MAX_REGISTRIES_TO_SEARCH)
+
+        if len(service_list) >= 1:
+            self._services = service_list
+        else:
+            is_service_initialised = False
+            error_message = "no services matching search parameters"
+            Logger.create_action_log(
+                Logger.ACTION_ERROR,
+                Logger.ACTION_TYPE_ARCHIVE_CONNECTION,
+                error_message)
+
+        return is_service_initialised, error_message
+
     def _set_query(self):
 
         qs = 'query_section'
         qsl = 'query_selection'
+        csts = 'cone_search_target_selection'
+        cs = 'cone_section'
+        ts = 'target_selection'
+        con = 'cone_object_name'
 
         if self._query_type == 'obscore_query':
 
@@ -517,6 +618,34 @@
             order_by = \
                 self._json_parameters[qs][qsl]['order_by']
 
+            if self._json_parameters[qs][qsl][cs][csts][ts] == 'coordinates':
+                ra = self._json_parameters[qs][qsl][cs][csts]['ra']
+                dec = self._json_parameters[qs][qsl][cs][csts]['dec']
+            else:
+                obs_target = self._json_parameters[qs][qsl][cs][csts][con]
+
+                if obs_target != 'none' and obs_target is not None:
+                    target = CelestialObject(obs_target)
+                    target_coordinates = target.get_coordinates_in_degrees()
+
+                    ra = target_coordinates['ra']
+                    dec = target_coordinates['dec']
+                else:
+                    ra = None
+                    dec = None
+
+            radius = self._json_parameters[qs][qsl][cs]['radius']
+
+            if (ra != '' and ra is not None)\
+                    and (dec != '' and dec is not None)\
+                    and (radius != '' and radius is not None):
+                cone_condition = \
+                    ADQLConeSearchQuery.get_search_circle_condition(ra,
+                                                                    dec,
+                                                                    radius)
+            else:
+                cone_condition = None
+
             obscore_query_object = ADQLObscoreQuery(dataproduct_type,
                                                     obs_collection,
                                                     obs_title,
@@ -531,6 +660,7 @@
                                                     calibration_level,
                                                     t_min,
                                                     t_max,
+                                                    cone_condition,
                                                     order_by)
 
             self._adql_query = obscore_query_object.get_query()
@@ -558,6 +688,33 @@
         else:
             self._adql_query = ADQLObscoreQuery.base_query
 
+    def _set_cone_query(self):
+
+        qs = 'query_section'
+        qsl = 'query_selection'
+        csts = 'cone_search_target_selection'
+        ts = 'target_selection'
+        con = 'cone_object_name'
+
+        search_radius = self._json_parameters[qs][qsl]['radius']
+        time = None
+
+        if self._json_parameters[qs][qsl][csts][ts] == 'coordinates':
+            ra = self._json_parameters[qs][qsl][csts]['ra']
+            dec = self._json_parameters[qs][qsl][csts]['dec']
+            time = self._json_parameters[qs][qsl][csts]['time']
+        else:
+            target = CelestialObject(self._json_parameters[qs][qsl][csts][con])
+
+            target_coordinates = target.get_coordinates_in_degrees()
+
+            ra = target_coordinates['ra']
+            dec = target_coordinates['dec']
+
+        cone_query_object = ADQLConeSearchQuery(ra, dec, search_radius, time)
+
+        self._adql_query = cone_query_object.get_query()
+
     def _set_output(self):
         self._number_of_files = \
             int(
@@ -594,12 +751,20 @@
                 self._archive_type)
 
             for archive in self._archives:
-                _file_url, error_message = archive.get_resources(
-                    self._adql_query,
-                    self._number_of_files,
-                    self._url_field)
+                try:
+                    _file_url, error_message = archive.get_resources(
+                        self._adql_query,
+                        self._number_of_files,
+                        self._url_field)
 
-                file_url.extend(_file_url)
+                    file_url.extend(_file_url)
+                except TimeoutException:
+                    error_message = \
+                        "Archive is taking too long to respond (timeout)"
+                    Logger.create_action_log(
+                        Logger.ACTION_ERROR,
+                        Logger.ACTION_TYPE_ARCHIVE_CONNECTION,
+                        error_message)
 
                 if len(file_url) >= int(self._number_of_files):
                     file_url = file_url[:int(self._number_of_files)]
@@ -708,12 +873,11 @@
 
                 FileHandler.write_file_to_output(summary_file,
                                                  self._output_error)
-
         else:
             summary_file = Logger.create_log_file("Archive",
                                                   self._adql_query)
 
-            summary_file += "Unable to initialize archive"
+            summary_file += "Unable to initialize archives"
 
             FileHandler.write_file_to_output(summary_file,
                                              self._output_error)
@@ -745,6 +909,7 @@
                  calibration_level,
                  t_min,
                  t_max,
+                 cone_condition,
                  order_by):
 
         super().__init__()
@@ -770,6 +935,11 @@
         if dataproduct_type == 'none' or dataproduct_type is None:
             dataproduct_type = ''
 
+        if cone_condition is not None:
+            self.cone_condition = cone_condition
+        else:
+            self.cone_condition = None
+
         self.parameters = {
             'dataproduct_type': dataproduct_type,
             'obs_collection': obs_collection,
@@ -782,9 +952,9 @@
             'target_name': target_name,
             'obs_publisher_id': obs_publisher_id,
             's_fov': s_fov,
-            'calibration_level': calibration_level,
+            'calib_level': calibration_level,
             't_min': t_min,
-            't_max': t_max
+            't_max': t_max,
         }
 
         self.order_by = order_by
@@ -807,11 +977,21 @@
         return super()._get_order_by_clause(obscore_order_type)
 
     def get_where_statement(self):
-        return self._get_where_clause(self.parameters)
+        where_clause = self._get_where_clause(self.parameters)
+
+        if where_clause == '' and self.cone_condition is not None:
+            where_clause = 'WHERE ' + self.get_cone_condition()
+        elif where_clause != '' and self.cone_condition is not None:
+            where_clause += 'AND ' + self.get_cone_condition()
+
+        return where_clause
 
     def _get_where_clause(self, parameters):
         return super()._get_where_clause(parameters)
 
+    def get_cone_condition(self):
+        return self.cone_condition
+
 
 class ADQLTapQuery(BaseADQLQuery):
     base_query = 'SELECT TOP '+str(MAX_ALLOWED_ENTRIES)+' * FROM '
@@ -833,6 +1013,69 @@
             return ADQLTapQuery.base_query + str(table)
 
 
+class ADQLConeSearchQuery:
+
+    base_query = "SELECT TOP 100 * FROM ivoa.obscore"
+
+    def __init__(self, ra, dec, radius, time=None):
+
+        self.ra = ra
+        self.dec = dec
+        self.radius = radius
+        self.time = time
+
+        self._query = ADQLObscoreQuery.base_query
+
+        if self.ra and self.dec and self.radius:
+            self._query += " WHERE "
+            self._query += self._get_search_circle(ra, dec, radius)
+
+            if self.time:
+                self._query += self._get_search_time()
+
+    def _get_search_circle(self, ra, dec, radius):
+        return "(CONTAINS" \
+               "(POINT('ICRS', s_ra, s_dec), " \
+               "CIRCLE('ICRS', "+str(ra)+", "+str(dec)+", "+str(radius)+")" \
+               ") = 1)"
+
+    def _get_search_time(self):
+        return " AND t_min <= "+self.time+" AND t_max >= "+self.time
+
+    def get_query(self):
+        return self._query
+
+    @staticmethod
+    def get_search_circle_condition(ra, dec, radius):
+        return "(CONTAINS" \
+               "(POINT('ICRS', s_ra, s_dec)," \
+               "CIRCLE('ICRS', "+str(ra)+", "+str(dec)+", "+str(radius)+")" \
+               ") = 1) "
+
+
+class CelestialObject:
+
+    def __init__(self, name):
+        self.name = name
+        self.coordinates = None
+
+        self.coordinates = SkyCoord.from_name(self.name)
+
+    def get_coordinates_in_degrees(self):
+
+        coordinates = {
+            'ra': '',
+            'dec': ''
+        }
+
+        ra_dec = self.coordinates.ravel()
+
+        coordinates['ra'] = ra_dec.ra.degree[0]
+        coordinates['dec'] = ra_dec.dec.degree[0]
+
+        return coordinates
+
+
 class HTMLReport:
     _html_report_base_header = ''
     _html_report_base_body = ''
--- a/astronomical_archives.xml	Mon Sep 04 14:20:34 2023 +0000
+++ b/astronomical_archives.xml	Tue Oct 24 06:38:22 2023 +0000
@@ -1,4 +1,4 @@
-<tool id="astronomical_archives" name="Astronomical Archives (IVOA)" version="0.9.0">
+<tool id="astronomical_archives" name="Astronomical Archives (IVOA)" version="0.9.1">
     <description>queries astronomical archives through Virtual Observatory protocols</description>
     <edam_operations>
         <edam_operation>operation_0224</edam_operation>
@@ -23,15 +23,12 @@
         <conditional name="archive_selection">
             <param name="archive_type" type="select" label="Archive Selection">
               <option value="archive">Query specific IVOA archive</option>
-              <option value="registry">Query IVOA registry</option>
+              <option value="registry">Query all matching IVOA archives</option>
             </param>
             <when value="registry">
               <param name="keyword" type="text" label="Keyword" />
               <param name="service_type" type="select" label="Service type">
                 <option value="TAP" selected="true">TAP: Tables</option>
-                <option value="SCS">SCS: Cone search</option>
-                <option value="SSA">SSA: Spectra</option>
-                <option value="SIA">SIA: Images</option>
               </param>
               <param name="wavebands" type="select" label="Wavebands">
                 <option value="all" selected="true">All</option>
@@ -58,11 +55,26 @@
               <option value="obscore_query">IVOA obscore table query builder</option>
               <option value="raw_query">Raw ADQL query builder</option>
             </param>
-            <!-- Parameters used in tests cannot be optional -->
             <when value="none"></when>
             <when value="obscore_query">
               <!-- Fields defined https://www.ivoa.net/documents/ObsCore/20160224/WD-ObsCore-v1.1-20160224.pdf -->
               <param name="target_name" type="text" label="Observation target name" help="Typically name of the astronomical object observed" />
+              <section name="cone_section" title="Cone search parameters">
+                <conditional name="cone_search_target_selection">
+                  <param name="target_selection" type="select" label="Search center">
+                    <option value="coordinates">Coordinates</option>
+                    <option value="object_name">Source name</option>
+                  </param>
+                  <when value="coordinates">
+                    <param name="ra" type="text" label="Right ascension" optional="false" help="In degree e.g. 27.1" />
+                    <param name="dec" type="text" label="Declination" optional="false" help="In degree e.g. 30.5" />
+                  </when>
+                  <when value="object_name">
+                    <param name="cone_object_name" type="text" label="Observation target name" optional="false" help="e.g. mrk 421" />
+                  </when>
+                </conditional>
+                <param name="radius" type="text" label="Search radius" optional="false" help="In degree e.g. 0.1"/>
+              </section>
               <param name="t_min" type="float" optional="true" label="Start time in MJD" />
               <param name="t_max" type="float" optional="true" label="Stop time in MJD" />
               <!-- Dataproduct_type field values definition https://www.ivoa.net/rdf/product-type/2023-06-26/product-type.html  -->
@@ -93,12 +105,13 @@
               <param name="obs_id" type="text" label="Observation Id" />
               <param name="facility_name" type="text" label="Facility name" help="Name of the facility used for this observation" />
               <param name="instrument_name" type="text" label="Instrument name" help="Name of the instrument used for this observation" />
-              <param name="em_min" type="float" optional="true" label="Start of the energy range, vacuum wavelength in meters" />
-              <param name="em_max" type="float" optional="true" label="Stop of the energy range, vacuum wavelength in meters" />
+              <param name="em_min" type="float" optional="true"  label="Start of the energy range, vacuum wavelength in meters" />
+              <param name="em_max" type="float" optional="true"  label="Stop of the energy range, vacuum wavelength in meters" />
               <param name="obs_publisher_id" type="text" label="Publisher dataset ID" />
               <param name="s_fov" type="text" label="Diameter (bounds) of the covered region (deg)" />
-              <param name="calibration_level" type="select" label="Calibration level (0, 1, 2, 3, 4, 5)" >
+              <param name="calibration_level" type="select" label="Calibration level (-1, 0, 1, 2, 3, 4, 5)" >
                 <option value="none">None</option>
+                <option value="-1">-1</option>
                 <option value="0">0</option>
                 <option value="1">1</option>
                 <option value="2">2</option>
@@ -132,7 +145,6 @@
             <option value="b">Return URL list as HTML</option>
           </param>
         </section>
-
     </inputs>
     <outputs>
         <data name="output" format="fits" label="${tool.name} -> File from ${archive_selection.archive_type} search:" >
@@ -671,7 +683,9 @@
 
 -----
 
-**ChiVO TAP** https://vo.chivo.cl/tap ChiVO TAP service
+.. workaround for temporary unavailable urls
+
+**ChiVO TAP** https vo.chivo.cl/tap ChiVO TAP service
 
 -----
 
@@ -925,7 +939,9 @@
 
 -----
 
-**FAI NVO DC TAP** http://vo.fai.kz/tap FAI archives TAP service
+.. Workaround for temporary unavailable urls
+
+*FAI NVO DC TAP** http vo.fai.kz/tap FAI archives TAP service
 
 -----
 
--- a/tool-data/astronomical_archives_gen.loc.sample	Mon Sep 04 14:20:34 2023 +0000
+++ b/tool-data/astronomical_archives_gen.loc.sample	Tue Oct 24 06:38:22 2023 +0000
@@ -137,4 +137,4 @@
 134	APPLAUSE - Archives of Photographic PLates for Astronomical USE  TAP Service	https://www.plate-archive.org/tap
 135	TAP interface of the 3XMM-DR5 XMM-Newton Catalogue	http://xcatdb.unistra.fr/3xmmdr5/tap
 136	TAP interface of the 3XMM-dr7 XMM-Newton Catalogue	http://xcatdb.unistra.fr/3xmmdr7/tap
-137	TAP interface of the 4XMM dr13 XMM-Newton Catalogue	https://xcatdb.unistra.fr/xtapdb
+137	TAP interface of the 4XMM dr13 XMM-Newton Catalogue	https://xcatdb.unistra.fr/xtapdb
\ No newline at end of file