Mercurial > repos > astroteam > astronomical_archives
changeset 2:7398ea3d9ac4 draft default tip
planemo upload for repository https://github.com/esg-epfl-apc/tools-astro/tree/main/tools/ commit b77ceb5085f378a8bef9b202b80e3ca7ef1e9a8e
| author | astroteam |
|---|---|
| date | Tue, 26 Mar 2024 10:03:55 +0000 |
| parents | 667fc28d803c |
| children | |
| files | astronomical_archives.py astronomical_archives.xml |
| diffstat | 2 files changed, 99 insertions(+), 24 deletions(-) [+] |
line wrap: on
line diff
--- a/astronomical_archives.py Tue Oct 24 06:38:22 2023 +0000 +++ b/astronomical_archives.py Tue Mar 26 10:03:55 2024 +0000 @@ -2,6 +2,7 @@ import functools import json import os +import re import signal import sys import urllib @@ -17,6 +18,10 @@ MAX_ALLOWED_ENTRIES = 100 MAX_REGISTRIES_TO_SEARCH = 100 +ARCHIVES_TIMEOUT_BYPASS = [ + "https://datalab.noirlab.edu/tap" +] + class TimeoutException(Exception): pass @@ -217,28 +222,34 @@ self.tables = [] - for table in self.archive_service.tables: - archive_table = { - 'name': table.name, - 'type': table.type, - 'fields': None - } - - fields = [] - - for table_field in table.columns: - field = { - 'name': table_field.name, - 'description': table_field.description, - 'unit': table_field.unit, - 'datatype': table_field.datatype.content + try: + for table in self.archive_service.tables: + archive_table = { + 'name': table.name, + 'type': table.type, + 'fields': None } - fields.append(field) + fields = [] + + for table_field in table.columns: + field = { + 'name': table_field.name, + 'description': table_field.description, + 'unit': table_field.unit, + 'datatype': table_field.datatype.content + } - archive_table['fields'] = fields + fields.append(field) + + archive_table['fields'] = fields - self.tables.append(archive_table) + self.tables.append(archive_table) + + # Exception is raised when a table schema is missing + # Missing table will be omitted so no action needed + except DALServiceError: + pass def _is_query_valid(self, query) -> bool: is_valid = True @@ -507,6 +518,20 @@ self._archives.append( TapArchive(access_url=self._service_access_url)) + elif self._archive_type == 'custom': + self._service_access_url = \ + self._json_parameters['archive_selection']['access_url'] + + if Utils.is_valid_url(self._service_access_url): + self._archives.append( + TapArchive(access_url=self._service_access_url)) + else: + error_message = "archive access url is not a valid url" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_ARCHIVE_CONNECTION, + error_message) + else: keyword = \ self._json_parameters['archive_selection']['keyword'] @@ -752,6 +777,11 @@ for archive in self._archives: try: + + if archive.access_url in ARCHIVES_TIMEOUT_BYPASS: + archive.get_resources = \ + timeout(40)(TapArchive.get_resources.__get__(archive)) # noqa: E501 + _file_url, error_message = archive.get_resources( self._adql_query, self._number_of_files, @@ -1250,9 +1280,9 @@ with open(output, "w") as file_output: for url in urls: try: - file_output.write(url[access_url] + ',') + file_output.write(str(url[access_url]) + ',') except Exception: - error_message = "url field not found for url" + error_message = f"url field {access_url} not found for url" Logger.create_action_log( Logger.ACTION_ERROR, Logger.ACTION_TYPE_WRITE_URL, @@ -1305,6 +1335,11 @@ resource_keys.append(key) return resource_keys + @staticmethod + def is_valid_url(url: str) -> bool: + regex_url = re.compile(r'^https?://(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,6}(?::\d+)?(?:/[^\s]*)?$') # noqa: E501 + return re.match(regex_url, url) is not None + class Logger: _logs = []
--- a/astronomical_archives.xml Tue Oct 24 06:38:22 2023 +0000 +++ b/astronomical_archives.xml Tue Mar 26 10:03:55 2024 +0000 @@ -1,4 +1,4 @@ -<tool id="astronomical_archives" name="Astronomical Archives (IVOA)" version="0.9.1"> +<tool id="astronomical_archives" name="Astronomical Archives (IVOA)" version="0.10.0"> <description>queries astronomical archives through Virtual Observatory protocols</description> <edam_operations> <edam_operation>operation_0224</edam_operation> @@ -24,6 +24,7 @@ <param name="archive_type" type="select" label="Archive Selection"> <option value="archive">Query specific IVOA archive</option> <option value="registry">Query all matching IVOA archives</option> + <option value="custom">Query custom TAP archive</option> </param> <when value="registry"> <param name="keyword" type="text" label="Keyword" /> @@ -47,6 +48,11 @@ <options from_data_table="astronomical_archives" /> </param> </when> + <when value="custom"> + <param name="access_url" type="text" label="TAP archive access url"> + <validator type="regex" message="URL seems invalid">^https?://[A-Za-z0-9]([A-Za-z0-9-\.]{0,61}[A-Za-z0-9])?\.[A-Za-z]{2,6}(:\d+)?(/[^\s]*)?$</validator> + </param> + </when> </conditional> <section name="query_section" title="Query selection" expanded="true"> <conditional name="query_selection"> @@ -255,6 +261,40 @@ </assert_contents> </output> </test> + <test expect_num_outputs="2"> + <param name="output_selection" value="c"/> + <param name="number_of_files" value="1"/> + <conditional name="archive_selection"> + <param name="archive_type" value="custom"/> + <param name="access_url" value="https://datalab.noirlab.edu/tap"/> + </conditional> + <conditional name="query_selection"> + <param name="query_type" value="raw_query" /> + <param name="table" value="allwise.source" /> + <param name="url_field" value="random_id" /> + </conditional> + <output name="output_csv" count="1"> + <assert_contents> + <has_text_matching expression=".*\S+.*" /> + </assert_contents> + </output> + </test> + <test expect_num_outputs="2"> + <param name="output_selection" value="c"/> + <param name="number_of_files" value="1"/> + <conditional name="archive_selection"> + <param name="archive_type" value="custom"/> + <param name="access_url" value="http://voparis-tap-he.obspm.fr/tap"/> + </conditional> + <conditional name="query_selection"> + <param name="query_type" value="obscore_query" /> + </conditional> + <output name="output_csv" count="1"> + <assert_contents> + <has_text_matching expression=".*\S+.*" /> + </assert_contents> + </output> + </test> </tests> <help> @@ -460,7 +500,7 @@ ----- -The MAST Archive at STScI TAP end point for the TESS Input Catalog. The TIC is used to help identify two-minute cadence target selection for the TESS mission, and to calculate physical and observational properties of planet candidates. It is for use by both the TESS science team and the public, and it is periodically updated – the current version is TIC-8. TIC-8 uses the GAIA DR2 catalog as a base and merges a large number of other photometric catalogs, including 2MASS, UCAC4, APASS, SDSS, WISE, etc. There are roughly 1.5 billion stellar and extended sources in TIC-8, containing compiled magnitudes including B, V, u, g, r, i, z, J, H, K, W1-W4, and G. +The MAST Archive at STScI TAP end point for the TESS Input Catalog.The TIC is used to help identify two-minute cadence target selection for the TESS mission, and to calculate physical and observational properties of planet candidates. It is for use by both the TESS science team and the public, and it is periodically updated – the current version is TIC-8. TIC-8 uses the GAIA DR2 catalog as a base and merges a large number of other photometric catalogs, including 2MASS, UCAC4, APASS, SDSS, WISE, etc. There are roughly 1.5 billion stellar and extended sources in TIC-8, containing compiled magnitudes including B, V, u, g, r, i, z, J, H, K, W1-W4, and G. The TIC can be directly accessed through the Mikulski Archive for Space Telescopes (MAST), using either queries or bulk download. The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. @@ -543,7 +583,7 @@ ----- -**ArVO Byu TAP** http://arvo-registry.sci.am/tap ArVO Byurakan TAP service +**ArVO Byu TAP** arvo-registry.sci.am/tap ArVO Byurakan TAP service ----- @@ -612,7 +652,7 @@ CSHP_DV_130_01_LORES_OBJ.OBJ. The service provides the cosine between the normal of each face (in the same order as the faces defined in the shape model) and the Sun direction; both numerical values and images of the illumination are available. Each map is defined for a given position of the Sun in the frame of 67P (67P/C-G_CK). Longitude 0 is at the center of each map. The code is developed by A. Beth, - Imperial College London, UK and the service is provided by CDPP (http://cdpp.eu). Acknowlegment: The illumination models + Imperial College London, UK and the service is provided by CDPP (cdpp.eu). Acknowlegment: The illumination models have been developed at the Department of Physics at Imperial College London (UK) under the financial support of STFC grant of UK ST/N000692/1 and ESA contract 4000119035/16/ES/JD (Rosetta RPC-PIU). We would also like to warmly thank Bernhard Geiger (ESA) for his support in validating the 2D-illumination maps.