# HG changeset patch # User astroteam # Date 1711447435 0 # Node ID 7398ea3d9ac482b28474c3797ce1dbdd4a779fc9 # Parent 667fc28d803ce6c69fe26b3c0b125886fe9ad182 planemo upload for repository https://github.com/esg-epfl-apc/tools-astro/tree/main/tools/ commit b77ceb5085f378a8bef9b202b80e3ca7ef1e9a8e diff -r 667fc28d803c -r 7398ea3d9ac4 astronomical_archives.py --- a/astronomical_archives.py Tue Oct 24 06:38:22 2023 +0000 +++ b/astronomical_archives.py Tue Mar 26 10:03:55 2024 +0000 @@ -2,6 +2,7 @@ import functools import json import os +import re import signal import sys import urllib @@ -17,6 +18,10 @@ MAX_ALLOWED_ENTRIES = 100 MAX_REGISTRIES_TO_SEARCH = 100 +ARCHIVES_TIMEOUT_BYPASS = [ + "https://datalab.noirlab.edu/tap" +] + class TimeoutException(Exception): pass @@ -217,28 +222,34 @@ self.tables = [] - for table in self.archive_service.tables: - archive_table = { - 'name': table.name, - 'type': table.type, - 'fields': None - } - - fields = [] - - for table_field in table.columns: - field = { - 'name': table_field.name, - 'description': table_field.description, - 'unit': table_field.unit, - 'datatype': table_field.datatype.content + try: + for table in self.archive_service.tables: + archive_table = { + 'name': table.name, + 'type': table.type, + 'fields': None } - fields.append(field) + fields = [] + + for table_field in table.columns: + field = { + 'name': table_field.name, + 'description': table_field.description, + 'unit': table_field.unit, + 'datatype': table_field.datatype.content + } - archive_table['fields'] = fields + fields.append(field) + + archive_table['fields'] = fields - self.tables.append(archive_table) + self.tables.append(archive_table) + + # Exception is raised when a table schema is missing + # Missing table will be omitted so no action needed + except DALServiceError: + pass def _is_query_valid(self, query) -> bool: is_valid = True @@ -507,6 +518,20 @@ self._archives.append( TapArchive(access_url=self._service_access_url)) + elif self._archive_type == 'custom': + self._service_access_url = \ + self._json_parameters['archive_selection']['access_url'] + + if Utils.is_valid_url(self._service_access_url): + self._archives.append( + TapArchive(access_url=self._service_access_url)) + else: + error_message = "archive access url is not a valid url" + Logger.create_action_log( + Logger.ACTION_ERROR, + Logger.ACTION_TYPE_ARCHIVE_CONNECTION, + error_message) + else: keyword = \ self._json_parameters['archive_selection']['keyword'] @@ -752,6 +777,11 @@ for archive in self._archives: try: + + if archive.access_url in ARCHIVES_TIMEOUT_BYPASS: + archive.get_resources = \ + timeout(40)(TapArchive.get_resources.__get__(archive)) # noqa: E501 + _file_url, error_message = archive.get_resources( self._adql_query, self._number_of_files, @@ -1250,9 +1280,9 @@ with open(output, "w") as file_output: for url in urls: try: - file_output.write(url[access_url] + ',') + file_output.write(str(url[access_url]) + ',') except Exception: - error_message = "url field not found for url" + error_message = f"url field {access_url} not found for url" Logger.create_action_log( Logger.ACTION_ERROR, Logger.ACTION_TYPE_WRITE_URL, @@ -1305,6 +1335,11 @@ resource_keys.append(key) return resource_keys + @staticmethod + def is_valid_url(url: str) -> bool: + regex_url = re.compile(r'^https?://(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,6}(?::\d+)?(?:/[^\s]*)?$') # noqa: E501 + return re.match(regex_url, url) is not None + class Logger: _logs = [] diff -r 667fc28d803c -r 7398ea3d9ac4 astronomical_archives.xml --- a/astronomical_archives.xml Tue Oct 24 06:38:22 2023 +0000 +++ b/astronomical_archives.xml Tue Mar 26 10:03:55 2024 +0000 @@ -1,4 +1,4 @@ - + queries astronomical archives through Virtual Observatory protocols operation_0224 @@ -24,6 +24,7 @@ + @@ -47,6 +48,11 @@ + + + ^https?://[A-Za-z0-9]([A-Za-z0-9-\.]{0,61}[A-Za-z0-9])?\.[A-Za-z]{2,6}(:\d+)?(/[^\s]*)?$ + +
@@ -255,6 +261,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -460,7 +500,7 @@ ----- -The MAST Archive at STScI TAP end point for the TESS Input Catalog.

The TIC is used to help identify two-minute cadence target selection for the TESS mission, and to calculate physical and observational properties of planet candidates. It is for use by both the TESS science team and the public, and it is periodically updated – the current version is TIC-8. TIC-8 uses the GAIA DR2 catalog as a base and merges a large number of other photometric catalogs, including 2MASS, UCAC4, APASS, SDSS, WISE, etc. There are roughly 1.5 billion stellar and extended sources in TIC-8, containing compiled magnitudes including B, V, u, g, r, i, z, J, H, K, W1-W4, and G. +The MAST Archive at STScI TAP end point for the TESS Input Catalog.The TIC is used to help identify two-minute cadence target selection for the TESS mission, and to calculate physical and observational properties of planet candidates. It is for use by both the TESS science team and the public, and it is periodically updated – the current version is TIC-8. TIC-8 uses the GAIA DR2 catalog as a base and merges a large number of other photometric catalogs, including 2MASS, UCAC4, APASS, SDSS, WISE, etc. There are roughly 1.5 billion stellar and extended sources in TIC-8, containing compiled magnitudes including B, V, u, g, r, i, z, J, H, K, W1-W4, and G. The TIC can be directly accessed through the Mikulski Archive for Space Telescopes (MAST), using either queries or bulk download. The Table Access Protocol (TAP) lets you execute queries against our database tables, and inspect various metadata. Upload is not currently supported. @@ -543,7 +583,7 @@ ----- -**ArVO Byu TAP** http://arvo-registry.sci.am/tap ArVO Byurakan TAP service +**ArVO Byu TAP** arvo-registry.sci.am/tap ArVO Byurakan TAP service ----- @@ -612,7 +652,7 @@ CSHP_DV_130_01_LORES_OBJ.OBJ. The service provides the cosine between the normal of each face (in the same order as the faces defined in the shape model) and the Sun direction; both numerical values and images of the illumination are available. Each map is defined for a given position of the Sun in the frame of 67P (67P/C-G_CK). Longitude 0 is at the center of each map. The code is developed by A. Beth, - Imperial College London, UK and the service is provided by CDPP (http://cdpp.eu). Acknowlegment: The illumination models + Imperial College London, UK and the service is provided by CDPP (cdpp.eu). Acknowlegment: The illumination models have been developed at the Department of Physics at Imperial College London (UK) under the financial support of STFC grant of UK ST/N000692/1 and ESA contract 4000119035/16/ES/JD (Rosetta RPC-PIU). We would also like to warmly thank Bernhard Geiger (ESA) for his support in validating the 2D-illumination maps.