adql query parser function

cdci_data_analysis/analysis/ivoa_helper.py

@@ -0,0 +1,93 @@
+import os.path
+
+from queryparser.adql import ADQLQueryTranslator
+from queryparser.exceptions import QuerySyntaxError
+
+from psycopg2 import connect, DatabaseError
+
+from ..app_logging import app_logging
+from ..analysis.exceptions import RequestNotUnderstood
+
+logger = app_logging.getLogger('ivoa_helper')
+
+
+def parse_adql_query(query):
+    try:
+        adt = ADQLQueryTranslator(query)
+
+        output_obj = dict(
+            mysql_query=None,
+            psql_query=adt.to_postgresql()
+        )
+
+    except QuerySyntaxError as qe:
+        logger.error(f'Error while parsing the ADQL query: {str(qe)}')
+        raise RequestNotUnderstood(f"Error while parsing the ADQL query: {str(qe)}")
+    return output_obj
+
+
+def run_ivoa_query(query, **kwargs):
+    parsed_query_obj = parse_adql_query(query)
+
+    logger.info('Performing query on the product_gallery')
+    vo_psql_pg_host = kwargs.get('vo_psql_pg_host', None)
+    vo_psql_pg_user = kwargs.get('vo_psql_pg_user', None)
+    vo_psql_pg_password = kwargs.get('vo_psql_pg_password', None)
+    vo_psql_pg_db = kwargs.get('vo_psql_pg_db', None)
+    product_gallery_url = kwargs.get('product_gallery_url', None)
+    result_list = run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                                      vo_psql_pg_host=vo_psql_pg_host,
+                                                      vo_psql_pg_user=vo_psql_pg_user,
+                                                      vo_psql_pg_password=vo_psql_pg_password,
+                                                      vo_psql_pg_db=vo_psql_pg_db,
+                                                      product_gallery_url=product_gallery_url)
+    return result_list
+
+
+def run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                        vo_psql_pg_host,
+                                        vo_psql_pg_user,
+                                        vo_psql_pg_password,
+                                        vo_psql_pg_db,
+                                        product_gallery_url=None
+                                        ):
+    result_list = []
+    connection = None
+    try:
+        with connect(
+            host=vo_psql_pg_host,
+            database=vo_psql_pg_db,
+            user=vo_psql_pg_user,
+            password=vo_psql_pg_password
+        ) as connection:
+            db_query = parsed_query_obj.get('psql_query')
+            with connection.cursor() as cursor:
+                cursor.execute(db_query)
+                for row in cursor:
+                    list_row = list(row)
+                    if product_gallery_url is not None:
+                        for index, value in enumerate(list_row):
+                            description = cursor.description[index]
+                            if description.name in {'file_uri', 'file_name'} and value is not None and isinstance(value, str):
+                                file_name_list = [v.strip() for v in value.split(',')]
+                                if description.name == 'file_uri':
+                                    for id, file_name in enumerate(file_name_list):
+                                        file_name_list[id] = os.path.join(product_gallery_url, 'sites/default/files/', file_name.strip())
+                                list_row[index] = file_name_list
+                            if description.name in {'path', 'path_alias'} and value is not None and isinstance(value, str):
+                                if value.startswith('/'):
+                                    value = value[1:]
+                                list_row[index] = os.path.join(product_gallery_url, value)
+                    result_list.append(list_row)
+
+    except (Exception, DatabaseError) as e:
+        logger.error(f"Error when querying to the Postgresql server: {str(e)}")
+        raise e
+
+    finally:
+        if connection is not None:
+            cursor.close()
+            connection.close()
+            logger.info('Database connection closed')
+
+    return result_list

cdci_data_analysis/config_dir/conf_env.yml.example

@@ -124,4 +124,15 @@ dispatcher:
         # url for the conversion of a given time, in UTC format, to the correspondent REVNUM
         converttime_revnum_service_url: COVERTTIME_REVNUM_SERVICE_URL
 
-
+    # virtual observatory related configurations (eg mysql credentials)
+    vo_options:
+        # mysql credentials
+        vo_mysql_pg_host: MYSQL_PG_HOST
+        vo_mysql_pg_user: MYSQL_PG_USER
+        vo_mysql_pg_password: MYSQL_PG_PASSWORD
+        vo_mysql_pg_db: MYSQL_PG_DB
+        # postgresql credentials
+        vo_psql_pg_host: PSQL_PG_HOST
+        vo_psql_pg_user: PSQL_PG_USER
+        vo_psql_pg_password: PSQL_PG_PASSWORD
+        vo_psql_pg_db: PSQL_PG_DB

cdci_data_analysis/configurer.py

@@ -269,6 +269,14 @@ def __init__(self, cfg_dict, origin=None):
                                      disp_dict.get('renku_options', {}).get('renku_gitlab_repository_url', None),
                                      disp_dict.get('renku_options', {}).get('renku_base_project_url', None),
                                      disp_dict.get('renku_options', {}).get('ssh_key_path', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_db', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_db', None)
                                      )
 
         # not used?
@@ -348,6 +356,14 @@ def set_conf_dispatcher(self,
                             renku_gitlab_repository_url,
                             renku_base_project_url,
                             renku_gitlab_ssh_key_path,
+                            vo_mysql_pg_host,
+                            vo_mysql_pg_user,
+                            vo_mysql_pg_password,
+                            vo_mysql_pg_db,
+                            vo_psql_pg_host,
+                            vo_psql_pg_user,
+                            vo_psql_pg_password,
+                            vo_psql_pg_db
                             ):
         # Generic to dispatcher
         #print(dispatcher_url, dispatcher_port)
@@ -400,6 +416,14 @@ def set_conf_dispatcher(self,
         self.renku_gitlab_repository_url = renku_gitlab_repository_url
         self.renku_gitlab_ssh_key_path = renku_gitlab_ssh_key_path
         self.renku_base_project_url = renku_base_project_url
+        self.vo_mysql_pg_host = vo_mysql_pg_host
+        self.vo_mysql_pg_user = vo_mysql_pg_user
+        self.vo_mysql_pg_password = vo_mysql_pg_password
+        self.vo_mysql_pg_db = vo_mysql_pg_db
+        self.vo_psql_pg_host = vo_psql_pg_host
+        self.vo_psql_pg_user = vo_psql_pg_user
+        self.vo_psql_pg_password = vo_psql_pg_password
+        self.vo_psql_pg_db = vo_psql_pg_db
 
     def get_data_serve_conf(self, instr_name):
         if instr_name in self.data_server_conf_dict.keys():

cdci_data_analysis/flask_app/app.py

@@ -29,7 +29,7 @@
 import time as _time
 from urllib.parse import urlencode, urlparse
 
-from cdci_data_analysis.analysis import drupal_helper, tokenHelper, renku_helper, email_helper, matrix_helper
+from cdci_data_analysis.analysis import drupal_helper, tokenHelper, renku_helper, email_helper, matrix_helper, ivoa_helper
 from .logstash import logstash_message
 from .schemas import QueryOutJSON, dispatcher_strict_validate
 from marshmallow.exceptions import ValidationError
@@ -269,7 +269,7 @@
                                                  'product_gallery_secret_key',
                                                  'matrix_sender_access_token', 'matrix_incident_report_sender_personal_access_token',
                                                  'matrix_bcc_receivers_room_ids', 'matrix_incident_report_receivers_room_ids',
-                                                 'smtp_server_password'])
+                                                 'smtp_server_password', 'vo_psql_pg_db', 'vo_psql_pg_host', 'vo_psql_pg_password', 'vo_psql_pg_user'])
     }
 
     plugins = {}
@@ -430,6 +430,57 @@
                                            "Our team is notified and is working on it.")
 
 
+@app.route('/run_adql_query')
+def run_adql_query():
+    try:
+        par_dic = request.values.to_dict()
+        sanitized_request_values = sanitize_dict_before_log(par_dic)
+        logger.info('\033[32m===========================> run_adql_query\033[0m')
+
+        logger.info('\033[33m raw request values: %s \033[0m', dict(sanitized_request_values))
+
+        token = par_dic.get('token', None)
+        app_config = app.config.get('conf')
+        secret_key = app_config.secret_key
+
+        output, output_code = tokenHelper.validate_token_from_request(token=token, secret_key=secret_key,
+                                                                      required_roles=['ivoa_user'],
+                                                                      action="run an ADQL query")
+
+        if output_code is not None:
+            return make_response(output, output_code)
+
+        adql_query = par_dic.get('adql_query', None)
+        vo_psql_pg_host = app_config.vo_psql_pg_host
+        vo_psql_pg_user = app_config.vo_psql_pg_user
+        vo_psql_pg_password = app_config.vo_psql_pg_password
+        vo_psql_pg_db = app_config.vo_psql_pg_db
+        product_gallery_url = app_config.product_gallery_url
+
+        result_query = ivoa_helper.run_ivoa_query(adql_query,
+                                                  vo_psql_pg_host=vo_psql_pg_host,
+                                                  vo_psql_pg_user=vo_psql_pg_user,
+                                                  vo_psql_pg_password=vo_psql_pg_password,
+                                                  vo_psql_pg_db=vo_psql_pg_db,
+                                                  product_gallery_url=product_gallery_url)
+
+        return jsonify(result_query)
+    except APIerror as api_e:
+        error_message = f"Error while running an ADQL query: "
+        if hasattr(api_e, 'message') and api_e.message is not None:
+            error_message += api_e.message
+        else:
+            error_message += f"{repr(api_e)}"
+        logging.getLogger().error(error_message)
+        sentry.capture_message(error_message)
+
+        return make_response(error_message)
+    except Exception as e:
+        error_message = f"Error while running an ADQL query: {str(e)}\n{traceback.format_exc()}"
+        logging.getLogger().error(error_message)
+        sentry.capture_message(error_message)
+        return make_response(f"Internal error while running an ADQL query. Our team is notified and is working on it.")
+
 
 @app.route('/run_analysis', methods=['POST', 'GET'])
 def run_analysis():

requirements.txt

@@ -4,6 +4,7 @@ pyyaml
 simplejson
 flask==2.0.3
 astropy>=5.0.1
+pytest-xdist<=3.5.0
 pylogstash_context>=0.1.19
 gunicorn
 decorator
@@ -29,6 +30,7 @@ GitPython
 nbformat
 sentry-sdk
 pytest-sentry
+queryparser-python3
 -e git+https://github.com/oda-hub/oda_api.git#egg=oda_api
 
 MarkupSafe==2.0.1

setup.py

@@ -45,11 +45,13 @@
     "black>=22.10.0",
     "bs4",
     "GitPython",
+    "queryparser-python3",
     "nbformat",
     "giturlparse",
     "sentry-sdk",
     "validators==0.28.3",
-    "jsonschema<=4.17.3"
+    "jsonschema<=4.17.3",
+    'psycopg2'
 ]
 
 test_req = [

tests/test_server_basic.py

@@ -2458,6 +2458,7 @@ def test_example_config(dispatcher_test_conf):
     example_config = yaml.load(open(example_config_fn), Loader=yaml.SafeLoader)['dispatcher']
     example_config.pop('product_gallery_options', None)
     example_config.pop('matrix_options', None)
+    example_config.pop('vo_options', None)
 
     mapper = lambda x, y: ".".join(map(str, x))
     example_config_keys = flatten_nested_structure(example_config, mapper)
@@ -2479,6 +2480,7 @@ def test_example_config_with_gallery(dispatcher_test_conf_with_gallery):
 
     example_config = yaml.load(open(example_config_fn), Loader=yaml.SafeLoader)['dispatcher']
     example_config.pop('matrix_options', None)
+    example_config.pop('vo_options', None)
 
     mapper = lambda x, y: ".".join(map(str, x))
     example_config_keys = flatten_nested_structure(example_config, mapper)
@@ -2500,6 +2502,7 @@ def test_example_config_with_matrix_options(dispatcher_test_conf_with_matrix_opt
     with open(example_config_fn) as example_config_fn_f:
         example_config = yaml.load(example_config_fn_f, Loader=yaml.SafeLoader)['dispatcher']
     example_config.pop('product_gallery_options', None)
+    example_config.pop('vo_options', None)
 
     mapper = lambda x, y: ".".join(map(str, x))
     example_config_keys = flatten_nested_structure(example_config, mapper)
@@ -2511,6 +2514,28 @@ def test_example_config_with_matrix_options(dispatcher_test_conf_with_matrix_opt
     assert set(example_config_keys) == set(test_config_keys)
 
 
+def test_example_config_with_vo_options(dispatcher_test_conf_with_vo_options):
+    import cdci_data_analysis.config_dir
+
+    example_config_fn = os.path.join(
+        os.path.dirname(cdci_data_analysis.__file__),
+        "config_dir/conf_env.yml.example"
+    )
+    with open(example_config_fn) as example_config_fn_f:
+        example_config = yaml.load(example_config_fn_f, Loader=yaml.SafeLoader)['dispatcher']
+    example_config.pop('product_gallery_options', None)
+    example_config.pop('matrix_options', None)
+
+    mapper = lambda x, y: ".".join(map(str, x))
+    example_config_keys = flatten_nested_structure(example_config, mapper)
+    test_config_keys = flatten_nested_structure(dispatcher_test_conf_with_vo_options, mapper)
+
+    print("\n\n\nexample_config_keys", example_config_keys)
+    print("\n\n\ntest_config_keys", test_config_keys)
+
+    assert set(example_config_keys) == set(test_config_keys)
+
+
 def test_image(dispatcher_live_fixture):
     server = dispatcher_live_fixture