adql query parser function

cdci_data_analysis/analysis/ivoa_helper.py

@@ -0,0 +1,94 @@
+import os.path
+
+from queryparser.adql import ADQLQueryTranslator
+from queryparser.postgresql import PostgreSQLQueryProcessor
+from queryparser.exceptions import QuerySyntaxError
+
+from psycopg2 import connect, DatabaseError
+
+from ..flask_app.sentry import sentry
+from ..app_logging import app_logging
+
+logger = app_logging.getLogger('ivoa_helper')
+
+
+def parse_adql_query(query):
+    try:
+        adt = ADQLQueryTranslator(query)
+
+        output_obj = dict(
+            mysql_query=None,
+            psql_query=adt.to_postgresql()
+        )
+
+    except QuerySyntaxError as qe:
+        logger.error(f'Error parsing ADQL query: {qe}')
+        output_obj = dict(
+            mysql_query=None,
+            psql_query=None
+        )
+    return output_obj
+
+
+def run_ivoa_query(query, **kwargs):
+    parsed_query_obj = parse_adql_query(query)
+
+    # TODO use a specific dedicated table and schema to refer to the product_gallery DB ?
+    # tables = parsed_query_obj.get('tables', [])
+    # if len(tables) == 1 and tables[0] == 'product_gallery':
+    logger.info('Performing query on the product_gallery')
+    vo_psql_pg_host = kwargs.get('vo_psql_pg_host', None)
+    vo_psql_pg_user = kwargs.get('vo_psql_pg_user', None)
+    vo_psql_pg_password = kwargs.get('vo_psql_pg_password', None)
+    vo_psql_pg_db = kwargs.get('vo_psql_pg_db', None)
+    product_gallery_url = kwargs.get('product_gallery_url', None)
+    result_list = run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                                      vo_psql_pg_host=vo_psql_pg_host,
+                                                      vo_psql_pg_user=vo_psql_pg_user,
+                                                      vo_psql_pg_password=vo_psql_pg_password,
+                                                      vo_psql_pg_db=vo_psql_pg_db,
+                                                      product_gallery_url=product_gallery_url)
+    return result_list
+
+
+def run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                        vo_psql_pg_host,
+                                        vo_psql_pg_user,
+                                        vo_psql_pg_password,
+                                        vo_psql_pg_db,
+                                        product_gallery_url=None
+                                        ):
+    result_list = []
+
+    try:
+        with connect(
+            host=vo_psql_pg_host,
+            database=vo_psql_pg_db,
+            user=vo_psql_pg_user,
+            password=vo_psql_pg_password
+        ) as connection:
+            db_query = parsed_query_obj.get('psql_query')
+            with connection.cursor() as cursor:
+                cursor.execute(db_query)
+                for row in cursor:
+                    list_row = list(row)
+                    if product_gallery_url is not None:
+                        for index, value in enumerate(list_row):
+                            description = cursor.description[index]
+                            if description.name in {'path', 'path_alias'} and value is not None and isinstance(value, str):
+                                if value.startswith('/'):
+                                    value = value[1:]
+                                list_row[index] = os.path.join(product_gallery_url, value)
+                    result_list.append(list_row)
+
+    except (Exception, DatabaseError) as e:
+        sentry.capture_message(f"Error when querying to the Postgresql server: {str(e)}")
+        logger.error(f"Error when querying to the Postgresql server: {str(e)}")
+
+    finally:
+        if connection is not None:
+            cursor.close()
+            connection.close()
+            logger.info('Database connection closed')
+
+    return result_list

cdci_data_analysis/config_dir/conf_env.yml.example

@@ -122,4 +122,10 @@ dispatcher:
         # url for the conversion of a given time, in UTC format, to the correspondent REVNUM
         converttime_revnum_service_url: COVERTTIME_REVNUM_SERVICE_URL
 
-
+    # virtual observatory related configurations (eg mysql credentials)
+    vo_options:
+        # mysql credentials
+        vo_mysql_pg_host: MYSQL_PG_HOST
+        vo_mysql_pg_user: MYSQL_PG_USER
+        vo_mysql_pg_password: MYSQL_PG_PASSWORD
+        vo_mysql_pg_db: MYSQL_PG_DB

cdci_data_analysis/configurer.py

@@ -266,6 +266,14 @@ def __init__(self, cfg_dict, origin=None):
                                      disp_dict.get('renku_options', {}).get('renku_gitlab_repository_url', None),
                                      disp_dict.get('renku_options', {}).get('renku_base_project_url', None),
                                      disp_dict.get('renku_options', {}).get('ssh_key_path', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_db', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_db', None)
                                      )
 
         # not used?
@@ -344,6 +352,14 @@ def set_conf_dispatcher(self,
                             renku_gitlab_repository_url,
                             renku_base_project_url,
                             renku_gitlab_ssh_key_path,
+                            vo_mysql_pg_host,
+                            vo_mysql_pg_user,
+                            vo_mysql_pg_password,
+                            vo_mysql_pg_db,
+                            vo_psql_pg_host,
+                            vo_psql_pg_user,
+                            vo_psql_pg_password,
+                            vo_psql_pg_db
                             ):
         # Generic to dispatcher
         #print(dispatcher_url, dispatcher_port)
@@ -395,6 +411,14 @@ def set_conf_dispatcher(self,
         self.renku_gitlab_repository_url = renku_gitlab_repository_url
         self.renku_gitlab_ssh_key_path = renku_gitlab_ssh_key_path
         self.renku_base_project_url = renku_base_project_url
+        self.vo_mysql_pg_host = vo_mysql_pg_host
+        self.vo_mysql_pg_user = vo_mysql_pg_user
+        self.vo_mysql_pg_password = vo_mysql_pg_password
+        self.vo_mysql_pg_db = vo_mysql_pg_db
+        self.vo_psql_pg_host = vo_psql_pg_host
+        self.vo_psql_pg_user = vo_psql_pg_user
+        self.vo_psql_pg_password = vo_psql_pg_password
+        self.vo_psql_pg_db = vo_psql_pg_db
 
     def get_data_serve_conf(self, instr_name):
         if instr_name in self.data_server_conf_dict.keys():

cdci_data_analysis/flask_app/app.py

@@ -29,7 +29,7 @@
 import time as _time
 from urllib.parse import urlencode, urlparse
 
-from cdci_data_analysis.analysis import drupal_helper, tokenHelper, renku_helper, email_helper, matrix_helper
+from cdci_data_analysis.analysis import drupal_helper, tokenHelper, renku_helper, email_helper, matrix_helper, ivoa_helper
 from .logstash import logstash_message
 from .schemas import QueryOutJSON, dispatcher_strict_validate
 from marshmallow.exceptions import ValidationError
@@ -430,6 +430,41 @@ def push_renku_branch():
                                            "Our team is notified and is working on it.")
 
 
+@app.route('/run_adql_query')
+def run_adql_query():
+    par_dic = request.values.to_dict()
+    sanitized_request_values = sanitize_dict_before_log(par_dic)
+    logger.info('\033[32m===========================> run_adql_query\033[0m')
+
+    logger.info('\033[33m raw request values: %s \033[0m', dict(sanitized_request_values))
+
+    token = par_dic.get('token', None)
+    app_config = app.config.get('conf')
+    secret_key = app_config.secret_key
+
+    output, output_code = tokenHelper.validate_token_from_request(token=token, secret_key=secret_key,
+                                                                  required_roles=['ivoa_user'],
+                                                                  action="run an ADQL query")
+
+    if output_code is not None:
+        return make_response(output, output_code)
+
+    adql_query = par_dic.get('adql_query', None)
+    vo_psql_pg_host = app_config.vo_psql_pg_host
+    vo_psql_pg_user = app_config.vo_psql_pg_user
+    vo_psql_pg_password = app_config.vo_psql_pg_password
+    vo_psql_pg_db = app_config.vo_psql_pg_db
+    product_gallery_url = app_config.product_gallery_url
+
+    result_query = ivoa_helper.run_ivoa_query(adql_query,
+                                              vo_psql_pg_host=vo_psql_pg_host,
+                                              vo_psql_pg_user=vo_psql_pg_user,
+                                              vo_psql_pg_password=vo_psql_pg_password,
+                                              vo_psql_pg_db=vo_psql_pg_db,
+                                              product_gallery_url=product_gallery_url)
+
+    return jsonify(result_query)
+
 
 @app.route('/run_analysis', methods=['POST', 'GET'])
 def run_analysis():

requirements.txt

@@ -29,6 +29,8 @@ GitPython
 nbformat
 sentry-sdk
 pytest-sentry
+sqlparse
+queryparser-python3
 -e git+https://github.com/oda-hub/oda_api.git#egg=oda_api
 
 MarkupSafe==2.0.1

setup.py

@@ -45,11 +45,14 @@
     "black>=22.10.0",
     "bs4",
     "GitPython",
+    "sqlparse",
+    "queryparser-python3",
     "nbformat",
     "giturlparse",
     "sentry-sdk",
     "validators==0.28.3",
-    "jsonschema<=4.17.3"
+    "jsonschema<=4.17.3",
+    'psycopg2'
 ]
 
 test_req = [