adql query parser function

cdci_data_analysis/analysis/ivoa_helper.py

@@ -0,0 +1,112 @@
+import json
+import os.path
+
+from black.lines import append_leaves
+from queryparser.adql import ADQLQueryTranslator
+from queryparser.mysql import MySQLQueryProcessor
+from queryparser.exceptions import QuerySyntaxError
+
+from mysql.connector import connect, Error
+
+from ..flask_app.sentry import sentry
+from ..app_logging import app_logging
+
+logger = app_logging.getLogger('ivoa_helper')
+
+
+def parse_adql_query(query):
+    try:
+        # queryparser
+        adt = ADQLQueryTranslator(query)
+        qp = MySQLQueryProcessor()
+        qp.set_query(adt.to_mysql())
+        qp.process_query()
+
+        output_obj = dict(
+            columns=qp.display_columns,
+            tables=qp.tables,
+            rest=qp,
+            mysql_query=qp.query
+        )
+
+    except QuerySyntaxError as qe:
+        logger.error(f'Error parsing ADQL query: {qe}')
+        output_obj = dict(
+            tables=None,
+            columns=None,
+            rest=None,
+            mysql_query=None
+        )
+    return output_obj
+
+
+def run_ivoa_query(query, **kwargs):
+    parsed_query_obj = parse_adql_query(query)
+
+    # TODO use a specific dedicated table and schema to refer to the product_gallery DB ?
+    # tables = parsed_query_obj.get('tables', [])
+    # if len(tables) == 1 and tables[0] == 'product_gallery':
+    logger.info('Performing query on the product_gallery')
+    vo_mysql_pg_host = kwargs.get('vo_mysql_pg_host', None)
+    vo_mysql_pg_user = kwargs.get('vo_mysql_pg_user', None)
+    vo_mysql_pg_password = kwargs.get('vo_mysql_pg_password', None)
+    vo_mysql_pg_db = kwargs.get('vo_mysql_pg_db', None)
+    product_gallery_url = kwargs.get('product_gallery_url', None)
+    result_list = run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                                      vo_mysql_pg_host=vo_mysql_pg_host,
+                                                      vo_mysql_pg_user=vo_mysql_pg_user,
+                                                      vo_mysql_pg_password=vo_mysql_pg_password,
+                                                      vo_mysql_pg_db=vo_mysql_pg_db,
+                                                      product_gallery_url=product_gallery_url)
+    return result_list
+
+
+def run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                        vo_mysql_pg_host,
+                                        vo_mysql_pg_user,
+                                        vo_mysql_pg_password,
+                                        vo_mysql_pg_db,
+                                        product_gallery_url=None
+                                        ):
+    result_list = []
+
+    try:
+        with connect(
+                host=vo_mysql_pg_host,
+                user=vo_mysql_pg_user,
+                password=vo_mysql_pg_password,
+                database=vo_mysql_pg_db
+        ) as connection:
+            create_db_query = parsed_query_obj.get('mysql_query')
+            with connection.cursor(dictionary=True) as cursor:
+                cursor.execute(create_db_query)
+                for row in cursor:
+                    if product_gallery_url is not None:
+                        path = row.get('path', None)
+                        if path is not None:
+                            if path.startswith('/'):
+                                path = path[1:]
+                            row['path'] = os.path.join(product_gallery_url, path)
+                        path_alias = row.get('path_alias', None)
+                        if path_alias is not None:
+                            if path_alias.startswith('/'):
+                                path_alias = path_alias[1:]
+                            row['path_alias'] = os.path.join(product_gallery_url, path_alias)
+                    result_list.append(row)
+                # result_obj = cursor.fetchall()
+
+
+    except Error as e:
+        sentry.capture_message(f"Error when connecting to MySQL: {str(e)}")
+        logger.error(f"Error when connecting to MySQL: {str(e)}")
+
+    except Exception as e:
+        sentry.capture_message(f"Error when performing the mysql query to the product_gallery DB: {str(e)}")
+        logger.error(f"Error when performing the mysql query to the product_gallery DB: {str(e)}")
+
+    finally:
+        if connection is not None and connection.is_connected():
+            connection.close()
+            logger.info('MySQL connection closed')
+
+    return result_list

cdci_data_analysis/config_dir/conf_env.yml.example

@@ -122,4 +122,10 @@ dispatcher:
         # url for the conversion of a given time, in UTC format, to the correspondent REVNUM
         converttime_revnum_service_url: COVERTTIME_REVNUM_SERVICE_URL
 
-
+    # virtual observatory related configurations (eg mysql credentials)
+    vo_options:
+        # mysql credentials
+        vo_mysql_pg_host: MYSQL_PG_HOST
+        vo_mysql_pg_user: MYSQL_PG_USER
+        vo_mysql_pg_password: MYSQL_PG_PASSWORD
+        vo_mysql_pg_db: MYSQL_PG_DB

cdci_data_analysis/configurer.py

@@ -266,6 +266,10 @@ def __init__(self, cfg_dict, origin=None):
                                      disp_dict.get('renku_options', {}).get('renku_gitlab_repository_url', None),
                                      disp_dict.get('renku_options', {}).get('renku_base_project_url', None),
                                      disp_dict.get('renku_options', {}).get('ssh_key_path', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_db', None),
                                      )
 
         # not used?
@@ -344,6 +348,10 @@ def set_conf_dispatcher(self,
                             renku_gitlab_repository_url,
                             renku_base_project_url,
                             renku_gitlab_ssh_key_path,
+                            vo_mysql_pg_host,
+                            vo_mysql_pg_user,
+                            vo_mysql_pg_password,
+                            vo_mysql_pg_db
                             ):
         # Generic to dispatcher
         #print(dispatcher_url, dispatcher_port)
@@ -395,6 +403,10 @@ def set_conf_dispatcher(self,
         self.renku_gitlab_repository_url = renku_gitlab_repository_url
         self.renku_gitlab_ssh_key_path = renku_gitlab_ssh_key_path
         self.renku_base_project_url = renku_base_project_url
+        self.vo_mysql_pg_host = vo_mysql_pg_host
+        self.vo_mysql_pg_user = vo_mysql_pg_user
+        self.vo_mysql_pg_password = vo_mysql_pg_password
+        self.vo_mysql_pg_db = vo_mysql_pg_db
 
     def get_data_serve_conf(self, instr_name):
         if instr_name in self.data_server_conf_dict.keys():

cdci_data_analysis/flask_app/app.py

@@ -27,7 +27,7 @@
 import time as _time
 from urllib.parse import urlencode, urlparse
 
-from cdci_data_analysis.analysis import drupal_helper, tokenHelper, renku_helper, email_helper, matrix_helper
+from cdci_data_analysis.analysis import drupal_helper, tokenHelper, renku_helper, email_helper, matrix_helper, ivoa_helper
 from .logstash import logstash_message
 from .schemas import QueryOutJSON, dispatcher_strict_validate
 from marshmallow.exceptions import ValidationError
@@ -415,6 +415,37 @@
                                            "Our team is notified and is working on it.")
 
 
+@app.route('/run_adql_query')
+def run_adql_query():
+    logger.info("request.args: %s ", request.args)
+
+    token = request.args.get('token', None)
+    app_config = app.config.get('conf')
+    secret_key = app_config.secret_key
+
+    output, output_code = tokenHelper.validate_token_from_request(token=token, secret_key=secret_key,
+                                                                  required_roles=['ivoa_user'],
+                                                                  action="run an ADQL query")
+
+    if output_code is not None:
+        return make_response(output, output_code)
+
+    adql_query = request.args.get('adql_query', None)
+    vo_mysql_pg_host = app_config.vo_mysql_pg_host
+    vo_mysql_pg_user = app_config.vo_mysql_pg_user
+    vo_mysql_pg_password = app_config.vo_mysql_pg_password
+    vo_mysql_pg_db = app_config.vo_mysql_pg_db
+    product_gallery_url = app_config.product_gallery_url
+
+    result_query = ivoa_helper.run_ivoa_query(adql_query,
+                                              vo_mysql_pg_host=vo_mysql_pg_host,
+                                              vo_mysql_pg_user=vo_mysql_pg_user,
+                                              vo_mysql_pg_password=vo_mysql_pg_password,
+                                              vo_mysql_pg_db=vo_mysql_pg_db,
+                                              product_gallery_url=product_gallery_url)
+
+    return jsonify(result_query)
+
 
 @app.route('/run_analysis', methods=['POST', 'GET'])
 def run_analysis():

requirements.txt

@@ -29,6 +29,8 @@ GitPython
 nbformat
 sentry-sdk
 pytest-sentry
+sqlparse
+queryparser-python3
 -e git+https://github.com/oda-hub/oda_api.git#egg=oda_api
 
 MarkupSafe==2.0.1

setup.py

@@ -45,6 +45,8 @@
     "black>=22.10.0",
     "bs4",
     "GitPython",
+    "sqlparse",
+    "queryparser-python3",
     "nbformat",
     "giturlparse",
     "sentry-sdk",