properly handling post request and sanitize request values before logging

cdci_data_analysis/flask_app/app.py

@@ -15,7 +15,7 @@
 import random
 import hashlib
 import validators
-
+import re
 import logging
 
 from raven.contrib.flask import Sentry
@@ -223,6 +223,19 @@
     return D
 
 
+def sanitize_dict_before_log(dict_to_sanitize):
+    sensitive_keys = ['token']  # Add any other sensitive keys here
+    allowed_characters = r'[^A-Za-z0-9 ]'
+    replacement_character = ''
+    sanitized_values = {}
+    for key, value in dict_to_sanitize.items():
+        if key not in sensitive_keys:
+            # value = str(value).replace('\n', '').replace('\r', '')
+            value = re.sub(allowed_characters, replacement_character, str(value))
+            sanitized_values[key] = value
+    return sanitized_values
+
+
 def common_exception_payload():
     payload = {}
 
@@ -441,9 +454,11 @@
     request_summary = log_run_query_request()
 
     try:
+        sanitized_request_values = sanitize_dict_before_log(request.values)
+
         logger.info('\033[32m===> dataserver_call_back\033[0m')
         logger.info('\033[33m raw request values: %s \033[0m',
-                    dict(request.values))
+                    dict(sanitized_request_values))
 
         query_id = hashlib.sha224(str(request.values).encode()).hexdigest()[:8]
 
@@ -522,9 +537,11 @@
 
 @app.route('/call_back', methods=['POST', 'GET'])
 def dataserver_call_back():
+    sanitized_request_values = sanitize_dict_before_log(request.values)
+
     logger.info('\033[32m===========================> dataserver_call_back\033[0m')
 
-    logger.info('\033[33m raw request values: %s \033[0m', dict(request.values))
+    logger.info('\033[33m raw request values: %s \033[0m', dict(sanitized_request_values))
 
     query_id = hashlib.sha224(str(request.values).encode()).hexdigest()[:8]
 
@@ -930,13 +947,17 @@
     return output_list
 
 
-
 @app.route('/post_astro_entity_to_gallery', methods=['POST'])
 def post_astro_entity_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -948,9 +969,6 @@
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
                                                         content_type="astrophysical_entity",
                                                         disp_conf=app_config,
@@ -962,10 +980,15 @@
 
 @app.route('/post_observation_to_gallery', methods=['POST'])
 def post_observation_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -977,9 +1000,6 @@
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
                                                         content_type="observation",
                                                         disp_conf=app_config,
@@ -991,10 +1011,15 @@
 
 @app.route('/post_product_to_gallery', methods=['POST'])
 def post_product_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -1006,9 +1031,6 @@
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
                                                         disp_conf=app_config,
                                                         files=request.files,
@@ -1019,10 +1041,15 @@
 
 @app.route('/delete_product_to_gallery', methods=['POST'])
 def delete_product_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -1034,9 +1061,6 @@
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.delete_content_gallery(decoded_token=decoded_token,
                                                        disp_conf=app_config,
                                                        files=request.files,
@@ -1047,10 +1071,15 @@
 
 @app.route('/post_revolution_processing_log_to_gallery', methods=['POST'])
 def post_revolution_processing_log_to_gallery():
-    logger.info("request.args: %s ", request.args)
+    par_dic = request.values.to_dict()
+    sanitized_par_dic = sanitize_dict_before_log(par_dic)
+
+    logger.info("request.values: %s ", sanitized_par_dic)
     logger.info("request.files: %s ", request.files)
 
-    token = request.args.get('token', None)
+    token = par_dic.get('token', None)
+    par_dic.pop('token')
+
     app_config = app.config.get('conf')
     secret_key = app_config.secret_key
 
@@ -1062,9 +1091,6 @@
         return make_response(output, output_code)
     decoded_token = output
 
-    par_dic = request.values.to_dict()
-    par_dic.pop('token')
-
     output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
                                                         disp_conf=app_config,
                                                         files=request.files,

tests/test_server_basic.py

@@ -28,6 +28,7 @@
 from cdci_data_analysis.analysis.renku_helper import clone_renku_repo, checkout_branch_renku_repo, check_job_id_branch_is_present, get_repo_path, generate_commit_request_url, create_new_notebook_with_code, generate_nb_hash, create_renku_ini_config_obj, generate_ini_file_hash
 from cdci_data_analysis.analysis.drupal_helper import execute_drupal_request, get_drupal_request_headers, get_revnum, get_observations_for_time_range, generate_gallery_jwt_token, get_user_id, get_source_astrophysical_entity_id_by_source_name
 from cdci_data_analysis.plugins.dummy_plugin.data_server_dispatcher import DataServerQuery, ReturnProgressProductQuery
+from cdci_data_analysis.flask_app.app import sanitize_dict_before_log
 
 # logger
 logger = logging.getLogger(__name__)
@@ -70,6 +71,24 @@ def remove_args_from_dic(arg_dic, remove_keys):
     tem=0,
 )
 
+@pytest.mark.fast
+def test_sanitize_dict_before_log():
+
+    test_dict = {
+        'token': 'mytoken',
+        'field': 'myfield\n\r',
+        'username': 'myusername',
+        'email': '[email protected]'
+    }
+
+    expected_dict = {
+        'field': 'myfield',
+        'username': 'myusername',
+        'email': 'myemailexamplecom'
+    }
+
+    sanitized_dict = sanitize_dict_before_log(test_dict)
+    assert sanitized_dict == expected_dict
 
 @pytest.mark.fast
 def test_js9(dispatcher_live_fixture):
@@ -2800,7 +2819,7 @@ def test_product_gallery_data_product_with_period_of_observation(dispatcher_live
         params['T2'] = now.strftime('%Y-%m-%dT%H:%M:%S')
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
 
@@ -3122,7 +3141,7 @@ def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_
         }
 
         c = requests.post(os.path.join(server, "post_astro_entity_to_gallery"),
-                          params={**source_params},
+                          data=source_params,
                           )
 
         assert c.status_code == 200
@@ -3141,7 +3160,7 @@ def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_
             'T2': '2022-08-23T05:29:11'
         }
         c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                          params={**product_params}
+                          data=product_params
                           )
 
         assert c.status_code == 200
@@ -3270,7 +3289,7 @@ def test_product_gallery_get_data_products_list_for_given_source(dispatcher_live
         }
 
         c = requests.post(os.path.join(server, "post_astro_entity_to_gallery"),
-                          params={**source_params},
+                          data=source_params,
                           )
 
         assert c.status_code == 200
@@ -3286,7 +3305,7 @@ def test_product_gallery_get_data_products_list_for_given_source(dispatcher_live
             'insert_new_source': True
         }
         c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                          params={**product_params}
+                          data=product_params
                           )
 
         assert c.status_code == 200
@@ -3434,7 +3453,7 @@ def test_product_gallery_get_period_of_observation_attachments(dispatcher_live_f
 
 
         c = requests.post(os.path.join(server, "post_observation_to_gallery"),
-                          params={**params},
+                          data=params,
                           files=file_obj
                           )
 
@@ -3522,7 +3541,7 @@ def test_product_gallery_post_period_of_observation(dispatcher_live_fixture_with
         params['T2'] = now.strftime('%Y-%m-%dT%H:%M:%S')
 
     c = requests.post(os.path.join(server, "post_observation_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
 
@@ -3621,7 +3640,7 @@ def test_revolution_processing_log_gallery_post(dispatcher_live_fixture_with_gal
     }
 
     c = requests.post(os.path.join(server, "post_revolution_processing_log_to_gallery"),
-                      params={**params},
+                      data=params,
                       )
 
     assert c.status_code == 200
@@ -3740,7 +3759,7 @@ def test_product_gallery_post(dispatcher_live_fixture_with_gallery, dispatcher_t
                 'fits_file_1': open('data/dummy_prods/query_catalog.fits', 'rb')}
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
 
@@ -3852,7 +3871,7 @@ def test_post_data_product_with_multiple_sources(dispatcher_live_fixture_with_ga
         'insert_new_source': insert_new_source
     }
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params}
+                      data=params
                       )
 
     assert c.status_code == 200
@@ -3982,7 +4001,7 @@ def test_product_gallery_update(dispatcher_live_fixture_with_gallery, dispatcher
                 'fits_file_1': open('data/dummy_prods/query_catalog.fits', 'rb')}
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
 
@@ -4028,7 +4047,7 @@ def test_product_gallery_update(dispatcher_live_fixture_with_gallery, dispatcher
                 'fits_file_0': open('data/dummy_prods/isgri_query_lc.fits', 'rb')}
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       files=file_obj
                       )
     assert c.status_code == 200
@@ -4084,7 +4103,7 @@ def test_product_gallery_delete(dispatcher_live_fixture_with_gallery, dispatcher
                   token=encoded_token)
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       )
 
     assert c.status_code == 200
@@ -4096,31 +4115,37 @@ def test_product_gallery_delete(dispatcher_live_fixture_with_gallery, dispatcher
     assert 'field_product_id' in drupal_res_obj
     assert drupal_res_obj['field_product_id'][0]['value'] == product_id
 
-    params = {
-        'product_id': product_id,
+    params_products_list = {
+        'product_id_value': product_id,
         'content_type': 'data_product',
         'token': encoded_token
     }
 
     c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
-                     params=params
+                     params=params_products_list
                      )
 
     assert c.status_code == 200
     drupal_res_obj = c.json()
     assert len(drupal_res_obj) == 1
     assert drupal_res_obj[0]['nid'] == str(nid_creation)
 
+    params = {
+        'product_id': product_id,
+        'content_type': 'data_product',
+        'token': encoded_token
+    }
+
     c = requests.post(os.path.join(server, "delete_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       )
     assert c.status_code == 200
 
     drupal_res_obj = c.json()
     assert drupal_res_obj == {}
 
     c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
-                     params=params
+                     params=params_products_list
                      )
 
     assert c.status_code == 200
@@ -4155,7 +4180,7 @@ def test_product_gallery_error_message(dispatcher_live_fixture_with_gallery):
     }
 
     c = requests.post(os.path.join(server, "post_product_to_gallery"),
-                      params={**params},
+                      data=params,
                       )
 
     assert c.status_code == 500