Merge pull request #666 from oda-hub/generic-file-uplaod

Generic file upload

.gitignore

@@ -70,4 +70,6 @@ data/dummy_prods**
 .venv
 js9.fits
 test-dispatcher-conf-with-gallery.yaml
-tests/oda-ontology.ttl
+tests/oda-ontology.ttl
+tests/request_files
+tests/local_request_files

cdci_data_analysis/analysis/hash.py

@@ -1,4 +1,5 @@
 import hashlib
+import subprocess
 import json
 from collections import OrderedDict
 
@@ -34,3 +35,10 @@ def format_hash(x): return hashlib.md5(
 
     # this takes care of various strange objects which can not be properly represented
     return format_hash(json.dumps(o))
+
+
+def make_hash_file(file_path):
+    command = ["md5sum", file_path]
+
+    output = subprocess.check_output(command)
+    return output.decode().split()[0]

cdci_data_analysis/analysis/instrument.py

@@ -27,18 +27,20 @@
 import string
 import json
 import logging
-
 import yaml
+import validators
 
 import numpy as np
 from astropy.table import Table
+from urllib.parse import urlencode
 
 from cdci_data_analysis.analysis.queries import _check_is_base_query
 from ..analysis import tokenHelper, parameters
 from .catalog import BasicCatalog
 from .products import QueryOutput
 from .queries import ProductQuery, SourceQuery, InstrumentQuery
-from .io_helper import upload_file
+from .io_helper import upload_file, upload_files_request
+
 from .exceptions import RequestNotUnderstood, RequestNotAuthorized, InternalError
 from ..flask_app.sentry import sentry
 
@@ -243,6 +245,10 @@ def parse_inputs_files(self,
                            temp_dir,
                            verbose,
                            use_scws,
+                           upload_dir,
+                           products_url,
+                           request_files_dir,
+                           decoded_token,
                            sentry_dsn=None):
         error_message = 'Error while {step} {temp_dir_content_msg}{additional}'
         # TODO probably exception handling can be further improved and/or optmized
@@ -263,6 +269,20 @@ def parse_inputs_files(self,
                                                                       temp_dir=temp_dir)
             step = 'setting input scw_list file'
             self.set_input_products_from_fronted(input_file_path=input_file_path, par_dic=par_dic, verbose=verbose)
+
+            # any other file
+            step = 'uploading other files'
+            uploaded_files_obj = upload_files_request(request=request,
+                                                       upload_dir=upload_dir)
+            step = 'updating par_dic with the uploaded files'
+            self.update_par_dic_with_uploaded_files(par_dic=par_dic,
+                                                    uploaded_files_obj=uploaded_files_obj,
+                                                    products_url=products_url)
+            step = 'updating ownership files'
+            self.update_ownership_files(uploaded_files_obj,
+                                        request_files_dir=request_files_dir,
+                                        decoded_token=decoded_token)
+
         except RequestNotUnderstood as e:
             error_message = error_message.format(step=step,
                                                  temp_dir_content_msg='',
@@ -684,6 +704,50 @@ def set_input_products_from_fronted(self, input_file_path, par_dic, verbose=Fals
             else:
                 raise RuntimeError
 
+    def update_par_dic_with_uploaded_files(self, par_dic, uploaded_files_obj, products_url):
+        if validators.url(products_url):
+            basepath = os.path.join(products_url, 'dispatch-data/download_file')
+        else:
+            basepath = os.path.join(products_url, 'download_file')
+        for f in uploaded_files_obj:
+            dpars = urlencode(dict(file_list=uploaded_files_obj[f],
+                                   return_archive=False))
+            download_file_url = f"{basepath}?{dpars}"
+            par_dic[f] = download_file_url
+
+    def update_ownership_files(self, uploaded_files_obj, request_files_dir, decoded_token=None):
+        if decoded_token is not None:
+            user_email = tokenHelper.get_token_user_email_address(decoded_token)
+            user_roles = tokenHelper.get_token_roles(decoded_token)
+        else:
+            user_email = 'public'
+            user_roles = []
+
+        update_file = False
+        for file_name in uploaded_files_obj:
+            file_hash = uploaded_files_obj[file_name]
+            ownership_file_path = os.path.join(request_files_dir, f'{file_hash}_ownerships.json')
+            if not os.path.exists(ownership_file_path):
+                ownerships = dict(
+                    user_emails=[user_email],
+                    user_roles=user_roles
+                )
+                update_file = True
+            else:
+                with open(ownership_file_path) as ownership_file:
+                    ownerships = json.load(ownership_file)
+            if user_email not in ownerships['user_emails']:
+                ownerships['user_emails'].append(user_email)
+                update_file = True
+            if not all(role in ownerships['user_roles'] for role in user_roles):
+                set_user_roles = set(ownerships['user_roles'])
+                set_user_roles |= set(user_roles)
+                ownerships['user_roles'] = list(set_user_roles)
+                update_file = True
+            if update_file:
+                with open(ownership_file_path, 'w') as ownership_file:
+                    json.dump(ownerships, ownership_file)
+
     def set_input_products(self, par_dic, input_file_path,input_prod_list_name):
         if input_file_path is None:
             #if no file we pass OK condition
@@ -716,6 +780,7 @@ def set_input_products(self, par_dic, input_file_path,input_prod_list_name):
 
         return has_prods
 
+
     def upload_catalog_from_fronted(self, par_dic, request, temp_dir):
         cat_file_path = None
         if request.method == 'POST':

cdci_data_analysis/analysis/io_helper.py

@@ -37,6 +37,9 @@
 from flask import request
 from werkzeug.utils import secure_filename
 import decorator
+
+from .hash import make_hash_file
+
 # Dependencies
 # eg numpy 
 # absolute import eg: import numpy as np
@@ -206,6 +209,22 @@ def upload_file(name, dir):
         return file_path
 
 
+def upload_files_request(request, upload_dir):
+    uploaded_files_obj = {}
+    if request.method == 'POST':
+        for f in request.files:
+            # TODO needed since those two files are extracted in a previous step
+            if f != 'user_scw_list_file' and f != 'user_catalog_file':
+                f_path = upload_file(f, upload_dir)
+                if f_path is not None:
+                    file_hash = make_hash_file(f_path)
+                    new_file_name = file_hash
+                    new_file_path = os.path.join(upload_dir, new_file_name)
+                    uploaded_files_obj[f] = new_file_name
+                    os.rename(f_path, new_file_path)
+    return uploaded_files_obj
+
+
 def format_size(size_bytes, format_returned='M'):
     size_bytes = float(size_bytes)
     size_kb = size_bytes / 1024

cdci_data_analysis/analysis/parameters.py

@@ -586,6 +586,15 @@ class LongString(String):
 class Name(String):
     owl_uris = String.owl_uris + ("http://odahub.io/ontology#AstrophysicalObject",)
 
+class FileReference(String):
+    owl_uris = String.owl_uris + ("http://odahub.io/ontology#FileReference",)
+
+class POSIXPath(FileReference):
+    owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#POSIXPath",)
+
+class FileURL(FileReference):
+    owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#FileURL",)
+
 class NumericParameter(Parameter):
     owl_uris = ("http://odahub.io/ontology#NumericParameter")
 
@@ -1052,6 +1061,7 @@ def check_value(self):
     def get_default_value(self):
         return json.dumps(self.value, sort_keys=True)
 
+
 class PhosphorosFiltersTable(StructuredParameter):
     owl_uris = ('http://odahub.io/ontology#PhosphorosFiltersTable')
 

cdci_data_analysis/flask_app/app.py

@@ -150,8 +150,30 @@ def meta_data_src():
 
 @app.route("/download_products", methods=['POST', 'GET'])
 def download_products():
-    query = InstrumentQueryBackEnd(app, download_products=True)
-    return query.download_products()
+    from_request_files_dir = request.args.get('from_request_files_dir', 'False') == 'True'
+    download_file = request.args.get('download_file', 'False') == 'True'
+    download_products = request.args.get('download_products', 'True') == 'True'
+    query = InstrumentQueryBackEnd(app, download_products=download_products, download_files=download_file)
+    return query.download_file(from_request_files_dir=from_request_files_dir)
+
+
+@app.route("/download_file", methods=['POST', 'GET'])
+def download_file():
+    if app.config['conf'].products_url is not None and validators.url(app.config['conf'].products_url):
+        redirection_url = os.path.join(app.config['conf'].products_url, 'dispatch-data/download_products')
+        if request.args:
+            args_request = urlencode(request.args)
+            redirection_url = f'{redirection_url}?{args_request}'
+
+    else:
+        parsed_request_url = urlparse(request.url)
+        path_request_url = parsed_request_url.path.replace('/download_file', '/download_products')
+        parsed_request_url = parsed_request_url._replace(path=path_request_url)
+        redirection_url = parsed_request_url.geturl()
+
+    redirection_url += f'&from_request_files_dir=True&download_file=True&download_products=False'
+
+    return redirect(redirection_url)
 
 
 class UnknownDispatcherException(Exception):