diff --git a/efelg/tools/resources.py b/efelg/tools/resources.py index e9c17d07..38053922 100755 --- a/efelg/tools/resources.py +++ b/efelg/tools/resources.py @@ -2,12 +2,9 @@ import sys import json import collections -import neo import pprint from datetime import datetime -import requests import logging -import bluepyefe.formats.axon as fa # set logging up logging.basicConfig(stream=sys.stdout) diff --git a/efelg/views.py b/efelg/views.py old mode 100755 new mode 100644 index d7b608a9..3f30310e --- a/efelg/views.py +++ b/efelg/views.py @@ -188,7 +188,7 @@ def get_data(request, cellname=""): trace_info['coefficient'] = coefficient trace_info['disp_sampling_rate'] = disp_sampling_rate - trace_info['md5'] = content['md5'] + trace_info['md5'] = content.get('md5') trace_info['sampling_rate'] = content['sampling_rate'] trace_info['etype'] = content['etype'] @@ -618,7 +618,7 @@ def file_formats_docs(request): """ Render Guidebook file formats page """ - logger.info(LOG_ACTION.format(request.session['username'], 'access FILE_FORMATS page')) + logger.info(LOG_ACTION.format(request.session.get('username', 'anonymous'), 'access FILE_FORMATS page')) return render(request, 'efelg/docs/file_formats.html') @@ -638,7 +638,7 @@ def dataset(request): """ Return Guidebook dataset page """ - logger.info(LOG_ACTION.format(request.session['username'], 'access DATASET page')) + logger.info(LOG_ACTION.format(request.session.get('username', 'anonymous'), 'access DATASET page')) return render(request, 'efelg/docs/dataset.html') diff --git a/hh_neuron_builder/config/dev_conf.py.example b/hh_neuron_builder/config/dev_conf.py.example index 93ba4bc3..dc4d1008 100755 --- a/hh_neuron_builder/config/dev_conf.py.example +++ b/hh_neuron_builder/config/dev_conf.py.example @@ -36,6 +36,6 @@ DATABASES = { } } -MEDIA_ROOT = os.path.join('/apps', 'media') +MEDIA_ROOT = os.path.join(BASE_DIR, '../app', 'media') -LOG_ROOT_PATH = './' \ No newline at end of file +LOG_ROOT_PATH = os.path.join(BASE_DIR, '../log/') diff --git a/hh_neuron_builder/config/hhf_template/parameters.json b/hh_neuron_builder/config/hhf_template/parameters.json deleted file mode 100644 index 428c8b7c..00000000 --- a/hh_neuron_builder/config/hhf_template/parameters.json +++ /dev/null @@ -1,60 +0,0 @@ -{ -"model_key": { - "mechanisms": { - "all": ["pas","kdrb", "na3"], - "somatic": ["kdb", "kmb", "kap", "hd", "can", "cal", "cat", "cagk", "kca", "cacum"], - "axonal": ["kmb", "kap"], - "alldend":["hd", "can", "cal", "cat", "cagk", "kca", "cacum", "kad"] - }, - "distributions": { - "exp_na_dend": "math.exp((-{distance})/50)*{value}" - }, - "fixed": { - "global": [["v_init", -80], ["celsius", 34]], - "all": [["cm", 1, "secvar"],["ena", 50, "secvar"],["ek", -90, "secvar"]] - }, - "optimized": { - "all": [ - ], - "axonal": [ - ["gbar_na3", 0.025, 0.15, "uniform" ], - ["gkdrbar_kdrb", 0.01, 0.08, "uniform" ], - ["gkabar_kap", 0.001, 0.04, "uniform" ], - ["gbar_kmb", 0.004, 0.05, "uniform" ], - ["Ra", 50, 300, "secvar"], - ["g_pas", 1e-6, 8e-5, "uniform" ], - ["e_pas", -95, -75, "uniform" ] - ], - "somatic": [ - ["gbar_na3", 0.02, 0.06, "uniform" ], - ["gkdrbar_kdrb", 0.001, 0.08, "uniform" ], - ["gkabar_kap", 0.005, 0.04, "uniform" ], - ["gkdbar_kdb", 1e-5, 10e-5, "uniform" ], - ["gbar_kmb", 0.001, 0.02, "uniform" ], - ["ghdbar_hd", 1e-6, 5e-4, "uniform" ], - ["gcalbar_cal", 0.1e-5, 3e-5, "uniform" ], - ["gcanbar_can", 0.1e-5, 3e-5, "uniform" ], - ["gcatbar_cat", 0.1e-5, 3e-5, "uniform" ], - ["gbar_kca", 1e-6, 1e-5, "uniform" ], - ["gbar_cagk", 0.5e-5, 3e-4, "uniform" ], - ["Ra", 50, 400, "secvar"], - ["g_pas", 1e-6, 20e-5, "uniform" ], - ["e_pas", -120, -75, "uniform" ] - ], - "alldend": [ - ["gbar_na3", 0.02, 0.06, "exp_na_dend" ], - ["gkdrbar_kdrb", 0.001, 0.8, "uniform" ], - ["gkabar_kad", 0.005, 0.1, "uniform" ], - ["ghdbar_hd", 1e-6, 5e-4, "uniform" ], - ["gcalbar_cal", 0.1e-5, 3e-5, "uniform" ], - ["gcanbar_can", 0.1e-5, 3e-5, "uniform" ], - ["gcatbar_cat", 0.1e-5, 3e-5, "uniform" ], - ["gbar_kca", 1e-6, 1e-5, "uniform" ], - ["gbar_cagk", 0.5e-5, 3e-4, "uniform" ], - ["Ra", 50, 400, "secvar"], - ["g_pas", 1e-6, 20e-5, "uniform" ], - ["e_pas", -120, -80, "uniform" ] - ] - } -} -} diff --git a/hh_neuron_builder/config/hhf_template/parameters/interneuron/parameters.json b/hh_neuron_builder/config/hhf_template/parameters/interneuron/parameters.json new file mode 100644 index 00000000..7e27f0a4 --- /dev/null +++ b/hh_neuron_builder/config/hhf_template/parameters/interneuron/parameters.json @@ -0,0 +1,276 @@ +{ + "W_20221102163317": { + "mechanisms": { + "all": [ + "pas", + "kdrb", + "na3" + ], + "somatic": [ + "kdb", + "kmb", + "kap", + "hd", + "can", + "cal", + "cat", + "cagk", + "kca", + "cacum" + ], + "axonal": [ + "kmb", + "kap" + ], + "alldend": [ + "hd", + "can", + "cal", + "cat", + "cagk", + "kca", + "cacum", + "kad" + ] + }, + "distributions": { + "exp_na_dend": "math.exp((-{distance})/50)*{value}" + }, + "fixed": { + "global": [ + [ + "v_init", + -80 + ], + [ + "celsius", + 34 + ] + ], + "all": [ + [ + "cm", + 1, + "secvar" + ], + [ + "ena", + 50, + "secvar" + ], + [ + "ek", + -90, + "secvar" + ] + ] + }, + "optimized": { + "all": [], + "axonal": [ + [ + "gbar_na3", + 0.025, + 0.15, + "uniform" + ], + [ + "gkdrbar_kdrb", + 0.01, + 0.08, + "uniform" + ], + [ + "gkabar_kap", + 0.001, + 0.04, + "uniform" + ], + [ + "gbar_kmb", + 0.004, + 0.05, + "uniform" + ], + [ + "Ra", + 50, + 300, + "secvar" + ], + [ + "g_pas", + 1e-06, + 8e-05, + "uniform" + ], + [ + "e_pas", + -95, + -75, + "uniform" + ] + ], + "somatic": [ + [ + "gbar_na3", + 0.02, + 0.06, + "uniform" + ], + [ + "gkdrbar_kdrb", + 0.001, + 0.08, + "uniform" + ], + [ + "gkabar_kap", + 0.005, + 0.04, + "uniform" + ], + [ + "gkdbar_kdb", + 1e-05, + 0.0001, + "uniform" + ], + [ + "gbar_kmb", + 0.001, + 0.02, + "uniform" + ], + [ + "ghdbar_hd", + 1e-06, + 0.0005, + "uniform" + ], + [ + "gcalbar_cal", + 1e-06, + 3e-05, + "uniform" + ], + [ + "gcanbar_can", + 1e-06, + 3e-05, + "uniform" + ], + [ + "gcatbar_cat", + 1e-06, + 3e-05, + "uniform" + ], + [ + "gbar_kca", + 1e-06, + 1e-05, + "uniform" + ], + [ + "gbar_cagk", + 5e-06, + 0.0003, + "uniform" + ], + [ + "Ra", + 50, + 400, + "secvar" + ], + [ + "g_pas", + 1e-06, + 0.0002, + "uniform" + ], + [ + "e_pas", + -120, + -75, + "uniform" + ] + ], + "alldend": [ + [ + "gbar_na3", + 0.02, + 0.06, + "exp_na_dend" + ], + [ + "gkdrbar_kdrb", + 0.001, + 0.8, + "uniform" + ], + [ + "gkabar_kad", + 0.005, + 0.1, + "uniform" + ], + [ + "ghdbar_hd", + 1e-06, + 0.0005, + "uniform" + ], + [ + "gcalbar_cal", + 1e-06, + 3e-05, + "uniform" + ], + [ + "gcanbar_can", + 1e-06, + 3e-05, + "uniform" + ], + [ + "gcatbar_cat", + 1e-06, + 3e-05, + "uniform" + ], + [ + "gbar_kca", + 1e-06, + 1e-05, + "uniform" + ], + [ + "gbar_cagk", + 5e-06, + 0.0003, + "uniform" + ], + [ + "Ra", + 50, + 400, + "secvar" + ], + [ + "g_pas", + 1e-06, + 0.0002, + "uniform" + ], + [ + "e_pas", + -120, + -80, + "uniform" + ] + ] + } + } +} \ No newline at end of file diff --git a/hh_neuron_builder/config/hhf_template/parameters/pyramidal/parameters.json b/hh_neuron_builder/config/hhf_template/parameters/pyramidal/parameters.json new file mode 100644 index 00000000..5e4c27be --- /dev/null +++ b/hh_neuron_builder/config/hhf_template/parameters/pyramidal/parameters.json @@ -0,0 +1,258 @@ +{ + "W_20221102163317": { + "mechanisms": { + "all": [ + "pas", + "kdr", + "nax" + ], + "axonal": [ + "kmb", + "kap" + ], + "somatic": [ + "kmb", + "kap", + "hd", + "can", + "cal", + "cat", + "kca", + "cagk", + "cacum" + ], + "alldend": [ + "kad", + "hd", + "can", + "cal", + "cat", + "kca", + "cagk", + "cacum" + ] + }, + "distributions": { + "linear_hd_apic": "(1. + 3./100. * {distance})*{value}", + "sigmoid_kad_apic": "(15./(1. + math.exp((300-{distance})/50)))*{value}", + "linear_e_pas_apic": "({value}-5*{distance}/150)" + }, + "fixed": { + "global": [ + [ + "v_init", + -70 + ], + [ + "celsius", + 34 + ] + ], + "all": [ + [ + "cm", + 1, + "secvar" + ], + [ + "ena", + 50, + "secvar" + ], + [ + "ek", + -90, + "secvar" + ] + ] + }, + "optimized": { + "axonal": [ + [ + "gbar_nax", + 0.1, + 0.4, + "uniform" + ], + [ + "gkdrbar_kdr", + 0.01, + 0.04, + "uniform" + ], + [ + "gbar_kmb", + 0.002, + 0.06, + "uniform" + ], + [ + "gkabar_kap", + 0.025, + 0.3, + "uniform" + ], + [ + "Ra", + 50, + 100, + "secvar" + ], + [ + "g_pas", + 2e-05, + 0.0002, + "uniform" + ], + [ + "e_pas", + -80, + -60, + "uniform" + ] + ], + "somatic": [ + [ + "gkabar_kap", + 0.025, + 0.1, + "uniform" + ], + [ + "gbar_nax", + 0.02, + 0.08, + "uniform" + ], + [ + "gbar_kmb", + 0.0015, + 0.06, + "uniform" + ], + [ + "Ra", + 100, + 400, + "secvar" + ], + [ + "g_pas", + 5e-06, + 0.0001, + "uniform" + ], + [ + "e_pas", + -80, + -65, + "linear_e_pas_apic" + ] + ], + "allnoaxon": [ + [ + "gkdrbar_kdr", + 0.001, + 0.008, + "uniform" + ], + [ + "ghdbar_hd", + 5e-06, + 2e-05, + "linear_hd_apic" + ], + [ + "gcalbar_cal", + 1e-06, + 2e-05, + "uniform" + ], + [ + "gcanbar_can", + 1e-06, + 2e-05, + "uniform" + ], + [ + "gcatbar_cat", + 1e-06, + 2e-05, + "uniform" + ], + [ + "gbar_kca", + 2e-05, + 0.0002, + "uniform" + ], + [ + "gbar_cagk", + 1e-06, + 0.0004, + "uniform" + ] + ], + "basal": [ + [ + "gbar_nax", + 0.002, + 0.02, + "uniform" + ], + [ + "Ra", + 100, + 400, + "secvar" + ], + [ + "g_pas", + 5e-06, + 0.0001, + "uniform" + ], + [ + "e_pas", + -80, + -65, + "linear_e_pas_apic" + ] + ], + "apical": [ + [ + "gbar_nax", + 0.02, + 0.08, + "uniform" + ], + [ + "Ra", + 100, + 400, + "secvar" + ], + [ + "g_pas", + 5e-06, + 0.0001, + "uniform" + ], + [ + "e_pas", + -80, + -65, + "linear_e_pas_apic" + ] + ], + "alldend": [ + [ + "gkabar_kad", + 0.005, + 0.05, + "sigmoid_kad_apic" + ] + ] + } + } +} \ No newline at end of file diff --git a/hh_neuron_builder/settings.py b/hh_neuron_builder/settings.py index a78eb526..bc688593 100755 --- a/hh_neuron_builder/settings.py +++ b/hh_neuron_builder/settings.py @@ -166,12 +166,13 @@ MEDIA_ROOT = conf.MEDIA_ROOT HHF_TEMPLATE_DIR = os.path.join(BASE_DIR, 'hh_neuron_builder', 'config', 'hhf_template', 'hhf') +HHF_PARAMETERS_TEMPLATE_DIR = os.path.join(BASE_DIR, 'hh_neuron_builder', 'config', 'hhf_template', 'parameters') if not os.path.exists(HHF_TEMPLATE_DIR): os.makedirs(HHF_TEMPLATE_DIR) TMP_DIR = os.path.join(MEDIA_ROOT, 'hhnb', 'tmp') if not os.path.exists(TMP_DIR): - os.mkdir(TMP_DIR) + os.makedirs(TMP_DIR) MODEL_CATALOG_FILTER = { @@ -219,12 +220,15 @@ OIDC_OP_TOKEN_ENDPOINT = 'https://iam.ebrains.eu/auth/realms/hbp/protocol/openid-connect/token' OIDC_OP_USER_ENDPOINT = 'https://iam.ebrains.eu/auth/realms/hbp/protocol/openid-connect/userinfo' + +# OIDC_TIMEOUT = 1 + + LOGIN_REDIRECT_URL = "/hh-neuron-builder" LOGOUT_REDIRECT_URL = "/hh-neuron-builder" LOGIN_URL = 'oidc_authentication_init' - # Logging # Set LOG_ROOT_PATH for logging files diff --git a/hhnb/core/conf/__init__.py b/hhnb/core/conf/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hhnb/core/conf/exec_files_conf.py b/hhnb/core/conf/exec_files_conf.py index 65e91a8d..287bfcf3 100644 --- a/hhnb/core/conf/exec_files_conf.py +++ b/hhnb/core/conf/exec_files_conf.py @@ -1,4 +1,5 @@ import os +import json class ExecFileConf: @@ -8,26 +9,32 @@ class ExecFileConf: @staticmethod - def write_nsg_exec(dst_dir, max_gen, offspring): + def write_nsg_exec(dst_dir, max_gen, offspring, job_name, mode='start'): """ - Write the excecution script for the NSG system. + Write the execution script for the NSG system. Parameters ---------- dst_dir : str - destination folder where the file will be writen. + destination folder where the file will be written. max_gen : int maximum number of generations used by opt_neuron. offspring : int - number of individuals in offspring useb by opt_neuron. + number of individuals in offspring used by opt_neuron. + job_name : str + set job_name + mode : str, optional, default='start' + select the action mode to run the job on the HPC. Can be 'start' or 'resume'. """ buffer = \ f""" import os +import json -os.system('python3 opt_neuron.py --max_ngen={max_gen} --offspring_size={offspring} --start --checkpoint ./checkpoints/checkpoint.pkl') - +os.system('python3 opt_neuron.py --max_ngen={max_gen} --offspring_size={offspring} --{mode} --checkpoint ./checkpoints/checkpoint.pkl') +with open('resume_job_settings.json', 'w') as fd: + json.dump({json.dumps({'offspring_size': offspring, 'max_gen': max_gen, 'job_name': job_name, 'hpc': 'nsg',})}, fd) """ try: with open(os.path.join(dst_dir, 'init.py'), 'w') as fd: @@ -35,22 +42,25 @@ def write_nsg_exec(dst_dir, max_gen, offspring): except Exception as e: raise e - @staticmethod - def write_daint_exec(dst_dir, folder_name, offspring, max_gen, ): + def write_daint_exec(dst_dir, folder_name, offspring, max_gen, job_name, mode='start'): """ - Write the excecution script for the Piz-Daint (UNICORE) system. + Write the execution script for the Piz-Daint (UNICORE) system. Parameters ---------- dst_dir : str - destination folder where the file will be writen. + destination folder where the file will be written. folder_name : str the model root folder name. max_gen : int maximum number of generations used by opt_neuron. offspring : int - number of individuals in offspring useb by opt_neuron. + number of individuals in offspring used by opt_neuron. + job_name : str + set job_name + mode : str, optional, default='start' + select the action mode to run the job on the HPC. Can be 'start' or 'resume'. """ buffer_zipfolder = \ f""" @@ -115,7 +125,8 @@ def zipdir(path, ziph): sleep 30 srun ipengine --profile=${IPYTHON_PROFILE} & CHECKPOINTS_DIR="checkpoints" -BLUEPYOPT_SEED=1 python opt_neuron.py --offspring_size={offspring} --max_ngen={max_gen} --start --checkpoint "${CHECKPOINTS_DIR}/checkpoint.pkl" +BLUEPYOPT_SEED=1 python opt_neuron.py --offspring_size={offspring} --max_ngen={max_gen} --{mode} --checkpoint "${CHECKPOINTS_DIR}/checkpoint.pkl" +echo '{json.dumps({'offspring_size': offspring, 'max_gen': max_gen, 'job_name': job_name, 'hpc': 'cscs'})}' > resume_job_settings.json python zipfolder.py """ try: diff --git a/hhnb/core/job_handler.py b/hhnb/core/job_handler.py index b5303c2d..24b8163b 100644 --- a/hhnb/core/job_handler.py +++ b/hhnb/core/job_handler.py @@ -70,7 +70,7 @@ def is_job_expired(job_details): class JobHandler: """ - Useful class to easly handle jobs, and the relative files, + Useful class to easily handle jobs, and the relative files, on the selected HPC system. This class is intended to be used by calling its static methods and for this it is not recommended to instantiate a JobHandler object @@ -91,14 +91,13 @@ class HPCException(Exception): def __init__(self): self._SA_ROOT_URL = 'https://bspsa.cineca.it/' - self._SA_DAINT_JOB_URL = self._SA_ROOT_URL + 'jobs/pizdaint/hhnb_daint_cscs/' - self._SA_DAINT_FILES_URL = self._SA_ROOT_URL + 'files/pizdaint/hhnb_daint_cscs/' - self._SA_NSG_JOB_URL = self._SA_ROOT_URL + 'jobs/nsg/hhnb_nsg/' - self._SA_NSG_FILES_URL = self._SA_ROOT_URL + 'files/nsg/hhnb_nsg/' + self._SA_JOBS_URL = self._SA_ROOT_URL + 'jobs/{}/{}/' + self._SA_FILES_URL = self._SA_ROOT_URL + 'files/{}/{}/' self._NSG_URL = 'https://nsgr.sdsc.edu:8443/cipresrest/v1' self._DAINT_URL = 'https://brissago.cscs.ch:8080/DAINT-CSCS/rest/core' self._DAINT_CSCS = 'DAINT-CSCS' + self._SA = 'SA' self._SA_CSCS = 'SA-CSCS' self._NSG = 'NSG' self._SA_NSG = 'SA-NSG' @@ -167,8 +166,7 @@ def _submit_on_nsg(self, username, password, zip_file, settings): hhnb.core.response.ResponseUtil the result of the submission. """ - zip_name = os.path.split(zip_file)[1] - payload = self._get_nsg_payload(job_name=zip_name.split('.')[0], + payload = self._get_nsg_payload(job_name=settings['job_name'], core_num=settings['core-num'], node_num=settings['node-num'], runtime=settings['runtime']) @@ -198,7 +196,7 @@ def _submit_on_nsg(self, username, password, zip_file, settings): if not r.status_code == 200: return ResponseUtil.ko_response(r.text) - return ResponseUtil.ok_response(messages.JOB_SUBMITTED.format('NSG')) + return ResponseUtil.ok_response(messages.JOB_SUBMITTED.format('NSG')) return ResponseUtil.ko_response(r.text) @@ -216,7 +214,7 @@ def _get_nsg_jobs(self, username, password): Returns ------- dict - a dictionaire of all submitted jobs. + a dictionary of all submitted jobs. Raises ------ @@ -315,7 +313,7 @@ def _get_nsg_job_results(self, username, password, job_id): logger.debug(f'requests: {r.url} with headers: {r.headers}') if r.status_code != 200: logger.error(f'CODE: {r.status_code}, CONTENT: {r.content}') - raise self.HPCException(messages.JOB_RESULTS_FETCH_ERRROR) + raise self.HPCException(messages.JOB_RESULTS_FETCH_ERROR) file_list = {} root = xml.etree.ElementTree.fromstring(r.text) @@ -351,7 +349,7 @@ def _get_unicore_job_description(self, command, job_name, node_num, Returns the UNICORE job description. The job description is a payload that is formed by the command to run, the name of the job, and the resources list to - reserve for the job that are substracted from the project. + reserve for the job that are subtracted from the project. Parameters ---------- @@ -455,7 +453,7 @@ def _submit_on_unicore(self, hpc, token, zip_file, settings): zip_name = os.path.split(zip_file)[1] job_description = self._get_unicore_job_description( command=self._get_unicore_command(zip_name), - job_name=zip_name.split('.')[0], + job_name=settings['job_name'], node_num=settings['node-num'], core_num=settings['core-num'], runtime=settings['runtime'], @@ -464,7 +462,60 @@ def _submit_on_unicore(self, hpc, token, zip_file, settings): client = self._initialize_unicore_client(hpc, token) job = client.new_job(job_description=job_description, inputs=[zip_file]) logger.info(f'job submitted on UNICORE Client: {job}') - return ResponseUtil.ok_response(messages.JOB_SUBMITTED.format(hpc)) + return ResponseUtil.ok_response(messages.JOB_SUBMITTED.format('' + hpc + '')) + + def _reoptimize_model_on_unicore(self, job_id, job_name, hpc, max_gen, + node_num, core_num, runtime, token): + """ + Submit job to reoptimize model on unicore. The job executes a dedicated script on + CSCS-PizDAINT that make a copy of the output of the previous optimization and then + resume the optimization starting from its checkpoint. + + Args: + job_id : str + the job id. + job_name : str + the job name. + hpc : str + the HPC in which submit the job. + max_gen : int + the new generation parameter for the optimization. + node_num : int + the node number allocated for the optimization. + core_num : int + the core number allocated for the optimization. + runtime : str + the maximum amount of time for the job to be completed. + token : str + the EBRAINS user token. + + Returns: + Response : (status_code, content) + the response of the submission request. + """ + job_description = { + 'User precommand': f'cp -r /scratch/snx3000/unicore/FILESPACE/{job_id}/{job_name}/ .', + 'Executable': f'/apps/hbp/ich002/cnr-software-utils/hhnb/reoptimize_model.sh {job_name} {max_gen}', + 'User postcommand': f'mv {job_name} reopt_{job_name}; cd reopt_{job_name}; ' + \ + f'sed -i "s/{job_name}/reopt_{job_name}/" zipfolder.py; ' + \ + f'python ./zipfolder.py', + 'Name': 'reopt_' + job_name, + 'Resources': { + 'Nodes': node_num, + 'CPUsPerNode': core_num, + 'Runtime': runtime, + 'NodeConstraints': 'mc', + 'Project': 'ich002' + }, + 'Tags': self._TAGS, + "haveClientStageIn": "false", + } + client = self._initialize_unicore_client(hpc, token) + job = client.new_job(job_description=job_description) + job.start + logger.info(f'reoptimize model job {job_id} submitted on UNICORE Client') + return ResponseUtil.ok_response(messages.JOB_SUBMITTED.format('' + hpc + '')) + def _get_unicore_jobs(self, hpc, token): """ @@ -582,7 +633,7 @@ def _get_service_account_headers(self, token, zip_name=None, payload=None): headers.update({'payload': json.dumps(payload)}) return headers - def _submit_on_service_account(self, hpc, token, zip_file, settings): + def _submit_on_service_account(self, hpc, project, token, zip_file, settings): """ Submit a job behind the Service Account in the selected HPC system. To submit a job, an available HPC system must be choose @@ -610,35 +661,33 @@ def _submit_on_service_account(self, hpc, token, zip_file, settings): """ zip_name = os.path.split(zip_file)[1] payload = self._get_service_account_payload( - command=self._get_unicore_command(zip_name) if hpc=='SA-CSCS' else None, - tool=self._NSG_TOOL if hpc=='SA-NSG' else None, + command=self._get_unicore_command(zip_name) if hpc=='pizdaint' else None, + tool=self._NSG_TOOL if hpc=='nsg' else None, node_num=settings['node-num'], core_num=settings['core-num'], runtime=settings['runtime'], - title=zip_name.split('.')[0] + title=settings['job_name'] ) headers = self._get_service_account_headers(token, zip_name, payload) job_file = {'file': open(zip_file, 'rb')} - if hpc == self._SA_CSCS: - sa_endpoint = self._SA_DAINT_JOB_URL - elif hpc == self._SA_NSG: - sa_endpoint =self._SA_NSG_JOB_URL + sa_endpoint = self._SA_JOBS_URL.format(hpc, project) + r = requests.post(url=sa_endpoint, headers=headers, files=job_file) logger.debug(f'requests: {r.url} with headers: {r.headers} and files: {job_file}') + print(f"SERVICE ACCOUNT RESPONSE: {r.status_code}, {r.content}") if r.status_code >= 400: logger.error(f'CODE: {r.status_code}, CONTENT: {r.content}') + if r.status_code == 500: + raise self.ServiceAccountException(r.content, r.status_code) return ResponseUtil.ko_response(r.text) - message = 'Job submitted' - if hpc == self._SA_CSCS: - message = messages.JOB_SUBMITTED.format('SA-CSCS') - elif hpc == self._SA_NSG: - message = messages.JOB_SUBMITTED.format('SA-NSG') + message = messages.JOB_SUBMITTED.format( + '' + hpc.upper() + ' using the Service Account project ' + project + '') return ResponseUtil.ok_response(message) - def _get_service_account_jobs(self, hpc, token): + def _get_service_account_jobs(self, hpc, project, token): """ Returns a list of all jobs submitted using the Service Account by the user to the selected HPC system. @@ -661,10 +710,7 @@ def _get_service_account_jobs(self, hpc, token): if something happens on the Service Account side """ headers = self._get_service_account_headers(token) - if hpc == self._SA_CSCS: - sa_endpoint = self._SA_DAINT_JOB_URL - elif hpc == self._SA_NSG: - sa_endpoint = self._SA_NSG_JOB_URL + sa_endpoint = self._SA_JOBS_URL.format(hpc, project) r = requests.get(url=sa_endpoint, headers=headers) logger.debug(f'requests: {r.url} with headers: {r.headers}') if r.status_code != 200: @@ -672,7 +718,7 @@ def _get_service_account_jobs(self, hpc, token): raise self.ServiceAccountException(r.content, r.status_code) return r.json() - def _get_service_account_job_results(self, hpc, token, job_id): + def _get_service_account_job_results(self, hpc, project, token, job_id): """ Returns a list of the files once the job ends. The list is formed by a json object per file @@ -697,17 +743,14 @@ def _get_service_account_job_results(self, hpc, token, job_id): self.JobsFilesNotFound if the job is not found for the selected HPC system. self.ServiceAccountException - if something happends on the Service Account side or + if something happens on the Service Account side or if it is down. """ headers = self._get_service_account_headers(token) - if hpc == self._SA_CSCS: - sa_endpoint = self._SA_DAINT_FILES_URL + job_id + '/' - elif hpc == self._SA_NSG: - sa_endpoint = self._SA_NSG_FILES_URL + job_id + '/' + sa_endpoint = self._SA_FILES_URL.format(hpc, project) - r = requests.get(url=sa_endpoint, headers=headers) + r = requests.get(url=sa_endpoint + job_id + '/', headers=headers) logger.debug(f'requests: {r.url} with headers: {r.headers}') if r.status_code >= 400: @@ -719,9 +762,9 @@ def _get_service_account_job_results(self, hpc, token, job_id): file_list = [] for f in r.json(): - if hpc == self._SA_CSCS: + if hpc == 'pizdaint': file_list.append({'id': f, 'name': f}) - elif hpc == self._SA_NSG: + elif hpc == 'nsg': file_list.append({'id': f['fileid'], 'name': f['filename']}) return file_list @@ -729,8 +772,8 @@ def _get_service_account_job_results(self, hpc, token, job_id): @classmethod def submit_job(cls, user, zip_file, settings): """ - A static method to easly submit a job in the HPC system. - A settings dictionaire must be provided with the "hpc" key + A static method to easily submit a job in the HPC system. + A settings dictionary must be provided with the "hpc" key that indicate in which HPC system the job should be submitted. A ResponseUtil object will be returned. @@ -757,15 +800,15 @@ def submit_job(cls, user, zip_file, settings): elif settings['hpc'] == job_handler._DAINT_CSCS: return job_handler._submit_on_unicore(job_handler._DAINT_CSCS, user.get_token(), zip_file, settings) - elif settings['hpc'] == job_handler._SA_CSCS or settings['hpc'] == job_handler._SA_NSG: - return job_handler._submit_on_service_account(settings['hpc'], user.get_token(), - zip_file, settings) + elif settings['hpc'] == job_handler._SA: + return job_handler._submit_on_service_account(settings['sa-hpc'], settings['sa-project'], + user.get_token(), zip_file, settings) return ResponseUtil.ko_response(messages.GENERAL_ERROR) @classmethod - def fetch_jobs_list(cls, hpc, user): + def fetch_jobs_list(cls, hpc, user, sa_hpc=None, sa_project=None): """ - A static method to easly fetch the jobs list from the selected HPC system. + A static method to easily fetch the jobs list from the selected HPC system. Parameters ---------- @@ -773,11 +816,17 @@ def fetch_jobs_list(cls, hpc, user): the HPC system according to the available ones user : hhnb.core.user.HhnbUser the user who wants to fetch the jobs - + sa_hpc : str, optional + select which HPC will be used behind the Service Account. Only works + with the "hpc" parameter set to "SA" + sa_project : str, optional + select which Service Account project will be used to fetch jobs. Only + works with the "hpc" parameter set to "SA" + Returns ------- dict - a dictionaire containing the list of job + a dictionary containing the list of job ordered by the date from the most recent """ logger.info(LOG_ACTION.format(user, 'fetch %s jobs list' % hpc)) @@ -800,8 +849,8 @@ def fetch_jobs_list(cls, hpc, user): } jobs.update(job) - elif hpc == job_handler._SA_CSCS or hpc == job_handler._SA_NSG: - raw_jobs = job_handler._get_service_account_jobs(hpc, user.get_token()) + elif hpc == job_handler._SA: + raw_jobs = job_handler._get_service_account_jobs(sa_hpc, sa_project, user.get_token()) for raw_job in raw_jobs: job = {raw_job['job_id']: { 'workflow_id': raw_job['title'], @@ -815,18 +864,19 @@ def fetch_jobs_list(cls, hpc, user): ordered_jobs = OrderedDict(sorted(jobs.items(), key=lambda x: x[1]['date'], reverse=True)) + logger.info(LOG_ACTION.format(user, 'jobs list: %s' % ordered_jobs)) return {'jobs': ordered_jobs} @classmethod - def fetch_job_files(cls, hpc, job_id, user): + def fetch_job_files(cls, hpc, job_id, user, sa_hpc=None, sa_project=None): """ - A static method to easly fetch the job's files that are produced + A static method to easily fetch the job's files that are produced once the HPC ends to process the job. To fetch the list a the parameters that must be passed are the HPC in which the job was submitted, the job id to identify the correct job, and the - user as owner of the job. The result will be a dictionaire + user as owner of the job. The result will be a dictionary containing the "root_url" that is required to download the files, the list of files and, optionally, an header that must be passed in the download request to correctly identify user when the HPC @@ -840,14 +890,21 @@ def fetch_job_files(cls, hpc, job_id, user): the job id user : hhnb.core.user.HhnbUser the owner of the job + sa_hpc : str, optional + select which HPC will be used behind the Service Account. Only + works with the "hpc" parameter set to "SA" + sa_project : str, optional + select the job'project where the job was submitted. Only works + with the "hpc" parameter set to "SA" Returns ------- dict - a dictionaire containing all required information to download the files + a dictionary containing all required information to download the files """ logger.info(LOG_ACTION.format(user, 'fetch files of job: %s in %s' % (job_id, hpc))) job_handler = cls() + file_list = None if hpc == job_handler._NSG: raw_file_list = job_handler._get_nsg_job_results(user.get_nsg_user().get_username(), user.get_nsg_user().get_password(), @@ -860,22 +917,22 @@ def fetch_job_files(cls, hpc, job_id, user): 'password': user.get_nsg_user().get_password() } - if hpc == job_handler._DAINT_CSCS: + elif hpc == job_handler._DAINT_CSCS: raw_file_list = job_handler._get_unicore_job_results(hpc, user.get_token(), job_id) file_list = { 'root_url': 'unicore', 'file_list': raw_file_list } - if hpc == job_handler._SA_CSCS or hpc == job_handler._SA_NSG: - raw_file_list = job_handler._get_service_account_job_results(hpc, user.get_token(), job_id) - if hpc == job_handler._SA_CSCS: - root_url = job_handler._SA_DAINT_FILES_URL + job_id - elif hpc == job_handler._SA_NSG: - root_url = job_handler._SA_NSG_FILES_URL + job_id + '/' + elif hpc == job_handler._SA: + raw_file_list = job_handler._get_service_account_job_results(sa_hpc, sa_project, user.get_token(), job_id) + root_url = job_handler._SA_FILES_URL.format(sa_hpc, sa_project) + job_id + if sa_hpc == 'nsg': + root_url += '/' file_list = { 'root_url': root_url, 'file_list': raw_file_list, 'headers': {'Authorization': 'Bearer ' + user.get_token()} } + return file_list diff --git a/hhnb/core/lib/exception/model_exception.py b/hhnb/core/lib/exception/model_exception.py index a9471af5..8699b8e7 100644 --- a/hhnb/core/lib/exception/model_exception.py +++ b/hhnb/core/lib/exception/model_exception.py @@ -14,4 +14,10 @@ class MorphologyConfigError(Exception): pass class InvalidMorphologyDirectory(Exception): + pass + +class InvalidMorphologyFile(Exception): + pass + +class InvalidMechanismFile(Exception): pass \ No newline at end of file diff --git a/hhnb/core/lib/exception/workflow_exception.py b/hhnb/core/lib/exception/workflow_exception.py index 2363cd62..0077e39e 100644 --- a/hhnb/core/lib/exception/workflow_exception.py +++ b/hhnb/core/lib/exception/workflow_exception.py @@ -19,8 +19,13 @@ class EmptyWorkflow(Exception): pass -class MechanismsProcessError(Exception): # CalledProcessError): +class MechanismsProcessError(CalledProcessError): pass -class AnalysisProcessError(Exception): # CalledProcessError): + +class AnalysisProcessError(CalledProcessError): pass + + +class UnknownParametersTemplate(Exception): + pass \ No newline at end of file diff --git a/hhnb/core/lib/model.py b/hhnb/core/lib/model.py index 9ca36f5c..353fe8cf 100644 --- a/hhnb/core/lib/model.py +++ b/hhnb/core/lib/model.py @@ -1,8 +1,6 @@ -""" This package provide some useful classes and methods to easly handle a model instance. """ +""" This package provide some useful classes and methods to easily handle a model instance. """ -from typing import Type from .exception.model_exception import * -from multipledispatch import dispatch import os import json @@ -37,10 +35,7 @@ def __str__(self): """ String representation of the Features instance. """ - print('', sep='\n') + return f'' def set_features(self, features): """ @@ -123,7 +118,7 @@ class Morphology: """ An easy way to handle the morphology of the model. """ - + def __init__(self, morphology=None, key=None): """ Instantiate a Morphology object. @@ -145,10 +140,7 @@ def __str__(self): """ String representation of the Features instance. """ - print('', sep='\n') + return f'' def set_morphology(self, morphology, key=None): """ @@ -162,7 +154,7 @@ def set_morphology(self, morphology, key=None): the morphology file path. The morphology must be in ".asc" format. key : str, optional the model global key to generate the config file, if set to None - a temporarely key will be used. + a temporarily key will be used. Raises ------ @@ -172,9 +164,12 @@ def set_morphology(self, morphology, key=None): if the path is pointing a directory. """ if not os.path.exists(morphology): - raise FileNotFoundError('%s not found' % morphology) + raise FileNotFoundError('%s not found.' % morphology) if not os.path.isfile(morphology): - raise IsADirectoryError('%s is a directory' % morphology) + raise IsADirectoryError('%s is a directory.' % morphology) + if not morphology.endswith('.asc'): + raise InvalidMorphologyFile('%s is not ".asc" morphology format.' % morphology) + self._morphology = morphology self._MORPHOLOGY = True if not key: @@ -245,12 +240,13 @@ def __init__(self, key, **kwargs): self._key = key self.set_features(Features()) self.set_morphology(Morphology()) + if 'features' in kwargs and kwargs['features']: self.set_features(features=kwargs['features']) if 'morphology' in kwargs and kwargs['morphology']: self.set_morphology(kwargs['morphology']) if 'protocols' in kwargs and kwargs['protocols']: - self.set_protocols(protocols=kwargs['protocols']) + self.set_features(protocols=kwargs['protocols']) if 'parameters' in kwargs and kwargs['parameters']: self.set_parameters(kwargs['parameters']) if 'mechanisms' in kwargs and kwargs['mechanisms']: @@ -259,7 +255,7 @@ def __init__(self, key, **kwargs): def set_features(self, *args, **kwargs): """ Set the model features and protocols. - This method accepts a unique Features instance as argoument, + This method accepts a unique Features instance as argument, or can be called by passing a "features" and/or "protocols" kwargs with the relative file Raises @@ -276,7 +272,7 @@ def set_features(self, *args, **kwargs): for k in kwargs.keys(): if k != 'features' and k != 'protocols': raise TypeError('set_features() got an unexpected\ - keyword argoument %s' % k) + keyword argument %s' % k) if 'features' in kwargs.keys(): self._features.set_features(kwargs['features']) if 'protocols' in kwargs.keys(): @@ -302,7 +298,7 @@ def set_morphology(self, morphology): def set_mechanisms(self, mechanisms): """ Set the mechanisms files for the model. - The mechanisms file can be a list of file, a directory cotaining the mechanisms file or a list of files. + The mechanisms file can be a list of file, a directory containing the mechanisms file or a list of files. Parameters ---------- @@ -316,20 +312,27 @@ def set_mechanisms(self, mechanisms): IsADirectoryError if it tries to access a file but is a directory. """ + self._mechanisms = [] + if type(mechanisms) == list: self._mechanisms = mechanisms - if type(mechanisms) == str: + + elif type(mechanisms) == str: if os.path.isdir(mechanisms): self._mechanisms = [ os.path.join(mechanisms, m) for m in os.listdir(mechanisms) ] else: - self._mechanisms = mechanisms + self._mechanisms.append(mechanisms) + for m in self._mechanisms: if not os.path.exists(m): raise FileNotFoundError('%s not found' % m) if not os.path.isfile(m): raise IsADirectoryError('%s is a directory' % m) + if not m.endswith('.mod'): + raise InvalidMechanismFile('%s is not a valid Mechanism file. \ + Only accept ".mod" extension.' % m) if not self._mechanisms: self._MECHANISMS = False else: @@ -364,7 +367,7 @@ def get_features(self): def get_mechanisms(self): """ Get the mechanism file list. """ - return self._mechansisms + return self._mechanisms def get_parameters(self): """ Get the parameters file path. """ diff --git a/hhnb/core/model.py b/hhnb/core/model.py index a44f3c00..64514975 100644 --- a/hhnb/core/model.py +++ b/hhnb/core/model.py @@ -5,7 +5,6 @@ """ -import datetime from hhnb.core.lib.exception.model_exception import * from hhnb.core.lib.model import * @@ -41,7 +40,7 @@ def _write_file_to_directory(src_file, dst_dir, dst_file=None): The destination file can be also a json file and in this case, the output will be formatted as json. In the other case the source file will be treated as a binary file. Furthermore - the copy can be explicitely named using the "dst_file" parameter + the copy can be explicitly named using the "dst_file" parameter otherwise the source file name will be used. Parameters @@ -84,7 +83,7 @@ def __init__(self, model_dir, **kwargs): """ Initialize the Model object by reading all the files present in the "model_dir" folder. - For keyword argouments read the hhnb.core.model.Model doc. + For keyword arguments read the hhnb.core.model.Model doc. Parameters ---------- @@ -100,7 +99,6 @@ def __init__(self, model_dir, **kwargs): super().__init__(key, **kwargs) self._ETRACES = False - def _check_if_file_exists(self, **kwargs): """ Private method that checks if a file already exists @@ -114,12 +112,12 @@ def _check_if_file_exists(self, **kwargs): Raises ------ TypeError - if the argoument passed is wrong + if the argument passed is wrong """ if len(kwargs) > 1: raise TypeError(f'{__name__} takes only 1 keyword argument') - keyword_list = ['paramters', 'protocols', 'features', 'morphology'] + keyword_list = ['parameters', 'protocols', 'features', 'morphology'] key = list(kwargs.keys())[0] if key not in keyword_list: @@ -138,49 +136,10 @@ def _check_if_file_exists(self, **kwargs): return False return c - # def set_features(self, features=None, protocols=None): - # f = self._features.get_features() - # if not self._check_if_file_exists(features=f): - # shutil.copy(f) - # super().set_features(features=features, protocols=protocols) - - # def set_features(self, features=None, protocols=None): - # if not features and not protocols: - # if os - - # def set_morphology(self, morphology=None): - # if not morphology: - # morphology = os.path.join( - # self._model_dir, - # os.listdir(os.path.join(self._model_dir, 'morphology'))[0] - # ) - - # super().set_morphology(morphology) - - # def set_mechanisms(self, mechansisms=None): - # if not mechansisms: - # mechansisms = os.path.join(self._model_dir, 'mechanisms') - # super().set_mechanisms(mechansisms=mechansisms) - - # def set_parameters(self, parameters=None): - # if not parameters: - # parameters = os.path.join(self._model_dir, 'config', 'parameters.json') - # super().set_parameters(parameters=parameters) - - - # @classmethod - # def from_zip(cls, zip_model): - # model = cls() - # # TODO: to be complete - # tmp_unzip_model = os.path.join(TMP_DIR, datetime.datetime.now()) - # shutil.unpack_archive(zip_model, tmp_unzip_model) - # pass - - @classmethod def from_dir(cls, model_dir, key): """ - Thie method is used to initialize automatically a Model object + This method is used to initialize automatically a Model object by passing the model root folder as parameter, reads all the files and the structure of the folder subtree and return a Model object with the "key" set as the model global key. @@ -261,9 +220,9 @@ def from_dir(cls, model_dir, key): def update_optimization_files(self, model_dir): """ This method update the current model optimization files using - the new ones in the "model_dir" folder passed as argoument. + the new ones in the "model_dir" folder passed as argument. - With "optimization files" is intendend any files that belog to + With "optimization files" is intended any files that belong to the following categories: ["parameters", "morphology", "mechanisms"]. Parameters @@ -276,30 +235,60 @@ def update_optimization_files(self, model_dir): FileNotFoundError if any optimization file is not found """ + + src_config_dir = os.path.join(model_dir, 'config') + src_morph_dir = os.path.join(model_dir, 'morphology') + src_mechans_dir = os.path.join(model_dir, 'mechanisms') + + dst_config_dir = os.path.join(self._model_dir, 'config') + dst_morph_dir = os.path.join(self._model_dir, 'morphology') + dst_mechans_dir = os.path.join(self._model_dir, 'mechanisms') + try: # parameters - parameters = shutil.copy(os.path.join(model_dir, 'config', 'parameters.json'), - os.path.join(self._model_dir, 'config')) - self.set_parameters(parameters) + if os.path.exists(src_config_dir) and \ + os.path.exists(os.path.join(src_config_dir, 'parameters.json')): + parameters = shutil.copy(os.path.join(src_config_dir, 'parameters.json'), + dst_config_dir) + self.set_parameters(parameters) + # morphology - for m in os.listdir(os.path.join(model_dir, 'morphology')): - morph = shutil.copy(os.path.join(model_dir, 'morphology', m), - os.path.join(self._model_dir, 'morphology')) - conf_file = shutil.copy(os.path.join(model_dir, 'config', 'morph.json'), - os.path.join(self._model_dir, 'config')) - self.set_morphology(morphology=morph) + if os.path.exists(src_morph_dir) and len(os.listdir(src_morph_dir)) == 1: + for m in os.listdir(dst_morph_dir): + os.remove(os.path.join(dst_morph_dir, m)) + try: + os.remove(os.path.join(dst_config_dir, 'morph.json')) + except FileNotFoundError: + pass + + for m in os.listdir(src_morph_dir): + morph = shutil.copy(os.path.join(src_morph_dir, m), + os.path.join(dst_morph_dir, m)) + + self.set_morphology(morph) + with open(os.path.join(dst_config_dir, 'morph.json'), 'w') as morph_conf: + json.dump(self.get_morphology().get_config(), morph_conf) + + # mechanisms - for m in os.listdir(os.path.join(model_dir, 'mechanisms')): - shutil.copy(os.path.join(model_dir, 'mechanisms', m), - os.path.join(self._model_dir, 'mechanisms')) - self.set_mechanisms(os.path.join(self._model_dir, 'mechanisms')) - # python's files + if os.path.exists(src_mechans_dir) and len(os.listdir(src_mechans_dir)) > 0: + for m in os.listdir(dst_mechans_dir): + os.remove(os.path.join(dst_mechans_dir, m)) + + for m in os.listdir(src_mechans_dir): + shutil.copy(os.path.join(src_mechans_dir, m), + os.path.join(dst_mechans_dir, m)) + + self.set_mechanisms(dst_mechans_dir) + + pass + except FileNotFoundError as e: raise FileNotFoundError(e) def get_optimization_files_raw_status(self): """ - Returns a dictionaire with the optimization files as keys and + Returns a dictionary with the optimization files as keys and their status (True if present, False otherwise) as a boolean value. """ return { @@ -310,7 +299,7 @@ def get_optimization_files_raw_status(self): def get_optimization_files_status(self): """ - Returns a dictionaire with the optimization files as keys and + Returns a dictionary with the optimization files as keys and their status as a message. """ return { @@ -338,7 +327,7 @@ class ModelUtil: @staticmethod def clone(model): """ - Static method that clone a Model object passed as argoument. + Static method that clone a Model object passed as argument. Parameters ---------- @@ -350,7 +339,7 @@ def clone(model): hhnb.core.model.Model a new Model object with the same files and properties of the cloned one """ - return Model( + return ModelBase( features=model.get_features(), parameters=model.get_parameters(), morphology=model.get_morphology(), @@ -358,6 +347,29 @@ def clone(model): key=model.get_key() ) + @staticmethod + def create_model_tree(model_dir): + """ + Create model tree folder by passing the model_dir path. + + Parameters + ---------- + model_dir : str + the path of where to create the model tree. + + Raise + ----- + FileExistError + if any subfolder already exists. + """ + + if not os.path.exists(model_dir): + os.mkdir(model_dir) + os.mkdir(os.path.join(model_dir, 'config')) + os.mkdir(os.path.join(model_dir, 'morphology')) + os.mkdir(os.path.join(model_dir, 'mechanisms')) + os.mkdir(os.path.join(model_dir, 'template')) + @staticmethod def write_to_workflow(model, workflow_id): """ @@ -385,8 +397,8 @@ def write_to_workflow(model, workflow_id): raise FileNotFoundError('%s path not found' % workflow_id) model_dir = os.path.join(workflow_id, 'model') if os.path.exists(model_dir): - os.rmtree(model_dir) - ModelUtil.create_model_subdir(model_dir) + shutil.rmtree(model_dir) + ModelUtil.create_model_tree(model_dir) _write_file_to_directory(model.get_features().get_features(), os.path.join(model_dir, 'config'), 'features.json') _write_file_to_directory(model.get_features().get_protocols(), @@ -463,7 +475,7 @@ def zip_model(model, dst_dir=None, zip_name=None): def update_key(model, key=None): """ This static method update the key of all Model's files with the - new one passed as argoument and then it will be set as the Model + new one passed as argument and then it will be set as the Model global key. Otherwise the files' keys are updated using the current Model global key. @@ -477,7 +489,7 @@ def update_key(model, key=None): Raises ------ TypeError - if the model object passed is not an istance of hhnb.core.model.Model + if the model object passed is not an instance of hhnb.core.model.Model shutil.Error is any error occurred when trying to update the files' key """ diff --git a/hhnb/core/response.py b/hhnb/core/response.py index edceec9b..db2dec5a 100644 --- a/hhnb/core/response.py +++ b/hhnb/core/response.py @@ -3,13 +3,8 @@ """ -from dis import dis -from genericpath import isfile -from os import stat -from typing import Tuple from django.http.response import JsonResponse, HttpResponse, FileResponse, HttpResponseNotAllowed from multipledispatch import dispatch -import json import os @@ -26,7 +21,7 @@ def _json_response(status_code, data, safe=False): This private function takes the status code, some data and return a JsonResponse object. Optionally the safe flag can be set True (False by default) to prevent the serialization - of non "dict" obejcts. + of non "dict" objects. """ return JsonResponse(data=data, status=status_code, safe=safe, content_type='application/json') @@ -162,7 +157,7 @@ def ko_json_response(data): @dispatch(int, str) def ko_json_response(code, data): """ - Returns an error JsonResponse with a custom erro code and json. + Returns a JsonResponse with a custom error code and json. """ return _json_response(code, data) @@ -170,7 +165,7 @@ def ko_json_response(code, data): @dispatch(int, dict) def ko_json_response(code, data): """ - Returns an error JsonResponse with a custom erro code and json. + Returns a JsonResponse with a custom error code and json. """ return _json_response(code, data) diff --git a/hhnb/core/security.py b/hhnb/core/security.py index 8508c1be..f32e63da 100644 --- a/hhnb/core/security.py +++ b/hhnb/core/security.py @@ -70,7 +70,7 @@ def decrypt(cipher_text, at_time=None): class Sign: """ This class is used to sign a file and to verify the integrity of the file - by verifing its sign. + by verifying its sign. """ def __init__(self): @@ -110,7 +110,7 @@ def get_data_sign(cls, data): @classmethod def verify_data_sign(cls, sign, data): """ - Verify if the sign is valide for the data. + Verify if the sign is valid for the data. Parameters ---------- diff --git a/hhnb/core/user.py b/hhnb/core/user.py index 23c390b7..a07d3faf 100644 --- a/hhnb/core/user.py +++ b/hhnb/core/user.py @@ -48,7 +48,7 @@ def __str__(self): def _get_user_info(self): """ - Returns a json containing all the user informations. + Returns a json containing all the user information. If any error occurred an UserInfoError will be raised. """ r = requests.get(url=OIDC_OP_USER_ENDPOINT, @@ -154,7 +154,6 @@ def validate_credentials(self): auth=(self._username, self._password), headers={'cipres-appkey': NSG_KEY}, verify=False) - print(r.status_code, r.content) if r.status_code == 200: return True return False diff --git a/hhnb/core/workflow.py b/hhnb/core/workflow.py index d67d3a72..813d0b26 100644 --- a/hhnb/core/workflow.py +++ b/hhnb/core/workflow.py @@ -2,7 +2,7 @@ Workspace utils classes """ -from hh_neuron_builder.settings import MEDIA_ROOT, HHF_TEMPLATE_DIR, TMP_DIR, LOG_ROOT_PATH +from hh_neuron_builder.settings import MEDIA_ROOT, HHF_TEMPLATE_DIR, HHF_PARAMETERS_TEMPLATE_DIR, TMP_DIR, LOG_ROOT_PATH from hhnb.core.conf.exec_files_conf import ExecFileConf @@ -13,6 +13,7 @@ from pyunicore.client import PathFile as UnicorePathFile from datetime import datetime from sys import prefix as env_prefix +from pathlib import Path import shutil import os @@ -51,6 +52,7 @@ def __init__(self, user_sub, workflow_id): self._optimization_settings = os.path.join(self._workflow_path, 'optimization_settings.json') + if os.path.exists(self._model_dir) and any(os.scandir(self._model_dir)): self._model = Model.from_dir(self._model_dir, key=workflow_id) @@ -160,7 +162,7 @@ def generate_user_workflow_from_zip(cls, user_sub, zip_file): """ Create a new workflow by generating a workflow id from the current timestamp and unpacking all files from the - zip passed as argoument. + zip passed as argument. Parameters ---------- @@ -203,7 +205,7 @@ def generate_user_workflow_from_path(cls, user_sub, path_to_clone): @classmethod def get_user_workflow_by_id(cls, user_sub, workflow_id): """ - Get a worfklow object by specifing the user sub id and + Get a workflow object by specifying the user sub id and the workflow id. This classmethod method is useful when the folders are already written on the disk but it is needed to reinitialize @@ -309,7 +311,7 @@ def make_workflow_dirs(self): for sub_path in [os.path.join(self._model_dir, f) for f in sub_dir]: if not os.path.exists(sub_path): os.mkdir(sub_path) - + os.mkdir(self._results_dir) os.mkdir(self._analysis_dir) os.mkdir(self._tmp_dir) @@ -401,41 +403,90 @@ def load_model_zip(self, model_zip): if os.path.exists(unzipped_tmp_model_dir): shutil.rmtree(unzipped_tmp_model_dir) os.mkdir(unzipped_tmp_model_dir) - shutil.unpack_archive(model_zip, unzipped_tmp_model_dir, 'zip') - if len(os.listdir(unzipped_tmp_model_dir)) == 1: - unzipped_tmp_model_dir = os.path.join(unzipped_tmp_model_dir, - os.listdir(unzipped_tmp_model_dir)[0]) + shutil.unpack_archive(model_zip, unzipped_tmp_model_dir) + + for f in os.listdir(unzipped_tmp_model_dir): + if 'output' in f: + shutil.unpack_archive(os.path.join(unzipped_tmp_model_dir, f), + unzipped_tmp_model_dir) + + for model_folder in os.listdir(unzipped_tmp_model_dir): + if os.path.isdir(os.path.join(unzipped_tmp_model_dir, model_folder)): + print(os.path.join(unzipped_tmp_model_dir, model_folder)) + else: + os.remove(os.path.join(unzipped_tmp_model_dir, model_folder)) + + unzipped_tmp_model_dir = os.path.join(unzipped_tmp_model_dir, + os.listdir(unzipped_tmp_model_dir)[0]) + + shutil.rmtree(self._model_dir) + if not os.path.exists(self._model_dir): + os.mkdir(self._model_dir) + shutil.copytree(unzipped_tmp_model_dir, self._model_dir, dirs_exist_ok=True) + + # The uploaded origin model case self._model.update_optimization_files(unzipped_tmp_model_dir) ModelUtil.update_key(model=self._model) + + def _set_optimization_settings(self, optimization_settings): + """ + Create the optimization settings file for the current workflow. + + Parameters + ---------- + optimization_settings : dict + the optimization settings + """ + with open(self._optimization_settings, 'w') as fd: + json.dump(optimization_settings, fd, indent=4) def get_optimization_settings(self): """ - Returns the workflow optimization settings as json if presents, - otherwise a FileNotFoundError will be raised. + Returns the optimization settings of the current workflow, + otherwise an empty dict will be returned. """ try: with open(self._optimization_settings, 'r') as fd: try: - return json.load(fd) + return json.load(fd) except JSONDecodeError: return {} except FileNotFoundError: - raise FileNotFoundError("%s not found" % self._optimization_settings) + return {} - def set_optimization_settings(self, optimization_settings, job_submitted_flag=False): + def add_optimization_settings(self, update_json): """ - Write/Overwrite the workflow optimization settings in the current workflow. + Update the optimization settings of the current workflow + with the new settings provided in the "update_json" dict. Parameters ---------- - optimization_settings : dict - the workflow optimization settings - job_submitted_flag : bool, optional - this flag tells if the job was already submitted or not, by default False + update_json : dict + the update dict """ - optimization_settings.update({'job_submitted': job_submitted_flag}) - with open(self._optimization_settings, 'w') as fd: - json.dump(optimization_settings, fd, indent=4) + settings = self.get_optimization_settings() + settings.update(update_json) + self._set_optimization_settings(settings) + + def get_resume_settings(self): + """ + Returns the resume settings for the current workflow, + otherwise an empty dict will be returned. + + Returns + ------- + dict + the resume settings, an empty dict. + """ + try: + with open(os.path.join(self._model_dir, 'resume_job_settings.json'), + 'r') as fd: + return json.load(fd) + except FileNotFoundError: + pass + except JSONDecodeError: + pass + return {} def remove_file(self, file_path): """ @@ -447,42 +498,45 @@ def remove_file(self, file_path): ---------- file_path : str the file to delete, can be also "some_path/*" to - delete all files insede the "some_path" folder + delete all files inside the "some_path" folder Raises ------ PermissionError - Raised if the file path passed as argoument points to a + Raised if the file path passed as argument points to a file that not belong to the application FileNotFoundError Raised if the file is not found """ directory, filename = os.path.split(file_path) - target_dir = os.path.abspath(os.path.join(self._model_dir, directory)) if os.path.commonpath([os.path.abspath(self._workflow_path), target_dir]) != \ os.path.abspath(self._workflow_path): raise PermissionError('Can\'t delete files on %s' % target_dir) - if not os.path.exists(target_dir): - raise FileNotFoundError('%s directory not exists' % directory) - if filename == '*': shutil.rmtree(target_dir) os.mkdir(target_dir) + + elif not filename: + if os.path.isdir(target_dir): + shutil.rmtree(target_dir) + elif directory.endswith("*"): + d = directory[:-1] # removed "*" chars + for dd in os.listdir(self._model_dir): + if dd.startswith(d): + shutil.rmtree(os.path.join(self._model_dir, dd)) else: full_file_path = os.path.join(self._model_dir, file_path) if os.path.exists(full_file_path): os.remove(full_file_path) + if not os.path.exists(target_dir): + raise FileNotFoundError('%s directory not exists' % directory) + def get_properties(self): """ Returns the workflow properties. """ - try: - job_submitted = self.get_optimization_settings()['job_submitted'] - except FileNotFoundError: - job_submitted = False - analysis_flag = False if len(os.listdir(self._analysis_dir)) == 1: analysis_model_dir = os.path.join(self._analysis_dir, @@ -491,14 +545,30 @@ def get_properties(self): ['mechanisms', 'morphology', 'checkpoints']: analysis_flag = True + optset_flag = (False, 'Optimization parameters NOT set') + if os.path.exists(self._optimization_settings): + with open(self._optimization_settings, 'r') as fd: + jj = json.load(fd) + if jj.get('hpc') == 'SA': + if jj.get('sa-hpc') and jj.get('sa-project'): + optset_flag = (True, '') + elif jj.get('hpc') == 'DAINT-CSCS': + if jj.get('project'): + optset_flag = (True, '') + elif jj.get('hpc') == 'NSG': + if not jj.get('username_submit') or not jj.get('password_submit'): + optset_flag = (False, 'NSG credentials required') + else: + optset_flag = (True, '') + props = { 'id': self._id, 'model': self._model.get_properties(), - 'optimization_settings': os.path.exists(self._optimization_settings), + 'optimization_settings': optset_flag, 'etraces': any(os.scandir(self._etraces_dir)), - 'job_submitted': job_submitted, + 'job_submitted': self.get_optimization_settings().get('job_submitted', False), 'results': any(os.scandir(self._results_dir)), - # 'analysis': any(os.scandir(self._analysis_dir)), + 'resume': os.path.exists(os.path.join(self._model_dir, 'checkpoints', 'checkpoint.pkl')), 'analysis': analysis_flag } return props @@ -533,10 +603,11 @@ def set_model_key(workflow, key=None): workflow : hhnb.core.workflow.Workflow the workflow key : str, optional - the new global model key, by default the current model key is choosen + the new global model key, by default the current model key is chosen """ ModelUtil.update_key(workflow.get_model(), key) + # deprecated method @staticmethod def set_default_parameters(workflow): """ @@ -576,7 +647,7 @@ def clone_workflow(workflow): @staticmethod def make_archive(workflow, zip_name, dir_name, file_list): """ - Create a zip archive of the listed file containend in the workflow. + Create a zip archive of the listed file contained in the workflow. Parameters ---------- @@ -643,7 +714,7 @@ def make_workflow_archive(workflow): @staticmethod def make_features_archive(workflow): """ - Create a zip archive contaning the features file of the workflow. + Create a zip archive containing the features file of the workflow. Parameters ---------- @@ -761,7 +832,7 @@ def make_optimization_model(workflow): dst=os.path.join(workflow.get_tmp_dir(), 'opt_model', workflow.get_model().get_key())) - # craeting directories and script + # creating directories and script try: os.mkdir(os.path.join(tmp_model_dir, 'checkpoints')) os.mkdir(os.path.join(tmp_model_dir, 'figures')) @@ -769,22 +840,28 @@ def make_optimization_model(workflow): pass settings = workflow.get_optimization_settings() - if settings['hpc'] == 'NSG' or settings['hpc'] == 'SA-NSG': + if settings['hpc'] == 'NSG' or \ + (settings['hpc'] == 'SA' and settings['sa-hpc'] == 'nsg'): ExecFileConf.write_nsg_exec(dst_dir=tmp_model_dir, max_gen=settings['gen-max'], - offspring=settings['offspring']) - elif settings['hpc'] == 'DAINT-CSCS' or settings['hpc'] == 'SA-CSCS': + offspring=settings['offspring'], + mode=settings['mode'], + job_name=settings['job_name']) + elif settings['hpc'] == 'DAINT-CSCS' or \ + (settings['hpc'] == 'SA' and settings['sa-hpc'] == 'pizdaint'): ExecFileConf.write_daint_exec(dst_dir=tmp_model_dir, folder_name=workflow.get_model().get_key(), offspring=settings['offspring'], - max_gen=settings['gen-max']) + max_gen=settings['gen-max'], + mode=settings['mode'], + job_name=settings['job_name']) return tmp_model_dir @staticmethod def download_from_hhf(workflow, hhf_dict): """ - Download all files containend in the "hhf_dict" dictionaire in + Download all files contained in the "hhf_dict" dictionary in the workflow that comes from the HippocampusHub. Parameters @@ -792,7 +869,7 @@ def download_from_hhf(workflow, hhf_dict): workflow : hhnb.core.workflow.Workflow the workflow in which to download the files hhf_dict : dict - the dictionaire containing all files that + the dictionary containing all files that comes from the HippocampusHub """ WorkflowUtil.set_default_parameters(workflow) @@ -812,7 +889,7 @@ def download_from_hhf(workflow, hhf_dict): for chunk in r.iter_content(chunk_size=4096): fd.write(chunk) workflow.get_model().set_morphology(morphology=file_path) - + for etrace in etraces: file_path = os.path.join(workflow.get_etraces_dir(), etrace['name']) r = requests.get(url=etrace['url'], verify=False) @@ -836,8 +913,8 @@ def download_from_hhf(workflow, hhf_dict): @staticmethod def list_model_files(workflow): """ - Returns a dictionaire containing all the model's files - within the workflow. The keys of the dictionaire + Returns a dictionary containing all the model's files + within the workflow. The keys of the dictionary represent the type of the model's files and can be one of the following values "config", "morphology", "model" and "root" that contains the root model folder path. @@ -848,7 +925,7 @@ def list_model_files(workflow): Returns ------- dict - a dictionaire containing the type of files as the key + a dictionary containing the type of files as the key and the path of the files as value. """ model_files = {} @@ -899,7 +976,7 @@ def download_job_result_files(workflow, data): workflow : hhnb.core.workflow.Workflow the workflow data : dict - the data object containing all the informations about + the data object containing all the information about the job result files to download from the HPC system Raises @@ -957,36 +1034,41 @@ def run_analysis(workflow, job_output): Parameters ---------- workflow : hhnb.core.workflow.Workflow - the workflow that will contain the analysis + the workflow that will contain the analysis. job_output : str - the path where the job to analyse is placed + the path where the job to analyse is placed. Raises ------ FileNotFoundError - if any required files is not found + if any required files is not found. AnalysisProcessError if the "opt_neuron.py" script ends with an error or if the script can't be run due to a required - file that is not found + file that is not found. MechanismsProcessError - if the "nrnivmodl" program ends with an error + if the "nrnivmodl" program ends with an error. """ analysis_dir = workflow.get_analysis_dir() shutil.unpack_archive(job_output, analysis_dir) + + output_dir = None for f in [os.path.join(analysis_dir, f) for f in os.listdir(analysis_dir)]: if os.path.isdir(f): output_dir = f else: os.remove(f) + if not output_dir: + raise FileNotFoundError('Output folder') + analysis_file = os.path.join(output_dir, 'model', 'analysis.py') if not os.path.exists(analysis_file): - raise FileNotFoundError('"analysis.py" not found') + raise FileNotFoundError('analysis.py') evaluator_file = os.path.join(output_dir, 'model', 'evaluator.py') if not os.path.exists(evaluator_file): - raise FileNotFoundError('"evaluator.py" not found') + raise FileNotFoundError('evaluator.py') figures_dir = os.path.join(output_dir, 'figures') if os.path.exists(figures_dir): @@ -1002,7 +1084,7 @@ def run_analysis(workflow, job_output): os.rename(os.path.join(checkpoint_dir, f), os.path.join(checkpoint_dir, 'checkpoint.pkl')) else: - raise AnalysisProcessError('Checkpoints folder not found! Maybe the optimization process failed.') + raise FileNotFoundError('checkpoint.pkl') opt_neuron_file = os.path.join(output_dir, 'opt_neuron.py') with open(opt_neuron_file, 'r') as fd: @@ -1020,33 +1102,27 @@ def run_analysis(workflow, job_output): compiled_mods_dir = os.path.join(output_dir, 'x86_64') if os.path.exists(compiled_mods_dir): shutil.rmtree(compiled_mods_dir) + curr_dir = os.getcwd() - curr_dir = os.getcwd() - log_file_path = os.path.join(LOG_ROOT_PATH, 'analysis', workflow.get_user()) if not os.path.exists(log_file_path): os.makedirs(log_file_path) - log_file = os.path.join(log_file_path, workflow.get_id() + '.log') + log_file = os.path.join(log_file_path, workflow.get_id() + '.log') + Path(log_file).touch() + + os.chdir(output_dir) build_mechanisms_command = f'source {env_prefix}/bin/activate; nrnivmodl mechanisms > {log_file}' - opt_neuron_analysis_command = f'source {env_prefix}/bin/activate; python ./opt_neuron.py --analyse --checkpoint ./checkpoints > {log_file}' - - os.chdir(output_dir) - p0 = subprocess.call(build_mechanisms_command, shell=True, executable='/bin/bash') - p1 = subprocess.call(opt_neuron_analysis_command, shell=True, executable='/bin/bash') + opt_neuron_analysis_command = f'source {env_prefix}/bin/activate; python ./opt_neuron.py --analyse --checkpoint ./checkpoints > {log_file}' + p0 = subprocess.run(build_mechanisms_command, shell=True, executable='/bin/bash', capture_output=True, text=True) + p1 = subprocess.run(opt_neuron_analysis_command, shell=True, executable='/bin/bash', capture_output=True, text=True) + os.chdir(curr_dir) - if p0 > 0: - raise MechanismsProcessError()#p0.returncode, build_mechanisms_command, stderr=p0.stderr) - if p1 > 0: - error = 'Can\'t identify the error.' - for f in os.listdir(os.path.join(output_dir, 'checkpoints')): - if not f.endswith('.pkl'): - error = 'Checkpoint not found! Maybe the optimization process failed.' - break - raise AnalysisProcessError(error)#p1.returncode, opt_neuron_analysis_command, stderr=p1.stderr) - - + if p0.returncode > 0: + raise MechanismsProcessError(p0.returncode, build_mechanisms_command, stderr=p0.stderr) + if p1.returncode > 0: + raise AnalysisProcessError(p1.returncode, opt_neuron_analysis_command, stderr=p1.stderr) @staticmethod def make_naas_archive(workflow): @@ -1078,10 +1154,13 @@ def make_naas_archive(workflow): os.remove(f_path) # rename .hoc file + hoc_file = None checkpoints_dir = os.path.join(tmp_analysis_dir, 'checkpoints') for f in os.listdir(checkpoints_dir): if f.endswith('.hoc'): hoc_file = f + if not hoc_file: + raise FileNotFoundError('".hoc" file not found') os.rename( src=os.path.join(checkpoints_dir, hoc_file), dst=os.path.join(checkpoints_dir, 'cell.hoc') @@ -1097,3 +1176,51 @@ def make_naas_archive(workflow): # moving naas archive to tmp dir return shutil.move(naas_archive, workflow.get_tmp_dir()) + + @staticmethod + def load_parameters_template(workflow, template_type): + """ + Load the chosen parameters template in the current workflow. + + Parameters + ---------- + workflow : hhnb.core.workflow.Workflow + the workflow in which load the parameters template. + template_type : str + the parameters template type. Can be "pyramidal" or "interneuron". + + Raises + ------ + UnknownParametersTemplate + if the "template_type" is of an unknown type. + """ + if template_type not in ['pyramidal', 'interneuron']: + raise UnknownParametersTemplate + + shutil.copy( + os.path.join(HHF_PARAMETERS_TEMPLATE_DIR, template_type, 'parameters.json'), + os.path.join(workflow.get_model_dir(), 'config') + ) + + @staticmethod + def clean_model(workflow): + """ + Clean the model folder in the workflow from all unnecessary folders and files. + + Parameters + ---------- + workflow : hhnb.core.workflow.Workflow + the workflow of the model to clean. + """ + try: + shutil.rmtree(os.path.join(workflow.get_model_dir(), 'checkpoints')) + shutil.rmtree(os.path.join(workflow.get_model_dir(), 'tools')) + shutil.rmtree(os.path.join(workflow.get_model_dir(), 'figures')) + for dd in os.listdir(workflow.get_model_dir()): + if dd.startswith('r_seed'): + shutil.rmtree(os.path.join(workflow.get_model_dir(), dd)) + shutil.rmtree(os.path.join(workflow.get_model_dir(), 'mod_nsgportal')) + shutil.rmtree(os.path.join(workflow.get_model_dir(), 'x86_64')) + except FileNotFoundError: + pass + diff --git a/hhnb/urls.py b/hhnb/urls.py index 858bceff..bcd8a3e7 100644 --- a/hhnb/urls.py +++ b/hhnb/urls.py @@ -33,11 +33,8 @@ register_converter(converters.JobIdConverter, 'jobid') register_converter(converters.FolderConverter, 'folder') register_converter(converters.ConfigFileConverter, 'config_file') - - register_converter(converters.FileTypeConverter, 'file_type') - urlpatterns = [ # session refresh path('session-refresh/', views.session_refresh), @@ -102,4 +99,9 @@ path('hhf-get-model-key/', views.hhf_get_model_key), path('hhf-apply-model-key/', views.hhf_apply_model_key), path('hhf-save-config-file///', views.hhf_save_config_file), + + path('hhf-load-parameters-template/', views.hhf_load_parameters_template), + + # get service-account content + path('get-service-account-content', views.get_service_account_content), ] diff --git a/hhnb/utils/converters.py b/hhnb/utils/converters.py index ef4d264b..d2af438f 100755 --- a/hhnb/utils/converters.py +++ b/hhnb/utils/converters.py @@ -13,7 +13,7 @@ def to_url(self, value): class CtxConverter: """ - Unsed due to uuid built-in converter + Unused due to uuid built-in converter """ regex = '[0-9a-zA-Z]{8}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{12}' @@ -144,4 +144,13 @@ def to_python(self, value): def to_url(self, value): return value - \ No newline at end of file + + +class JobModeConverters: + regex = 'start|resume|' + + def to_python(self, value): + return value + + def to_url(self, value): + return value \ No newline at end of file diff --git a/hhnb/utils/messages.py b/hhnb/utils/messages.py index d62a7d52..d40b63a2 100644 --- a/hhnb/utils/messages.py +++ b/hhnb/utils/messages.py @@ -1,21 +1,23 @@ # General error messages -CRITICAL_ERROR = 'Critical Error !

Please contact the EBRAINS support if the problem persists.' -MODEL_CATALOG_CREDENTIALS_NOT_FOUND = 'Error !

Invalid Model Catalog credentials. Set them in your configuraztion file under "/hh_neuron_builder/conf".' +CRITICAL_ERROR = 'Critical Error !

Please contact the EBRAINS support if the problem persists.' +MODEL_CATALOG_CREDENTIALS_NOT_FOUND = 'Error !

Invalid Model Catalog credentials. Set them in your configuration file under "/hh_neuron_builder/conf".' GENERAL_ERROR = 'Error !' +UNABLE_TO_FETCH_FILES = 'Something went wrong !

Unable to fetch files you choose in the HippocampusHub from their source.
Please try again later and if the problem persists over the day please contact the EBRAINS support.' + # Model Catalog error messages -MODEL_CATALOG_INVALID_CREDENTIALS = 'Error !

Invalid Model Catalog credentials.

Please, contanct the EBRAINS support.' +MODEL_CATALOG_INVALID_CREDENTIALS = 'Error !

Invalid Model Catalog credentials.

Please, contact the EBRAINS support.' MODEL_CATALOG_NOT_AVAILABLE = 'The Model Catalog is temporarily not available.
Please, try again later.' -# Files error messsages +# Files error messages NO_FILE_UPLOADED = 'No file was uploaded.' NO_MORE_THEN = 'You can upload only {}.' -WRONG_UPLAODED_FILE = 'The uploaded file/s is wrong.' +WRONG_UPLOADED_FILE = 'The uploaded file/s is/are wrong.' ONLY_ACCEPTED_FILE = 'You can upload only a {} file.' INVALID_FILE = 'Invalid file !

Upload a correct {} archive.' INVALID_SIGNATURE = 'Invalid signature !

Uploaded {} archive is corrupted or was modified.' -MARLFORMED_FILE = 'Error !

Malformed {} file.' +MALFORMED_FILE = 'Error !

Malformed "{}" file.' NO_FILE_TO_DOWNLOAD = 'No file selected to download.' NO_FILE_TO_DELETE = 'No file to delete.' FILE_NOT_FOUND_ERROR = 'Error !

File not found.' @@ -30,27 +32,28 @@ # HPC error messages NO_HPC_SELECTED = 'No HPC was selected.' NO_JOB_SELECTED = 'No job was selected.' -JOB_FETCH_ERROR = 'Error !

Some error occurred while fetching jobs on {}.
Please contact the EBRAINS support if the problem persists.' -JOB_RESULTS_FETCH_ERRROR = 'Error !

Error while fetching job results.' +JOB_FETCH_ERROR = 'Error !

Some error occurred while fetching jobs on {}.
Please contact the EBRAINS support if the problem persists.' +JOB_RESULTS_FETCH_ERROR = 'Error !

Error while fetching job results.' JOB_SUBMITTED = 'Job submitted on {}.' JOB_EXPIRED = 'Job {} has expired and no file is present' -HPC_NOT_AVAILABLE = '{} system not available at this moment.
Please, try again later...

If the problem persists contact the EBRAINS support.' +HPC_NOT_AVAILABLE = 'HPC not available

{} system not available at this moment.
Please, try again later...

If the problem persists contact the EBRAINS support.' # Analysis error messages -ANALYSIS_ERROR = 'Error !

Something went wrong while the analysis process was running.
Please contact the EBRAINS support if the problem persists.' -MECHANISMS_PROCESS_ERROR = 'Error !

Something went wrong while mechanisms file were building.
Please, take a look on the files and if the problem persists contact the EBRAINS support.' -ANALYSIS_PROCESS_ERROR = 'Error !

Analysis process was stopped due to the following error:

{}

If the problem persists contact the EBRAINS support.' +ANALYSIS_ERROR = 'Analysis Error !

Something went wrong while the analysis process was running.
Please contact the EBRAINS support if the problem persists.' +MECHANISMS_PROCESS_ERROR = 'Analysis Error !

The mechanisms building process was stopped due to the following error

{}

If the problem persists contact the EBRAINS support.' +ANALYSIS_PROCESS_ERROR = 'Analysis Error !

the analysis process was stopped due to the following error:

{}

If the problem persists contact the EBRAINS support.' +ANALYSIS_FILE_NOT_FOUND_ERROR = 'Analysis Error !

The analysis process could\'t start due to the following missing file:

{}

Download the optimization, check the stderr file, fix the optimization error and run the optimization again to go ahead.

If the problem persists contact the EBRAINS support.' BLUE_NAAS_NOT_AVAILABLE = 'The BlueNaas is temporarily not available.
Please, try again later.' -MODEL_SUCCESSFULLY_REGISTERED = 'Congratulation, the model was successfully registered in the Model Catalog !

Model\'s info and metadata can be shown here.

Once the page is opened in a new tab, if a welcome message is displayed instead of the model instance, please click on the "Authorize" button if requested.

Leave the current tab open in case you need to recollect the model url.' +MODEL_SUCCESSFULLY_REGISTERED = 'Congratulation, the model was successfully registered in the Model Catalog !

Model\'s info and metadata can be shown here.

Once the page is opened in a new tab, if a welcome message is displayed instead of the model instance, please click on the "Authorize" button if requested.

Leave the current tab open in case you need to recollect the model url.' MODEL_ALREADY_EXISTS = 'Model already exists on the Model Catalog.' SIGNATURE_README_DESCRIPTION = \ """ The "signature" file provides the ".zip" file sign generated from the Hodgikin-Huxley NeuronBuilder. -This signature is unique and is generated using the ".zip" file readed bitwise. +This signature is unique and is generated using the ".zip" file read bitwise. If you want to upload a previously downloaded ".zip" file, you must provide the original "signature.txt" otherwise the file will be rejected. """ diff --git a/hhnb/utils/misc.py b/hhnb/utils/misc.py index 78de1e7c..cf402545 100644 --- a/hhnb/utils/misc.py +++ b/hhnb/utils/misc.py @@ -6,7 +6,7 @@ import os import shutil import time - +import json class InvalidArchiveError(Exception): @@ -31,14 +31,14 @@ def validate_archive(archive): """ Validate a zip file previously download from the Hodgkin-Huxley Neuron Builder. This function ensures that the file is not - corrupted or modified with a malevolus intenction by verifing + corrupted or modified with a malicious intention by verifying its sign. - The validation process is done by recalcuting the hash of the - original zip archive. If it matchs with the provided one within + The validation process is done by recalculating the hash of the + original zip archive. If it matches with the provided one within the archive, then the archive will be accepted and its path will be returned from the function, otherwise an - InvlidSign error will be generated. + InvalidSign error will be generated. Parameters ---------- @@ -66,7 +66,7 @@ def validate_archive(archive): if f.endswith('.zip'): archive_name = f - # check if the archive zip is malfomed + # check if the archive zip is malformed if not archive_name: raise InvalidArchiveError(f'{archive} not valid.') @@ -82,11 +82,9 @@ def validate_archive(archive): # read zip data and validate the sign with open(archive_path, 'rb') as fd: Sign.verify_data_sign(signature, fd.read()) - return archive_path - def get_signed_archive(arch_file): """ Returns a new archive that include the archive itself and its sign. @@ -131,3 +129,28 @@ def get_signed_archive(arch_file): shutil.rmtree(tmp_dir) return signed_archive + +def validate_json_file(file): + # check the uploaded file if it is a json file + full_path = os.path.abspath(file) + if file.endswith('.json'): + fd = open(full_path, 'r') + jj = json.load(fd) + fd.close() + + if full_path.endswith('parameters.json'): + # check for wf_id key as main key + if len(jj.keys()) == 1: + main_key = list(jj.keys())[0] + jj = jj[main_key] + # check for distribution and add an empty field if not exists + if not 'distributions' in jj.keys(): + jj.update({'distributions': {}}) + + os.remove(full_path) + fd = open(full_path, 'w') + json.dump({main_key: jj} if main_key else jj, fd) + fd.close() + + # return True if everything is ok, otherwise some exception will be raised + return True \ No newline at end of file diff --git a/hhnb/views.py b/hhnb/views.py index 6d1c54ad..ea7d1e8b 100644 --- a/hhnb/views.py +++ b/hhnb/views.py @@ -6,6 +6,7 @@ from hh_neuron_builder.settings import MODEL_CATALOG_COLLAB_DIR, MODEL_CATALOG_COLLAB_URL, MODEL_CATALOG_CREDENTIALS, MODEL_CATALOG_FILTER, TMP_DIR from hhnb.core.lib.exception.workflow_exception import AnalysisProcessError, MechanismsProcessError, WorkflowExists +from hhnb.core.lib.exception.model_exception import * from hhnb.core.response import ResponseUtil from hhnb.core.workflow import Workflow, WorkflowUtil from hhnb.core.user import * @@ -19,7 +20,6 @@ from ebrains_drive.exceptions import ClientHttpError as EbrainsDriveClientError import ebrains_drive -from subprocess import CalledProcessError import requests import datetime import os @@ -27,13 +27,14 @@ import shutil import logging -from hhnb.utils.misc import InvalidArchiveError, get_signed_archive, validate_archive +from hhnb.utils.misc import InvalidArchiveError, get_signed_archive, validate_archive, validate_json_file logger = logging.getLogger(__name__) - LOG_ACTION = 'User: "{}"\t Action: {}' + + def status(request): """ Returns the status of the service if is up. """ return ResponseUtil.ok_json_response({'hh-neuron-builder-status': 1}) @@ -82,7 +83,7 @@ def index_docs(request): def home_page(request): """ - By default the home page is redered, but if the "old_workflow_path" + By default the home page is rendered, but if the "old_workflow_path" is found in the session stored keys, the old workflow is restored and the workflow page is rendered. """ @@ -182,7 +183,7 @@ def upload_workflow(request): fd.write(wf) try: - valide_wf_zip = validate_archive(wf_zip) + valid_wf_zip = validate_archive(wf_zip) except InvalidArchiveError: return ResponseUtil.ko_json_response({'response': 'KO', 'message': messages.INVALID_FILE.format(filename)}) @@ -196,7 +197,7 @@ def upload_workflow(request): try: workflow = Workflow.generate_user_workflow_from_zip(hhnb_user.get_sub(), - valide_wf_zip) + valid_wf_zip) except WorkflowExists as e: logger.error(e) return ResponseUtil.ko_json_response({'response': 'KO', 'message': str(e)}) @@ -214,14 +215,14 @@ def clone_workflow(request, exc): return ResponseUtil.no_exc_code_response() old_workflow, hhnb_user = get_workflow_and_user(request, exc) - new_worfklow = WorkflowUtil.clone_workflow(old_workflow) + new_workflow = WorkflowUtil.clone_workflow(old_workflow) new_exc = generate_exc_code(request) logger.info(LOG_ACTION.format( - hhnb_user, 'cloning old workflow %s to %s' % (old_workflow, new_worfklow)) + hhnb_user, 'cloning old workflow %s to %s' % (old_workflow, new_workflow)) ) - request.session[new_exc]['workflow_id'] = new_worfklow.get_id() + request.session[new_exc]['workflow_id'] = new_workflow.get_id() request.session.save() return ResponseUtil.ok_json_response({'exc': new_exc}) @@ -255,7 +256,11 @@ def get_workflow_properties(request, exc): workflow, _ = get_workflow_and_user(request, exc) if request.session[exc].get('hhf_dict') and \ not request.session[exc]['hhf_already_downloaded']: - WorkflowUtil.download_from_hhf(workflow, request.session[exc]['hhf_dict']) + try: + WorkflowUtil.download_from_hhf(workflow, request.session[exc]['hhf_dict']) + except requests.exceptions.ConnectionError as e: + logger.error(e) + return ResponseUtil.ko_response(messages.UNABLE_TO_FETCH_FILES) request.session[exc]['hhf_already_downloaded'] = True request.session.save() @@ -375,7 +380,7 @@ def upload_features(request, exc): for uploaded_file in uploaded_files: if uploaded_file.name != 'features.json' and uploaded_file.name != 'protocols.json': - return ResponseUtil.ko_response(messages.WRONG_UPLAODED_FILE) + return ResponseUtil.ko_response(messages.WRONG_UPLOADED_FILE) if uploaded_file.name == 'features.json': workflow.write_features(uploaded_file) elif uploaded_file.name == 'protocols.json': @@ -423,7 +428,7 @@ def upload_model(request, exc): workflow.load_model_zip(zip_path) except FileNotFoundError as e: logger.error(e) - return ResponseUtil.ko_response(messages.MARLFORMED_FILE.format('"model.zip"')) + return ResponseUtil.ko_response(messages.MALFORMED_FILE.format('model.zip')) return ResponseUtil.ok_response() @@ -497,7 +502,7 @@ def upload_analysis(request, exc): print(e) logger.error(e) - return ResponseUtil.ko_response(messages.MARLFORMED_FILE.format(f'"{uploaded_file.name}"')) + return ResponseUtil.ko_response(messages.MALFORMED_FILE.format(f'"{uploaded_file.name}"')) def upload_files(request, exc): @@ -519,19 +524,7 @@ def upload_files(request, exc): folder = request.POST.get('folder') uploaded_file = request.FILES.get('file') - if folder == 'morphology/': - if not uploaded_file.endswith('.asc'): - return ResponseUtil.ko_response('Morphology must be ".asc" file.') - elif folder == 'mechanisms/': - if not uploaded_file.endswith('.mod'): - return ResponseUtil.ko_response('Mechanisms must be ".mod" file.') - elif folder == 'config/': - if not uploaded_file.name in ['features.json', 'protocols.json', 'parameters.json']: - return ResponseUtil.ko_response( - 'Config file must be one of the fallowing files:
\ - "protocols.json", "features.json", "parameters.json"' - ) - + # store the uploaded file full_path = os.path.join(workflow.get_model_dir(), folder, uploaded_file.name) with open(full_path, 'wb') as fd: if uploaded_file.multiple_chunks(chunk_size=4096): @@ -540,19 +533,46 @@ def upload_files(request, exc): else: fd.write(uploaded_file.read()) + try: + if folder == 'morphology/': + workflow.get_model().set_morphology(full_path) + with open(os.path.join(workflow.get_model_dir(), 'config', 'morph.json'), 'w') as fd: + json.dump(workflow.get_model().get_morphology().get_config(), fd) + + elif folder == 'mechanisms/': + if not uploaded_file.name.endswith('.mod'): + return ResponseUtil.ko_response('Mechanisms must be ".mod" file.') + + elif folder == 'config/': + if not uploaded_file.name in ['features.json', 'protocols.json', 'parameters.json']: + os.remove(full_path) + return ResponseUtil.ko_response( + 'Config file must be one of the fallowing files:
\ + "protocols.json", "features.json", "parameters.json"' + ) + validate_json_file(full_path) + + except InvalidMorphologyFile as e: + os.remove(full_path) + return ResponseUtil.ko_response('File Format Error

' + str(e)) + + except json.decoder.JSONDecodeError as e: + os.remove(full_path) + return ResponseUtil.ko_response('JSON Decode Error:

' + str(e) + '.') + return ResponseUtil.ok_response('') def download_files(request, exc): """ This download API can works in two different ways depending - on wich parameter is passed through the GET request. + on which parameter is passed through the GET request. If the key "pack" is found in the GET request, then an archive zip will be downloaded depends on what the relative value is. The accepted "pack" values are: "model": that allows to download the whole model in the workflow, - "results": that allows to dowload the all results of a downloaded job, + "results": that allows to download the all results of a downloaded job, "analysis": that allows to download the files that are required by the blue naas to run the simulation. @@ -628,7 +648,9 @@ def delete_files(request, exc): except PermissionError as e: logger.error(e) return ResponseUtil.ko_response(messages.CRITICAL_ERROR) - + except FileNotFoundError: + pass + return ResponseUtil.ok_json_response() @@ -647,31 +669,67 @@ def optimization_settings(request, exc=None): if not exc in request.session.keys(): return ResponseUtil.no_exc_code_response() workflow, hhnb_user = get_workflow_and_user(request, exc) + if request.method == 'GET': logger.info(LOG_ACTION.format(hhnb_user, 'get optimization settings from %s' % workflow)) + settings = {'settings': {}, 'service-account': {}, 'resume': {}} + + # fetch service account hpc and projects + settings.update(json.loads(get_service_account_content(request).content)) + try: - return ResponseUtil.ok_json_response(workflow.get_optimization_settings()) + settings['settings'].update(workflow.get_optimization_settings()) + settings['resume'].update(workflow.get_resume_settings()) + + return ResponseUtil.ok_json_response(settings) except FileNotFoundError as e: - logger.error(e) - return ResponseUtil.ok_json_response({}) + return ResponseUtil.ok_json_response(settings) + elif request.method == 'POST': logger.info(LOG_ACTION.format(hhnb_user, 'set optimization settings from %s' % workflow)) optimization_settings = json.loads(request.body) + + if optimization_settings['mode'] == 'resume' and \ + not optimization_settings.get('offspring'): + offspring = workflow.get_resume_settings().get('offspring_size', '10') + optimization_settings.update({'offspring': offspring}) + + if not optimization_settings.get('hpc'): + return ResponseUtil.ko_response('Invalid settings') + if optimization_settings['hpc'] == 'NSG': nsg_username = optimization_settings.get('username_submit') nsg_password = optimization_settings.get('password_submit') if nsg_username and nsg_password: nsg_user = NsgUser(nsg_username, nsg_password) - if not nsg_user.validate_credentials(): - return ResponseUtil.ko_response(messages.AUTHENTICATION_INVALID_CREDENTIALS) + + # override nsg credentials + try: + if not nsg_user.validate_credentials(): + optimization_settings.update({ + 'username_submit': False, + 'password_submit': False + }) + else: + optimization_settings.update({ + 'username_submit': True, + 'password_submit': True + }) + request.session['nsg_username'] = nsg_username + request.session['nsg_password'] = nsg_password + request.session.save() + except requests.exceptions.ConnectionError: + return ResponseUtil.ko_response(messages.HPC_NOT_AVAILABLE.format('NSG')) + + workflow.add_optimization_settings(optimization_settings) - request.session['nsg_username'] = nsg_username - request.session['nsg_password'] = nsg_password - request.session.save() + if optimization_settings['hpc'] == 'NSG': + if not optimization_settings['username_submit'] and not optimization_settings['password_submit']: + return ResponseUtil.ko_response(messages.AUTHENTICATION_INVALID_CREDENTIALS) - workflow.set_optimization_settings(optimization_settings) return ResponseUtil.ok_json_response() + else: return ResponseUtil.method_not_allowed() @@ -682,17 +740,12 @@ def run_optimization(request, exc): according to the stored optimization settings of the current workflow. """ + if exc not in request.session.keys(): return ResponseUtil.no_exc_code_response() workflow, hhnb_user = get_workflow_and_user(request, exc) logger.info(LOG_ACTION.format(hhnb_user, 'run optimization of %s' % workflow)) - opt_model_dir = WorkflowUtil.make_optimization_model(workflow) - zip_dst_dir = os.path.join(workflow.get_tmp_dir(), - os.path.split(opt_model_dir)[1]) - zip_file = shutil.make_archive(base_name=zip_dst_dir, - format='zip', - root_dir=os.path.split(opt_model_dir)[0]) optimization_settings = workflow.get_optimization_settings() @@ -700,13 +753,27 @@ def run_optimization(request, exc): 'nsg_username': request.session.get('nsg_username'), 'nsg_password': request.session.get('nsg_password'), }) - + + workflow.add_optimization_settings(optimization_settings) + + if optimization_settings.get('mode') == 'start': + WorkflowUtil.clean_model(workflow) + + opt_model_dir = WorkflowUtil.make_optimization_model(workflow) + zip_dst_dir = os.path.join(workflow.get_tmp_dir(), + os.path.split(opt_model_dir)[1]) + zip_file = shutil.make_archive(base_name=zip_dst_dir, + format='zip', + root_dir=os.path.split(opt_model_dir)[0]) + try: response = JobHandler.submit_job(hhnb_user, zip_file, optimization_settings) except JobHandler.HPCException as e: return ResponseUtil.ko_response(str(e)) + except JobHandler.ServiceAccountException as e: + return ResponseUtil.ko_response(messages.HPC_NOT_AVAILABLE.format('SERVICE ACCOUNT')) if response.status_code == 200: - workflow.set_optimization_settings(optimization_settings, job_submitted_flag=True) + workflow.add_optimization_settings({'job_submitted': True}) return response @@ -719,6 +786,9 @@ def fetch_jobs(request, exc): return ResponseUtil.no_exc_code_response() hpc = request.GET.get('hpc') + sa_hpc = request.GET.get('saHPC') + sa_project = request.GET.get('saProject') + if not hpc: return ResponseUtil.ko_response(messages.NO_HPC_SELECTED) @@ -726,7 +796,7 @@ def fetch_jobs(request, exc): logger.info(LOG_ACTION.format(hhnb_user, 'fetch all %s jobs' % hpc)) try: - data = JobHandler.fetch_jobs_list(hpc, hhnb_user) + data = JobHandler.fetch_jobs_list(hpc, hhnb_user, sa_hpc, sa_project) return ResponseUtil.ok_json_response(data) except JobHandler.ServiceAccountException as e: logger.error(e) @@ -752,8 +822,10 @@ def fetch_job_results(request, exc): if exc not in request.session.keys(): return ResponseUtil.no_exc_code_response() - hpc = request.GET.get('hpc') job_id = request.GET.get('job_id') + hpc = request.GET.get('hpc') + sa_hpc = request.GET.get('saHPC') + sa_project = request.GET.get('saProject') if not hpc: return ResponseUtil.ko_response(messages.NO_HPC_SELECTED) @@ -766,7 +838,7 @@ def fetch_job_results(request, exc): )) try: - file_list = JobHandler.fetch_job_files(hpc, job_id, hhnb_user) + file_list = JobHandler.fetch_job_files(hpc, job_id, hhnb_user, sa_hpc, sa_project) WorkflowUtil.download_job_result_files(workflow, file_list) return ResponseUtil.ok_response() @@ -778,7 +850,7 @@ def fetch_job_results(request, exc): except Exception as e: logger.error(e) - return ResponseUtil.ko_response(messages.JOB_RESULTS_FETCH_ERRROR) + return ResponseUtil.ko_response(messages.JOB_RESULTS_FETCH_ERROR) def run_analysis(request, exc): @@ -803,17 +875,17 @@ def run_analysis(request, exc): except MechanismsProcessError as e: logger.error(e) workflow.clean_analysis_dir() - return ResponseUtil.ko_response(498, messages.MECHANISMS_PROCESS_ERROR) + return ResponseUtil.ko_response(498, messages.MECHANISMS_PROCESS_ERROR.format(e.stderr)) except AnalysisProcessError as e: logger.error(e) workflow.clean_analysis_dir() - return ResponseUtil.ko_response(499, messages.ANALYSIS_PROCESS_ERROR.format(e)) + return ResponseUtil.ko_response(499, messages.ANALYSIS_PROCESS_ERROR.format(e.stderr)) except FileNotFoundError as e: logger.error(e) workflow.clean_analysis_dir() - return ResponseUtil.ko_response(404, str(e)) + return ResponseUtil.ko_response(404, messages.ANALYSIS_FILE_NOT_FOUND_ERROR.format(e)) except Exception as e: logger.error(e) @@ -872,7 +944,7 @@ def get_model_catalog_attribute_options(request): options = mc.get_attribute_options() return ResponseUtil.ok_json_response(options) except Exception as e: - logger.error('get_model_catalog_attriute_options(): %s' % e) + logger.error('get_model_catalog_attribute_options(): %s' % e) return ResponseUtil.ko_response(messages.GENERAL_ERROR) @@ -960,10 +1032,8 @@ def register_model(request, exc): return ResponseUtil.ok_response(messages.MODEL_SUCCESSFULLY_REGISTERED.format(model_path_on_mc)) except EbrainsDriveClientError as e: - code = e.code - message = e.message logger.error(e) - return ResponseUtil.ko_response(code, message) + return ResponseUtil.ko_response(e.code, e.message) except FileExistsError as e: logger.error(e) @@ -1117,21 +1187,27 @@ def hhf_get_files_content(request, folder, exc): if not exc in request.session.keys(): return ResponseUtil.no_exc_code_response() - workflow, _ = get_workflow_and_user(request, exc) + workflow, user = get_workflow_and_user(request, exc) if not os.path.join(workflow.get_model_dir(), folder): return ResponseUtil.ko_response() folder = folder.split('Folder')[0] hhf_files_content = {} - for f in os.listdir(os.path.join(workflow.get_model_dir(), folder)): - with open(os.path.join(workflow.get_model_dir(), folder, f), 'r') as fd: + file_path = os.path.join(workflow.get_model_dir(), folder) + for f in os.listdir(file_path): + with open(os.path.join(file_path, f), 'r') as fd: if f.endswith('.json'): - jj = json.load(fd) - hhf_files_content[f] = json.dumps(jj, indent=8) - else: + try: + jj = json.load(fd) + hhf_files_content[f] = json.dumps(jj, indent=8) + except json.decoder.JSONDecodeError as e: + logger.warning(LOG_ACTION.format(user, 'JSON Decode error on file: "%s".' % os.path.join(file_path, f))) + fd.seek(0) hhf_files_content[f] = fd.read() - + else: + hhf_files_content[f] = fd.read() + return ResponseUtil.ok_json_response(hhf_files_content) @@ -1173,13 +1249,82 @@ def hhf_save_config_file(request, folder, config_file, exc): config_file) with open(file_path, 'w') as fd: json.dump(file_content, fd, indent=4) + + # automatically add empty distributions field in parameters.json + if file_path.endswith('parameters.json'): + fd = open(file_path, 'r') + jj = json.load(fd) + fd.close() + + # check if there is only the wf_id key + if len(jj.keys()) == 1: + main_key = list(jj.keys())[0] + jj = jj[main_key] + + # check for distributions key and add an empty field if there isn't + if not 'distributions' in jj.keys(): + jj.update({'distributions': {}}) + + fd = open(file_path, 'w') + json.dump({main_key: jj} if main_key else jj, fd) + fd.close() + return ResponseUtil.ok_response('') except json.JSONDecodeError: - print('Malformed json') - return ResponseUtil.ko_response(messages.MARLFORMED_FILE.format(config_file + '.json')) + return ResponseUtil.ko_response(messages.MALFORMED_FILE.format(config_file)) except FileNotFoundError: - print('File not found') return ResponseUtil.ko_response(404, messages.CRITICAL_ERROR) except Exception as e: print(str(e)) return ResponseUtil.ko_response(400, messages.GENERAL_ERROR) + + +def hhf_load_parameters_template(request, exc): + """ + Load parameters template. + """ + if request.method != 'POST': + return ResponseUtil.method_not_allowed('POST') + + if not exc in request.session.keys(): + return ResponseUtil.no_exc_code_response() + + workflow, _ = get_workflow_and_user(request, exc) + + parameters_type = request.POST.get('type') + if not parameters_type in ['pyramidal', 'interneuron']: + return ResponseUtil.ko_response(f"Parameters type {parameters_type} unknown") + + WorkflowUtil.load_parameters_template(workflow, parameters_type) + return ResponseUtil.ok_response() + + +def get_service_account_content(request): + """ + Get Service Account HPC and relative projects. + """ + if request.method != 'GET': + return ResponseUtil.method_not_allowed('GET') + + SERVICE_ACCOUNT_ROOT_URL = 'https://bspsa.cineca.it/' + APP_CONTEXT_TAG = 'hhnb' + + sa_content = {'service-account': {}} + + try: + r0 = requests.get(SERVICE_ACCOUNT_ROOT_URL + 'hpc/', timeout=30) + r1 = requests.get(SERVICE_ACCOUNT_ROOT_URL + 'projects/', timeout=30) + if r0.status_code == 200 and r1.status_code == 200: + for hpc in r0.json(): + h = hpc['id'] + projects = [] + for p in r1.json(): + if p['hpc'] == h and APP_CONTEXT_TAG in p['name']: + projects.append(p['name']) + sa_content['service-account'].update({h: projects}) + + except requests.RequestException as e: + logger.error('Service-Account connection error: "%s".' % str(e)) + sa_content = {'service-account': False} + + return ResponseUtil.ok_json_response(sa_content, True) diff --git a/requirements.txt b/requirements.txt index 3174ab45..39f09925 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,108 +1,81 @@ -appdirs==1.4.4 asgiref==3.5.2 -asttokens==2.0.8 +asttokens==2.2.1 backcall==0.2.0 -backports.zoneinfo==0.2.1 -bluepyefe==0.3.131 -bluepyopt==1.12.63 -CacheControl==0.12.11 -certifi==2022.6.15 +bluepyefe==0.4.10 +bluepyopt==1.13.86 +certifi==2022.12.7 cffi==1.15.1 -chardet==5.0.0 charset-normalizer==2.1.1 -colorama==0.4.5 -contextlib2==21.6.0 -cryptography==37.0.4 +comm==0.1.2 +contourpy==1.0.6 +cryptography==38.0.4 cycler==0.11.0 deap==1.3.3 -debugpy==1.6.3 +debugpy==1.6.4 decorator==5.1.1 -Deprecated==1.2.13 -distlib==0.3.5 -distro==1.7.0 -Django==4.1 +Django==4.1.4 django-sslserver==0.22 -djangorestframework==3.13.1 ebrains-drive==0.5.0 -efel==4.0.42 +efel==4.1.49 entrypoints==0.4 -executing==0.10.0 -fonttools==4.37.1 +executing==1.2.0 +fonttools==4.38.0 future==0.18.2 -greenlet==1.1.2 -hbp-seafile==0.1.1 -hbp-service-client==1.1.1 -hbp-validation-framework==0.6.4 -html5lib==1.1 -idna==3.3 +h5py==3.7.0 +hbp-validation-framework==0.7.1 +idna==3.4 igor==0.3 -importlib-metadata==4.12.0 -ipaddr==2.2.0 -ipykernel==6.15.1 +ipykernel==6.19.3 ipyparallel==8.4.1 -ipython==8.4.0 -ipython-genutils==0.2.0 -jedi==0.18.1 +ipython==8.7.0 +jedi==0.18.2 Jinja2==3.1.2 josepy==1.13.0 -jupyter-client==7.3.4 -jupyter-core==4.11.1 +jupyter_client==7.4.8 +jupyter_core==5.1.0 kiwisolver==1.4.4 -lockfile==0.12.2 MarkupSafe==2.1.1 -matplotlib==3.5.3 +matplotlib==3.6.2 matplotlib-inline==0.1.6 -mozilla-django-oidc==2.0.0 -msgpack==1.0.4 +mozilla-django-oidc==3.0.0 multipledispatch==0.6.0 -nameparser==1.1.1 -neo==0.10.2 -nest-asyncio==1.5.5 -NEURON==8.2.1 -numpy==1.23.2 -packaging==21.3 -pandas==1.4.3 +nameparser==1.1.2 +neo==0.11.1 +nest-asyncio==1.5.6 +NEURON==8.2.2 +numpy==1.24.0 +packaging==22.0 +pandas==1.5.2 parso==0.8.3 -Pebble==4.6.3 -pep517==0.13.0 +Pebble==5.0.3 pexpect==4.8.0 pickleshare==0.7.5 -Pillow==9.2.0 -progress==1.6 -prompt-toolkit==3.0.30 -psutil==5.9.1 -psycopg2-binary==2.9.3 +Pillow==9.3.0 +platformdirs==2.6.0 +prompt-toolkit==3.0.36 +psutil==5.9.4 +psycopg2-binary==2.9.5 ptyprocess==0.7.0 pure-eval==0.2.2 pycparser==2.21 Pygments==2.13.0 -PyJWT==2.4.0 -pyOpenSSL==22.0.0 +PyJWT==2.6.0 +pyOpenSSL==22.1.0 pyparsing==3.0.9 python-dateutil==2.8.2 -pytoml==0.1.21 -pytz==2022.2.1 -pyunicore==0.9.19 -pyzmq==23.2.1 +pytz==2022.7 +pyunicore==0.12.0 +pyzmq==24.0.1 quantities==0.13.0 requests==2.28.1 -retrying==1.3.3 -scipy==1.9.0 -scoop==0.7.2.0 +scipy==1.9.3 sh==1.14.3 -simplejson==3.17.6 +simplejson==3.18.0 six==1.16.0 -sqlparse==0.4.2 -stack-data==0.4.0 -toml==0.10.2 -tomli==2.0.1 +sqlparse==0.4.3 +stack-data==0.6.2 tornado==6.2 -tqdm==4.64.0 -traitlets==5.3.0 -typing_extensions==4.3.0 -urllib3==1.26.12 -validators==0.20.0 -wcwidth==0.2.5 -webencodings==0.5.1 -wrapt==1.14.1 -zipp==3.8.1 +tqdm==4.64.1 +traitlets==5.8.0 +urllib3==1.26.13 +wcwidth==0.2.5 \ No newline at end of file diff --git a/static/assets/animation/loading_animation/frame_0.png b/static/assets/animation/loading_animation/frame_0.png deleted file mode 100755 index c975f5ca..00000000 Binary files a/static/assets/animation/loading_animation/frame_0.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_1.png b/static/assets/animation/loading_animation/frame_1.png deleted file mode 100755 index 43a1fe35..00000000 Binary files a/static/assets/animation/loading_animation/frame_1.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_2.png b/static/assets/animation/loading_animation/frame_2.png deleted file mode 100755 index 51810986..00000000 Binary files a/static/assets/animation/loading_animation/frame_2.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_3.png b/static/assets/animation/loading_animation/frame_3.png deleted file mode 100755 index cbadc92a..00000000 Binary files a/static/assets/animation/loading_animation/frame_3.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_4.png b/static/assets/animation/loading_animation/frame_4.png deleted file mode 100755 index 63e2790b..00000000 Binary files a/static/assets/animation/loading_animation/frame_4.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_5.png b/static/assets/animation/loading_animation/frame_5.png deleted file mode 100755 index 335a6a59..00000000 Binary files a/static/assets/animation/loading_animation/frame_5.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_6.png b/static/assets/animation/loading_animation/frame_6.png deleted file mode 100755 index f3d2f379..00000000 Binary files a/static/assets/animation/loading_animation/frame_6.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_7.png b/static/assets/animation/loading_animation/frame_7.png deleted file mode 100755 index 0189b1d5..00000000 Binary files a/static/assets/animation/loading_animation/frame_7.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_8.png b/static/assets/animation/loading_animation/frame_8.png deleted file mode 100755 index eae6706b..00000000 Binary files a/static/assets/animation/loading_animation/frame_8.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/frame_9.png b/static/assets/animation/loading_animation/frame_9.png deleted file mode 100755 index feff33ab..00000000 Binary files a/static/assets/animation/loading_animation/frame_9.png and /dev/null differ diff --git a/static/assets/animation/loading_animation/loading_animation.css b/static/assets/animation/loading_animation/loading_animation.css index 0fd0b8ee..73c2da33 100755 --- a/static/assets/animation/loading_animation/loading_animation.css +++ b/static/assets/animation/loading_animation/loading_animation.css @@ -1,5 +1,4 @@ .loading-animation.sublayer { - /* display: none; */ position: absolute; top: 0; left: 0; @@ -23,19 +22,16 @@ bottom: 0; box-shadow: inset 0 0 200px rgba(255, 255, 255, 0.822); filter: blur(100px); - /* margin: -20px; */ padding: 50px; } .loading-animation.container-glass { - /* display: none; */ position: relative; margin: auto; top: 50%; transform: translateY(-50%); width: max-content; height: max-content; - /* background-color: rgba(255, 255, 255, 0.144); */ border: solid 1px black; padding: 50px; text-align: center; @@ -47,51 +43,192 @@ .loading-animation.text { text-align: center; - /* font-size: 1.5rem; */ + font-size: 1.2em; font-weight: bold; - /* color: rgb(11, 0, 109); */ margin: auto; width: 345px; margin-bottom: 30px; - /* margin-top: 25px; */ +} + +.loading-animation { + display: none; +} + +.loading-animation.show { + display: block; +} + +svg > path, svg > circle, svg > ellipse, svg > rect { + transition: all ease 1s; +} + +.start-animation { + fill: #ffd4b7; + animation: loading ease-in-out 0.3s forwards; +} +#termination.start-animation { + fill: #ff5a5a; + animation-name: termination-loading; +} + +#head { + animation-delay: 0.3s; +} + +#axon-0 { + animation-delay: 0.6s; +} + +#axon-1 { + animation-delay: 0.9s; +} + +#axon-2 { + animation-delay: 1.2s; +} + +#axon-3 { + animation-delay: 1.5s; +} + +#axon-4 { + animation-delay: 1.8s; +} + +#axon-5 { + animation-delay: 2.1s; +} + +#axon-6 { + animation-delay: 2.4s; +} + +#termination { + animation-delay: 2.7s; +} + +.reverse-animation { + fill: #ffff00; + animation-delay: 0s !important; + animation: reverse-loading ease-in-out 0.3s forwards; +} +#termination.reverse-animation { + fill: #ffff00; + animation-name: reverse-termination-loading; +} + +@keyframes loading { + from { fill: #ffd4b7;} + to {fill: #ffff00;} +} +@keyframes reverse-loading { + from {fill: #ffff00;} + to { fill: #ffd4b7;} +} + + +@keyframes termination-loading { + from {fill: #ff5a5a;} + to {fill: #ffff00;} +} +@keyframes reverse-termination-loading { + from {fill: #ffff00;} + to {fill: #ff5a5a;} } -.loading-animation.frames { - /* display: block; */ - /* visibility: hidden; */ + + + + + + + + + + + + + + + + +/* +/* .neuron-loader { + height: 4rem; + display: inline-block; + width: 100%; +} *//* + +.line { + /* position: relative; *//* + width: 80%; + margin: auto; + top: 1.75rem; + border: solid .25rem black; + border-radius: 1rem; + z-index: 1; +} + + +.neuron-loader.container > .row { + align-self: center; + border: 1px blue solid; +} +.neuron-loader.container > .row > .col { + border: 1px red solid; + align-self: center; + padding: 0 .2rem; + height: 2rem; +} + +.neuron { + /* position: relative; *//* + z-index: 2; + border: .25rem black solid; + background-color: rgb(255, 90, 90); +} + +.neuron > .core { + background-color: rgb(255, 212, 183); + border: .2rem black solid; position: relative; + top: 0.25rem; margin: auto; - /* margin-top: 10px; */ - height: 84px; - width: 345px; - z-index: 100; - animation: loading-animation; - animation-duration: 3s; - animation-iteration-count: infinite; } -.loading-animation.show { - /* visibility: visible; */ - display: block; - animation-play-state: initial; +.root { + height: 3rem; + width: 3.2rem; + border-radius: 1.5rem; +} +.root > .core { + width: 2rem; + height: 2rem; + border-radius: 1rem; + left: 0.25rem } -.loading-animation.hide { - /* visibility: hidden; */ - display: none; - animation-play-state: unset; -} - -@keyframes loading-animation { - 0% {background-image: url('frame_0.png');} - 10% {background-image: url('frame_1.png');} - 20% {background-image: url('frame_2.png');} - 30% {background-image: url('frame_3.png');} - 40% {background-image: url('frame_4.png');} - 50% {background-image: url('frame_5.png');} - 60% {background-image: url('frame_6.png');} - 70% {background-image: url('frame_7.png');} - 80% {background-image: url('frame_8.png');} - 90% {background-image: url('frame_9.png');} - 100% {background-image: url('frame_0.png');} +.line { + /* position: relative; *//* + height: 0.25rem; + width: 120%; + left: -10%; + top: 1rem; + /* margin: auto; *//* } + +.axon { + height: 1.5rem; + background-color: rgb(255, 212, 183); + border-radius: 1rem; + margin: auto; + /* top: 0.5rem; *//* +} + +.termination { + height: 2.5rem; + width: 2.5rem; + border-radius: 2rem; + top: 0.175rem; + margin: auto; +} */ \ No newline at end of file diff --git a/static/assets/animation/loading_animation/loading_animation.js b/static/assets/animation/loading_animation/loading_animation.js index d3166086..08f202eb 100755 --- a/static/assets/animation/loading_animation/loading_animation.js +++ b/static/assets/animation/loading_animation/loading_animation.js @@ -1,15 +1,40 @@ +class Log { + + static enable = false; + + static debug(message) { + if (this.enable) { + console.log(message); + } + } +} + +if (window.location.href.startsWith("https://127.0.0.1")) { + Log.enable = true; +} + function showLoadingAnimation(message="") { - console.log("show loading animation"); + Log.debug("showLoadingAnimation(\"" + message + "\") called"); + setLoadingAnimationText(message); - $(".loading-animation").removeClass("hide"); $(".loading-animation").addClass("show"); + $(".neuron").addClass("start-animation"); + + $("#termination")[0].addEventListener("animationend", async event => { + if (event.animationName == "termination-loading") { + $(".neuron").removeClass("start-animation").addClass("reverse-animation"); + } else if (event.animationName == "reverse-termination-loading") { + $(".neuron").removeClass("reverse-animation").addClass("start-animation"); + } + }) } function hideLoadingAnimation() { + Log.debug("hideLoadingAnimation() called"); $(".loading-animation").removeClass("show"); - $(".loading-animation").addClass("hide"); + $(".neuron").removeClass("start-animation reverse-animation"); } function setLoadingAnimationText(message) { $("#loading-text-div").html(message); -} \ No newline at end of file +} diff --git a/static/assets/animation/loading_animation/neuron_animation_compressed.svg b/static/assets/animation/loading_animation/neuron_animation_compressed.svg new file mode 100644 index 00000000..78d6dcdd --- /dev/null +++ b/static/assets/animation/loading_animation/neuron_animation_compressed.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/static/assets/img/add-file-white.svg b/static/assets/img/add-file-white.svg new file mode 100644 index 00000000..3e87b71c --- /dev/null +++ b/static/assets/img/add-file-white.svg @@ -0,0 +1,38 @@ + + + + + + + diff --git a/static/assets/img/interneuron.jpeg b/static/assets/img/interneuron.jpeg new file mode 100644 index 00000000..6e0f0fde Binary files /dev/null and b/static/assets/img/interneuron.jpeg differ diff --git a/static/assets/img/pyramidal.jpeg b/static/assets/img/pyramidal.jpeg new file mode 100644 index 00000000..787ea990 Binary files /dev/null and b/static/assets/img/pyramidal.jpeg differ diff --git a/static/assets/themes/hhnb.css b/static/assets/themes/hhnb.css index b26b8874..c450b43c 100644 --- a/static/assets/themes/hhnb.css +++ b/static/assets/themes/hhnb.css @@ -24,15 +24,6 @@ html { border: 1px solid black; } -.accordion-button:not(collapsed):focus { - border-color: #daa520ff; - box-shadow: none; -} -.accordion-button:focus { - border-color: unset; -} - - ::-webkit-scrollbar { width: 6px; } @@ -389,9 +380,9 @@ body { cursor: help; } -.workflow-content > .row > .col { - /* padding: 0 4px; */ -} +/* .workflow-content > .row > .col { + padding: 0 4px; +} */ .secondary-content { border: solid 1px #cfd8dc; @@ -572,25 +563,27 @@ body { } .accordion-button.collapsed { background-color: rgba(255, 255, 255, .5); - color: 37474f; + color: 37474f; } -.accordion-button.disable { - background-color: rgb(210, 210, 210); - color: rgb(50, 50, 50); - cursor: unset; +.accordion-button.active { + background-color: #37474f; + color: white; box-shadow: unset; - border-color: unset; } .accordion-item { background-color: rgba(255, 255, 255, .3); border-color: #37474f; } -.accordion-button.collapsed::after { - background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' style='fill: rgb(121, 56, 2)' %3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e"); -} -.accordion-button:not(.collapsed)::after { +.accordion-button:not(.collapsed)::after, accordion-button.active::after { background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='white'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e"); } +.accordion-button.disabled, .accordion-button.disable { + background-color: lightgray; + color: gray; + cursor: unset; + box-shadow: unset; + border-color: unset; +} /* choose opt model page content */ .opt-model-container { @@ -650,6 +643,9 @@ body { .progress-bar-animated { animation-duration: .5s; } +.progress-bar { + background-color: #78909c; +} #progressBarFetchJob.s40 { transition: width 40s linear; } @@ -729,7 +725,13 @@ body { z-index: 100; overflow-y: auto; padding: 50px; - transition: all ease 0.3s; + -webkit-transition: all ease .5s; + -moz-transition: all ease .5s;; + transition: all ease .5s; + opacity: 0; +} +.overlay-wrapper.show { + opacity: 1; } .overlay-content:before { @@ -749,9 +751,8 @@ body { display: none; position: relative; margin: auto; - top: calc(50%); + top: 30%; width: max-content; - /* min-width: 800px; */ height: max-content; padding: 2rem; text-align: center; @@ -759,36 +760,29 @@ body { background-color: rgb(255, 255, 255, 0.8); box-shadow: 0 0 1rem 0 rgb(0, 0, 0, .8); z-index: 100; - transition: all ease-out 0.3s; + transition: all ease .5s; transform: translateY(-50%); + + /* adding posterior */ + opacity: 0; } +.overlay-content.show { + top: 50%; + opacity: 1; +} + .overlay-content.overflow { overflow-y: auto; } -/* -.reg-model-content { - /* transform: translateY( -20% ); */ - /* width: 850px; */ -} - */ + .error-content { - /* max-width: 800px; */ font-size: 1rem; font-weight: 600; width: 100%; padding: 1rem; } -/* .info-content > button { - border-color: blue; - color: bl; -} -.info-content > button:hover { - background-color: blue; - color: white; -} */ - #overlayupload { width: 520px; min-width: unset; @@ -839,7 +833,6 @@ body { min-width: 170px; } - .form-control.username { margin-right: 12px; border-top-right-radius: .25rem; @@ -887,7 +880,6 @@ body { .list-group-item:hover { background-color: #cfd8dc; - color: white; } .list-group-item.disabled { background-color: rgba(220, 220, 220, .5); @@ -896,6 +888,14 @@ body { .list-group-item.fetch-jobs { width: 100%; } +.list-group-item.fetch-jobs:first-child { + border-top-left-radius: 0.5rem; + border-bottom-left-radius: 0.5rem; +} +.list-group-item.fetch-jobs:last-child { + border-top-right-radius: 0.5rem; + border-bottom-right-radius: 0.5rem; +} .list-group-item.disable { background-color: rgb(210, 210, 210); color: rgb(50, 50, 50); @@ -909,12 +909,6 @@ body { min-width: 500px; } -.progress-bar { - background-color: #78909c; - animation-duration: 5s; - transition-duration: 5s; -} - .job-list-table { overflow: auto; background-color: rgba(255, 255, 255, .5); @@ -923,7 +917,7 @@ body { padding: 8px 8px; } -#nsgLoginRow, #spinnerRow, #progressRow, #tableRow { +#nsgLoginRow, #saChoiseRow, #spinnerRow, #progressRow, #tableRow { display: none; } @@ -937,20 +931,36 @@ td { vertical-align: middle; } -.job-download-button { - width: auto; - margin: auto; +.job-button { + width: 48px; + height: 48px; + margin: auto .5rem; + padding: 0px; +} + +.job-button.left { + border-bottom-right-radius: 0px; + border-top-right-radius: 0px; + margin-right: 0px; + border-right-width: 0px; +} +.job-button.right { + border-bottom-left-radius: 0px; + border-top-left-radius: 0px; + margin-left: -2px; + border-left-width: 1px; } .alert { - top: 10vh; - z-index: -999; - position: absolute; - width: 80%; - left: 10%; + z-index: 1100; + transition: all ease 0.5s; + position: fixed; + top: 0vh; + padding: 1.5rem 1rem; + width: fit-content; } .alert.show { - z-index: 999; + top: 10vh; } @@ -1016,7 +1026,6 @@ td { border-radius: inherit; } - .file-manager-container { /* min-width: 1084px; */ line-height: 1.5; @@ -1088,6 +1097,8 @@ td { .file-group { background-color: none; } + +.file-group > .list-group-item.active, .file-group > .list-group-item:hover { background-color: #cfd8dc; color: #212529; @@ -1096,9 +1107,9 @@ td { .file-selector > .list-group > .list-group-item:hover { background-color:#cfd8dc; } -.file-group > .list-group-item.active, + .file-group > .list-group-item.active:hover { - background-color: #cfd8dc; + background-color: #adb4b7; color: #212529; } @@ -1107,10 +1118,6 @@ td { } .overlay-content.file-manager-content { - /* vertically centered */ - top: 50%; - transform: translateY(-50%); - /***********************/ padding: 0rem; width: 100%; height: 100%; @@ -1159,12 +1166,7 @@ td { display: flex; justify-content: center; cursor: default; -}/* -.file-item.empty:hover, .file-item.empty.active { - background-color: unset; - border: unset; - color: rgb(180, 180, 180); -}*/ +} .spinner-border.file-item-spinner { display: none; @@ -1178,7 +1180,7 @@ td { } .model-key-flex-col { - display: flex; + display: none; } .model-key-group { align-self: end; @@ -1247,6 +1249,7 @@ td { bottom: 5%; z-index: 1; } + #saveFileButtonSpinner { display: none; width: 1.5rem; @@ -1616,7 +1619,6 @@ i.disabled:hover { } - .modal-content.sublayer { display: flex; position: absolute; @@ -1722,7 +1724,7 @@ i.disabled:hover { } .floating-btn.mc.show { - z-index: 100000; + z-index: 1056; opacity: .7; top: 2rem; } @@ -1746,6 +1748,10 @@ i.disabled:hover { width: 70%; } +.modal-btn:disabled { + background-color: white; +} + #errordynamictext { max-width: 750px; } @@ -1759,7 +1765,6 @@ i.disabled:hover { #dialogtext { min-width: 300px; - /* font-weight: bold; */ padding: 1rem; } @@ -1775,9 +1780,13 @@ i.disabled:hover { } #overlayjobs { - transition: unset; - max-width: 1300px; - width: unset; + min-width: 1000px; +} +#overlayjobprocessing { + top: 70%; +} +#overlayjobprocessing.show { + top: 50%; } #overlaycontentdialog { @@ -1823,4 +1832,208 @@ i.disabled:hover { } .dimmed-background-fade-out { opacity: 0; -} \ No newline at end of file +} + +.sa-dropdown, .sa-dropdown-menu { + width: 100%; +} + +.gone { + display: none; +} + +.card.resume-job { + background-color: rgba(255, 255, 255, .5); + border-radius: .5rem; +} + +.shadow-layer { + display: none; + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + z-index: 100; + background-color: rgb(0, 0, 0, .7); + transition: all ease-out 0.3s; + opacity: 0; +} + +.shadow-layer.show { + opacity: 1; +} + +#tableRow { + overflow: auto; + border-radius: .5rem; + margin-bottom: 1rem; +} + +.input-group > span { + justify-content: center; +} + +.input-group.job-name{ + height: 3rem; + margin-bottom: 1rem; +} +.input-group.job-name > span { + font-weight: bold; + background-color: #37474f; + color: white; + border-top-left-radius: .5rem; + border-bottom-left-radius: .5rem; + border: solid 1px #37474f; +} +.input-group.job-name > input { + font-weight: bold; + border-top-right-radius: .5rem; + border-bottom-right-radius: .5rem; + border-color: #37474f; +} + +.input-group.job-name > button { + background-color: #37474f; + color: white; +} + +.input-group.job-name > button.dropdown-toggle { + border-top-right-radius: .5rem; + border-bottom-right-radius: .5rem; +} + +.input-group.job-name > ul.dropdown-menu { + border: solid 1px #37474f; +} + +#job-action { + width: 100px; +} +#job-action:active { + box-shadow: unset; +} + +#overlayparameterstemplate { + z-index: 101; + position: relative; +} + +#parametersTemplateChooser > div.row > div.col { + padding: 1rem; +} +#parametersTemplateChooser > div.row > div.col > div.card { + margin: auto; + border-radius: 20px; + padding: 10px; + transition: all ease .5s; +} +#parametersTemplateChooser > div.row > div.col > div.card:hover { + cursor: pointer; + box-shadow: 0px 10px 20px -10px; + transform: scale(1.05, 1.05); +} +#parametersTemplateChooser > div.row > div.col > div.card > img { + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -o-user-select: none; + user-select: none; +} + +#overlaycontentdialog.overlay-content { + top: 50%; + transform: scale(0.5, 0.5) translateY(-50%); + transition-duration: .3s; +} + +#overlaycontentdialog.overlay-content.show { + transform: scale(1, 1) translateY(-50%); +} + + +#alert-button{ + margin: auto; + margin-right: 1rem; + margin-top: .5rem; +} + +#alert-text{ + margin: auto; + margin-left: 2rem; + margin-right: 2rem; + text-align: center; + width: 100%; + padding: .5rem; +} +#alert-text > p { + margin: auto; +} + +#alert-svg { + margin: auto; + margin-left: 1rem; + margin-top: .5rem; + /* margin: .5rem; */ +} + +#alert-svg.center { + margin-top: auto; +} +#alert-button.center { + margin-top: auto; +} + +#alert { + box-shadow: 0px 20px 10px -10px rgba(0, 0, 0, .6); +} + +#alert-dialog { + display: block; + position: relative; + left: unset; + margin: auto; + padding: 2rem 4rem; + width: fit-content; + max-width: 800px; + max-height: 800px; + overflow-y: auto; + border-radius: 1rem; + top: 50vh; + transform: translateY(-50%); + /* background-color: rgb(255, 255, 255, .8); */ + box-shadow: 0 0 1rem 0 rgb(255, 255, 255, .3); +} + +.alert-dialog-button { + width: 50%; + margin: auto; + margin-top: .5rem; + background-color: inherit; + border-color: inherit; + color: inherit; +} + +#alert-dialog.alert-success > div > button:hover { + color: #d1e7dd; + background-color: #0f5132; + border-color: #d1e7dd; +} + +#alert-dialog.alert-danger > div > button:hover { + background-color: #842029; + border-color: #f8d7da; + color: #f8d7da; +} + +#alert-dialog.alert-info > div > button:hover { + color: #cff4fc; + background-color: #055160; + border-color: #cff4fc; +} + +#alert-dialog.alert-warning > div > button:hover { + color: #fff3cd; + background-color: #664d03; + border-color: #fff3cd; +} diff --git a/static/bspganalytics.js b/static/bspganalytics.js deleted file mode 100755 index 7469da15..00000000 --- a/static/bspganalytics.js +++ /dev/null @@ -1,9 +0,0 @@ - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-91794319-3', 'auto'); - ga('send', 'pageview'); - - diff --git a/static/hhnb-motomo.js b/static/hhnb-matomo.js similarity index 100% rename from static/hhnb-motomo.js rename to static/hhnb-matomo.js diff --git a/static/hhnb/js/app.js b/static/hhnb/js/app.js index f0da2a62..3bde5fd4 100644 --- a/static/hhnb/js/app.js +++ b/static/hhnb/js/app.js @@ -2,16 +2,18 @@ import Log from "./utils/logger.js"; import Workflow from "./workflow/workflow.js"; import { UploadFileDialog, OptimizationSettingsDialog, MessageDialog, ModelRegistrationDialog } from "./ui/components/dialog.js"; -Log.enabled = true; +if (window.location.href.startsWith("https://127.0.0.1")) { + Log.enabled = true; +} const exc = sessionStorage.getItem("exc"); const hhf_dict = sessionStorage.getItem("hhf_dict"); const workflow = new Workflow(exc, hhf_dict); - function checkRefreshSession(response) { - console.log(response); + Log.debug(response); if (response.status === 403 && response.responseJSON.refresh_url) { + $("#overlaywrapper").css("display", "none"); showLoadingAnimation("Session expired.
Refreshing session automatically..."); $.ajax({ url: "/hh-neuron-builder/session-refresh/" + exc, @@ -28,8 +30,7 @@ function checkRefreshSession(response) { } } - -$(document).ready(() => { +$(window).on("load", () => { if (workflow.getProps().hhf_flag) { $("#modalHHF").modal("show"); } @@ -53,7 +54,6 @@ $("#wf-btn-new-wf").on("click", () => { // Clone workflow button callback $("#wf-btn-clone-wf").on("click", () => { showLoadingAnimation("Cloning current workflow..."); - let new_exc = null; $.ajax({ url: "/hh-neuron-builder/clone-workflow/" + exc, method: "GET", @@ -65,11 +65,11 @@ $("#wf-btn-clone-wf").on("click", () => { error: (error) => { checkRefreshSession(error); Log.error("Status: " + error.status + " > " + error.responseText); - let message = error.responseText; if (error.status == 404) { - message = "Something goes wrong.
Try to reload the application."; + MessageDialog.openReloadDialog("Something goes wrong.
Try to reload the application."); + } else { + MessageDialog.openErrorDialog(error.responseText); } - MessageDialog.openErrorDialog(message); } }).always(() => { hideLoadingAnimation() }); }); @@ -113,7 +113,7 @@ $("#uploadForm").submit(function (e) { for (let v of formFileData.values()) { Log.debug(v); } - + workflow.uploadFile(formFileData); }); @@ -129,18 +129,16 @@ $("#cancelUploadFormButton").on("click", () => { }) // display upload dialog and set file type to upload -$(".upload-btn").on("click", (button) => { - Log.debug(button); +$(".upload-btn").on("click", async event => { $("#formFile").val(""); // clean formFile input - switch (button.target.id) { + switch (event.currentTarget.id) { case "feat-up-btn": UploadFileDialog.openFeatures(); workflow.setUploadFileType("features"); break; case "opt-up-btn": - UploadFileDialog.openModel(); - workflow.setUploadFileType("model"); + preOperationOnOptUp(); break; case "opt-res-up-btn": @@ -152,6 +150,23 @@ $(".upload-btn").on("click", (button) => { } }); +function onOptUp() { + $("#confirmationDialogModalOkButton").off("click"); + UploadFileDialog.openModel(); + workflow.setUploadFileType("model"); +} + +function preOperationOnOptUp() { + if (workflow.getUIFlags().features) { + $("#confirmationDialogModalTitle").text("Warning") + $("#confirmationDialogModalBody").html("This operation will overwrite features and protocols files.
Do you want to continue anyway ?"); + $("#confirmationDialogModalCancelButton").text("Cancel"); + $("#confirmationDialogModalOkButton").text("Yes").on("click", onOptUp); + $("#confirmationDialogModal").modal("show"); + } else { + return onOptUp(); + } +} // delete features $(".delete-btn").on("click", (button) => { @@ -179,7 +194,20 @@ $(".delete-btn").on("click", (button) => { "config/parameters.json", "config/morph.json", "morphology/*", - "mechanisms/*" + "mechanisms/*", + "x86_64/", + "zipfolder.py", + "job_parameters.json", + "ipyparallel.sbatch", + "checkpoints/", + "mod_nsgportal/", + "figures/", + "resume_job_settings.json", + "makelib.err", + "makelib.out", + "tools/", + "figures/", + "r_seed*/" ] }); break; @@ -228,13 +256,29 @@ $(".download-btn").on("click", (button) => { // display and close settings dialog -$("#opt-set-btn").on("click", () => { - Log.debug("Set parameters button clicked"); - let settings = workflow.getOptimizationSettings(); - Log.debug(settings); - OptimizationSettingsDialog.loadSettings(settings); - OptimizationSettingsDialog.open(); -}) +$("#opt-set-btn").on("click", openOptimizationSettings); + +async function openOptimizationSettings(event) { + let btn = event.currentTarget.id; + $("#" + btn).blur(); + + Log.debug(btn + " clicked, optimization settings opening..."); + + await workflow.getOptimizationSettingsAsPromise() + .then(settings => { + if (!workflow.getProps().resume) { + $("#job-action-resume").addClass("disabled"); + } else { + $("#job-action-resume").removeClass("disabled"); + } + OptimizationSettingsDialog.loadSettings(settings); + OptimizationSettingsDialog.open(); + }).catch(error => { + checkRefreshSession(error); + Log.error("Error on getting settings"); + Log.debug(error); + }); +} $("#cancel-param-btn").on("click", () => { Log.debug("Close parameters button clicked"); @@ -243,8 +287,8 @@ $("#cancel-param-btn").on("click", () => { $("#apply-param").on("click", () => { Log.debug("Uploading optimization settings"); - let formData = OptimizationSettingsDialog.getJsonData(); - if (formData.hpc == "DAINT-CSCS" || formData.hpc == "SA-CSCS" || "SA-NSG" ) { + let formData = OptimizationSettingsDialog.getJsonData(); + if (formData.hpc == "DAINT-CSCS" || formData.hpc == "SA" ) { showLoadingAnimation("Checking login..."); $.get("/hh-neuron-builder/get-authentication") .done(() => { @@ -252,8 +296,9 @@ $("#apply-param").on("click", () => { workflow.uploadOptimizationSettings(formData); }).fail((error) => { checkRefreshSession(error); - $("#hpcAuthAlert").addClass("show"); - }).always(() => { hideLoadingAnimation() }); + showHpcAuthAlert(); + hideLoadingAnimation(); + }) } else { OptimizationSettingsDialog.close(); workflow.uploadOptimizationSettings(formData); @@ -297,15 +342,16 @@ $("#save-feature-files").on("click", () => { $("#modalNFEContainer").removeClass("show"); workflow.updateProperties(); }).fail((error) => { - alert("Something goes wrong. Please download the Features files and upload them manually."); - }).always(() => { hideLoadingAnimation() }); + hideLoadingAnimation(); + showErrorAlert(makeAlertText("Error", "Something goes wrong. Please download the Features files and upload them manually.")); + }); }); /* **** */ /* Model Catalog */ -$("#opt-db-hpc-btn").on("click", chooseOptModel); +$("#opt-db-hpc-btn").on("click", preOperationOnChooseOpt); $("#closeModalMCButton").on("click", closeModelCatalog); function closeModelCatalog() { @@ -314,8 +360,22 @@ function closeModelCatalog() { } +function preOperationOnChooseOpt() { + if (workflow.getUIFlags().features) { + $("#confirmationDialogModalTitle").text("Warning") + $("#confirmationDialogModalBody").html("This operation will overwrite features and protocols files.
Do you want to continue anyway ?"); + $("#confirmationDialogModalCancelButton").text("Cancel"); + $("#confirmationDialogModalOkButton").text("Yes").on("click", chooseOptModel); + $("#confirmationDialogModal").modal("show"); + } else { + return chooseOptModel(); + } +} + + var modelsReady = false; function chooseOptModel() { + $("#confirmationDialogModalOkButton").off("click") if (modelsReady) { $("#modalMC").modal("show"); $("#closeModalMCButton").css("display", "block").addClass("show"); @@ -364,7 +424,6 @@ function chooseOptModel() { $("#modalMC").modal("show"); $("#closeModalMCButton").css("display", "block").addClass("show"); $(".mc-model").on("click", (button) => { - console.log(button.target.getAttribute("id")); if (button.target.getAttribute("id") != "external_link") { closeModelCatalog(); let optimization_id = button.currentTarget.getAttribute("id"); @@ -378,7 +437,8 @@ function chooseOptModel() { }).fail((error) => { checkRefreshSession(error); Log.error(error); - }).always(() => { hideLoadingAnimation() }) + MessageDialog.openInfoDialog(error.responseText); + }).always( () => hideLoadingAnimation() ); } }); }).done(() => { @@ -449,7 +509,6 @@ function formatDescription(meta = "") { $("#launch-opt-btn").on("click", () => { workflow.runOptimization(); - }) $("#closeFileManagerButton").on("click", () => { @@ -459,28 +518,30 @@ $("#closeFileManagerButton").on("click", () => { }) - - - /* FETCH JOBS */ - -$("#opt-fetch-btn").on("click", () => { +$("#opt-fetch-btn").on("click", async () => { Log.debug('Fetch job'); - $("#overlayjobs").css("display", "block"); - // $("#overlaywrapper").css("z-index", "100").addClass("show"); $("#overlaywrapper").css("display", "block"); + $("#overlayjobs").css("display", "block"); + await sleep(10); + $("#overlaywrapper").addClass("show"); + $("#overlayjobs").addClass("show"); $(".list-group-item.fetch-jobs").attr("aria-disabled", "false").removeClass("disabled active"); }); $("#cancel-job-list-btn").on("click", closeJobFetchDiv); -function closeJobFetchDiv() { +async function closeJobFetchDiv() { Log.debug('Close job fetch'); - resetJobFetchDiv(); + $("#overlayjobs").removeClass("show scroll-long-content"); + $("#overlaywrapper").removeClass("show"); + await sleep(500); $("#overlayjobs").css("display", "none"); $("#overlaywrapper").css("display", "none"); + + resetJobFetchDiv(); } $("#refresh-job-list-btn").on("click", () => { @@ -493,26 +554,22 @@ $("#refresh-job-list-btn").on("click", () => { }); -// var is_user_authenticated = sessionStorage.getItem("isUserAuthenticated"); - -function dismissAlert(el) { - console.log($(el).parent().removeClass("show")); -} - function resetJobFetchDiv() { $("#overlayjobs").removeClass("scroll-long-content"); $("#nsgLoginRow").css("display", "none"); + $("#saChoiseRow").css("display", "none"); $("#spinnerRow").css("display", "none"); $("#progressRow").css("display", "none"); $("#tableRow").css("display", "none"); $("#job-list-body").empty(); - $("#jobsAuthAlert").removeClass("show"); $("#cancel-job-list-btn").prop("disabled", false); $("#refresh-job-list-btn").prop("disabled", true); $(".list-group-item.fetch-jobs").removeClass("disabled active clicked").attr("aria-disabled", "false"); $("#checkNsgSpinnerButton").css("opacity", "0"); $("#usernameNsg").removeClass("is-invalid"); $("#passwordNsg").removeClass("is-invalid"); + $("#sa-project-dropdown-jobs-btn").prop("disabled", true); + $("#sa-fetch-jobs").prop("disabled", true); resetProgressBar(); } @@ -524,27 +581,67 @@ $(".jobs-unicore").on("click", (button) => { return false; } resetJobFetchDiv(); + $("#spinnerRow").css("display", "flex"); + jButton.addClass("clicked active"); $.get("/hh-neuron-builder/get-authentication") .done(() => { - jButton.addClass("clicked"); - displayJobList(jButton); + if (button.currentTarget.id == "jobsDaint") { + displayJobList(jButton); + } else if (button.currentTarget.id == "jobsSA") { + jButton.addClass("active clicked"); + loadSAContent(); + } }) .fail((error) => { checkRefreshSession(error); - $("#jobsAuthAlert").addClass("show"); + $("#spinnerRow").css("display", "none"); + showJobsAuthAlert(); + jButton.removeClass("clicked active"); }); }); + +function loadSAContent() { + $("#spinnerRow").css("display", "flex"); + $.getJSON("/hh-neuron-builder/get-service-account-content") + .done((data) => { + Log.debug(data); + if (!data["service-account"]) { + showServiceAccountAlert() + $("#jobsSA").removeClass("clicked active").blur(); + return; + } + populateServiceAccountSettings(data["service-account"], "jobs"); + $("#tableRow").css("display", "none"); + $("#saChoiseRow").css("display", "flex"); + if ($("#sa-hpc-dropdown-jobs-btn").text().toLowerCase() != "select hpc") { + $(".dropdown-item.project." + $("#sa-hpc-dropdown-jobs-btn").text().toLowerCase()).removeClass("gone"); + $("#sa-project-dropdown-jobs-btn").prop("disabled", false); + $("#sa-fetch-jobs").prop("disabled", false); + } + }) + .then(() => { + $("#spinnerRow").css("display", "none"); + }) +} + + +$("#sa-fetch-jobs").on("click", () => { + // resetJobFetchDiv(); + $("#saChoiseRow").css("display", "none"); + displayJobList($("#jobsSA")); +}) + + $("#jobsNSG").on("click", (button) => { let jButton = $("#" + button.currentTarget.id); if (jButton.hasClass("clicked")) { return false; } resetJobFetchDiv(); - $("#jobsNSG").addClass("active"); + $("#jobsNSG").addClass("active clicked"); $("#tableRow").css("display", "none"); $("#nsgLoginRow").css("display", "flex"); - jButton.addClass("clicked"); }); @@ -561,29 +658,39 @@ function displayJobList(button) { button.attr("aria-disabled", "false").removeClass("disabled").addClass("active"); let hpc = button.attr("name"); - $.getJSON("/hh-neuron-builder/fetch-jobs/" + exc, { hpc }) + let saHPC, saProject; + if (hpc == "SA") { + saHPC = $("#sa-hpc-dropdown-jobs-btn").text().toLowerCase(); + saProject = $("#sa-project-dropdown-jobs-btn").text().toLowerCase(); + } + + $.getJSON("/hh-neuron-builder/fetch-jobs/" + exc, { hpc, saHPC, saProject }) .done((results) => { let jobs = results.jobs; - Log.debug(jobs); if ($.isEmptyObject(jobs)) { closeJobFetchDiv(); MessageDialog.openInfoDialog("No jobs on " + hpc + " system"); return false; } + // jobs = $.extend({'-1': {}, '-2': {}, '-3': {}}, jobs); for (let job_id of Object.keys(jobs)) { + if (job_id < 0) { + $("#job-list-body").append("1"); + continue; + } let job = jobs[job_id]; let statusColor = ""; - let disabled = "disabled"; - if (job.status == "COMPLETED" || job.status == "SUCCESSFUL" || job.status == "FAILED") { + let downloadDisabled = "disabled"; + if (job.status == "COMPLETED" || job.status == "SUCCESSFUL") { statusColor = "#00802b" - disabled = ""; - if (job.status == "FAILED") { - statusColor = "#DD9900"; - } + downloadDisabled = ""; + } else if (job.status == "FAILED") { + statusColor = "#DF0000"; + downloadDisabled = "" } else { statusColor = "#DD9900"; - disabled = "disabled"; } + $("#overlayjobs").removeClass("scroll-long-content"); $("#job-list-body").append( "" @@ -592,21 +699,17 @@ function displayJobList(button) { + "" + job.status + "" + "" + job.date + "" + "" - + "" + + "
" + + "
" + + "" + + "
" + + "
" + "" + "" - ) + ); } - $(".job-download-button").on("click", (button) => { - var jobId = button.target.id; - var jobStatus = jobs[jobId].status - if (jobStatus == "SUCCESSFUL" || jobStatus == "COMPLETED") { - downloadJobAndRunAnalysis(jobId); - } else { - downloadJobOnly(jobId); - } - }); + $(".job-button.download").on("click", downloadJobButtonCallback); $("#spinnerRow").css("display", "none"); $("#progressRow").css("display", "none"); @@ -615,10 +718,9 @@ function displayJobList(button) { $("#cancel-job-list-btn").prop("disabled", false); $(".list-group-item.fetch-jobs").attr("aria-disabled", "false").removeClass("disabled clicked"); - let windowHeight = $(window).height(); - let overlayJobsHeight = $("#overlayjobs").height(); - if (overlayJobsHeight > (windowHeight - (windowHeight / 10))) { - $("#overlayjobs").addClass("scroll-long-content"); + let maxHeight = $(window).height() - $(window).height() * 30 / 100; + if ($("#tableRow").height() > maxHeight) { + $("#tableRow").css("max-height", maxHeight.toString() + "px"); } }).fail((error) => { checkRefreshSession(error); @@ -629,44 +731,61 @@ function displayJobList(button) { jobListButtonClicked = false; } -function animateProgressBar(progress) { - let current_progress = parseFloat($(".progress-bar").attr("aria-valuenow")); - let next_progress = current_progress + progress; - $(".progress-bar").css("width", next_progress + "%").attr("aria-valuenow", next_progress); +function downloadJobButtonCallback(button) { + let rowElement = button.currentTarget.parentElement.parentElement.parentElement.parentElement; + let jobId = button.currentTarget.parentElement.parentElement.id; + let jobName = rowElement.children[0].innerText; + let jobStatus = rowElement.children[2].innerText; + if (jobStatus == "SUCCESSFUL" || jobStatus == "COMPLETED") { + downloadJobAndRunAnalysis(jobId, jobName); + } else { + downloadJobOnly(jobId); + } } function setProgressBarValue(value) { - $(".progress-bar").css("width", parseInt(value) + "%").attr("aria-valuenow", parseInt(value)); + $(".progress-bar").width(parseInt(value) + "%").attr("aria-valuenow", parseInt(value)); } function resetProgressBar() { - $(".progress-bar").css("width", "0%").attr("aria-valuenow", "0"); + $(".progress-bar").width("0%").attr("aria-valuenow", "0"); } function setJobProcessingTitle(message) { $("#jobProcessingTitle").html(message); } -function openJobProcessingDiv() { - $("#overlaywrapper").css("display", "block"); +async function openJobProcessingDiv() { + resetJobFetchDiv(); + $("#overlayjobs").removeClass("show"); + await sleep(500); + $("#overlayjobs").css("display", "none"); $("#overlayjobprocessing").css("display", "block"); - setProgressBarValue(0); + await sleep(10); + $("#overlayjobprocessing").addClass("show"); } -function closeJobProcessingDiv() { +async function closeJobProcessingDiv() { + $("#overlayjobprocessing").removeClass("show"); + $("#overlaywrapper").removeClass("show"); + await sleep(500); $("#overlaywrapper").css("display", "none"); $("#overlayjobprocessing").css("display", "none"); - setProgressBarValue(0); + resetProgressBar(); } async function downloadJobOnly(jobId) { Log.debug("Downloading " + jobId); // disable all buttons - const data = { "job_id": jobId, "hpc": $("button.fetch-jobs.active").attr("name") } + const data = { + "job_id": jobId, + "hpc": $("button.fetch-jobs.active").attr("name"), + "saHPC": $("#sa-hpc-dropdown-jobs-btn").text().toLowerCase(), + "saProject": $("#sa-project-dropdown-jobs-btn").text().toLowerCase() + } $("#jobProcessingTitle").html("Downloading job:
" + jobId.toUpperCase() + "
"); - closeJobFetchDiv(); openJobProcessingDiv(); await sleep(500); @@ -685,52 +804,58 @@ async function downloadJobOnly(jobId) { checkRefreshSession(downloadError); closeJobProcessingDiv(); Log.error("Status: " + downloadError.status + " > " + downloadError.responseText); - MessageDialog.openErrorDialog(downloadError.responseText) + MessageDialog.openErrorDialog(downloadError.responseText); workflow.updateProperties(); }); } -async function downloadJobAndRunAnalysis(jobId) { +async function downloadJobAndRunAnalysis(jobId, jobName) { Log.debug("Downloading " + jobId); // disable all buttons - const data = { "job_id": jobId, "hpc": $("button.fetch-jobs.active").attr("name") } + const data = { + "job_id": jobId, + "job_name": jobName, + "hpc": $("button.fetch-jobs.active").attr("name"), + "saHPC": $("#sa-hpc-dropdown-jobs-btn").text().toLowerCase(), + "saProject": $("#sa-project-dropdown-jobs-btn").text().toLowerCase() + } $("#jobProcessingTitle").html("Downloading job:
" + jobId.toUpperCase() + "
"); - closeJobFetchDiv(); openJobProcessingDiv(); await sleep(500); - $("#progressBarFetchJob").addClass("s40").removeClass("s20 s4 s2"); + $("#progressBarFetchJob").addClass("s20").removeClass("s40 s4 s2"); setProgressBarValue(40); $.get("/hh-neuron-builder/fetch-job-result/" + exc, data) .done((downloadResult) => { Log.debug(downloadResult); setJobProcessingTitle("Running Analysis...
"); + $("#progressBarFetchJob").addClass("s40").removeClass("s20 s4 s2"); setProgressBarValue(80); $.get("/hh-neuron-builder/run-analysis/" + exc) .done(async (analysisResult) => { setJobProcessingTitle("Completing...
"); - $("#progressBarFetchJob").addClass("s4").removeClass("s40 s20 s2"); + $("#progressBarFetchJob").addClass("s2").removeClass("s40 s20 s4"); setProgressBarValue(100); - await sleep(4000); + await sleep(2000); closeJobProcessingDiv(); - workflow.updateProperties(); }).fail((analysisError) => { checkRefreshSession(analysisError); Log.error("Status: " + analysisError.status + " > " + analysisError.responseText); closeJobProcessingDiv(); MessageDialog.openErrorDialog(analysisError.responseText); + }).always(() => { workflow.updateProperties(); }) }).fail((downloadError) => { checkRefreshSession(downloadError); closeJobProcessingDiv(); Log.error("Status: " + downloadError.status + " > " + downloadError.responseText); - MessageDialog.openErrorDialog(downloadError.responseText) + MessageDialog.openErrorDialog(downloadError.responseText); workflow.updateProperties(); - }); + }) } $("#checkNsgLoginButton").on("click", () => { @@ -760,7 +885,7 @@ $("#checkNsgLoginButton").on("click", () => { var blueNaasModel = ""; $("#run-sim-btn").on("click", () => { showLoadingAnimation("Uploading to BlueNaas..."); - $("#run-sim-btn").prop("disabled", true); + $("#run-sim-btn").prop("disabled", true).blur(); $.get("/hh-neuron-builder/upload-to-naas/" + exc) .done((data) => { if (blueNaasModel != data) { @@ -795,8 +920,9 @@ $("#back-to-wf-btn").on("click", () => { /* Registration Model */ -$("#reg-mod-main-btn").on("click", () => { +$("#reg-mod-main-btn").on("click", async () => { $("#modalBlueNaasContainer").removeClass("show"); + // await sleep(300); ModelRegistrationDialog.open(); }) @@ -825,5 +951,35 @@ $("#register-model-btn").on("click", () => { workflow.registerModel(formData); }) - /* ************************************* */ + +$("#modalHHFCloseButton").on("click", async () => { + if (workflow.getProps().model.optimization_files.parameters !== "") { + $("#overlaywrapper").css("display", "block"); + $("#overlayparameterstemplate").css("display", "block"); + await sleep(200); + $("#overlaywrapper").addClass("show"); + $("#overlayparameterstemplate").addClass("show"); + } else { + openFileManager(true); + } +}) + +$(".parametersTemplate").on("click", (event) => { + showLoadingAnimation("Loading paramteres template..."); + let parametersType = $(event.currentTarget).attr("name"); + $.post("/hh-neuron-builder/hhf-load-parameters-template/" + exc, {type: parametersType}) + .fail((error) => { + Log.error(error); + MessageDialog.openErrorDialog("Please upload a parameters file manually.", "Something went wrong!"); + }).always(async () => { + workflow.updateProperties(); + hideLoadingAnimation(); + $("#overlaywrapper").removeClass("show"); + $("#overlayparameterstemplate").removeClass("show"); + await sleep(500); + $("#overlaywrapper").css("display", "none"); + $("#overlayparameterstemplate").css("display", "none"); + openFileManager(true); + }) +}) diff --git a/static/hhnb/js/ui/components/dialog.js b/static/hhnb/js/ui/components/dialog.js index 6da41732..183b5ab7 100644 --- a/static/hhnb/js/ui/components/dialog.js +++ b/static/hhnb/js/ui/components/dialog.js @@ -3,84 +3,107 @@ import Log from "../../utils/logger.js"; class MessageDialog { - static #overlayWrapper = $("#overlaywrapperdialog"); - static #overlayContent = $("#overlaycontentdialog"); - static #overlayDialog = $("#overlaydialog"); - static #dialogMessage = $("#dialogtext"); - static #dialogButton = $("#dialog-btn"); - - - static #openMessageDialog(msg) { - if (!msg.startsWith("{\"refresh_url\"")) { - this.#overlayWrapper.css("display", "block"); - this.#overlayContent.css("display", "block"); - this.#dialogMessage.html(msg); + static #createAlertDialog(level, title, msg) { + console.log(level, title, msg); + let alert = $("
"); + let classes = "alert alert-dismissable fade "; + let button; + if (level != "warning") { + button = ""; + } else { + button = ""; } + switch (level) { + case "danger": + if (!title) { + title = "Error !"; + } + classes += "alert-danger"; + break; + + case "warning": + if (!title) { + title = "Unexpected behavior occurred !" + } + classes += "alert-warning"; + break; + + case "success": + if (!title) { + title = "Success !"; + } + classes += "alert-success"; + break; + + case "info": + default: + if (!title) { + title = "Info:"; + } + classes += "alert-info"; + } + alert.addClass(classes); + alert.append("\ +

" + title + "

\ +
\ +

" + msg + "

\ +
\ +
" + button + "
" + ); + return alert; } - static #closeMessageDialog() { - this.#overlayWrapper.css("display", "none"); - this.#overlayContent.css("display", "none"); + static async #openDialog(level, text) { + if (text.startsWith("{\"refresh_url\"")) { + return false; + } + let alertDialog; + let sm = splitTitleAndMessage(text); + $("#shadow-layer").css("display", "block").addClass("show"); + if (sm) { + alertDialog = this.#createAlertDialog(level, sm.title, sm.message); + } else { + alertDialog = this.#createAlertDialog(level, undefined, text); + } + $("body").append(alertDialog); + alertDialog.addClass("show"); } static openSuccessDialog(msg) { - this.#overlayContent.css("box-shadow", "0 0 1rem 1rem rgba(0, 128, 0, .8)") - .css("border-color", "green"); - this.#dialogButton.text("Ok") - .addClass("green").removeClass("blue red fill-background") - .on("click", () => { this.#closeMessageDialog() }); - this.#openMessageDialog(msg); + this.#openDialog("success", msg) } static openErrorDialog(msg) { - this.#overlayContent.css("box-shadow", "0 0 1rem 1rem rgba(255, 0, 0, .8)") - .css("border-color", "red"); - this.#dialogButton.text("Ok") - .addClass("red").removeClass("blue green fill-background") - .on("click", () => { this.#closeMessageDialog() }); - this.#openMessageDialog(msg); + this.#openDialog("danger", msg); } static openInfoDialog(msg) { - this.#overlayContent.css("box-shadow", "0 0 1rem 1rem rgba(0, 0, 255, .8)") - .css("border-color", "blue"); - this.#dialogButton.text("Ok") - .addClass("blue").removeClass("red green fill-background") - .on("click", () => { this.#closeMessageDialog() }); - this.#openMessageDialog(msg); + this.#openDialog("info", msg); } - static openReloadDialog(href, msg) { - this.#overlayDialog.addClass("reload-content") - .removeClass("error-content info-content"); - this.#dialogButton.addClass("fill-background"); - this.#dialogButton.text("Reload").on("click", () => { - if (href == null) { - window.location.href = "/hh-neuron-builder"; - } else { - window.location.href = href; - } - }); - if (msg == null) { - msg = "Something goes wrong !
Please reload the application."; - } - this.#openMessageDialog(msg); + static openReloadDialog(msg) { + this.#openDialog("warning", msg); } - } class UploadFileDialog { - static #open(label) { - $("#overlayupload").css("display", "block"); + static async #open(label) { $("#overlaywrapper").css("display", "block"); + $("#overlayupload").css("display", "block"); + await sleep(10); + $("#overlaywrapper").addClass("show"); + $("#overlayupload").addClass("show"); $("#uploadFormLabel").html("" + label + ""); $("#uploadFormButton").prop("disabled", true); } - static close() { + static async close() { Log.debug("Close upload box") + $("#overlayupload").removeClass("show"); + $("#overlaywrapper").removeClass("show"); + await sleep(500); $("#overlayupload").css("display", "none"); $("#overlaywrapper").css("display", "none"); } @@ -113,118 +136,232 @@ class UploadFileDialog { // Enable apply button hpc selection -$(".accordion-button.hpc").on("click", (button) => { - Log.debug(button); - let isAlreadyOpened = $("#" + button.currentTarget.id).hasClass("active"); - $(".accordion-button.hpc").removeClass("active"); - if (!isAlreadyOpened) { - $("#" + button.currentTarget.id).addClass("active"); +$(".accordion-button.hpc").on("click",async event => { + let btn = $(event.currentTarget); + + if (btn.parent().siblings().hasClass("show")) { + return false; + } + + if (btn.hasClass("active")) { + $(".accordion-button.hpc").removeClass("active").blur(); + } else { + $(".accordion-button.hpc").removeClass("active").blur(); + btn.addClass("active"); + } + + if ($(".accordion-button.hpc.active").length == 1) { + Log.debug("Accordion button active = 1... Enabling ApplyParam button"); $("#apply-param").prop("disabled", false); } else { $("#apply-param").prop("disabled", true); } + + if ($(".accordion-collapse.collapsing").length >= 3) { + Log.debug("Accordion collapsing >= 3... Disabling ApplyParam button"); + await sleep(400); + $(".accordion-collapse.show").each((i, e) => { + console.log($(e).children().hasClass("active")); + // if (!$(e).siblings().children().hasClass("active")) { + // $(e).removeClass("show"); + // } + let b = $(e).siblings().children(); + if (!b.hasClass("active")) { + b.trigger("click").removeClass("active"); + } + }); + } +}); + +$("#username_submit").on("input", () => { + $("#username_submit").removeClass("is-valid is-invalid"); }) +$("#password_submit").on("input", () => { + $("#password_submit").removeClass("is-valid is-invalid"); +}) + +$("#job-action-start").on("click", () => { + $("#job-action").text("Start"); + + let jobName = $("#job-name").val().toString(); + if (jobName.endsWith("_resume")) { + $("#job-name").val($("#job-name").val().toString().split("_resume")[0]); + } + + let jObj = JSON.parse(window.sessionStorage.getItem("job_settings")); + OptimizationSettingsDialog.loadSettings(jObj, "start"); +}) + +$("#job-action-resume").on("click", () => { + $("#job-action").text("Resume"); + let jObj = JSON.parse(window.sessionStorage.getItem("job_settings")); + OptimizationSettingsDialog.loadSettings(jObj, "resume"); +}) class OptimizationSettingsDialog { + static #resetSettingsDialog() { + $(".accordion-button").removeClass("active").addClass("collapsed").attr("aria-expanded", "false"); + $(".accordion-collapse").removeClass("show"); + } + static #setDefaultValue() { Log.debug("setting deafault values"); $("#daint_project_id").val(""); - $("#daint-gen-max").val(2) - $("#daint-offspring").val(10) + $("#daint-gen-max").val(2); + $("#daint-offspring").val(10).prop("disabled", false) $("#daint-node-num").val(6); $("#daint-core-num").val(24); $("#daint-runtime").val("120m"); - $("#sa-daint-gen-max").val(2) - $("#sa-daint-offspring").val(10) - $("#sa-daint-node-num").val(3); - $("#sa-daint-core-num").val(24); - $("#sa-daint-runtime").val(2); - $("#username_submit").val(""); - $("#password_submit").val(""); $("#nsg-gen-max").val(2); - $("#nsg-offspring").val(10); + $("#nsg-offspring").val(10).prop("disabled", false); $("#nsg-node-num").val(1); $("#nsg-core-num").val(2); $("#nsg-runtime").val(2); - $("#sa-nsg-gen-max").val(2) - $("#sa-nsg-offspring").val(10) - $("#sa-nsg-node-num").val(1); - $("#sa-nsg-core-num").val(2); - $("#sa-nsg-runtime").val(2); + + $("#sa-gen-max").val(2); + $("#sa-offspring").val(10).prop("disabled", false); + $("#sa-runtime").val(2); } - static loadSettings(jObj) { - Log.debug("settings value"); - this.#setDefaultValue(); - if (!$.isEmptyObject(jObj)) { - Log.debug("open default accordion") - if (jObj.hpc == "DAINT-CSCS") { - $("#daint_project_id").val(jObj.project); - $("#daint-gen-max").val(jObj["gen-max"]); - $("#daint-offspring").val(jObj.offspring); - $("#daint-node-num").val(jObj["node-num"]); - $("#daint-core-num").val(jObj["core-num"]); - $("#daint-runtime").val(jObj.runtime); - } else if (jObj.hpc == "SA-CSCS") { - $("#sa-daint-gen-max").val(jObj["gen-max"]); - $("#sa-daint-offspring").val(jObj.offspring); - $("#sa-daint-node-num").val(jObj["node-num"]); - $("#sa-daint-core-num").val(jObj["core-num"]); - $("#sa-daint-runtime").val(jObj.runtime); - } else if (jObj.hpc == "NSG") { - $("#nsg-gen-max").val(jObj["gen-max"]); - $("#nsg-offspring").val(jObj.offspring); - $("#nsg-node-num").val(jObj["node-num"]); - $("#nsg-core-num").val(jObj["core-num"]); - $("#nsg-runtime").val(jObj.runtime); - } else if (jObj.hpc == "SA-NSG") { - $("#sa-nsg-gen-max").val(jObj["gen-max"]); - $("#sa-ngs-offspring").val(jObj.offspring); - $("#sa-nsg-node-num").val(jObj["node-num"]); - $("#sa-nsg-core-num").val(jObj["core-num"]); - $("#sa-nsg-runtime").val(jObj.runtime); + static loadSettings(jObj, mode="start") { + window.sessionStorage.setItem("job_settings", JSON.stringify(jObj)); + Log.debug(jObj); + + if (!jObj["service-account"]) { + $("#accordionSA").addClass("disabled").prop("disabled", true); + if (!$("#accordionSA").text().includes("unreachable")) { + $("#accordionSA").append(" *( temporarily unreachable )"); } } + populateServiceAccountSettings(jObj["service-account"], "optset"); + + let settings = jObj.settings; + let resume = jObj.resume; + + $("#job-name").val($("#wf-title").text().split("Workflow ID: ")[1]); + + if (mode == "start") { + if ($.isEmptyObject(settings)) { + this.#setDefaultValue(); + } else { + $("#apply-param").prop("disabled", false) + // $(".accordion-button.hpc").removeClass("active").addClass("collapsed"); + // $(".accordion-collapse").removeClass("show"); + if (settings.hpc == "DAINT-CSCS") { + $("#accordionDaint").addClass("active"); + $("#daintCollapse").addClass("show"); + $("#daint_project_id").val(settings.project); + $("#daint-gen-max").val(settings["gen-max"]); + $("#daint-offspring").val(settings.offspring); + $("#daint-node-num").val(settings["node-num"]); + $("#daint-core-num").val(settings["core-num"]); + $("#daint-runtime").val(settings.runtime); + } else if (settings.hpc == "NSG") { + $("#accordionNSG").addClass("active"); + $("#nsgCollapse").addClass("show"); + $("#nsg-gen-max").val(settings["gen-max"]) + $("#nsg-offspring").val(settings.offspring); + $("#nsg-node-num").val(settings["node-num"]); + $("#nsg-core-num").val(settings["core-num"]); + $("#nsg-runtime").val(settings.runtime); + if (settings["username_submit"]) { + $("#username_submit").addClass("is-valid").removeClass("is-invalid"); + } else { + $("#username_submit").addClass("is-invalid").removeClass("is-valid"); + } + if (settings["password_submit"]) { + $("#password_submit").addClass("is-valid").removeClass("is-invalid"); + } else { + $("#password_submit").addClass("is-invalid").removeClass("is-valid"); + } + } else if (settings.hpc == "SA") { + $("#accordionSA").addClass("active"); + $("#saCollapse").addClass("show"); + $("#sa-gen-max").val(settings["gen-max"]); + $("#sa-offspring").val(settings.offspring); + $("#sa-node-num").val(settings["node-num"]); + $("#sa-core-num").val(settings["core-num"]); + $("#sa-runtime").val(settings.runtime); + Log.debug(Object.keys(settings)); + if (Object.keys(settings).includes("sa-hpc")) { + $("#sa-hpc-dropdown-optset > button").html(settings["sa-hpc"].toUpperCase()); + $("#sa-project-dropdown-optset > button").html(settings["sa-project"]).prop("disabled", false); + $(".dropdown-item.project." + settings["sa-hpc"]).removeClass("gone"); + } + if ($("#sa-hpc-dropdown-optset-btn").text().toLowerCase() == "select hpc" || + $("#sa-project-dropdown-optset-btn").text().toLowerCase() == "select project") { + $("#sa-project-dropdown-optset-btn").prop("disabled", true); + $("#apply-param").prop("disabled", true); + } + } + } + } else if (mode == "resume") { + if (!$.isEmptyObject(resume)) { + $("#sa-offspring").val(resume["offspring_size"]).prop("disabled", true); + $("#nsg-offspring").val(resume["offspring_size"]).prop("disabled", true); + $("#daint-offspring").val(resume["offspring_size"]).prop("disabled", true); + $("#sa-gen-max").val("").attr("placeholder", "Last gen: " + resume["max_gen"].toString()); + $("#nsg-gen-max").val("").attr("placeholder", "Last gen: " + resume["max_gen"].toString()); + $("#daint-gen-max").val("").attr("placeholder", "Last gen: " + resume["max_gen"].toString()); + // $(".accordion-button").removeClass("active"); + // $(".accordion-collapse").removeClass("show"); + } + $("#job-name").val(resume.job_name ? resume.job_name + "_resume" : $("#job-name").val() + "_resume"); + } } static getJsonData() { const data = new Object(); let hpc = $(".accordion-button.active").attr("name"); data["hpc"] = hpc; + data["job_name"] = $("#job-name").val(); if (hpc == "DAINT-CSCS") { if ($("#daint_project_id").val() == "") { $("#daint_project_id").removeClass("is-valid").addClass("is-invalid"); - alert('Please fill "Project ID" to apply settings'); + showWarningAlert('Please fill "Project ID" to apply settings and cotinue your workflow.'); throw "daint_project empty"; } - data["gen-max"] = $("#daint-gen-max").val(); - data["offspring"] = $("#daint-offspring").val(); - data["node-num"] = $("#daint-node-num").val(); - data["core-num"] = $("#daint-core-num").val(); + data["gen-max"] = parseInt($("#daint-gen-max").val()); + if (!$("#daint-offspring").prop("disabled")) { + data["offspring"] = parseInt($("#daint-offspring").val()); + } + data["node-num"] = parseInt($("#daint-node-num").val()); + data["core-num"] = parseInt($("#daint-core-num").val()); data["runtime"] = $("#daint-runtime").val(); data["project"] = $("#daint_project_id").val(); } - if (hpc == "SA-CSCS") { - data["gen-max"] = $("#sa-daint-gen-max").val(); - data["offspring"] = $("#sa-daint-offspring").val(); - data["node-num"] = $("#sa-daint-node-num").val(); - data["core-num"] = $("#sa-daint-core-num").val(); - data["runtime"] = $("#sa-daint-runtime").val(); - } if (hpc == "NSG") { - data["gen-max"] = $("#nsg-gen-max").val(); - data["offspring"] = $("#nsg-offspring").val(); - data["node-num"] = $("#nsg-node-num").val(); - data["core-num"] = $("#nsg-core-num").val(); - data["runtime"] = $("#nsg-runtime").val(); + data["gen-max"] = parseInt($("#nsg-gen-max").val()); + if (!$("#nsg-offspring").prop("disabled")) { + data["offspring"] = parseInt($("#nsg-offspring").val()); + } + data["node-num"] = parseInt($("#nsg-node-num").val()); + data["core-num"] = parseInt($("#nsg-core-num").val()); + data["runtime"] = parseFloat($("#nsg-runtime").val()); let nsgUser = $("#username_submit"); let nsgPass = $("#password_submit"); - if (nsgUser.val() == "" && nsgPass.val() == "") { + + if (nsgUser.val() == "") { + nsgUser.addClass("is-invalid"); + } + if (nsgPass.val() == "") { + nsgPass.addClass("is-invalid"); + } + if (nsgUser.val() == "" || nsgPass.val() == "") { + showWarningAlert("Please fill \"username\" and/or \"password\" to apply settings and continue your workflow."); + throw "credentials empty"; + } + + + if (nsgUser.val() == "" || nsgPass.val() == "") { + if (nsgUser.val()) + nsgUser.addClass("is-invalid") + if (!nsgUser.hasClass("is-valid") && !nsgPass.hasClass("is-valid")) { - alert("Please fill \"username\" and \"password\" to apply settings"); + showWarningAlert("Please fill \"username\" and/or \"password\" to apply settings and continue your workflow."); throw "credentials empty"; } } else { @@ -232,21 +369,42 @@ class OptimizationSettingsDialog { data["password_submit"] = nsgPass.val(); } } - if (hpc == "SA-NSG") { - data["gen-max"] = $("#sa-nsg-gen-max").val(); - data["offspring"] = $("#sa-nsg-offspring").val(); - data["node-num"] = $("#sa-nsg-node-num").val(); - data["core-num"] = $("#sa-nsg-core-num").val(); - data["runtime"] = $("#sa-nsg-runtime").val(); + if (hpc == "SA") { + if ($("#sa-hpc-dropdown-optset > button").text().toLowerCase() == "select hpc") { + $("#sa-hpc-dropdown-optset > button").html("Select HPC"); + $("#sa-project-dropdown-optset > button").html("Select Project").prop("disabled", true); + } else { + data["sa-hpc"] = $("#sa-hpc-dropdown-optset > button").text().toLowerCase(); + data["sa-project"] = $("#sa-project-dropdown-optset > button").text().toLowerCase(); + } + data["gen-max"] = parseInt($("#sa-gen-max").val()); + if (!$("#sa-offspring").prop("disabled")) { + data["offspring"] = parseInt($("#sa-offspring").val()); + } + data["node-num"] = parseInt($("#sa-node-num").val()); + data["core-num"] = parseInt($("#sa-core-num").val()); + data["runtime"] = parseFloat($("#sa-runtime").val()); } - + data["mode"] = $("#job-action").text().toLowerCase(); + + + for (let x in data) { + console.log(typeof(data[x])) + if (typeof(data[x]) === 'number' && isNaN(data[x])) { + showWarningAlert("Missing correct value for " + x.toUpperCase() + ""); + throw Error("Not a number error on " + x); + } + } + return data; } static getFomData() { const formData = new FormData(); let hpc = $(".accordion-button.active").attr("name"); + let job_name = $("#job-name").val(); formData.append("csrfmiddlewaretoken", $("input[name=csrfmiddlewaretoken]").val()); + formData.append("job_name", $("#job-name").text()); formData.append("hpc", hpc); if (hpc == "DAINT-CSCS") { @@ -257,13 +415,6 @@ class OptimizationSettingsDialog { formData.append("runtime", $("#daint-runtime").val()); formData.append("project", $("#daint_project_id").val()); } - if (hpc == "SA-CSCS") { - formData.append("gen-max", $("#sa-daint-gen-max").val()); - formData.append("offspring", $("#sa-daint-offspring").val()); - formData.append("node-num", $("#sa-daint-node-num").val()); - formData.append("core-num", $("#sa-daint-core-num").val()); - formData.append("runtime", $("#sa-daint-runtime").val()); - } if (hpc == "NSG") { formData.append("gen-max", $("#nsg-gen-max").val()); formData.append("offspring", $("#nsg-offspring").val()); @@ -277,14 +428,19 @@ class OptimizationSettingsDialog { formData.append("password_submit", $("#password_submit").val()); } } - if (hpc == "SA-NSG") { - formData.append("gen-max", $("#sa-nsg-gen-max").val()); - formData.append("offspring", $("#sa-nsg-offspring").val()); - formData.append("node-num", $("#sa-nsg-node-num").val()); - formData.append("core-num", $("#sa-nsg-core-num").val()); - formData.append("runtime", $("#sa-nsg-runtime").val()); + if (hpc == "SA") { + if ($("#sa-hpc-dropdown-optset > button").text().toLowerCase() != "select hpc") { + formData.append("sa-hpc", $("#sa-hpc-dropdown-optset > button").text().toLowerCase()); + formData.append("sa-project", $("#sa-project-dropdown-optset > button").text().toLowerCase()); + } + formData.append("gen-max", $("#sa-gen-max").val()); + formData.append("offspring", $("#sa-offspring").val()); + formData.append("node-num", $("#sa-node-num").val()); + formData.append("core-num", $("#sa-core-num").val()); + formData.append("runtime", $("#sa-runtime").val()); } + if (Log.enabled) { for (let v of formData.values()) { Log.debug(v); @@ -293,22 +449,25 @@ class OptimizationSettingsDialog { return formData; } - static open() { + static async open() { Log.debug("open settings dialog"); - $("#overlayparam").css("display", "block"); $("#overlaywrapper").css("display", "block"); - $("#username_submit").removeClass("is-invalid"); - $("#password_submit").removeClass("is-invalid"); - $("#daint_project_id").removeClass("is-invalid"); + $("#overlayparam").css("display", "block"); + await sleep(10); + $("#overlaywrapper").addClass("show") + $("#overlayparam").addClass("show"); } - - static close() { + + static async close() { Log.debug("close"); + $("#overlayparam").removeClass("show"); + $("#overlaywrapper").removeClass("show"); + await sleep(500); $("#overlayparam").css("display", "none"); $("#overlaywrapper").css("display", "none"); - $("#hpcAuthAlert").removeClass("show"); + $("#apply-param").prop("disabled", true); + this.#resetSettingsDialog(); } - } @@ -323,16 +482,11 @@ function replaceWithSelectElement(id, options) { class ModelRegistrationDialog { - static open() { + static async open() { showLoadingAnimation("Loading options..."); let modelName = $("#wf-title").text().split("Workflow ID: ")[1]; - - - $.ajax({ - url: "/hh-neuron-builder/get-model-catalog-attribute-options", - method: "GET", - async: false, - success: options => { + $.getJSON("/hh-neuron-builder/get-model-catalog-attribute-options") + .then(async options => { Log.debug(options); for (let key of Object.keys(options)) { switch(key) { @@ -367,17 +521,26 @@ class ModelRegistrationDialog { default: } } - } - }) - Log.debug("modelName " + modelName); - $("#modelName").val(modelName) - - $("#overlaywrapper").css("display", "block"); - $("#overlaywrapmodreg").css("display", "block"); - hideLoadingAnimation(); + Log.debug("modelName " + modelName); + $("#modelName").val(modelName) + + $("#overlaywrapper").css("display", "block"); + $("#overlaywrapmodreg").css("display", "block"); + await sleep(10); + $("#overlaywrapper").addClass("show"); + $("#overlaywrapmodreg").addClass("show"); + }).catch(error => { + Log.error("Error while getting options from model catalog: ", error); + MessageDialog.openErrorDialog(error); + }).always(() => { + hideLoadingAnimation(); + }) } - static close() { + static async close() { + $("#overlaywrapmodreg").removeClass("show"); + $("#overlaywrapper").removeClass("show"); + await sleep(500); $("#overlaywrapper").css("display", "none"); $("#overlaywrapmodreg").css("display", "none"); } diff --git a/static/hhnb/js/ui/components/file_manager.js b/static/hhnb/js/ui/components/file_manager.js index 8caa59dd..d39f50f1 100644 --- a/static/hhnb/js/ui/components/file_manager.js +++ b/static/hhnb/js/ui/components/file_manager.js @@ -9,7 +9,16 @@ function checkRefreshSession(response) { .done((result) => { document.location.href = "/hh-neuron-builder/workflow/" + exc; }).fail((error) => { - alert("Can't refresh session automatically, please refresh page"); + showAlert( + makeAlertText( + "", + "Can't refresh session automatically.", + "Please refresh the page." + ), + "warning", + true, + true + ); }) } } @@ -18,12 +27,7 @@ function checkRefreshSession(response) { $("#show-opt-files-btn").on("click", openFileManager); -function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - - -function openFileManager(showConfig=false) { +async function openFileManager(showConfig=false) { refreshHHFFileList(); if (showConfig) { showFileList($("#configFolder")); @@ -31,7 +35,11 @@ function openFileManager(showConfig=false) { $("#overlaywrapper").css("display", "block"); $("#overlayfilemanager").css("display", "block"); - $("#overlayfilemanager").addClass("open"); + + await sleep(10); + $("#overlaywrapper").addClass("show"); + $("#overlayfilemanager").addClass("open show"); + $.ajax({ url: "/hh-neuron-builder/hhf-get-model-key/" + req_pattern, method: "GET", @@ -118,7 +126,10 @@ function refreshHHFFileList() { } -function closeFileManager() { +async function closeFileManager() { + $("#overlaywrapper").removeClass("show"); + $("#overlayfilemanager").removeClass("show"); + await sleep(500); $("#overlaywrapper").css("display", "none"); $("#overlayfilemanager").css("display", "none"); @@ -136,15 +147,19 @@ $("#deleteFileButton").click(function() { if ($(this).hasClass("disabled")) { return false; } - if ($(".file-item.active").length == 0 && $("editor-item.active").length == 0) { + + if ((editorMode && $(".editor-item.active").length == 0) || + (!editorMode && $(".file-item.active").length == 0)) { return false; } - if ($(".file-item.active").length > 1) { - $("#confirmationDialogModalTitle").text("Are you sure to delete these files ?"); + $("#confimationDialogModalTitle").text("Warning"); + if (!editorMode && $(".file-item.active").length > 1) { + $("#confirmationDialogModalBody").text("Are you sure to delete these files ?"); } else { - $("#confirmationDialogModalTitle").text("Are you sure to delete this file ?"); + $("#confirmationDialogModalBody").text("Are you sure to delete this file ?"); } + $("#confirmationDialogModalCancelButton").text("Cancel"); $("#confirmationDialogModalOkButton").text("Yes").attr("onclick", "deleteHHFFiles()"); $("#confirmationDialogModal").modal("show"); @@ -187,7 +202,12 @@ function deleteHHFFiles() { contentType: "application/json", data: JSON.stringify(data), error: (error) => { - // MessageDialog.openErrorDialog(error.responseText); + let sm = splitTitleAndMessage(error.responseText); + if (sm) { + MessageDialog.openErrorDialog(sm.message, sm.title); + } else { + MessageDialog.openErrorDialog(error.responseText); + } }, }).always(() => { hideLoadingAnimation(); @@ -215,16 +235,18 @@ $("#downloadFileButton").click(function() { } else { $(".file-item.active").each(function(i) { jj_ids.path.push($(".folder-item.active").attr("id").split("Folder")[0] + "/" + $(this).attr("id")); - + }); } } else { if ($(".editor-item.active").length == 0) { return false; } - jj_ids.path.push($(".editor-item.active").attr("name")); + jj_ids.path.push($(".folder-item.active").attr("id").split("Folder")[0] + "/" + $(".editor-item.active").attr("name")); } + console.log(jj_ids); + showLoadingAnimation("Downloading files..."); $.get("/hh-neuron-builder/download-files/" + req_pattern + "?file_list=" + encodeURIComponent(JSON.stringify(jj_ids))) @@ -270,9 +292,12 @@ $("#uploadFileButton").click(function() { contentType: false, processData: false, error: error => { - let text = error.responseText; - console.log(text); - openErrorMessage(text); + let sm = splitTitleAndMessage(error.responseText); + if (sm) { + MessageDialog.openErrorDialog(sm.message, sm.title); + } else { + MessageDialog.openErrorDialog(error.responseText); + } } }).always(() => { counter += 1; @@ -286,6 +311,7 @@ $("#uploadFileButton").click(function() { const onSelectFile = async function(x) { files = input.files.length; + $(".file-group").css("display", "none"); $("#fileItemSpinner").css("display", "flex"); showLoadingAnimation('Uploading files...'); @@ -299,13 +325,30 @@ $("#uploadFileButton").click(function() { $("#selectAllButton").click(function() { - if ($(this).hasClass("active")) { - $(this).removeClass("active"); - $(".file-item").removeClass("active"); - } else { - $(this).addClass("active"); - $(".file-group.active").children().addClass("active"); + let files = $(".file-group.active").children(); + let isActive = files.hasClass("active"); + let alreadyChanged = false; + + for (let i=0; i < files.length; i++) { + if (isActive != files[i].classList.contains("active")) { + if (isActive) { + files.addClass("active"); + } else { + files.removeClass("active"); + } + alreadyChanged = true; + break; + } } + + if (!alreadyChanged) { + if (!isActive) { + files.addClass("active"); + } else { + files.removeClass("active"); + } + } + }) @@ -403,7 +446,6 @@ function hideFloatingOpenFileButton() { var editorMode = false; - $("#editFileButton").mousedown(function() { $(this).addClass("clicked") }) @@ -427,11 +469,14 @@ $("#editFileButton").mouseout(function() { function resetEditorMode() { editorMode = false; + $("#folderselector").css("display", "block").removeClass("fade-out fade-in"); $("#editorselector").css("display", "none").removeClass("fade-out fade-in"); + $("#fileselector").css("display", "block").removeClass("fade-out fade-in"); $("#fileeditor").css("display", "none").removeClass("fade-out fade-in"); - $("#editFileButtonImage").removeClass("show zoomout").css("top", "8px").css("left", "2px").attr("src", "/static/assets/img/open-file-white.svg"); + + $("#editFileButtonImage").attr("src", "/static/assets/img/open-file-white.svg").removeClass("show zoomout back-arrow").css("top", "8px").css("left", "2px"); $("#editFileButton").attr("title", "Open/Edit files"); $("#editorfilelist").empty(); $("#saveFileButton").removeClass("show disabled"); @@ -441,8 +486,6 @@ function resetEditorMode() { function switchMode() { - console.log("CIAONE SWITCH") - let folderSelector = $("#folderselector"); let editorSelector = $("#editorselector"); @@ -509,7 +552,9 @@ function switchMode() { editorSelector.removeClass("fade-in").addClass("fade-out"); fileEditor.removeClass("fade-in").addClass("fade-out"); $("#saveFileButton").removeClass("show"); - $("#selectAllButton").removeClass("disabled"); + if (!$("#configFileList").hasClass("empty-list")) { + $("#selectAllButton").removeClass("disabled"); + } } }; @@ -695,20 +740,22 @@ function saveCurrentTextAreaVal() { url: "/hh-neuron-builder/hhf-save-config-file/" + currFolder + "/" + currFile + "/" + req_pattern, method: "POST", data: jj, - success: function(data) { + success: async function(data) { originalTextAreaVal = currValue; $("#saveFileButton").removeClass("show disabled"); $("#saveFileButtonSpinner").css("display", "none"); $("#saveFileButtonImage").css("display", "inline"); $("#editorAlertText").removeClass("error").addClass("info").html("File saved successfully"); $("#editorAlert").addClass("show"); + await sleep(300); }, - error: function(error) { + error: async function(error) { $("#saveFileButton").removeClass("disabled"); $("#saveFileButtonSpinner").css("display", "none"); $("#saveFileButtonImage").css("display", "inline"); $("#editorAlertText").removeClass("info").addClass("error").html(error.responseText); $("#editorAlert").addClass("show"); + await sleep(300); } }) } @@ -720,52 +767,76 @@ $("#editorAlert")[0].addEventListener("transitionend", async function(){ }); -function openErrorMessage(msg) { - console.log(msg); - $("#overlaywrapperdialog").css("display", "block"); - $("#overlaycontentdialog").css("display", "block").css("box-shadow", "0 0 1rem 1rem rgba(255, 0, 0, .8)") - .css("border-color", "red"); - $("#dialog-btn").text("Ok").addClass("red").removeClass("blue green fill-background") - .on("click", closeErrorMessage); - $("#dialogtext").html(msg); -} - -function closeErrorMessage() { - $("#overlaywrapperdialog").css("display", "none"); - $("#overlaycontentdialog").css("display", "none"); - -} +class MessageDialog { + static #createAlertDialog(level, title, msg) { + let alert = $("
"); + let classes = "alert alert-dismissable fade "; + let button; + if (level != "warning") { + button = ""; + } else { + button = ""; + } + switch (level) { + case "danger": + classes += "alert-danger"; + break; -class MessageDialog { + case "warning": + classes += "alert-warning"; + break; - static #overlayWrapper = $("#overlaywrapperdialog"); - static #overlayContent = $("#overlaycontentdialog"); - static #dialogMessage = $("#dialogtext"); - static #dialogButton = $("#dialog-btn"); + case "success": + classes += "alert-success"; + break; + case "info": + default: + classes += "alert-info"; + } + alert.addClass(classes); + alert.append("\ +

" + title + "

\ +
\ +

" + msg + "

\ +
\ +
" + button + "
" + ); + return alert; + } - static #openMessageDialog(msg) { + static async #openDialog(level, title, msg) { if (!msg.startsWith("{\"refresh_url\"")) { - this.#overlayWrapper.css("display", "block"); - this.#overlayContent.css("display", "block"); - this.#dialogMessage.html(msg); + $("#shadow-layer").css("display", "block"); + await sleep(10); + $("#shadow-layer").addClass("show"); + let alertDialog = this.#createAlertDialog(level, title, msg); + $("body").append(alertDialog); + await sleep(10); + alertDialog.addClass("show"); } } - static #closeMessageDialog() { - this.#overlayWrapper.css("display", "none"); - this.#overlayContent.css("display", "none"); + static async closeDialog() { + $("#shadow-layer").removeClass("show"); + await sleep(500); + $("#shadow-layer").css("display", "none"); + } + + static openSuccessDialog(msg, title="Success !") { + this.#openDialog("success", title, msg) } - static openErrorDialog(msg) { - console.log(msg); - this.#overlayContent.css("box-shadow", "0 0 1rem 1rem rgba(255, 0, 0, .8)") - .css("border-color", "red"); - this.#dialogButton.text("Ok") - .addClass("red").removeClass("blue green fill-background") - .on("click", () => { this.#closeMessageDialog() }); - this.#openMessageDialog(msg); + static openErrorDialog(msg, title="Error !") { + this.#openDialog("danger", title, msg); } + static openInfoDialog(msg, title="Info") { + this.#openDialog("info", title, msg); + } + + static openReloadDialog(msg, title="Unexpected behavior occurred !") { + this.#openDialog("warning", title, msg); + } } \ No newline at end of file diff --git a/static/hhnb/js/utils/misc.js b/static/hhnb/js/utils/misc.js index fac64ebb..876cb595 100644 --- a/static/hhnb/js/utils/misc.js +++ b/static/hhnb/js/utils/misc.js @@ -49,7 +49,6 @@ async function checkLastPage(iframe){ }; - $("#modalBlueNaas")[0].addEventListener("transitionstart", function (transition) { if (transition.target == $(this)[0]) { if ($(this).hasClass("show")) { @@ -76,10 +75,6 @@ $("#modalBlueNaasContainer")[0].addEventListener("transitionstart", function (tr }) -function dismissAlert(el) { - console.log($(el).parent().removeClass("show")); -} - $("#modelPrivate").on("click", (button) => { console.log(button.target.checked); if (button.target.checked) { @@ -87,4 +82,232 @@ $("#modelPrivate").on("click", (button) => { } else { $("#modelPrivateValue").text("Public"); } -}); \ No newline at end of file +}); + + +function setServiceAccountHPC(hpc, context) { + let hpcButton = $("#sa-hpc-dropdown-" + context + " > button"); + let projectButton = $("#sa-project-dropdown-" + context + " > button"); + + if (hpcButton.text() == hpc) { + return false; + } + + hpcButton.html(hpc); + projectButton.prop("disabled", false); + + if (hpc == "PIZDAINT") { + $("#sa-node-num").val(3); + $("#sa-core-num").val(24); + } else if (hpc == "NSG") { + $("#sa-node-num").val(1); + $("#sa-core-num").val(2); + } + + $(".dropdown-item.project").addClass("gone"); + $(".dropdown-item.project." + hpc.toLowerCase()).removeClass("gone"); + + // automatically set the project if there is only one project for the selecte HPC system + if ($(".dropdown-item.project." + context + "." + hpc.toLowerCase()).length == 1) { + setServiceAccountProject($(".dropdown-item.project." + context + "." + hpc.toLowerCase()).text(), context) + } +} + +function setServiceAccountProject(project, context) { + $("#sa-project-dropdown-" + context + " > button").html(project); + if (context == "optset") { + $("#apply-param").prop("disabled", false); + } else if (context == "jobs") { + $("#sa-fetch-jobs").prop("disabled", false); + } +} + +function populateServiceAccountSettings(jObj, context) { + var hpcList = $("#sa-hpc-dropdown-" + context + " > ul"); + var projectList = $("#sa-project-dropdown-" + context + " > ul"); + + hpcList.empty(); + projectList.empty(); + + let saHPC = Object.keys(jObj); + let dividerNum = saHPC.length - 1; + + for (let i=0; i < saHPC.length; i++) { + hpcList.append("
  • " + saHPC[i] + "
    • "); + if (dividerNum > 0) { + hpcList.append("
    • "); + dividerNum -= 1; + } + } + + for (var hpc in jObj) { + let projects = jObj[hpc]; + let dividerNum = projects.length - 1; + for (let i=0; i < projects.length; i++) { + projectList.append("
  • "+ projects[i] +"
    • "); + if (dividerNum > 0) { + projectList.append("
    • "); + dividerNumm -= 1; + } + } + } +} + + +async function closeAlertDialog() { + $("#shadow-layer").removeClass("show"); + await sleep(500); + $("#shadow-layer").css("display", "none"); + $("#alert-dialog").remove(); +} +/* Alerts functions */ + + +async function closeAlert(counter) { + let alert = $("#alert[counter='" + counter.toString() + "']"); + alert.removeClass("show"); + await sleep(500); + alert.css("display", "none"); + alert.remove(); +} + +var counter = 0; +/** + * Show the alert. + * + * @param {*} msg The message to be shown in the alert. + * @param {String} level Optional param. Can be "danger", "warning", "success", "info" (default). + */ +async function showAlert(msg, level="", timeout, showSymbol=true, showCloseButton=true) { + if ($("#alert[counter]:last > div#alert-text").html() == msg) { + return false; + } + + let alert = $("
    "); + let classes = "alert d-flex align-items-center alert-dismissable fade "; + let symbol = ''; + + switch (level) { + case "danger": + classes += "alert-danger"; + symbol += 'aria-label="Danger:">'; + break; + + case "warning": + classes += "alert-warning"; + symbol += 'aria-label="Danger:">'; + break; + + case "success": + classes += "alert-success"; + symbol += 'aria-label="Success:">'; + break; + + case "info": + default: + classes += "alert-info"; + symbol += 'aria-label="Info:">'; + } + + alert.addClass(classes); + alert.append(symbol); + alert.append("
    " + msg + "
    "); + alert.append(button); + + $("body").append(alert); + alert.css("left", (($(".page").width() - alert.width()) / 2).toString() + "px"); + counter += 1; + + await sleep(10); + alert.addClass("show"); + await sleep(timeout); + $("#alert-button").trigger("click"); +} + + +function makeAlertText(head="", strong="", content="") { + let msg = ""; + if (head) { + msg += "

    " + head + "

    "; + } + if (strong) { + msg += "" + strong + ""; + } + if (content) { + msg += "

    " + content + "

    "; + } + return msg; +} + + +function showInfoAlert(msg, timeout=10000) { + showAlert(msg, "info", timeout, true, false); +} + +function showSuccessAlert(msg, timeout=10000) { + showAlert(msg, "success", timeout, true, true); +} + +function showErrorAlert(msg, timeout=10000) { + showAlert(msg, "danger", timeout, true, true); +} + +function showWarningAlert(msg, timeout=10000) { + showAlert(msg, "warning", timeout, true, false); +} + +function showHpcAuthAlert() { + console.log("showHpcAuthAlert() called."); + showAlert( + makeAlertText( + head="", + strong="You need to be logged in to use this HPC system !", + content="Please, click \"Cancel\" and login with the button in the top right corner before doing this operation." + ), + "warning", + 5000, + true, + false + ); +} + +function showJobsAuthAlert() { + console.log("showJobsAuthAlert()"); + showAlert( + makeAlertText( + head="", + strong="You need to authenticate with Ebrains to fetch jobs on this HPC system !", + content="Please, click \"Cancel\" and login with the button in the top right corner before doing this operation." + ), + "warning", + 5000, + true, + false + ) +} + +function showServiceAccountAlert() { + showAlert( + makeAlertText( + head="", + strong="The Service Account is temporarily unreachable !", + content="Please, try again later or contact the support if the problem persists." + ), + "warning", + 5000, + true, + false + ) +} + +function splitTitleAndMessage(message) { + let splittedMessage = message.split("

    "); + if (splittedMessage.length == 2) { + return { + title: splittedMessage[0].replace("", ""), + message: splittedMessage[1] + }; + } + return null; +} \ No newline at end of file diff --git a/static/hhnb/js/workflow/workflow.js b/static/hhnb/js/workflow/workflow.js index add8a492..76ff2136 100644 --- a/static/hhnb/js/workflow/workflow.js +++ b/static/hhnb/js/workflow/workflow.js @@ -7,7 +7,7 @@ const UPLOAD_FILES_BASE_URL = "/hh-neuron-builder/upload-files/"; function checkRefreshSession(response) { - console.log(response); + Log.debug(response); if (response.status === 403 && response.responseJSON.refresh_url) { showLoadingAnimation("Session expired.
    Refreshing session automatically..."); $.ajax({ @@ -38,7 +38,7 @@ function disable(jObj) { export default class Workflow { #exc = null; - #props = null; + #props = {}; #uploadFileType = null; @@ -83,11 +83,12 @@ export default class Workflow { if (error.status == 404) { MessageDialog.openReloadDialog(); } + MessageDialog.openErrorDialog(error.responseText); } }).done(() => { this.updateUI() }) .fail((error) => { if (error.status == 500) { - return MessageDialog.openReloadDialog("/hh-neuron-builder", "A critical error occurred.
    Please restart the application and if the the problem persists contact us."); + return MessageDialog.openReloadDialog("A critical error occurred !

    Please restart the application and try again.
    If the problem persists try to start a new workflow, otherwise contact us through the EBRAINS support."); } }) .always(() => { hideLoadingAnimation() }); @@ -165,15 +166,15 @@ export default class Workflow { #updateSettingsBlock() { let bar = $("#opt-param-bar"); - if (this.#props.optimization_settings) { + Log.debug(this.#props.optimization_settings); + if (this.#props.optimization_settings[0]) { bar.addClass("green").removeClass("red"); - bar.text(""); this.#optsettingsBlock = true; } else { bar.addClass("red").removeClass("green"); - bar.text("Optimization parameters NOT set"); this.#optsettingsBlock = false; } + bar.text(this.#props.optimization_settings[1]); } #updateCellOptimizationBlock() { @@ -223,7 +224,7 @@ export default class Workflow { disable(downloadAnalysisLink); enable(deleteLink); } else { - bar.text("Fetch job results to run analysis or upload it"); + bar.text("Fetch job results or upload a previously downloaded \"zip\" to run analysis"); enable(fetchButton); enable(uploadButton); disable(downloadResultLink); @@ -277,8 +278,9 @@ export default class Workflow { error: (error) => { Log.error("Status: " + error.status + " > " + error.responseText); MessageDialog.openErrorDialog(error.responseText); + hideLoadingAnimation() }, - }).always(() => { hideLoadingAnimation() }); + }); } setUploadFileType(fileType) { @@ -298,34 +300,49 @@ export default class Workflow { data: formFileData, contentType: false, // need to upload files correctly processData: false, - async: false, success: (result) => { Log.debug(result); - this.updateProperties(); }, error: (error) => { Log.error("Status: " + error.status + " > " + error.responseText); MessageDialog.openErrorDialog(error.responseText); + hideLoadingAnimation(); }, - }).always(() => { hideLoadingAnimation(); }); + }).always(() => { + this.updateProperties(); + }); } + getOptimizationSettingsAsPromise() { + showLoadingAnimation("Loading settings..."); + return new Promise((resolve, reject) => { + $.getJSON("/hh-neuron-builder/optimization-settings/" + this.#exc) + .done(results => { + resolve(results); + }).fail(error => { + reject(error); + }).always(() => { + hideLoadingAnimation(); + }) + }) + } + // will be deprecated getOptimizationSettings() { let settings = null; - showLoadingAnimation("Getting settings..."); + showLoadingAnimation("Loading settings..."); $.ajax({ url: "/hh-neuron-builder/optimization-settings/" + this.#exc, method: "GET", async: false, headers: { "Accept": "application/json" }, success: (result) => { - // Log.debug(result); settings = result; if (settings.hpc == "NSG") { $("#username_submit").removeClass("is-invalid").addClass("is-valid"); $("#password_submit").removeClass("is-invalid").addClass("is-valid"); } + return settings; }, error: (error) => { checkRefreshSession(error); @@ -345,24 +362,24 @@ export default class Workflow { data: JSON.stringify(jData), processData: false, contentType: false, - async: false, success: (result) => { - Log.debug(result) if (jData.hpc == "NSG") { $("#username_submit").removeClass("is-invalid").addClass("is-valid"); $("#password_submit").removeClass("is-invalid").addClass("is-valid"); } - this.updateProperties(); }, error: (error) => { + hideLoadingAnimation(); Log.error("Status: " + error.status + " > " + error.responseText); MessageDialog.openErrorDialog(error.responseText); - if (error.responseText == "Invalid credentials") { + if (error.responseText == "Invalid credentials.") { $("#username_submit").removeClass("is-valid").addClass("is-invalid"); $("#password_submit").removeClass("is-valid").addClass("is-invalid"); } } - }).always(() => { hideLoadingAnimation() }); + }).always(() => { + this.updateProperties(); + }); } runOptimization() { @@ -371,17 +388,17 @@ export default class Workflow { $.ajax({ url: "/hh-neuron-builder/run-optimization/" + this.#exc, method: "GET", - async: false, success: (result) => { Log.debug(result); + this.updateProperties(false); MessageDialog.openSuccessDialog(result); - this.updateProperties(); }, error: (error) => { Log.error("Status: " + error.status + " > " + error.responseText); MessageDialog.openErrorDialog(error.responseText); + hideLoadingAnimation(); } - }).always(() => { hideLoadingAnimation() }); + }); } registerModel(formData) { @@ -389,7 +406,6 @@ export default class Workflow { $.ajax({ url: "/hh-neuron-builder/register-model/" + this.#exc, method: "POST", - async: false, data: formData, processData: false, contentType: false, @@ -399,6 +415,7 @@ export default class Workflow { }, error: (error) => { Log.error("Status: " + error.status + " > " + error.responseText); + if (error.status == 500) { // ModelRegistrationDialog.close(); } diff --git a/templates/efelg/docs/dataset.html b/templates/efelg/docs/dataset.html index 3f0607c5..df751163 100644 --- a/templates/efelg/docs/dataset.html +++ b/templates/efelg/docs/dataset.html @@ -58,7 +58,6 @@

    The data provided in the Trace selection page are available at the following - diff --git a/templates/efelg/docs/file_formats.html b/templates/efelg/docs/file_formats.html index 77272261..7815bebc 100755 --- a/templates/efelg/docs/file_formats.html +++ b/templates/efelg/docs/file_formats.html @@ -134,6 +134,4 @@

    JSON

    - - diff --git a/templates/efelg/docs/index.html b/templates/efelg/docs/index.html index 3523f180..126fc363 100755 --- a/templates/efelg/docs/index.html +++ b/templates/efelg/docs/index.html @@ -217,6 +217,4 @@
    Results
    - - diff --git a/templates/efelg/error_space_left.html b/templates/efelg/error_space_left.html index 6a1bf29e..f8603d63 100644 --- a/templates/efelg/error_space_left.html +++ b/templates/efelg/error_space_left.html @@ -11,7 +11,7 @@ - + @@ -56,6 +56,5 @@

    - \ No newline at end of file diff --git a/templates/efelg/overview.html b/templates/efelg/overview.html index 3fbfb18a..0bdc3082 100755 --- a/templates/efelg/overview.html +++ b/templates/efelg/overview.html @@ -11,7 +11,7 @@ - + @@ -275,6 +275,5 @@

    Terms & Conditions

    - \ No newline at end of file diff --git a/templates/efelg/results.html b/templates/efelg/results.html index 634b7aae..09defb77 100755 --- a/templates/efelg/results.html +++ b/templates/efelg/results.html @@ -11,7 +11,7 @@ - + @@ -88,6 +88,5 @@

    Result

    - diff --git a/templates/efelg/select_features.html b/templates/efelg/select_features.html index 6040e8ea..6389e78f 100755 --- a/templates/efelg/select_features.html +++ b/templates/efelg/select_features.html @@ -11,7 +11,7 @@ - + @@ -79,6 +79,5 @@

    Feature selection

    - diff --git a/templates/efelg/show_traces.html b/templates/efelg/show_traces.html index bf536a5e..b35a0999 100755 --- a/templates/efelg/show_traces.html +++ b/templates/efelg/show_traces.html @@ -11,7 +11,7 @@ - + @@ -379,7 +379,6 @@

    - - - + diff --git a/templates/hhnb/home.html b/templates/hhnb/home.html index 05555632..4b1033f9 100755 --- a/templates/hhnb/home.html +++ b/templates/hhnb/home.html @@ -10,7 +10,6 @@
    Welcome to the
    Hodgkin-Huxley Neuron Builder

    This tool makes an intensive use of JavaScript.
    Please, make sure you have not disabled JavaScript for this domain.

    -
    @@ -65,16 +64,7 @@
    Welcome to the
    Hodgkin-Huxley
    - -
    -
    -
    - Loading... -
    -
    -
    -
    - + - - - + + - - - + + - +
    @@ -69,7 +52,6 @@

    Neuron Builder

    {% if user.is_authenticated %} - user @@ -81,7 +63,6 @@

    Neuron Builder

    {% else %}     - {% endif %}
    @@ -90,13 +71,21 @@

    Neuron Builder

    {% block content %}{% endblock %} - - + {% include "hhnb/loading_animation.html" %} + {% include "hhnb/alert_symbols.html" %} + + + + diff --git a/templates/hhnb/loading_animation.html b/templates/hhnb/loading_animation.html new file mode 100644 index 00000000..cb331a79 --- /dev/null +++ b/templates/hhnb/loading_animation.html @@ -0,0 +1,6 @@ +
    +
    +
    Loading...
    + +
    +
    \ No newline at end of file diff --git a/templates/hhnb/workflow.html b/templates/hhnb/workflow.html index 05002277..77d192c5 100644 --- a/templates/hhnb/workflow.html +++ b/templates/hhnb/workflow.html @@ -17,6 +17,7 @@
    +
    Cell Optimization
    @@ -31,7 +32,7 @@
    + title='Click the "Extract features" button to compute electrophysiological features (from the HBP dataset or your own experimental files) to be used in your optimization process. You can also upload your own features.json and protocols.json files by clicking on "Upload file"'>
    @@ -68,7 +69,7 @@
    + title='Choose a model from the HBP dataset by clicking the "Choose from database" button. Alternative, you can upload a properly formatted .zip file containing the model folders/files or a previously downloaded job results or analysis from the "Single Cell Simulation Run" section to resume the job from its checkpoint. For further details, please refer to the HBP Brain Simulation Platform Guidebook'>
    @@ -106,7 +107,7 @@
    + title='Click the "Set parameters/Choose HPC button" to choose the HPC system in which you want to optimize the model and set the optimization (i.e. BluePyOpt) and execution (e.g. number of cores, runtime, etc.) parameters. You can also select an action ("Start" or "Resume") to optimize the job. The "Start" action start the optimization from scratch while the "Resume" action continue the optimization from its checkpoint.'>
    @@ -145,7 +146,7 @@
    + title='Fetch optimization results from HPC systems by clicking the "Fetch results" button. Alternative, you can upload a properly formatted .zip file containing the optimized model folders/files ready for simulation. For further details, please refer to the HBP Brain Simulation Platform Guidebook'>
    @@ -160,7 +161,7 @@
    -
    @@ -177,7 +178,7 @@
    - NO simulation .zip file of NO correct output drom optimization + NO simulation .zip file of NO correct output from optimization
    @@ -212,7 +213,23 @@
    {% csrf_token %} -
    + {% comment %}
    {% endcomment %} +
    +
    + Job Name + + + +
      +
    • Start
      • +
      • +
    • Resume
      • +
    +
    +
    +
    @@ -225,9 +242,7 @@

    - - Project ID - + Project ID
    @@ -238,16 +253,12 @@

    Algorithm Parameters
    - - # Gen - + # Gen

    - - Offspring - + Offspring

    @@ -257,7 +268,6 @@

    -

    @@ -266,23 +276,17 @@

    Optimization Run Parameters
    - - # Nodes - + # Nodes

    - - # Cores - + # Cores

    - - Runtime - + Runtime
    @@ -292,74 +296,6 @@

    -
    -

    - -

    -
    -
    -
    -
    -
    -
    Algorithm Parameters
    -
    -
    - - # Gen - - -
    -
    -
    - - Offspring - - -
    -
    -
    - - - - -
    - -
    -
    -
    -
    -
    -
    Optimization Run Parameters
    -
    -
    - - # Nodes - - -
    -
    -
    - - # Cores - - -
    -
    -
    - - Runtime - - -
    -
    -
    -
    -
    -
    -
    -

    -
    +
    +
    +
    +
    +
    HPC System
    +
    +
    + +
      +
      +
      +
      +
      +
      +
      +
      Service Account Project
      +
      +
      + +
        +
        +
        +
        +
        +
        +
        Algorithm Parameters
        - # Gen - + # Gen +

        - Offspring - + Offspring +

        - +
        @@ -477,18 +422,18 @@

        Optimization Run Parameters
        - # Nodes - + # Nodes +

        - # Cores - + # Cores +

        - Runtime - + Runtime +

        @@ -508,7 +453,7 @@

        -
        + {% comment %}
        {% endcomment %}
        @@ -518,34 +463,61 @@

        - - +
        -
        +
        Check NSG Login
        - - Username - - - - Password - - - + Username + + Password +

        + +
        +
        +
        +
        +
        +
        Service Account
        +
        +
        +
        +
        +
        HPC System
        +
        +
        + +
          +
          +
          +
          +
          +
          +
          +
          Service Account Project
          +
          +
          + +
            +
            +
            +
            +
            +
            +
            +
            @@ -554,17 +526,12 @@

            -
            -
            -
            -
            -
            - + @@ -574,6 +541,7 @@

            Workflow IDJob name Job ID Status Date/Time (UTC)
            +
            @@ -585,17 +553,6 @@

            - -
            - You need to be logged in to use this HPC system !
            Please, click Cancel and login via the top-right button before doing this operation.             - -
            - -
            - You need to authenticate with Ebrains to fetch jobs on this HPC system !
            Please, click Cancel and login via the top-right button before doing this operation.             - -
            -
            @@ -658,6 +615,7 @@

            +
            @@ -669,6 +627,32 @@

            +
            +
            +
            +

            Parameters not found !
            Please choose one of the following templates and modify it according to your needs.
            You can also delete it from the file manager and upload a new one.

            +
            +
            +
            +
            + pyramidal +
            +

            Pyramical cell

            +
            +
            +
            +
            +
            + interneuron +
            +

            Interneuron cell

            +
            +
            +
            +
            +
            +
            +
            {% csrf_token %} @@ -837,19 +821,16 @@

            -
            ![Screenshot from 2021-12-16 10-49-09](https://user-images.githubusercontent.com/44343476/146348432-fdb4a866-de34-4175-bfd5-7dca51c14596.png) - +
            - -
            Model file manager
            - +

            Model file manager

            The single cell model optimization process of the Hodgkin Huxley Neuron Builder relies on the Python-based BluePyOpt optimization suite.

            The model to be optimized is made up of several files and folders relative to both the NEURON model itself and the BluePyOpt optimization process.

            More specifically, the NEURON model comprises the mechanisms (i.e., NEURON .mod files) and a morphology file (in .asc format). These files combines with the BluePyOpt specific routines (needed to finalize the model optimization) that are implemented following this workflow: the opt_neuron.py script is launched and imports the template.py and evaluator.py modules, which in turn read the parameters.json, features.json and protocols.json files.

            In order to customize the model to be optimized, you can click on 'Manage files' in the 'Optimization files' panel and upload your own files (a filter is applied to exclusively allow the upload of relevant files -based on their extension- and prevent users to modify sensitive scripts) and/or modifying them when possible. From the same GUI, you will also be able to download any model file.

            Please bear in mind that these are advanced functionalities for power users. If you have any doubt on how to use these features, do not hesitate to contact us at support@ebrains.eu.
            - +
            @@ -861,6 +842,9 @@
            Model file manager
            +
            +

            +

            @@ -930,37 +914,19 @@
            -
            -
            -
            -
            -
            - -
            -
            -
            - -
            -
            -
            - Loading workspace... -
            -
            -
            -
            +
            - - + + - {% endblock %}