From 3976ce62cda04c057047d88d47ce978df661ebd2 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Sun, 28 Aug 2022 14:00:02 -0500 Subject: [PATCH 01/25] Corrected serialization for use of hed-python DictionaryDef --- hedweb/events.py | 7 +++---- hedweb/routes.py | 13 ------------- hedweb/schema.py | 2 +- hedweb/sidecar.py | 3 +-- hedweb/spreadsheet.py | 2 +- tests/test_services.py | 5 +++-- 6 files changed, 9 insertions(+), 23 deletions(-) diff --git a/hedweb/events.py b/hedweb/events.py index d1a9b491..80504ac3 100644 --- a/hedweb/events.py +++ b/hedweb/events.py @@ -3,14 +3,13 @@ from werkzeug.utils import secure_filename import pandas as pd -from hed.models import Sidecar, TabularInput from hed import schema as hedschema from hed.errors import get_printable_issue_string, HedFileError +from hed.tools import assemble_hed, BidsTabularSummary, generate_filename, generate_sidecar_entry, search_tabular +from hed.models import DefinitionDict, Sidecar, TabularInput from hed.validator import HedValidator from hedweb.constants import base_constants from hedweb.columns import create_column_selections, create_columns_included -from hed.util import generate_filename -from hed.tools import BidsTabularSummary, assemble_hed, generate_sidecar_entry, search_tabular from hedweb.web_util import form_has_option, get_hed_schema_from_pull_down app_config = current_app.config @@ -116,7 +115,7 @@ def assemble(hed_schema, events, columns_included=None, expand_defs=True): file_name = generate_filename(display_name, name_suffix='_expanded', extension='.tsv') return {base_constants.COMMAND: base_constants.COMMAND_ASSEMBLE, base_constants.COMMAND_TARGET: 'events', - 'data': csv_string, 'output_display_name': file_name, 'definitions': defs, + 'data': csv_string, 'output_display_name': file_name, 'definitions': DefinitionDict.get_as_strings(defs), 'schema_version': schema_version, 'msg_category': 'success', 'msg': 'Events file successfully expanded'} diff --git a/hedweb/routes.py b/hedweb/routes.py index d01a9697..e90373ec 100644 --- a/hedweb/routes.py +++ b/hedweb/routes.py @@ -13,19 +13,6 @@ app_config = current_app.config route_blueprint = Blueprint(route_constants.ROUTE_BLUEPRINT, __name__) -# with app.app_context(): -# from hedweb.routes import route_blueprint -# -# app.register_blueprint(route_blueprint, url_prefix=app.config['URL_PREFIX']) -# os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True) -# -# app.config['VERSIONS'] = get_version_dict() -# print(f"Versions: {app.config['VERSIONS']}") -# print(f"Using cache directory {app.config['HED_CACHE_FOLDER']}") -# -# hedschema.set_cache_directory(app.config['HED_CACHE_FOLDER']) -# -# hedschema.set_cache_directory(app_config[]) @route_blueprint.route(route_constants.COLUMNS_INFO_ROUTE, methods=['POST']) def columns_info_results(): diff --git a/hedweb/schema.py b/hedweb/schema.py index d4dfa1cd..1dc4d545 100644 --- a/hedweb/schema.py +++ b/hedweb/schema.py @@ -7,7 +7,7 @@ from hed import schema as hedschema from hed.errors import get_exception_issue_string, get_printable_issue_string from hed.errors import HedFileError -from hed.util import generate_filename +from hed.tools import generate_filename from hedweb.web_util import form_has_file, form_has_option, form_has_url from hedweb.constants import base_constants, file_constants diff --git a/hedweb/sidecar.py b/hedweb/sidecar.py index e61b5ecf..cfffa9e1 100644 --- a/hedweb/sidecar.py +++ b/hedweb/sidecar.py @@ -9,8 +9,7 @@ from hed.errors import HedFileError, get_printable_issue_string from hed.models import SpreadsheetInput, Sidecar -from hed.tools import df_to_hed, hed_to_df, merge_hed_dict -from hed.util import generate_filename +from hed.tools import df_to_hed, generate_filename, hed_to_df, merge_hed_dict from hedweb.constants import base_constants, file_constants from hedweb.web_util import form_has_option, get_hed_schema_from_pull_down diff --git a/hedweb/spreadsheet.py b/hedweb/spreadsheet.py index e98e299e..99864ee6 100644 --- a/hedweb/spreadsheet.py +++ b/hedweb/spreadsheet.py @@ -4,7 +4,7 @@ from hed import schema as hedschema from hed.errors import get_printable_issue_string, HedFileError from hed.models import SpreadsheetInput -from hed.util import generate_filename +from hed.tools import generate_filename from hed.validator import HedValidator from hedweb.constants import base_constants, file_constants diff --git a/tests/test_services.py b/tests/test_services.py index 227daf7b..ff07cdde 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -103,7 +103,7 @@ def test_services_get_sidecar(self): data_upper = json.load(f) with open(sidecar_path_lower2) as f: data_lower2 = json.load(f) - params2= {base_constants.JSON_LIST: [json.dumps(data_upper), json.dumps(data_lower2)]} + params2 = {base_constants.JSON_LIST: [json.dumps(data_upper), json.dumps(data_lower2)]} arguments2 = {} get_sidecar(arguments2, params2) self.assertIn(base_constants.JSON_SIDECAR, arguments2, 'get_sidecar arguments should have a sidecar') @@ -115,7 +115,7 @@ def test_services_get_sidecar(self): with open(sidecar_path_lower3) as f: data_lower3 = json.load(f) - params3= {base_constants.JSON_LIST: [json.dumps(data_upper), json.dumps(data_lower3)]} + params3 = {base_constants.JSON_LIST: [json.dumps(data_upper), json.dumps(data_lower3)]} arguments3 = {} get_sidecar(arguments3, params3) self.assertIn(base_constants.JSON_SIDECAR, arguments3, 'get_sidecar arguments should have a sidecar') @@ -125,5 +125,6 @@ def test_services_get_sidecar(self): self.assertNotIn('event_type', data_lower3, "get_sidecar lower3 does not have event_type") self.assertIn('event_type', sidecar3.loaded_dict, "get_sidecar merged sidecar has event_type") + if __name__ == '__main__': unittest.main() From e57c6372ad8f66cb2ff65cd130cbf7deeadbf834 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Mon, 12 Sep 2022 16:27:19 -0500 Subject: [PATCH 02/25] Starting working on hed_dev deployment --- deploy_hed/base_config.py | 2 + deploy_hed_dev/Dockerfile | 24 +++ deploy_hed_dev/base_config.py | 45 ++++ deploy_hed_dev/deploy.sh | 130 +++++++++++ deploy_hed_dev/httpd.conf | 15 ++ deploy_hed_dev/requirements.txt | 43 ++++ deploy_hed_dev/web.wsgi | 3 + hedweb/columns.py | 4 +- hedweb/constants/base_constants.py | 4 +- hedweb/events.py | 57 ++++- hedweb/services.py | 20 ++ hedweb/static/resources/services.json | 20 ++ hedweb/templates/actions.html | 11 +- hedweb/templates/events.html | 11 +- hedweb/templates/js/events-form.js | 17 +- hedweb/templates/js/remodel-input.js | 37 ++++ hedweb/templates/options.html | 2 +- hedweb/templates/remodel-input.html | 14 ++ tests/data/bad_reorder_remdl.json | 46 ++++ tests/data/simple_reorder_remdl.json | 19 ++ ...b-002_task-FacePerception_run-1_events.tsv | 201 ++++++++++++++++++ tests/data/task-FacePerception_events.json | 138 ++++++++++++ tests/test_events.py | 44 ++++ tests/test_routes/test_routes_events.py | 60 ++++++ 24 files changed, 956 insertions(+), 11 deletions(-) create mode 100644 deploy_hed_dev/Dockerfile create mode 100644 deploy_hed_dev/base_config.py create mode 100644 deploy_hed_dev/deploy.sh create mode 100644 deploy_hed_dev/httpd.conf create mode 100644 deploy_hed_dev/requirements.txt create mode 100644 deploy_hed_dev/web.wsgi create mode 100644 hedweb/templates/js/remodel-input.js create mode 100644 hedweb/templates/remodel-input.html create mode 100644 tests/data/bad_reorder_remdl.json create mode 100644 tests/data/simple_reorder_remdl.json create mode 100644 tests/data/sub-002_task-FacePerception_run-1_events.tsv create mode 100644 tests/data/task-FacePerception_events.json diff --git a/deploy_hed/base_config.py b/deploy_hed/base_config.py index 5b23de10..03f12c4a 100644 --- a/deploy_hed/base_config.py +++ b/deploy_hed/base_config.py @@ -26,6 +26,8 @@ class Config(object): class DevelopmentConfig(Config): DEBUG = False TESTING = False + URL_PREFIX = '/heddev' + STATIC_URL_PATH = '/heddev/hedweb/static' class ProductionConfig(Config): diff --git a/deploy_hed_dev/Dockerfile b/deploy_hed_dev/Dockerfile new file mode 100644 index 00000000..53823d99 --- /dev/null +++ b/deploy_hed_dev/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.9-slim-buster +COPY requirements.txt /root/ +WORKDIR /root +RUN apt-get update && apt-get install -y gcc \ +musl-dev \ +openrc \ +libxslt-dev \ +libxml2-dev \ +apache2 \ +apache2-dev && \ +pip3 install --upgrade pip && \ +pip3 install --no-cache-dir -r requirements.txt && \ +pip3 install git+https://github.com/hed-standard/hed-python/@develop \ +mkdir -p /var/www/localhost/htdocs && \ +cp /etc/mime.types /var/www/mime.types && \ +mkdir -p /var/log/hedtools_dev && \ +chown -R www-data:www-data /var/log/hedtools_dev && \ +mkdir -p /var/cache/schema_cache && \ +chown -R www-data:www-data /var/cache/schema_cache +COPY httpd.conf /etc/apache2/apache2.conf +COPY ./hedtools /var/www/hedtools_dev/ +COPY ./hedtools/hedweb /var/www/hedtools_dev/hedweb/ +ENTRYPOINT /usr/sbin/apache2 -D FOREGROUND -f /etc/apache2/apache2.conf +ENV HEDTOOLS_CONFIG_CLASS=config.ProductionConfig diff --git a/deploy_hed_dev/base_config.py b/deploy_hed_dev/base_config.py new file mode 100644 index 00000000..5bde8029 --- /dev/null +++ b/deploy_hed_dev/base_config.py @@ -0,0 +1,45 @@ +""" +This module contains the configurations for the HEDTools application. +""" + +import os +import tempfile + + +class Config(object): + LOG_DIRECTORY = '/var/log/hedtools' + LOG_FILE = os.path.join(LOG_DIRECTORY, 'error.log') + if not os.path.exists('/var/log/hedtools/tmp.txt'): + f = open('/var/log/hedtools/tmp.txt', 'w+') + f.write(str(os.urandom(24))) + f.close() + f = open('/var/log/hedtools/tmp.txt', 'r') + SECRET_KEY = f.read() # os.getenv('SECRET_KEY') # os.urandom(24) + f.close() + STATIC_URL_PATH = None + STATIC_URL_PATH_ATTRIBUTE_NAME = 'STATIC_URL_PATH' + UPLOAD_FOLDER = os.path.join(tempfile.gettempdir(), 'hedtools_uploads') + URL_PREFIX = None + HED_CACHE_FOLDER = '/var/cache/schema_cache' + + +class DevelopmentConfig(Config): + DEBUG = False + TESTING = False + + +class ProductionConfig(Config): + DEBUG = False + TESTING = False + URL_PREFIX = '/hed_dev' + STATIC_URL_PATH = '/hed_dev/hedweb/static' + + +class TestConfig(Config): + DEBUG = False + TESTING = True + + +class DebugConfig(Config): + DEBUG = True + TESTING = False diff --git a/deploy_hed_dev/deploy.sh b/deploy_hed_dev/deploy.sh new file mode 100644 index 00000000..55a5aeb1 --- /dev/null +++ b/deploy_hed_dev/deploy.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# deploy.sh - A script used to _build and deploy a docker container for the HEDTools online validator + +if [ $# -eq 0 ]; then + BRANCH="master" +else + BRANCH="$1" +fi +##### Constants + +DEPLOY_DIR=${PWD} +IMAGE_NAME="hedtools_dev:latest" +CONTAINER_NAME="hedtools_dev" +GIT_WEB_REPO_URL="https://github.com/hed-standard/hed-web" +GIT_HED_WEB_DIR="${DEPLOY_DIR}/hed-web" +GIT_WEB_REPO_BRANCH=${BRANCH} +HOST_PORT=33004 +CONTAINER_PORT=80 + +CODE_DEPLOY_DIR="${DEPLOY_DIR}/hedtools" +SOURCE_DEPLOY_DIR="${DEPLOY_DIR}/hed-web/deploy_hed" +BASE_CONFIG_FILE="${SOURCE_DEPLOY_DIR}/base_config.py" +CONFIG_FILE="${CODE_DEPLOY_DIR}/config.py" +SOURCE_WSGI_FILE="${SOURCE_DEPLOY_DIR}/web.wsgi" +SOURCE_DOCKERFILE="${SOURCE_DEPLOY_DIR}/Dockerfile" +SOURCE_REQUIREMENTS_FILE="${SOURCE_DEPLOY_DIR}/requirements.txt" +SOURCE_HTTPD_CONF="${SOURCE_DEPLOY_DIR}/httpd.conf" +WEB_CODE_DIR="${DEPLOY_DIR}/hed-web/hedweb" + +##### Functions + +clone_github_repos(){ +echo "Deploy dir: ${DEPLOY_DIR}" +cd "${DEPLOY_DIR}" || exit_error +echo "Cloning repo ${GIT_WEB_REPO_URL} in ${DEPLOY_DIR} using ${GIT_WEB_REPO_BRANCH} branch" +git clone "${GIT_WEB_REPO_URL}" -b "${GIT_WEB_REPO_BRANCH}" +} + +create_web_directory() +{ +echo Creating hedweb directory... +echo "Make ${CODE_DEPLOY_DIR}" +mkdir "${CODE_DEPLOY_DIR}" +echo "Copy ${BASE_CONFIG_FILE} to ${CONFIG_FILE}" +cp "${BASE_CONFIG_FILE}" "${CONFIG_FILE}" +echo "Copy ${SOURCE_WSGI_FILE} to ${CODE_DEPLOY_DIR}" +cp "${SOURCE_WSGI_FILE}" "${CODE_DEPLOY_DIR}/." +echo "Copy ${SOURCE_DOCKERFILE} to ${DEPLOY_DIR}" +cp "${SOURCE_DOCKERFILE}" "${DEPLOY_DIR}/." +echo "Copy ${SOURCE_REQUIREMENTS_FILE} to ${DEPLOY_DIR}" +cp "${SOURCE_REQUIREMENTS_FILE}" "${DEPLOY_DIR}/." +echo "Copy ${SOURCE_HTTPD_CONF} to ${DEPLOY_DIR}" +cp "${SOURCE_HTTPD_CONF}" "${DEPLOY_DIR}/." +echo "Copy ${WEB_CODE_DIR} directory to ${CODE_DEPLOY_DIR}" +cp -r "${WEB_CODE_DIR}" "${CODE_DEPLOY_DIR}" +} + +switch_to_web_directory() +{ +echo Switching to hedweb directory... +cd "${DEPLOY_DIR}" || error_exit "Cannot access $DEPLOY_DIR" +} + +build_new_container() +{ +echo "Building new container ${IMAGE_NAME} ..." +docker build -t $IMAGE_NAME . +} + +delete_old_container() +{ +echo "Deleting old container ${CONTAINER_NAME} ..." +docker rm -f $CONTAINER_NAME +} + +run_new_container() +{ +echo "Running new container $CONTAINER_NAME ..." +docker run --restart=always --name $CONTAINER_NAME -d -p 127.0.0.1:$HOST_PORT:$CONTAINER_PORT $IMAGE_NAME +} + +cleanup_directory() +{ +echo "Cleaning up directory ${GIT_HED_WEB_DIR} ..." +rm -rf "${GIT_HED_WEB_DIR}" +echo "Cleaning up ${CODE_DEPLOY_DIR}" +rm -rf "${CODE_DEPLOY_DIR}" +} + +error_exit() +{ + echo "$1" 1>&2 + exit 1 +} + +output_paths() +{ +echo "The relevant deployment information is:" +echo "Deploy directory: ${DEPLOY_DIR}" +echo "Docker image name: ${IMAGE_NAME}" +echo "Docker container name: ${CONTAINER_NAME}" +echo "Git tools repo: ${GIT_TOOLS_REPO_URL}" +echo "Git web repo: ${GIT_WEB_REPO_URL}" +echo "Git web repo branch: ${GIT_WEB_REPO_BRANCH}" +echo "Git hed web dir: ${GIT_HED_WEB_DIR}" +echo "Host port: ${HOST_PORT}" +echo "Container port: ${CONTAINER_PORT}" +echo "Local deployment directory: ${DEPLOY_DIR}" +echo "Local deploy code dir: ${DEPLOY_CODE_DIR}" +echo "Local code deployment directory: ${CODE_DEPLOY_DIR}" +echo "Configuration file: ${CONFIG_FILE}" +echo "Base configuration file: ${BASE_CONFIG_FILE}" +echo "Source WSGI file: ${SOURCE_WSGI_FILE}" +echo "Source web code directory: ${WEB_CODE_DIR}" +} + +##### Main +echo "Starting...." +output_paths +echo "....." +echo "Cleaning up directories before deploying..." +cleanup_directory +clone_github_repos || error_exit "Cannot clone repo ${GIT_WEB_REPO_URL}" +create_web_directory +switch_to_web_directory +build_new_container +delete_old_container +run_new_container +cleanup_directory diff --git a/deploy_hed_dev/httpd.conf b/deploy_hed_dev/httpd.conf new file mode 100644 index 00000000..b327968a --- /dev/null +++ b/deploy_hed_dev/httpd.conf @@ -0,0 +1,15 @@ +DocumentRoot /var/www/localhost/htdocs +ErrorLog /dev/stderr +Transferlog /dev/stdout +Listen 80 +ServerName localhost +ServerRoot /var/www +User www-data +Group www-data +LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so +LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so +LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so +LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so +#LoadModule unixd_module /usr/lib/apache2/modules/mod_unixd.so +LoadModule wsgi_module /usr/local/lib/python3.9/site-packages/mod_wsgi/server/mod_wsgi-py39.cpython-39-x86_64-linux-gnu.so +WSGIScriptAlias / /var/www/hedtools_dev/web.wsgi diff --git a/deploy_hed_dev/requirements.txt b/deploy_hed_dev/requirements.txt new file mode 100644 index 00000000..deda73e9 --- /dev/null +++ b/deploy_hed_dev/requirements.txt @@ -0,0 +1,43 @@ +attrs==21.4.0 +Pygments==2.12.0 +click==8.1.3 +coverage>=6.3.2 +defusedxml==0.7.1 +et-xmlfile==1.1.0 +Flask==2.1.2 +Flask-WTF==1.0.1 +hedtools==0.1.0 +inflect>=5.5.2 +itsdangerous==2.1.2 +jdcal==1.4.1 +Jinja2>=3.0.0 +jupyter==1.0.0 +MarkupSafe==2.1.1 +mod_wsgi==4.9.0 +numpy>=1.20.3 +numpydoc==1.3.1 +openpyxl>=3.0.9 +pandas>=1.3.5 +portalocker==2.4.0 +python-dateutil==2.8.2 +pytz>=2022.1 +semantic_version>=2.9.0 +six==1.16.0 +Sphinx>=4,<5 +SphinxExtensions==0.2.0 +sphinx_rtd_theme==1.0.0 +Werkzeug==2.1.2 +WTForms==3.0.1 +xlrd==2.0.1 +myst-parser>=0.17.0 + + + + + + + + + + + diff --git a/deploy_hed_dev/web.wsgi b/deploy_hed_dev/web.wsgi new file mode 100644 index 00000000..f8266178 --- /dev/null +++ b/deploy_hed_dev/web.wsgi @@ -0,0 +1,3 @@ +import sys +sys.path.insert(0, "/var/www/hedtools_dev") +from hedweb.runserver import app as application diff --git a/hedweb/columns.py b/hedweb/columns.py index a3f678c4..19f32d5e 100644 --- a/hedweb/columns.py +++ b/hedweb/columns.py @@ -5,7 +5,7 @@ from pandas import DataFrame, read_csv from hed.errors import HedFileError -from hed.tools import BidsTabularSummary +from hed.tools import TabularSummary from hedweb.constants import base_constants, file_constants from hedweb.web_util import form_has_file, form_has_option @@ -88,7 +88,7 @@ def _create_columns_info(columns_file, has_column_names: True, sheet_name: None) raise HedFileError('BadFileExtension', f'File {filename} extension does not correspond to an Excel or tsv file', '') col_list = list(dataframe.columns) - col_dict = BidsTabularSummary() + col_dict = TabularSummary() col_dict.update(dataframe) col_counts = col_dict.get_number_unique() columns_info = {base_constants.COLUMNS_FILE: filename, base_constants.COLUMN_LIST: col_list, diff --git a/hedweb/constants/base_constants.py b/hedweb/constants/base_constants.py index 08afad25..ed95f1da 100644 --- a/hedweb/constants/base_constants.py +++ b/hedweb/constants/base_constants.py @@ -26,6 +26,7 @@ COMMAND_MERGE_SPREADSHEET = 'merge_spreadsheet' COMMAND_OPTION = 'command_option' COMMAND_REMAP = 'remap' +COMMAND_REMODEL = 'remodel' COMMAND_SEARCH = 'search' COMMAND_TARGET = 'command_target' COMMAND_TO_LONG = 'to_long' @@ -70,7 +71,8 @@ OUTPUT_DISPLAY_NAME = 'output_display_name' QUERY = 'query' - +REMODEL_FILE = 'remodel_file' +REMODEL_COMMANDS = 'remodel_commands' REMOVE_DEFS = 'remove_defs' REQUIRED_COLUMN_INDICES = 'required_column_indices' diff --git a/hedweb/events.py b/hedweb/events.py index 80504ac3..bcb2f2ed 100644 --- a/hedweb/events.py +++ b/hedweb/events.py @@ -5,7 +5,7 @@ from hed import schema as hedschema from hed.errors import get_printable_issue_string, HedFileError -from hed.tools import assemble_hed, BidsTabularSummary, generate_filename, generate_sidecar_entry, search_tabular +from hed.tools import assemble_hed, Dispatcher, TabularSummary, generate_filename, generate_sidecar_entry, search_tabular from hed.models import DefinitionDict, Sidecar, TabularInput from hed.validator import HedValidator from hedweb.constants import base_constants @@ -43,6 +43,12 @@ def get_events_form_input(request): f = request.files[base_constants.JSON_FILE] json_sidecar = Sidecar(files=f, name=secure_filename(f.filename)) arguments[base_constants.JSON_SIDECAR] = json_sidecar + remodel = None + if base_constants.REMODEL_FILE in request.files: + f = request.files[base_constants.REMODEL_FILE] + name = secure_filename(f.filename) + remodel = {'name': name, 'commands': json.load(f)} + arguments[base_constants.REMODEL_COMMANDS] = remodel if base_constants.EVENTS_FILE in request.files: f = request.files[base_constants.EVENTS_FILE] arguments[base_constants.EVENTS] = \ @@ -72,6 +78,7 @@ def process(arguments): raise HedFileError('BadHedSchema', "Please provide a valid HedSchema for event processing", "") events = arguments.get(base_constants.EVENTS, None) sidecar = arguments.get(base_constants.JSON_SIDECAR, None) + remodeler = arguments.get(base_constants.REMODEL_COMMANDS, None) query = arguments.get(base_constants.QUERY, None) columns_included = arguments.get(base_constants.COLUMNS_INCLUDED, None) if not events or not isinstance(events, TabularInput): @@ -86,6 +93,8 @@ def process(arguments): arguments.get(base_constants.EXPAND_DEFS, False)) elif command == base_constants.COMMAND_GENERATE_SIDECAR: results = generate_sidecar(events, arguments.get(base_constants.COLUMNS_SELECTED, None)) + elif command == base_constants.COMMAND_REMODEL: + results = remodel(hed_schema, events, sidecar, remodeler) else: raise HedFileError('UnknownEventsProcessingMethod', f'Command {command} is missing or invalid', '') return results @@ -131,7 +140,7 @@ def generate_sidecar(events, columns_selected): """ - columns_info = BidsTabularSummary.get_columns_info(events.dataframe) + columns_info = TabularSummary.get_columns_info(events.dataframe) hed_dict = {} for column_name, column_type in columns_selected.items(): if column_name not in columns_info: @@ -151,6 +160,50 @@ def generate_sidecar(events, columns_selected): 'msg': 'JSON sidecar generation from event file complete'} +def remodel(hed_schema, events, sidecar, remodeler): + """ Remodel a given events file. + + Args: + hed_schema (HedSchema, HedSchemaGroup or None): A HED schema or HED schema group. + events (EventsInput): An events input object. + sidecar (Sidecar or None): A sidecar object. + remodeler (dict): Remodeling file. + + Returns: + dict: A dictionary pointing to results or errors. + + """ + + display_name = events.name + if hed_schema: + schema_version = hed_schema.version + else: + schema_version = None + remodeler_name = remodeler['name'] + remodeler_commands = remodeler['commands'] + command_list, errors = Dispatcher.parse_commands(remodeler_commands) + if errors: + issue_str = Dispatcher.errors_to_str(errors) + file_name = generate_filename(remodeler_name, name_suffix='_command_parse_errors', extension='.txt') + return {base_constants.COMMAND: base_constants.COMMAND_REMODEL, + base_constants.COMMAND_TARGET: 'events', + 'data': issue_str, "output_display_name": file_name, + base_constants.SCHEMA_VERSION: schema_version, "msg_category": "warning", + 'msg': f"Remodeling command file for {display_name} had validation errors"} + df = events.dataframe + dispatch = Dispatcher(remodeler_commands, data_root=None, hed_versions=schema_version) + df = dispatch.prep_events(df) + for operation in dispatch.parsed_ops: + df = operation.do_op(dispatch, df, display_name, sidecar=sidecar) + df = df.fillna('n/a') + csv_string = df.to_csv(None, sep='\t', index=False, header=True) + file_name = generate_filename(display_name, name_suffix='_remodeled', extension='.tsv') + return {base_constants.COMMAND: base_constants.COMMAND_REMODEL, + base_constants.COMMAND_TARGET: 'events', 'data': csv_string, "output_display_name": file_name, + base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', + 'msg': f"Command parsing for {display_name} remodeling was successful"} + + def search(hed_schema, events, query, columns_included=None): """ Create a three-column tsv file with event number, matched string, and assembled strings for matched events. diff --git a/hedweb/services.py b/hedweb/services.py index a57a5d24..4322f163 100644 --- a/hedweb/services.py +++ b/hedweb/services.py @@ -29,6 +29,7 @@ def get_input_from_request(request): arguments = get_service_info(service_request) arguments[base_constants.SCHEMA] = get_input_schema(service_request) get_column_parameters(arguments, service_request) + get_remodeler(arguments, service_request) get_sidecar(arguments, service_request) get_input_objects(arguments, service_request) arguments[base_constants.QUERY] = service_request.get('query', None) @@ -114,6 +115,25 @@ def get_input_objects(arguments, params): arguments[base_constants.STRING_LIST] = s_list +def get_remodeler(arguments, params): + """ Update arguments with the remodeler information if any. + + Args: + arguments (dict): A dictionary with the extracted parameters that are to be processed. + params (dict): The service request dictionary extracted from the Request object. + + Updates the arguments dictionary with the sidecars. + + """ + + remodel = None + if base_constants.REMODEL_FILE in params: + f = io.StringIO(base_constants.REMODEL_FILE) + name = 'remodel_commands.json' + remodel = {'name': name, 'commands': json.load(f)} + arguments[base_constants.REMODEL_COMMANDS] = remodel + + def get_service_info(params): """ Get a dictionary with the service request command information filled in.. diff --git a/hedweb/static/resources/services.json b/hedweb/static/resources/services.json index a8539006..0314d547 100644 --- a/hedweb/static/resources/services.json +++ b/hedweb/static/resources/services.json @@ -70,6 +70,25 @@ ], "Returns": "A JSON sidecar (template) in string form or a list of errors." }, + "events_remodel": { + "Name": "events_remodel", + "Description": "Restructure and events file. Returns: remodeled events or error list.", + "Parameters": [ + "events_string", + "remodel_string", + [ + "json_list", + "json_string" + ], + [ + "schema_string", + "schema_url", + "schema_version" + ], + "expand_defs" + ], + "Returns": "A string containing the text of remodeled events file or a list of errors." + }, "sidecar_validate": { "Description": "Validate a BIDS JSON sidecar (in string form) and return errors.", "Parameters": [ @@ -232,6 +251,7 @@ "json_list": "A list of BIDS JSON sidecars as strings.", "json_string": "A JSON sidecar as a string.", "query_list": "A list of query strings for searching.", + "remodel_string": "JSON remodel commands as a string", "schema_string": "HED XML schema as a string.", "schema_url": "A URL from which a HED schema can be downloaded.", "schema_version": "Version of HED to used in processing.", diff --git a/hedweb/templates/actions.html b/hedweb/templates/actions.html index 162b99c4..61507730 100644 --- a/hedweb/templates/actions.html +++ b/hedweb/templates/actions.html @@ -1,5 +1,5 @@ {% macro create_actions(title,assemble=False,convert_schema=False,generate_sidecar=False, -extract_spreadsheet=False,merge_spreadsheet=False,to_long=False,to_short=False,validate=False) %} +extract_spreadsheet=False,merge_spreadsheet=False,remodel=False,to_long=False,to_short=False,validate=False) %}

{{ title }}

{% if validate %} @@ -73,5 +73,14 @@

{{ title }}

{% endif %} + {% if remodel %} +
+ + +
+ {% endif %} + {% endmacro %} \ No newline at end of file diff --git a/hedweb/templates/events.html b/hedweb/templates/events.html index f4abe446..cbf23144 100644 --- a/hedweb/templates/events.html +++ b/hedweb/templates/events.html @@ -1,6 +1,7 @@ {% extends "layout.html" %} {% from "schema-pulldown.html" import create_schema_pulldown %} {% from "json-input.html" import create_json_input %} +{% from "remodel-input.html" import create_remodel_input %} {% from "column-info.html" import create_column_info %} {% from "actions.html" import create_actions %} {% from "options.html" import create_options %} @@ -12,11 +13,11 @@

Process a BIDS-style event
- {{ create_actions('Pick an action:',assemble=True,generate_sidecar=True,validate=True) }} + {{ create_actions('Pick an action:',assemble=True,generate_sidecar=True,validate=True,remodel=True) }} {{ create_options('Check applicable options if any:',check_for_warnings=True,expand_defs=True) }} -

Upload BIDS-style events file:

+

Upload events file: