diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 03372c58..6f651f67 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,9 +6,11 @@ on: pull_request: branches: ["*"] +env: + BRANCH_NAME: ${{ github.event.pull_request.base.ref || github.ref_name }} + jobs: build: - strategy: matrix: platform: [ubuntu-latest] @@ -17,19 +19,33 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - uses: actions/cache@v3 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}-${{ env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' }} + - name: Install dependencies(using github hed-python) + if: ${{ env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' }} + run: | + python -m pip install --upgrade pip + pip install flake8 + pip install coverage + pip install git+https://github.com/hed-standard/hed-python/@${{env.BRANCH_NAME}} + pip install -r requirements.txt + pip install -r docs/requirements.txt + + - name: Install dependencies(using pip) + if: ${{ env.BRANCH_NAME != 'develop' && env.BRANCH_NAME != 'master' }} run: | python -m pip install --upgrade pip pip install flake8 pip install coverage - echo Using ${CI_COMMIT_BRANCH} pip install hedtools pip install -r requirements.txt pip install -r docs/requirements.txt @@ -38,19 +54,18 @@ jobs: run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --show-source --statistics --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with unittest run: | cp -f ./config_template.py ./config.py coverage run -m unittest - - name: publish-coverages - uses: paambaati/codeclimate-action@v2.7.5 - env: - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} + if: ${{ env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' }} + continue-on-error: true with: - coverageCommand: coverage xml - debug: true + coverageCommand: coverage xml + debug: true + uses: paambaati/codeclimate-action@v3.2.0 + env: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID_WEB }} \ No newline at end of file diff --git a/README.md b/README.md index c834827c..b5a430c9 100644 --- a/README.md +++ b/README.md @@ -72,4 +72,30 @@ to your `deploy_hed` directory. ``` The `deploy.sh` script will download the latest versions of the `hed-python` -and the `hed-web` repositories and deploy. \ No newline at end of file +and the `hed-web` repositories and deploy. + +### Branches and versions + +The web tools are built on the `hedtools` package housed in the `hed-python` +GitHub repository. +The tools are related to the `hed-specification` and `hed-schemas` repositories. +The branches correspond as follows: + +| Branch | Meaning | Synchronized with | +| ------ | -------- | ------------------ | +| stable | Tagged as a released version - will not change. | `stable@hed-python`
`stable@hed-specification`
`stable@hed-examples` | +| master | Most recent usable version.
[https://hedtools/edu/hed](https://hedtools/edu/hed). | `master@hed-python`
`master@hed-specification`
`main@hed-examples` | +| develop | Experimental and evolving.
[https://hedtools/edu/hed_dev](https://hedtools/edu/hed_dev). | `develop@hed-python`
`develop@hed-specification`
`develop@hed-examples` | + +As features are integrated, they first appear in the `develop` branches of the +repositories. +The `develop` branches of the repositories will be kept in sync as much as possible +If an interface change in `hed-python` triggers a change in `hed-web` or `hed-examples`, +every effort will be made to get the `master`/`main` branches of the respective repositories in +sync. +The `stable` version refers to the last officially released version. +It generally refers to a version without the latest features. + +API documentation is generated on ReadTheDocs when a new version is +pushed on any of the three branches. For example, the API documentation for the +`latest` branch can be found on [hed-python.readthedocs.io/en/latest/](hed-python.readthedocs.io/en/latest/). diff --git a/deploy_hed/Dockerfile b/deploy_hed/Dockerfile index 534b55a2..c26df9c9 100644 --- a/deploy_hed/Dockerfile +++ b/deploy_hed/Dockerfile @@ -10,7 +10,7 @@ apache2 \ apache2-dev && \ pip3 install --upgrade pip && \ pip3 install --no-cache-dir -r requirements.txt && \ -pip3 install hedtools && \ +pip3 install git+https://github.com/hed-standard/hed-python/@master && \ mkdir -p /var/www/localhost/htdocs && \ cp /etc/mime.types /var/www/mime.types && \ mkdir -p /var/log/hedtools && \ diff --git a/deploy_hed/base_config.py b/deploy_hed/base_config.py index 5b23de10..03f12c4a 100644 --- a/deploy_hed/base_config.py +++ b/deploy_hed/base_config.py @@ -26,6 +26,8 @@ class Config(object): class DevelopmentConfig(Config): DEBUG = False TESTING = False + URL_PREFIX = '/heddev' + STATIC_URL_PATH = '/heddev/hedweb/static' class ProductionConfig(Config): diff --git a/deploy_hed/deploy.sh b/deploy_hed/deploy.sh index 950afc7f..75c7027f 100644 --- a/deploy_hed/deploy.sh +++ b/deploy_hed/deploy.sh @@ -77,7 +77,7 @@ docker rm -f $CONTAINER_NAME run_new_container() { echo "Running new container $CONTAINER_NAME ..." -docker run --restart=always --name $CONTAINER_NAME -d -p 127.0.0.1:$HOST_PORT:$CONTAINER_PORT $IMAGE_NAME +docker run --restart=always --name $CONTAINER_NAME -d -p 127.0.0.1:$HOST_PORT:$CONTAINER_PORT $IMAGE_NAME --log-opt max-size=50m } cleanup_directory() diff --git a/deploy_hed/httpd.conf b/deploy_hed/httpd.conf index 1b358209..85573f99 100644 --- a/deploy_hed/httpd.conf +++ b/deploy_hed/httpd.conf @@ -13,3 +13,4 @@ LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so #LoadModule unixd_module /usr/lib/apache2/modules/mod_unixd.so LoadModule wsgi_module /usr/local/lib/python3.9/site-packages/mod_wsgi/server/mod_wsgi-py39.cpython-39-x86_64-linux-gnu.so WSGIScriptAlias / /var/www/hedtools/web.wsgi +WSGIApplicationGroup %{GLOBAL} diff --git a/deploy_hed_dev/Dockerfile b/deploy_hed_dev/Dockerfile new file mode 100644 index 00000000..a18c6943 --- /dev/null +++ b/deploy_hed_dev/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.9-slim-buster +COPY requirements.txt /root/ +WORKDIR /root +RUN apt-get update && apt-get install -y gcc \ +git \ +musl-dev \ +openrc \ +libxslt-dev \ +libxml2-dev \ +apache2 \ +apache2-dev && \ +pip3 install --upgrade pip && \ +pip3 install --no-cache-dir -r requirements.txt && \ +pip3 install git+https://github.com/hed-standard/hed-python/@develop && \ +mkdir -p /var/www/localhost/htdocs && \ +cp /etc/mime.types /var/www/mime.types && \ +mkdir -p /var/log/hedtools && \ +chown -R www-data:www-data /var/log/hedtools && \ +mkdir -p /var/cache/schema_cache && \ +chown -R www-data:www-data /var/cache/schema_cache +COPY httpd.conf /etc/apache2/apache2.conf +COPY ./hedtools /var/www/hedtools/ +COPY ./hedtools/hedweb /var/www/hedtools/hedweb/ +ENTRYPOINT /usr/sbin/apache2 -D FOREGROUND -f /etc/apache2/apache2.conf +ENV HEDTOOLS_CONFIG_CLASS=config.ProductionConfig diff --git a/deploy_hed_dev/base_config.py b/deploy_hed_dev/base_config.py new file mode 100644 index 00000000..5bde8029 --- /dev/null +++ b/deploy_hed_dev/base_config.py @@ -0,0 +1,45 @@ +""" +This module contains the configurations for the HEDTools application. +""" + +import os +import tempfile + + +class Config(object): + LOG_DIRECTORY = '/var/log/hedtools' + LOG_FILE = os.path.join(LOG_DIRECTORY, 'error.log') + if not os.path.exists('/var/log/hedtools/tmp.txt'): + f = open('/var/log/hedtools/tmp.txt', 'w+') + f.write(str(os.urandom(24))) + f.close() + f = open('/var/log/hedtools/tmp.txt', 'r') + SECRET_KEY = f.read() # os.getenv('SECRET_KEY') # os.urandom(24) + f.close() + STATIC_URL_PATH = None + STATIC_URL_PATH_ATTRIBUTE_NAME = 'STATIC_URL_PATH' + UPLOAD_FOLDER = os.path.join(tempfile.gettempdir(), 'hedtools_uploads') + URL_PREFIX = None + HED_CACHE_FOLDER = '/var/cache/schema_cache' + + +class DevelopmentConfig(Config): + DEBUG = False + TESTING = False + + +class ProductionConfig(Config): + DEBUG = False + TESTING = False + URL_PREFIX = '/hed_dev' + STATIC_URL_PATH = '/hed_dev/hedweb/static' + + +class TestConfig(Config): + DEBUG = False + TESTING = True + + +class DebugConfig(Config): + DEBUG = True + TESTING = False diff --git a/deploy_hed_dev/deploy.sh b/deploy_hed_dev/deploy.sh new file mode 100644 index 00000000..61c216dc --- /dev/null +++ b/deploy_hed_dev/deploy.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# deploy.sh - A script used to _build and deploy a docker container for the HEDTools online validator + +if [ $# -eq 0 ]; then + BRANCH="master" +else + BRANCH="$1" +fi +##### Constants + +DEPLOY_DIR=${PWD} +IMAGE_NAME="hedtools_dev:latest" +CONTAINER_NAME="hedtools_dev" +GIT_WEB_REPO_URL="https://github.com/hed-standard/hed-web" +GIT_HED_WEB_DIR="${DEPLOY_DIR}/hed-web" +GIT_WEB_REPO_BRANCH=${BRANCH} +HOST_PORT=33004 +CONTAINER_PORT=80 + +CODE_DEPLOY_DIR="${DEPLOY_DIR}/hedtools" +SOURCE_DEPLOY_DIR="${DEPLOY_DIR}/hed-web/deploy_hed_dev" +BASE_CONFIG_FILE="${SOURCE_DEPLOY_DIR}/base_config.py" +CONFIG_FILE="${CODE_DEPLOY_DIR}/config.py" +SOURCE_WSGI_FILE="${SOURCE_DEPLOY_DIR}/web.wsgi" +SOURCE_DOCKERFILE="${SOURCE_DEPLOY_DIR}/Dockerfile" +SOURCE_REQUIREMENTS_FILE="${SOURCE_DEPLOY_DIR}/requirements.txt" +SOURCE_HTTPD_CONF="${SOURCE_DEPLOY_DIR}/httpd.conf" +WEB_CODE_DIR="${DEPLOY_DIR}/hed-web/hedweb" + +##### Functions + +clone_github_repos(){ +echo "Deploy dir: ${DEPLOY_DIR}" +cd "${DEPLOY_DIR}" || exit_error +echo "Cloning repo ${GIT_WEB_REPO_URL} in ${DEPLOY_DIR} using ${GIT_WEB_REPO_BRANCH} branch" +git clone "${GIT_WEB_REPO_URL}" -b "${GIT_WEB_REPO_BRANCH}" +} + +create_web_directory() +{ +echo Creating hedweb directory... +echo "Make ${CODE_DEPLOY_DIR}" +mkdir "${CODE_DEPLOY_DIR}" +echo "Copy ${BASE_CONFIG_FILE} to ${CONFIG_FILE}" +cp "${BASE_CONFIG_FILE}" "${CONFIG_FILE}" +echo "Copy ${SOURCE_WSGI_FILE} to ${CODE_DEPLOY_DIR}" +cp "${SOURCE_WSGI_FILE}" "${CODE_DEPLOY_DIR}/." +echo "Copy ${SOURCE_DOCKERFILE} to ${DEPLOY_DIR}" +cp "${SOURCE_DOCKERFILE}" "${DEPLOY_DIR}/." +echo "Copy ${SOURCE_REQUIREMENTS_FILE} to ${DEPLOY_DIR}" +cp "${SOURCE_REQUIREMENTS_FILE}" "${DEPLOY_DIR}/." +echo "Copy ${SOURCE_HTTPD_CONF} to ${DEPLOY_DIR}" +cp "${SOURCE_HTTPD_CONF}" "${DEPLOY_DIR}/." +echo "Copy ${WEB_CODE_DIR} directory to ${CODE_DEPLOY_DIR}" +cp -r "${WEB_CODE_DIR}" "${CODE_DEPLOY_DIR}" +} + +switch_to_web_directory() +{ +echo Switching to hedweb directory... +cd "${DEPLOY_DIR}" || error_exit "Cannot access $DEPLOY_DIR" +} + +build_new_container() +{ +echo "Building new container ${IMAGE_NAME} ..." +docker build -t $IMAGE_NAME . +} + +delete_old_container() +{ +echo "Deleting old container ${CONTAINER_NAME} ..." +docker rm -f $CONTAINER_NAME +} + +run_new_container() +{ +echo "Running new container $CONTAINER_NAME ..." +docker run --restart=always --name $CONTAINER_NAME -d -p 127.0.0.1:$HOST_PORT:$CONTAINER_PORT $IMAGE_NAME --log-opt max-size=50m +} + +cleanup_directory() +{ +echo "Cleaning up directory ${GIT_HED_WEB_DIR} ..." +rm -rf "${GIT_HED_WEB_DIR}" +echo "Cleaning up ${CODE_DEPLOY_DIR}" +rm -rf "${CODE_DEPLOY_DIR}" +} + +error_exit() +{ + echo "$1" 1>&2 + exit 1 +} + +output_paths() +{ +echo "The relevant deployment information is:" +echo "Deploy directory: ${DEPLOY_DIR}" +echo "Docker image name: ${IMAGE_NAME}" +echo "Docker container name: ${CONTAINER_NAME}" +echo "Git tools repo: ${GIT_TOOLS_REPO_URL}" +echo "Git web repo: ${GIT_WEB_REPO_URL}" +echo "Git web repo branch: ${GIT_WEB_REPO_BRANCH}" +echo "Git hed web dir: ${GIT_HED_WEB_DIR}" +echo "Host port: ${HOST_PORT}" +echo "Container port: ${CONTAINER_PORT}" +echo "Local deployment directory: ${DEPLOY_DIR}" +echo "Local deploy code dir: ${DEPLOY_CODE_DIR}" +echo "Local code deployment directory: ${CODE_DEPLOY_DIR}" +echo "Configuration file: ${CONFIG_FILE}" +echo "Base configuration file: ${BASE_CONFIG_FILE}" +echo "Source WSGI file: ${SOURCE_WSGI_FILE}" +echo "Source web code directory: ${WEB_CODE_DIR}" +} + +##### Main +echo "Starting...." +output_paths +echo "....." +echo "Cleaning up directories before deploying..." +cleanup_directory +clone_github_repos || error_exit "Cannot clone repo ${GIT_WEB_REPO_URL}" +create_web_directory +switch_to_web_directory +build_new_container +delete_old_container +run_new_container +cleanup_directory diff --git a/deploy_hed_dev/httpd.conf b/deploy_hed_dev/httpd.conf new file mode 100644 index 00000000..85573f99 --- /dev/null +++ b/deploy_hed_dev/httpd.conf @@ -0,0 +1,16 @@ +DocumentRoot /var/www/localhost/htdocs +ErrorLog /dev/stderr +Transferlog /dev/stdout +Listen 80 +ServerName localhost +ServerRoot /var/www +User www-data +Group www-data +LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so +LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so +LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so +LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so +#LoadModule unixd_module /usr/lib/apache2/modules/mod_unixd.so +LoadModule wsgi_module /usr/local/lib/python3.9/site-packages/mod_wsgi/server/mod_wsgi-py39.cpython-39-x86_64-linux-gnu.so +WSGIScriptAlias / /var/www/hedtools/web.wsgi +WSGIApplicationGroup %{GLOBAL} diff --git a/deploy_hed_dev/requirements.txt b/deploy_hed_dev/requirements.txt new file mode 100644 index 00000000..b60d7402 --- /dev/null +++ b/deploy_hed_dev/requirements.txt @@ -0,0 +1,42 @@ +attrs==21.4.0 +Pygments==2.12.0 +click==8.1.3 +coverage>=6.3.2 +defusedxml==0.7.1 +et-xmlfile==1.1.0 +Flask==2.1.2 +Flask-WTF==1.0.1 +inflect>=5.5.2 +itsdangerous==2.1.2 +jdcal==1.4.1 +Jinja2>=3.0.0 +jupyter==1.0.0 +MarkupSafe==2.1.1 +mod_wsgi==4.9.0 +numpy>=1.20.3 +numpydoc==1.3.1 +openpyxl>=3.0.9 +pandas>=1.3.5 +portalocker==2.4.0 +python-dateutil==2.8.2 +pytz>=2022.1 +semantic_version>=2.9.0 +six==1.16.0 +Sphinx>=4,<5 +SphinxExtensions==0.2.0 +sphinx_rtd_theme==1.0.0 +Werkzeug==2.1.2 +WTForms==3.0.1 +xlrd==2.0.1 +myst-parser>=0.17.0 + + + + + + + + + + + diff --git a/deploy_hed_dev/web.wsgi b/deploy_hed_dev/web.wsgi new file mode 100644 index 00000000..13923de3 --- /dev/null +++ b/deploy_hed_dev/web.wsgi @@ -0,0 +1,3 @@ +import sys +sys.path.insert(0, "/var/www/hedtools") +from hedweb.runserver import app as application diff --git a/hedweb/columns.py b/hedweb/columns.py index a3f678c4..162d5c83 100644 --- a/hedweb/columns.py +++ b/hedweb/columns.py @@ -5,7 +5,7 @@ from pandas import DataFrame, read_csv from hed.errors import HedFileError -from hed.tools import BidsTabularSummary +from hed.tools.analysis.tabular_summary import TabularSummary from hedweb.constants import base_constants, file_constants from hedweb.web_util import form_has_file, form_has_option @@ -88,7 +88,7 @@ def _create_columns_info(columns_file, has_column_names: True, sheet_name: None) raise HedFileError('BadFileExtension', f'File {filename} extension does not correspond to an Excel or tsv file', '') col_list = list(dataframe.columns) - col_dict = BidsTabularSummary() + col_dict = TabularSummary() col_dict.update(dataframe) col_counts = col_dict.get_number_unique() columns_info = {base_constants.COLUMNS_FILE: filename, base_constants.COLUMN_LIST: col_list, diff --git a/hedweb/constants/base_constants.py b/hedweb/constants/base_constants.py index 08afad25..1beacc5c 100644 --- a/hedweb/constants/base_constants.py +++ b/hedweb/constants/base_constants.py @@ -26,6 +26,7 @@ COMMAND_MERGE_SPREADSHEET = 'merge_spreadsheet' COMMAND_OPTION = 'command_option' COMMAND_REMAP = 'remap' +COMMAND_REMODEL = 'remodel' COMMAND_SEARCH = 'search' COMMAND_TARGET = 'command_target' COMMAND_TO_LONG = 'to_long' @@ -46,6 +47,7 @@ EXPAND_DEFS = 'expand_defs' +FILE_LIST = 'file_list' FORMAT_OPTION = 'format_option' FORMAT_TO_EXCEL = 'to_excel' FORMAT_TO_JSON = 'to_json' @@ -56,21 +58,19 @@ INCLUDE_DEFINITION_TAGS = 'include_definition_tags' INCLUDE_DESCRIPTION_TAGS = 'include_description_tags' +INCLUDE_SUMMARIES = 'include_summaries' ISSUE_STRING = 'issue_string' -JSON_DISPLAY_NAME = 'json_display_name' -JSON_FILE = 'json_file' -JSON_LIST = 'json_list' -JSON_PATH = 'json_path' -JSON_SIDECAR = 'json_sidecar' -JSON_SIDECARS = 'json_sidecars' -JSON_STRING = 'json_string' +MSG = 'msg' +MSG_CATEGORY = 'msg_category' OTHER_VERSION_OPTION = 'Other' OUTPUT_DISPLAY_NAME = 'output_display_name' QUERY = 'query' - +REMODEL_FILE = 'remodel_file' +REMODEL_OPERATIONS = 'remodel_operations' +REMODEL_STRING = 'remodel_string' REMOVE_DEFS = 'remove_defs' REQUIRED_COLUMN_INDICES = 'required_column_indices' @@ -95,10 +95,16 @@ SCHEMA_URL_OPTION = 'schema_url_option' SCHEMA_VERSION = 'schema_version' SCHEMA_VERSION_LIST = 'schema_version_list' +SCHEMA_VERSION_STRING = 'schema_version_string' SERVICE = 'service' SERVICE_PARAMETERS = 'service_parameters' +SIDECAR = 'sidecar' +SIDECAR_DISPLAY_NAME = 'sidecar_display_name' +SIDECAR_FILE = 'sidecar_file' +SIDECAR_PATH = 'sidecar_path' +SIDECAR_STRING = 'sidecar_string' SPREADSHEET = 'spreadsheet' SPREADSHEET_DISPLAY_NAME = 'spreadsheet_display_name' @@ -119,7 +125,7 @@ WORKSHEET_NAMES = 'worksheet_names' WORKSHEET_SELECT = 'worksheet_select' WORKSHEET_SELECTED = 'worksheet_selected' - +ZIP_NAME = 'zip_name' # Type constants BOOLEAN = 'boolean' diff --git a/hedweb/events.py b/hedweb/events.py index d1a9b491..301ff2d3 100644 --- a/hedweb/events.py +++ b/hedweb/events.py @@ -3,15 +3,18 @@ from werkzeug.utils import secure_filename import pandas as pd -from hed.models import Sidecar, TabularInput from hed import schema as hedschema from hed.errors import get_printable_issue_string, HedFileError +from hed.models import DefinitionDict, Sidecar, TabularInput +from hed.tools.util.io_util import generate_filename +from hed.tools.remodeling.dispatcher import Dispatcher +from hed.tools.analysis.tabular_summary import TabularSummary +from hed.tools.analysis.annotation_util import generate_sidecar_entry +from hed.tools.analysis.analysis_util import search_tabular, assemble_hed from hed.validator import HedValidator from hedweb.constants import base_constants from hedweb.columns import create_column_selections, create_columns_included -from hed.util import generate_filename -from hed.tools import BidsTabularSummary, assemble_hed, generate_sidecar_entry, search_tabular -from hedweb.web_util import form_has_option, get_hed_schema_from_pull_down +from hedweb.web_util import filter_issues, form_has_option, get_hed_schema_from_pull_down app_config = current_app.config @@ -32,6 +35,7 @@ def get_events_form_input(request): base_constants.COMMAND: request.form.get(base_constants.COMMAND_OPTION, ''), base_constants.CHECK_FOR_WARNINGS: form_has_option(request, base_constants.CHECK_FOR_WARNINGS, 'on'), base_constants.EXPAND_DEFS: form_has_option(request, base_constants.EXPAND_DEFS, 'on'), + base_constants.INCLUDE_SUMMARIES: form_has_option(request, base_constants.INCLUDE_SUMMARIES, 'on'), base_constants.COLUMNS_SELECTED: create_column_selections(request.form), base_constants.COLUMNS_INCLUDED: create_columns_included(request.form) } @@ -39,15 +43,22 @@ def get_events_form_input(request): arguments[base_constants.COLUMNS_INCLUDED] = ['onset'] # TODO add user interface option to choose columns. if arguments[base_constants.COMMAND] != base_constants.COMMAND_GENERATE_SIDECAR: arguments[base_constants.SCHEMA] = get_hed_schema_from_pull_down(request) - json_sidecar = None - if base_constants.JSON_FILE in request.files: - f = request.files[base_constants.JSON_FILE] - json_sidecar = Sidecar(files=f, name=secure_filename(f.filename)) - arguments[base_constants.JSON_SIDECAR] = json_sidecar + sidecar = None + if base_constants.SIDECAR_FILE in request.files: + f = request.files[base_constants.SIDECAR_FILE] + sidecar = Sidecar(files=f, name=secure_filename(f.filename)) + arguments[base_constants.SIDECAR] = sidecar + remodel_operations = None + if arguments[base_constants.COMMAND] == base_constants.COMMAND_REMODEL and \ + base_constants.REMODEL_FILE in request.files: + f = request.files[base_constants.REMODEL_FILE] + name = secure_filename(f.filename) + remodel_operations = {'name': name, 'operations': json.load(f)} + arguments[base_constants.REMODEL_OPERATIONS] = remodel_operations if base_constants.EVENTS_FILE in request.files: f = request.files[base_constants.EVENTS_FILE] arguments[base_constants.EVENTS] = \ - TabularInput(file=f, sidecar=arguments.get(base_constants.JSON_SIDECAR, None), + TabularInput(file=f, sidecar=arguments.get(base_constants.SIDECAR, None), name=secure_filename(f.filename)) return arguments @@ -69,10 +80,12 @@ def process(arguments): command = arguments.get(base_constants.COMMAND, None) if command == base_constants.COMMAND_GENERATE_SIDECAR: pass - elif not hed_schema or not isinstance(hed_schema, hedschema.hed_schema.HedSchema): + elif not hed_schema or not \ + isinstance(hed_schema, (hedschema.hed_schema.HedSchema, hedschema.hed_schema_group.HedSchemaGroup)): raise HedFileError('BadHedSchema', "Please provide a valid HedSchema for event processing", "") events = arguments.get(base_constants.EVENTS, None) - sidecar = arguments.get(base_constants.JSON_SIDECAR, None) + sidecar = arguments.get(base_constants.SIDECAR, None) + remodel_operations = arguments.get(base_constants.REMODEL_OPERATIONS, None) query = arguments.get(base_constants.QUERY, None) columns_included = arguments.get(base_constants.COLUMNS_INCLUDED, None) if not events or not isinstance(events, TabularInput): @@ -87,6 +100,9 @@ def process(arguments): arguments.get(base_constants.EXPAND_DEFS, False)) elif command == base_constants.COMMAND_GENERATE_SIDECAR: results = generate_sidecar(events, arguments.get(base_constants.COLUMNS_SELECTED, None)) + elif command == base_constants.COMMAND_REMODEL: + results = remodel(hed_schema, events, sidecar, remodel_operations, + include_summaries=arguments.get(base_constants.INCLUDE_SUMMARIES, False)) else: raise HedFileError('UnknownEventsProcessingMethod', f'Command {command} is missing or invalid', '') return results @@ -106,18 +122,18 @@ def assemble(hed_schema, events, columns_included=None, expand_defs=True): """ - schema_version = hed_schema.version results = validate(hed_schema, events) if results['data']: return results df, defs = assemble_hed(events, columns_included=columns_included, expand_defs=expand_defs) csv_string = df.to_csv(None, sep='\t', index=False, header=True) display_name = events.name - file_name = generate_filename(display_name, name_suffix='_expanded', extension='.tsv') + file_name = generate_filename(display_name, name_suffix='_expanded', extension='.tsv', append_datetime=True) return {base_constants.COMMAND: base_constants.COMMAND_ASSEMBLE, base_constants.COMMAND_TARGET: 'events', - 'data': csv_string, 'output_display_name': file_name, 'definitions': defs, - 'schema_version': schema_version, 'msg_category': 'success', 'msg': 'Events file successfully expanded'} + 'data': csv_string, 'output_display_name': file_name, 'definitions': DefinitionDict.get_as_strings(defs), + 'schema_version': hed_schema.get_formatted_version(as_string=True), + 'msg_category': 'success', 'msg': 'Events file successfully expanded'} def generate_sidecar(events, columns_selected): @@ -132,7 +148,7 @@ def generate_sidecar(events, columns_selected): """ - columns_info = BidsTabularSummary.get_columns_info(events.dataframe) + columns_info = TabularSummary.get_columns_info(events.dataframe) hed_dict = {} for column_name, column_type in columns_selected.items(): if column_name not in columns_info: @@ -144,7 +160,7 @@ def generate_sidecar(events, columns_selected): hed_dict[column_name] = generate_sidecar_entry(column_name, column_values=column_values) display_name = events.name - file_name = generate_filename(display_name, name_suffix='_generated', extension='.json') + file_name = generate_filename(display_name, name_suffix='_generated', extension='.json', append_datetime=True) return {base_constants.COMMAND: base_constants.COMMAND_GENERATE_SIDECAR, base_constants.COMMAND_TARGET: 'events', 'data': json.dumps(hed_dict, indent=4), @@ -152,6 +168,61 @@ def generate_sidecar(events, columns_selected): 'msg': 'JSON sidecar generation from event file complete'} +def remodel(hed_schema, events, sidecar, remodel_operations, include_summaries=True): + """ Remodel a given events file. + + Args: + hed_schema (HedSchema, HedSchemaGroup or None): A HED schema or HED schema group. + events (EventsInput): An events input object. + sidecar (Sidecar or None): A sidecar object. + remodel_operations (dict): A dictionary with the name and list of operations in the remodeling file. + include_summaries (bool): If true and summaries exist, package event file and summaries in a zip file. + + Returns: + dict: A dictionary pointing to results or errors. + + """ + + display_name = events.name + remodel_name = remodel_operations['name'] + operations = remodel_operations['operations'] + operations_list, errors = Dispatcher.parse_operations(operations) + if errors: + issue_str = Dispatcher.errors_to_str(errors) + file_name = generate_filename(remodel_name, name_suffix='_operation_parse_errors', + extension='.txt', append_datetime=True) + return {base_constants.COMMAND: base_constants.COMMAND_REMODEL, + base_constants.COMMAND_TARGET: 'events', + 'data': issue_str, 'output_display_name': file_name, + 'msg_category': "warning", + 'msg': f"Remodeling operation list for {display_name} had validation errors"} + df = events.dataframe + dispatch = Dispatcher(operations, data_root=None, hed_versions=hed_schema) + + for operation in dispatch.parsed_ops: + df = dispatch.prep_data(df) + df = operation.do_op(dispatch, df, display_name, sidecar=sidecar) + df = dispatch.post_proc_data(df) + data = df.to_csv(None, sep='\t', index=False, header=True) + name_suffix = f"_remodeled_by_{remodel_name}" + file_name = generate_filename(display_name, name_suffix=name_suffix, extension='.tsv', append_datetime=True) + output_name = file_name + response = {base_constants.COMMAND: base_constants.COMMAND_REMODEL, + base_constants.COMMAND_TARGET: 'events', 'data': '', "output_display_name": output_name, + base_constants.SCHEMA_VERSION: hedschema.get_schema_versions(hed_schema, as_string=True), + base_constants.MSG_CATEGORY: 'success', + base_constants.MSG: f"Command parsing for {display_name} remodeling was successful"} + if dispatch.context_dict and include_summaries: + file_list = dispatch.get_summaries() + file_list.append({'file_name': output_name, 'file_format': '.tsv', 'file_type': 'tabular', 'content': data}) + response[base_constants.FILE_LIST] = file_list + response[base_constants.ZIP_NAME] = generate_filename(display_name, name_suffix=name_suffix + '_zip', + extension='.zip', append_datetime=True) + else: + response['data'] = data + return response + + def search(hed_schema, events, query, columns_included=None): """ Create a three-column tsv file with event number, matched string, and assembled strings for matched events. @@ -165,7 +236,7 @@ def search(hed_schema, events, query, columns_included=None): dict: A dictionary pointing to results or errors. """ - schema_version = hed_schema.version + results = validate(hed_schema, events) if results['data']: return results @@ -181,11 +252,12 @@ def search(hed_schema, events, query, columns_included=None): csv_string = '' msg = f"Events file has no events satisfying the query {query}." display_name = events.name - file_name = generate_filename(display_name, name_suffix='_query', extension='.tsv') + file_name = generate_filename(display_name, name_suffix='_query', extension='.tsv', append_datetime=True) return {base_constants.COMMAND: base_constants.COMMAND_SEARCH, base_constants.COMMAND_TARGET: 'events', 'data': csv_string, 'output_display_name': file_name, - 'schema_version': schema_version, 'msg_category': 'success', 'msg': msg} + 'schema_version': hed_schema.get_formatted_version(as_string=True), + base_constants.MSG_CATEGORY: 'success', base_constants.MSG: msg} def validate(hed_schema, events, sidecar=None, check_for_warnings=False): @@ -202,31 +274,36 @@ def validate(hed_schema, events, sidecar=None, check_for_warnings=False): """ - schema_version = hed_schema.version display_name = events.name validator = HedValidator(hed_schema=hed_schema) issue_str = '' if sidecar: issues = sidecar.validate_entries(validator, check_for_warnings=check_for_warnings) + issues = filter_issues(issues, check_for_warnings) if issues: issue_str = issue_str + get_printable_issue_string(issues, title="Sidecar definition errors:") if not issue_str: issues = events.validate_file(validator, check_for_warnings=check_for_warnings) + issues = filter_issues(issues, check_for_warnings) if issues: issue_str = get_printable_issue_string(issues, title="Event file errors:") if issue_str: - file_name = generate_filename(display_name, name_suffix='_validation_errors', extension='.txt') - return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, - base_constants.COMMAND_TARGET: 'events', - 'data': issue_str, "output_display_name": file_name, - base_constants.SCHEMA_VERSION: schema_version, "msg_category": "warning", - 'msg': f"Events file {display_name} had validation errors"} + data = issue_str + file_name = generate_filename(display_name, name_suffix='_validation_errors', + extension='.txt', append_datetime=True) + category = 'warning' + msg = f"Events file {display_name} had validation errors" else: - return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, - base_constants.COMMAND_TARGET: 'sidecar', 'data': '', - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', - 'msg': f"Events file {display_name} had no validation errors"} + data = '' + file_name = display_name + category = 'success' + msg = f"Events file {display_name} had validation errors" + + return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, base_constants.COMMAND_TARGET: 'events', + 'data': data, "output_display_name": file_name, + base_constants.SCHEMA_VERSION: hedschema.get_schema_versions(hed_schema, as_string=True), + base_constants.MSG_CATEGORY: category, base_constants.MSG: msg} def validate_query(hed_schema, query): @@ -241,19 +318,15 @@ def validate_query(hed_schema, query): """ - schema_version = hed_schema.version if not query: - display_name = 'empty_query' - issue_str = "Empty query could not be processed." - file_name = generate_filename(display_name, name_suffix='_validation_errors', extension='.txt') - return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, - base_constants.COMMAND_TARGET: 'query', - 'data': issue_str, "output_display_name": file_name, - base_constants.SCHEMA_VERSION: schema_version, "msg_category": "warning", - 'msg': f"Query {display_name} had validation errors"} + data = "Empty query could not be processed." + category = 'warning' + msg = f"Empty query could not be processed" else: - display_name = 'Nice_query' - return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, - base_constants.COMMAND_TARGET: 'query', 'data': '', - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', - 'msg': f"Events file {display_name} had no validation errors"} + data = '' + category = 'success' + msg = f"Query had no validation errors" + + return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, base_constants.COMMAND_TARGET: 'query', + 'data': data, base_constants.SCHEMA_VERSION: hedschema.get_schema_versions(hed_schema, as_string=True), + base_constants.MSG_CATEGORY: category, base_constants.MSG: msg} diff --git a/hedweb/routes.py b/hedweb/routes.py index d01a9697..ff8b0a6c 100644 --- a/hedweb/routes.py +++ b/hedweb/routes.py @@ -13,19 +13,6 @@ app_config = current_app.config route_blueprint = Blueprint(route_constants.ROUTE_BLUEPRINT, __name__) -# with app.app_context(): -# from hedweb.routes import route_blueprint -# -# app.register_blueprint(route_blueprint, url_prefix=app.config['URL_PREFIX']) -# os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True) -# -# app.config['VERSIONS'] = get_version_dict() -# print(f"Versions: {app.config['VERSIONS']}") -# print(f"Using cache directory {app.config['HED_CACHE_FOLDER']}") -# -# hedschema.set_cache_directory(app.config['HED_CACHE_FOLDER']) -# -# hedschema.set_cache_directory(app_config[]) @route_blueprint.route(route_constants.COLUMNS_INFO_ROUTE, methods=['POST']) def columns_info_results(): @@ -101,7 +88,7 @@ def schema_version_results(): f = request.files[base_constants.SCHEMA_PATH] hed_schema = hedschema.from_string(f.stream.read(file_constants.BYTE_LIMIT).decode('ascii'), file_type=secure_filename(f.filename)) - hed_info[base_constants.SCHEMA_VERSION] = hed_schema.version + hed_info[base_constants.SCHEMA_VERSION] = hed_schema.get_formatted_version(as_string=True) return json.dumps(hed_info) except Exception as ex: return handle_error(ex) diff --git a/hedweb/schema.py b/hedweb/schema.py index d4dfa1cd..5352fc31 100644 --- a/hedweb/schema.py +++ b/hedweb/schema.py @@ -5,9 +5,9 @@ from werkzeug.utils import secure_filename from hed import schema as hedschema -from hed.errors import get_exception_issue_string, get_printable_issue_string +from hed.errors import get_printable_issue_string from hed.errors import HedFileError -from hed.util import generate_filename +from hed.tools.util.io_util import generate_filename from hedweb.web_util import form_has_file, form_has_option, form_has_url from hedweb.constants import base_constants, file_constants @@ -38,9 +38,9 @@ def get_schema(arguments): else: file_found = False except HedFileError as e: - issues = e.issues + issues.append({'code': e.args[0], 'message': e.args[1]}) if not file_found: - raise HedFileError("NoSchemaProvided", "Must provide a loadable schema", "") + raise HedFileError("SCHEMA_NOT_FOUND", "Must provide a loadable schema", "") return hed_schema, issues @@ -76,7 +76,7 @@ def get_input_from_form(request): arguments[base_constants.SCHEMA_FILE_TYPE] = basename(url_parsed.path) arguments[base_constants.SCHEMA_DISPLAY_NAME] = basename(url_parsed.path) else: - raise HedFileError("NoSchemaProvided", "Must provide a loadable schema", "") + raise HedFileError("SCHEMA_NOT_FOUND", "Must provide a loadable schema", "") return arguments @@ -96,7 +96,7 @@ def process(arguments): display_name = arguments.get('schema_display_name', 'unknown_source') hed_schema, issues = get_schema(arguments) if issues: - issue_str = get_exception_issue_string(issues, f"Schema for {display_name} had these errors") + issue_str = get_issue_string(issues, f"Schema for {display_name} had these errors") file_name = generate_filename(arguments[base_constants.SCHEMA_DISPLAY_NAME], name_suffix='schema__errors', extension='.txt') return {'command': arguments[base_constants.COMMAND], @@ -128,7 +128,6 @@ def schema_convert(hed_schema, display_name): """ - schema_version = hed_schema.version schema_format = os.path.splitext(display_name)[1] if schema_format == file_constants.SCHEMA_XML_EXTENSION: data = hed_schema.get_as_mediawiki_string() @@ -141,7 +140,8 @@ def schema_convert(hed_schema, display_name): return {'command': base_constants.COMMAND_CONVERT_SCHEMA, base_constants.COMMAND_TARGET: 'schema', 'data': data, 'output_display_name': file_name, - 'schema_version': schema_version, 'msg_category': 'success', + 'schema_version': hed_schema.get_formatted_version(as_string=True), + 'msg_category': 'success', 'msg': 'Schema was successfully converted'} @@ -157,7 +157,6 @@ def schema_validate(hed_schema, display_name): """ - schema_version = hed_schema.version issues = hed_schema.check_compliance() if issues: issue_str = get_printable_issue_string(issues, f"Schema HED 3G compliance errors for {display_name}:") @@ -165,11 +164,44 @@ def schema_validate(hed_schema, display_name): return {'command': base_constants.COMMAND_VALIDATE, base_constants.COMMAND_TARGET: 'schema', 'data': issue_str, 'output_display_name': file_name, - 'schema_version': schema_version, 'msg_category': 'warning', + 'schema_version': hed_schema.get_formatted_version(as_string=True), + 'msg_category': 'warning', 'msg': 'Schema is not HED 3G compliant'} else: return {'command': base_constants.COMMAND_VALIDATE, base_constants.COMMAND_TARGET: 'schema', 'data': '', 'output_display_name': display_name, - 'schema_version': schema_version, 'msg_category': 'success', + 'schema_version': hed_schema.get_formatted_version(as_string=True), + 'msg_category': 'success', 'msg': 'Schema had no HED-3G validation errors'} + + +def get_issue_string(issues, title=None): + """ Return a string with issues list flatted into single string, one issue per line. + + Parameters: + issues (list): A list of strings containing issues to print. + title (str or None): An optional title that will always show up first if present. + + Returns: + str: A str containing printable version of the issues or ''. + + """ + + issue_str = '' + if issues: + issue_list = [] + for issue in issues: + if isinstance(issue, str): + issue_list.append(f"ERROR: {issue}.") + else: + this_str = f"{issue['message']}" + if 'code' in issue: + this_str = f"{issue['code']}:" + this_str + if 'line_number' in issue: + this_str = this_str + f"\n\tLine number {issue['line_number']}: {issue.get('line', '')} " + issue_list.append(this_str) + issue_str += '\n' + '\n'.join(issue_list) + if title: + issue_str = title + '\n' + issue_str + return issue_str diff --git a/hedweb/services.py b/hedweb/services.py index a57a5d24..e536e977 100644 --- a/hedweb/services.py +++ b/hedweb/services.py @@ -29,6 +29,7 @@ def get_input_from_request(request): arguments = get_service_info(service_request) arguments[base_constants.SCHEMA] = get_input_schema(service_request) get_column_parameters(arguments, service_request) + get_remodel_parameters(arguments, service_request) get_sidecar(arguments, service_request) get_input_objects(arguments, service_request) arguments[base_constants.QUERY] = service_request.get('query', None) @@ -72,17 +73,18 @@ def get_sidecar(arguments, params): """ sidecar_list = [] - if base_constants.JSON_STRING in params and params[base_constants.JSON_STRING]: - sidecar_list = [params[base_constants.JSON_STRING]] - elif base_constants.JSON_LIST in params and params[base_constants.JSON_LIST]: - sidecar_list = params[base_constants.JSON_LIST] + if base_constants.SIDECAR_STRING in params and params[base_constants.SIDECAR_STRING]: + sidecar_list = params[base_constants.SIDECAR_STRING] + if not isinstance(sidecar_list, list): + sidecar_list = [sidecar_list] if sidecar_list: file_list = [] for s_string in sidecar_list: file_list.append(io.StringIO(s_string)) - arguments[base_constants.JSON_SIDECAR] = Sidecar(files=file_list, name="Merged_JSON_Sidecar") + schema = arguments.get('schema', None) + arguments[base_constants.SIDECAR] = Sidecar(files=file_list, name="Merged_Sidecar", hed_schema=schema) else: - arguments[base_constants.JSON_SIDECAR] = None + arguments[base_constants.SIDECAR] = None def get_input_objects(arguments, params): @@ -96,24 +98,41 @@ def get_input_objects(arguments, params): """ + schema = arguments.get('schema', None) if base_constants.EVENTS_STRING in params and params[base_constants.EVENTS_STRING]: arguments[base_constants.EVENTS] = \ TabularInput(file=io.StringIO(params[base_constants.EVENTS_STRING]), - sidecar=arguments.get(base_constants.JSON_SIDECAR, None), name='Events') + sidecar=arguments.get(base_constants.SIDECAR, None), name='Events', hed_schema=schema) if base_constants.SPREADSHEET_STRING in params and params[base_constants.SPREADSHEET_STRING]: tag_columns, prefix_dict = spreadsheet.get_prefix_dict(params) has_column_names = arguments.get(base_constants.HAS_COLUMN_NAMES, None) arguments[base_constants.SPREADSHEET] = \ SpreadsheetInput(file=io.StringIO(params[base_constants.SPREADSHEET_STRING]), file_type=".tsv", tag_columns=tag_columns, has_column_names=has_column_names, - column_prefix_dictionary=prefix_dict, name='spreadsheet.tsv') + column_prefix_dictionary=prefix_dict, name='spreadsheet.tsv', hed_schema=schema) if base_constants.STRING_LIST in params and params[base_constants.STRING_LIST]: s_list = [] for s in params[base_constants.STRING_LIST]: - s_list.append(HedString(s)) + s_list.append(HedString(s, hed_schema=schema)) arguments[base_constants.STRING_LIST] = s_list +def get_remodel_parameters(arguments, params): + """ Update arguments with the remodeler information if any. + + Args: + arguments (dict): A dictionary with the extracted parameters that are to be processed. + params (dict): The service request dictionary extracted from the Request object. + + Updates the arguments dictionary with the sidecars. + + """ + + if base_constants.REMODEL_STRING in params: + arguments[base_constants.REMODEL_OPERATIONS] = \ + {'name': 'remodel_commands.json', 'operations': json.loads(params[base_constants.REMODEL_STRING])} + + def get_service_info(params): """ Get a dictionary with the service request command information filled in.. @@ -163,8 +182,9 @@ def get_input_schema(parameters): schema_url = parameters[base_constants.SCHEMA_URL] the_schema = hedschema.load_schema(schema_url) elif base_constants.SCHEMA_VERSION in parameters and parameters[base_constants.SCHEMA_VERSION]: - hed_file_path = hedschema.get_path_from_hed_version(parameters[base_constants.SCHEMA_VERSION]) - the_schema = hedschema.load_schema(hed_file_path) + # hed_file_path = hedschema.get_path_from_hed_version(parameters[base_constants.SCHEMA_VERSION]) + versions = parameters[base_constants.SCHEMA_VERSION] + the_schema = hedschema.load_schema_version(versions) except HedFileError: the_schema = None diff --git a/hedweb/sidecar.py b/hedweb/sidecar.py index e61b5ecf..90aa7b01 100644 --- a/hedweb/sidecar.py +++ b/hedweb/sidecar.py @@ -9,10 +9,10 @@ from hed.errors import HedFileError, get_printable_issue_string from hed.models import SpreadsheetInput, Sidecar -from hed.tools import df_to_hed, hed_to_df, merge_hed_dict -from hed.util import generate_filename +from hed.tools.analysis.annotation_util import df_to_hed, hed_to_df, merge_hed_dict +from hed.tools.util.io_util import generate_filename from hedweb.constants import base_constants, file_constants -from hedweb.web_util import form_has_option, get_hed_schema_from_pull_down +from hedweb.web_util import form_has_option, filter_issues, get_hed_schema_from_pull_down app_config = current_app.config @@ -28,7 +28,7 @@ def get_input_from_form(request): """ - arguments = {base_constants.SCHEMA: get_hed_schema_from_pull_down(request), base_constants.JSON_SIDECAR: None, + arguments = {base_constants.SCHEMA: get_hed_schema_from_pull_down(request), base_constants.SIDECAR: None, base_constants.COMMAND: request.form.get(base_constants.COMMAND_OPTION, None), base_constants.CHECK_FOR_WARNINGS: form_has_option(request, base_constants.CHECK_FOR_WARNINGS, 'on'), @@ -38,10 +38,10 @@ def get_input_from_form(request): form_has_option(request, base_constants.INCLUDE_DESCRIPTION_TAGS, 'on'), base_constants.SPREADSHEET_TYPE: file_constants.TSV_EXTENSION, } - if base_constants.JSON_FILE in request.files: - f = request.files[base_constants.JSON_FILE] + if base_constants.SIDECAR_FILE in request.files: + f = request.files[base_constants.SIDECAR_FILE] fb = io.StringIO(f.read(file_constants.BYTE_LIMIT).decode('ascii')) - arguments[base_constants.JSON_SIDECAR] = Sidecar(files=fb, name=secure_filename(f.filename)) + arguments[base_constants.SIDECAR] = Sidecar(files=fb, name=secure_filename(f.filename)) if base_constants.SPREADSHEET_FILE in request.files and \ request.files[base_constants.SPREADSHEET_FILE].filename: filename = request.files[base_constants.SPREADSHEET_FILE].filename @@ -72,10 +72,10 @@ def process(arguments): pass elif not hed_schema or not isinstance(hed_schema, hedschema.hed_schema.HedSchema): raise HedFileError('BadHedSchema', "Please provide a valid HedSchema", "") - sidecar = arguments.get(base_constants.JSON_SIDECAR, None) + sidecar = arguments.get(base_constants.SIDECAR, None) spreadsheet = arguments.get(base_constants.SPREADSHEET, 'None') if not sidecar: - raise HedFileError('MissingJSONFile', "Please give a valid JSON file to process", "") + raise HedFileError('MissingSidecarFile', "Please give a valid JSON sidecar file to process", "") check_for_warnings = arguments.get(base_constants.CHECK_FOR_WARNINGS, False) expand_defs = arguments.get(base_constants.EXPAND_DEFS, False) include_description_tags = arguments.get(base_constants.INCLUDE_DESCRIPTION_TAGS, False) @@ -106,10 +106,6 @@ def sidecar_convert(hed_schema, sidecar, command=base_constants.COMMAND_TO_SHORT """ - schema_version = hed_schema.version - # results = sidecar_validate(hed_schema, sidecar, check_for_warnings=False) - # if results['data']: - # return results if command == base_constants.COMMAND_TO_LONG: tag_form = 'long_tag' else: @@ -125,22 +121,22 @@ def sidecar_convert(hed_schema, sidecar, command=base_constants.COMMAND_TO_SHORT # issues = ErrorHandler.filter_issues_by_severity(issues, ErrorSeverity.ERROR) display_name = sidecar.name + issues = filter_issues(issues, False) if issues: - issue_str = get_printable_issue_string(issues, f"JSON conversion for {display_name} was unsuccessful") - file_name = generate_filename(display_name, name_suffix=f"_{tag_form}_conversion_errors", extension='.txt') - return {base_constants.COMMAND: command, - base_constants.COMMAND_TARGET: 'sidecar', - 'data': issue_str, 'output_display_name': file_name, - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'warning', - 'msg': f'JSON file {display_name} had validation errors'} + data = get_printable_issue_string(issues, f"JSON conversion for {display_name} was unsuccessful") + file_name = generate_filename(display_name, name_suffix=f"_{tag_form}_conversion_errors", + extension='.txt', append_datetime=True) + category = 'warning' + msg = f'Sidecar file {display_name} had validation errors' else: - file_name = generate_filename(display_name, name_suffix=f"_{tag_form}", extension='.json') + file_name = generate_filename(display_name, name_suffix=f"_{tag_form}", extension='.json', append_datetime=True) data = sidecar.get_as_json_string() - return {base_constants.COMMAND: command, - base_constants.COMMAND_TARGET: 'sidecar', - 'data': data, 'output_display_name': file_name, - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', - 'msg': f'JSON sidecar {display_name} was successfully converted'} + category = 'success' + msg = f'Sidecar file {display_name} was successfully converted' + return {base_constants.COMMAND: command, base_constants.COMMAND_TARGET: 'sidecar', + 'data': data, 'output_display_name': file_name, + base_constants.SCHEMA_VERSION: hedschema.get_schema_versions(hed_schema, as_string=True), + 'msg_category': category, 'msg': msg} def sidecar_extract(sidecar): @@ -159,7 +155,7 @@ def sidecar_extract(sidecar): df = hed_to_df(str_sidecar) data = df.to_csv(None, sep='\t', index=False, header=True) display_name = sidecar.name - file_name = generate_filename(display_name, name_suffix='_extracted', extension='.tsv') + file_name = generate_filename(display_name, name_suffix='_extracted', extension='.tsv', append_datetime=True) return {base_constants.COMMAND: base_constants.COMMAND_EXTRACT_SPREADSHEET, base_constants.COMMAND_TARGET: 'sidecar', 'data': data, 'output_display_name': file_name, @@ -189,7 +185,8 @@ def sidecar_merge(sidecar, spreadsheet, include_description_tags=False): merge_hed_dict(sidecar_dict, hed_dict) display_name = sidecar.name data = json.dumps(sidecar_dict, indent=4) - file_name = generate_filename(display_name, name_suffix='_extracted_merged', extension='.json') + file_name = generate_filename(display_name, name_suffix='_extracted_merged', + extension='.json', append_datetime=True) return {base_constants.COMMAND: base_constants.COMMAND_EXTRACT_SPREADSHEET, base_constants.COMMAND_TARGET: 'sidecar', 'data': data, 'output_display_name': file_name, @@ -209,20 +206,22 @@ def sidecar_validate(hed_schema, sidecar, check_for_warnings=False): """ - schema_version = hed_schema.version display_name = sidecar.name validator = HedValidator(hed_schema) issues = sidecar.validate_entries(validator, check_for_warnings=check_for_warnings) if issues: - issue_str = get_printable_issue_string(issues, f"JSON dictionary {sidecar.name} validation errors") - file_name = generate_filename(display_name, name_suffix='validation_errors', extension='.txt') - return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, - base_constants.COMMAND_TARGET: 'sidecar', - 'data': issue_str, 'output_display_name': file_name, - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'warning', - 'msg': f'JSON sidecar {display_name} had validation errors'} + data = get_printable_issue_string(issues, f"JSON dictionary {sidecar.name} validation errors") + file_name = generate_filename(display_name, name_suffix='validation_errors', + extension='.txt', append_datetime=True) + category = 'warning' + msg = f'JSON sidecar {display_name} had validation errors' else: - return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, - base_constants.COMMAND_TARGET: 'sidecar', 'data': '', - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', - 'msg': f'JSON file {display_name} had no validation errors'} + data = '' + file_name = display_name + category = 'success' + msg = f'JSON file {display_name} had no validation errors' + + return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, base_constants.COMMAND_TARGET: 'sidecar', + 'data': data, 'output_display_name': file_name, + base_constants.SCHEMA_VERSION: hedschema.get_schema_versions(hed_schema, as_string=True), + base_constants.MSG_CATEGORY: category, base_constants.MSG: msg} diff --git a/hedweb/spreadsheet.py b/hedweb/spreadsheet.py index e98e299e..7158eecd 100644 --- a/hedweb/spreadsheet.py +++ b/hedweb/spreadsheet.py @@ -4,12 +4,12 @@ from hed import schema as hedschema from hed.errors import get_printable_issue_string, HedFileError from hed.models import SpreadsheetInput -from hed.util import generate_filename +from hed.tools.util.io_util import generate_filename from hed.validator import HedValidator from hedweb.constants import base_constants, file_constants from hedweb.columns import get_prefix_dict -from hedweb.web_util import form_has_option, get_hed_schema_from_pull_down +from hedweb.web_util import filter_issues, form_has_option, get_hed_schema_from_pull_down app_config = current_app.config @@ -96,7 +96,6 @@ def spreadsheet_convert(hed_schema, spreadsheet, command=base_constants.COMMAND_ """ - schema_version = hed_schema.version results = spreadsheet_validate(hed_schema, spreadsheet, check_for_warnings=check_for_warnings) if results['data']: return results @@ -111,12 +110,13 @@ def spreadsheet_convert(hed_schema, spreadsheet, command=base_constants.COMMAND_ suffix = '_to_short' spreadsheet.convert_to_short(hed_schema) - file_name = generate_filename(display_name, name_suffix=suffix, extension=display_ext) + file_name = generate_filename(display_name, name_suffix=suffix, extension=display_ext, append_datetime=True) return {base_constants.COMMAND: command, base_constants.COMMAND_TARGET: 'spreadsheet', 'data': '', base_constants.SPREADSHEET: spreadsheet, 'output_display_name': file_name, - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', - 'msg': f'Spreadsheet {display_name} converted_successfully'} + base_constants.SCHEMA_VERSION: hedschema.get_schema_versions(hed_schema, as_string=True), + base_constants.MSG_CATEGORY: 'success', + base_constants.MSG: f'Spreadsheet {display_name} converted_successfully'} def spreadsheet_validate(hed_schema, spreadsheet, check_for_warnings=False): @@ -131,20 +131,25 @@ def spreadsheet_validate(hed_schema, spreadsheet, check_for_warnings=False): dict: A dictionary containing results of validation in standard format. """ - schema_version = hed_schema.version + validator = HedValidator(hed_schema=hed_schema) issues = spreadsheet.validate_file(validator, check_for_warnings=check_for_warnings) display_name = spreadsheet.name + issues = filter_issues(issues, check_for_warnings) if issues: - issue_str = get_printable_issue_string(issues, f"Spreadsheet {display_name} validation errors") - file_name = generate_filename(display_name, name_suffix='_validation_errors', extension='.txt') - return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, - base_constants.COMMAND_TARGET: 'spreadsheet', - 'data': issue_str, "output_display_name": file_name, - base_constants.SCHEMA_VERSION: schema_version, "msg_category": "warning", - 'msg': f"Spreadsheet {display_name} had validation errors"} + data = get_printable_issue_string(issues, f"Spreadsheet {display_name} validation errors") + file_name = generate_filename(display_name, name_suffix='_validation_errors', + extension='.txt', append_datetime=True) + category = "warning" + msg = f"Spreadsheet {file_name} had validation errors" else: - return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, - base_constants.COMMAND_TARGET: 'spreadsheet', 'data': '', - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', - 'msg': f'Spreadsheet {display_name} had no validation errors'} + data = '' + file_name = display_name + category = 'success' + msg = f'Spreadsheet {display_name} had no validation errors' + + return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, + base_constants.COMMAND_TARGET: 'spreadsheet', 'data': data, + base_constants.SCHEMA_VERSION: hedschema.get_schema_versions(hed_schema, as_string=True), + "output_display_name": file_name, + base_constants.MSG_CATEGORY: category, base_constants.MSG: msg} diff --git a/hedweb/static/img/temp2.zip b/hedweb/static/img/temp2.zip new file mode 100644 index 00000000..6fe9ca67 Binary files /dev/null and b/hedweb/static/img/temp2.zip differ diff --git a/hedweb/static/resources/services.json b/hedweb/static/resources/services.json index a8539006..bccd9ecd 100644 --- a/hedweb/static/resources/services.json +++ b/hedweb/static/resources/services.json @@ -10,10 +10,7 @@ "Description": "Validate a BIDS-style event file and JSON sidecar if provided. ", "Parameters": [ "events_string", - [ - "json_list", - "json_string" - ], + "sidecar_string", [ "schema_string", "schema_url", @@ -28,10 +25,7 @@ "Description": "Search a BIDS-style event file and return list of event numbers satisfying search.", "Parameters": [ "events_string", - [ - "json_list", - "json_string" - ], + "sidecar_string", [ "schema_string", "schema_url", @@ -47,10 +41,7 @@ "Parameters": [ "events_string", "columns_included", - [ - "json_list", - "json_string" - ], + "sidecar_string", [ "schema_string", "schema_url", @@ -70,10 +61,26 @@ ], "Returns": "A JSON sidecar (template) in string form or a list of errors." }, + "events_remodel": { + "Name": "events_remodel", + "Description": "Restructure and events file. Returns: remodeled events or error list.", + "Parameters": [ + "events_string", + "remodel_string", + "sidecar_string", + [ + "schema_string", + "schema_url", + "schema_version" + ], + "expand_defs" + ], + "Returns": "A string containing the text of remodeled events file or a list of errors." + }, "sidecar_validate": { "Description": "Validate a BIDS JSON sidecar (in string form) and return errors.", "Parameters": [ - "json_string", + "sidecar_string", [ "schema_string", "schema_url", @@ -86,7 +93,7 @@ "sidecar_to_long": { "Description": "Convert a JSON sidecar with all of its HED tags expressed in long form.", "Parameters": [ - "json_string", + "sidecar_string", [ "schema_string", "schema_url", @@ -99,7 +106,7 @@ "sidecar_to_short": { "Description": "Convert a JSON sidecar with all of its HED tags expressed in short form.", "Parameters": [ - "json_string", + "sidecar_string", [ "schema_string", "schema_url", @@ -112,14 +119,14 @@ "sidecar_extract_spreadsheet": { "Description": "Convert the HED portion of a JSON sidecar to a 4-column spreadsheet.", "Parameters": [ - "json_string" + "sidecar_string" ], "Returns": "A string containing a 4-column tab-separated value spreadsheet extracted from the JSON." }, "sidecar_merge_spreadsheet": { "Description": "Merge the information in a 4-column spreadsheet into the HED portion of a JSON sidecar.", "Parameters": [ - "json_string", + "sidecar_string", "spreadsheet_string", "include_description_tags" ], @@ -229,12 +236,12 @@ "has_column_names": "If true, interpret the first row of file as column names.", "hed_strings": "List of HED strings to be processed.", "include_description_tag": "Include the Description/XXX tag in the tag string", - "json_list": "A list of BIDS JSON sidecars as strings.", - "json_string": "A JSON sidecar as a string.", "query_list": "A list of query strings for searching.", + "remodel_string": "JSON remodel commands as a string", "schema_string": "HED XML schema as a string.", "schema_url": "A URL from which a HED schema can be downloaded.", "schema_version": "Version of HED to used in processing.", + "sidecar_string": "A JSON sidecar as a string or a list of JSON sidecar strings.", "spreadsheet_string": "A spreadsheet tsv as a string." }, "returns": { diff --git a/hedweb/strings.py b/hedweb/strings.py index 992838bd..ebe987d2 100644 --- a/hedweb/strings.py +++ b/hedweb/strings.py @@ -81,7 +81,6 @@ def convert(hed_schema, string_list, command=base_constants.COMMAND_TO_SHORT, ch """ - schema_version = hed_schema.version results = validate(hed_schema, string_list, check_for_warnings=check_for_warnings) if results['data']: return results @@ -100,12 +99,14 @@ def convert(hed_schema, string_list, command=base_constants.COMMAND_TO_SHORT, ch return {base_constants.COMMAND: command, base_constants.COMMAND_TARGET: 'strings', 'data': conversion_errors, 'additional_info': string_list, - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'warning', + base_constants.SCHEMA_VERSION: hed_schema.get_formatted_version(as_string=True), + 'msg_category': 'warning', 'msg': 'Some strings had conversion errors, results of conversion in additional_info'} else: return {base_constants.COMMAND: command, base_constants.COMMAND_TARGET: 'strings', 'data': strings, - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', + base_constants.SCHEMA_VERSION: hed_schema.get_formatted_version(as_string=True), + 'msg_category': 'success', 'msg': 'Strings converted successfully'} @@ -121,7 +122,6 @@ def validate(hed_schema, string_list, check_for_warnings=False): dict: The results in standard form. """ - schema_version = hed_schema.version hed_validator = HedValidator(hed_schema=hed_schema) validation_errors = [] @@ -132,10 +132,12 @@ def validate(hed_schema, string_list, check_for_warnings=False): if validation_errors: return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, base_constants.COMMAND_TARGET: 'strings', 'data': validation_errors, - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'warning', + base_constants.SCHEMA_VERSION: hed_schema.get_formatted_version(as_string=True), + 'msg_category': 'warning', 'msg': 'Strings had validation errors'} else: return {base_constants.COMMAND: base_constants.COMMAND_VALIDATE, base_constants.COMMAND_TARGET: 'strings', 'data': '', - base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', + base_constants.SCHEMA_VERSION: hed_schema.get_formatted_version(as_string=True), + 'msg_category': 'success', 'msg': 'Strings validated successfully...'} diff --git a/hedweb/templates/actions.html b/hedweb/templates/actions.html index 162b99c4..a7abbb09 100644 --- a/hedweb/templates/actions.html +++ b/hedweb/templates/actions.html @@ -1,5 +1,5 @@ {% macro create_actions(title,assemble=False,convert_schema=False,generate_sidecar=False, -extract_spreadsheet=False,merge_spreadsheet=False,to_long=False,to_short=False,validate=False) %} +extract_spreadsheet=False,merge_spreadsheet=False,remodel=False,to_long=False,to_short=False,validate=False) %}

{{ title }}

{% if validate %} @@ -73,5 +73,14 @@

{{ title }}

{% endif %} + {% if remodel %} +
+ + +
+ {% endif %} + {% endmacro %} \ No newline at end of file diff --git a/hedweb/templates/events.html b/hedweb/templates/events.html index f4abe446..4ebf253a 100644 --- a/hedweb/templates/events.html +++ b/hedweb/templates/events.html @@ -1,6 +1,7 @@ {% extends "layout.html" %} {% from "schema-pulldown.html" import create_schema_pulldown %} -{% from "json-input.html" import create_json_input %} +{% from "sidecar-input.html" import create_sidecar_input %} +{% from "remodel-input.html" import create_remodel_input %} {% from "column-info.html" import create_column_info %} {% from "actions.html" import create_actions %} {% from "options.html" import create_options %} @@ -12,11 +13,11 @@

Process a BIDS-style event
- {{ create_actions('Pick an action:',assemble=True,generate_sidecar=True,validate=True) }} + {{ create_actions('Pick an action:',assemble=True,generate_sidecar=True,validate=True,remodel=True) }} - {{ create_options('Check applicable options if any:',check_for_warnings=True,expand_defs=True) }} + {{ create_options('Check applicable options if any:',check_for_warnings=True,expand_defs=True,include_summaries=True) }} -

Upload BIDS-style events file:

+

Upload events file: