hed-standard · VisLab · Oct 23, 2023 · Oct 23, 2023
diff --git a/docs/source/FileRemodelingTools.md b/docs/source/FileRemodelingTools.md
@@ -236,9 +236,9 @@ The programs use a standard command-line argument list for specifying input as s
 ````{table} Summary of command-line arguments for the remodeling programs.
 | Script name | Arguments | Purpose | 
 | ----------- | -------- | ------- |
-|*run_remodel_backup* | *data_dir*<br/>*-e -\\-extensions*<br/>*-f -\\-file-suffix*<br/>*-n -\\-backup-name*<br/>*-t -\\-task-names*<br/>*-v -\\-verbose*<br/>*-w -\\-work-dir*<br/>*-x -\\-exclude-dirs*| Create a backup event files. |
-|*run_remodel* | *data_dir*<br/>*model_path*<br/>*-b -\\-bids-format*<br/>*-e -\\-extensions*<br/>*-f -\\-file-suffix*<br/>*-i -\\-individual-summaries*<br/>*-j -\\-json-sidecar*<br/>*-n -\\-backup-name*<br/>*-nb -\\-no-backup*<br/>*-ns -\\-no-summaries*<br/>*-nu -\\-no-update*<br/>*-r -\\-hed-version*<br/>*-s -\\-save-formats*<br/>*-t -\\-task-names*<br/>*-v -\\-verbose*<br/>*-w -\\-work-dir*<br/>*-x -\\-exclude-dirs* | Restructure or summarize the event files. |
-|*run_remodel_restore* | *data_dir*<br/>*-n -\\-backup-name*<br/>*-t -\\-task-names*<br/>*-v -\\-verbose*<br/>*-w -\\-work-dir*<br/> | Restore a backup of event files. |
+|*run_remodel_backup* | *data_dir*<br/>*-bd -\\-backup-dir*<br/>*-bn -\\-backup-name*<br/>*-e -\\-extensions*<br/>*-f -\\-file-suffix*<br/>*-t -\\-task-names*<br/>*-v -\\-verbose*<br/>*-x -\\-exclude-dirs*| Create a backup event files. |
+|*run_remodel* | *data_dir*<br/>*model_path*<br/>*-b -\\-bids-format*<br/>*-bd -\\-backup-dir*<br/>*-bn -\\-backup-name*<br/>*-e -\\-extensions*<br/>*-f -\\-file-suffix*<br/>*-i -\\-individual-summaries*<br/>*-j -\\-json-sidecar*<br/>*-nb -\\-no-backup*<br/>*-ns -\\-no-summaries*<br/>*-nu -\\-no-update*<br/>*-r -\\-hed-version*<br/>*-s -\\-save-formats*<br/>*-t -\\-task-names*<br/>*-v -\\-verbose*<br/>*-w -\\-work-dir*<br/>*-x -\\-exclude-dirs* | Restructure or summarize the event files. |
+|*run_remodel_restore* | *data_dir*<br/>*-bd -\\-backup-dir*<br/>*-bn -\\-backup-name*<br/>*-t -\\-task-names*<br/>*-v -\\-verbose*<br/> | Restore a backup of event files. |
 
 ````
 All the scripts have a required argument, which is the full path of the dataset root (*data_dir*).
@@ -278,6 +278,13 @@ Users are free to use either form.
 `-b`, `--bids-format`
 > If this flag present, the dataset is in BIDS format with sidecars. Tabular files and their associated sidecars are located using BIDS naming.
 
+`-bd`, `--backup-dir`
+> The path to the directory holding the backups (default: `[data_root]/derivatives/remodel/backups`).
+> Use the `-nb` option if you wish to omit the backup (in `run_remodel`).
+
+`-bn`, `--backup-name`
+> The name of the backup used for the remodeling (default: `default_back`).
+
 `-e`, `--extensions`
 > This option is followed by a list of file extension(s) of the data files to process.
 > The default is `.tsv`. Comma separated tabular files are not permitted.
@@ -298,9 +305,6 @@ Users are free to use either form.
 > This option is followed by the full path of the JSON sidecar with HED annotations to be
 > applied during the processing of HED-related remodeling operations.
 
-`-n`, `--backup-name`
-> The name of the backup used for the remodeling (default: `default_back`).
-
 `-nb`, `--no-backup`
 > If present, no backup is used. Rather operations are performed directly on the files.
 
@@ -346,7 +350,7 @@ Users are free to use either form.
 > are printed to standard output.
 
 `-w`, `--work-dir`
-> The path to the remodeling work root directory --both for backups and summaries (default: `[data_root]/derivatives/remodel`).
+> The path to the remodeling work root directory --both for summaries (default: `[data_root]/derivatives/remodel`).
 > Use the `-nb` option if you wish to omit the backup (in `run_remodel`).
 
 `-x`, `--exclude-dirs`

diff --git a/hedcode/jupyter_notebooks/remodeling/run_remodel.ipynb b/hedcode/jupyter_notebooks/remodeling/run_remodel.ipynb
diff --git a/hedcode/jupyter_notebooks/remodeling/run_remodel_backup.ipynb b/hedcode/jupyter_notebooks/remodeling/run_remodel_backup.ipynb
diff --git a/hedcode/matlab_scripts/data_cleaning/runEeglabImportEvents.m b/hedcode/matlab_scripts/data_cleaning/runEeglabImportEvents.m
@@ -1,88 +1,89 @@
 %% This imports the _events.tsv into the corresponding EEG.set file
 
 %% Set up the specifics for your dataset
-
-rootPath = '/XXX/SternbergWorkingPhaseTwo';
-setname = '';
-log_name = 'sternberg_12_import_events_log.txt';
-renameColumns = {'event_type', 'type'; 'onset', 'latency'};
-
-% rootPath = 'G:/AttentionShift/AttentionShiftWorkingPhaseTwo';
-% setname = 'Auditory Visual Attention Shift';
-% log_name = 'attention_shift_18_import_events_log.txt';
-
-% rootPath = 's:/bcit/AdvancedGuardDutyWorkingPhaseTwo';
-% setname = 'BCIT Advanced Guard Duty';
-% log_name = 'bcit_advanced_guard_duty_10_import_events_log.txt';
+filename = 'T:SummaryTests/ds004105-download/sub-01/ses-01/eeg/sub-01_ses-01_task-DriveRandomSound_run-1_events.tsv';
+eventstruct = importevent( filename, [], 1024, 'key1', 'value1', ...);
+% rootPath = '/XXX/SternbergWorkingPhaseTwo';
+% setname = '';
+% log_name = 'sternberg_12_import_events_log.txt';
+% renameColumns = {'event_type', 'type'; 'onset', 'latency'};
 % 
-% rootPath = 's:/bcit/AuditoryCueingWorkingPhaseTwo';
-% setname = 'BCIT Auditory Cueing';
-% log_name = 'bcit_auditory_cueing_10_import_events_log.txt';
-
-% rootPath = 's:/bcit/BaselineDrivingWorkingPhaseTwo';
-% setname = 'BCIT Baseline Driving';
-% log_name = 'bcit_baseline_driving_10_import_events_log.txt';
-
-% rootPath = 's:/bcit/BasicGuardDutyWorkingPhaseTwo';
-% setname = 'BCIT Basic Guard Duty';
-% log_name = 'bcit_basic_guard_duty_10_import_events_log.txt';
-
-% rootPath = 's:/bcit/CalibrationDrivingWorkingPhaseTwo';
-% setname = 'BCIT Calibration Driving';
-% log_name = 'bcit_calibration_driving_10_import_events_log.txt';
-
-% rootPath = 's:/bcit/MindWanderingWorkingPhaseTwo';
-% setname = 'BCIT Mind Wandering';
-% log_name = 'bcit_mind_wandering_10_import_events_log.txt';
-
-% rootPath = 's:/bcit/SpeedControlWorkingPhaseTwo';
-% setname = 'BCIT Speed Control';
-% log_name = 'bcit_speed_control_10_import_events_log.txt';
-
-% rootPath = 's:/bcit/TrafficComplexityWorkingPhaseTwo';
-% setname = 'BCIT Traffic Complexity';
-% log_name = 'bcit_traffic_complexity_10_import_events_log.txt';
-
-excludeDirs = {'sourcedata', 'code', 'stimuli'};
-namePrefix = '';
-nameSuffix = '_eeg';
-extensions = {'.set'};
-
-% Designate the columns that are numeric (rest are char)
-columnTypes = {'onset', 'double'; 'duration', 'double'; 'sample', 'int32'};
-
-convertLatency = true; 
-
-%% Open the log
-fid = fopen([rootPath filesep 'code/curation_logs', filesep log_name], 'w');
-fprintf(fid, 'Log of runEeglabEventsImport.m on %s\n', datetime('now'));
-
-%% Generate json file.
-fileList = getFileList(rootPath, namePrefix, nameSuffix, ...
-                           extensions, excludeDirs);
-for k = 1:length(fileList)
-   EEG = pop_loadset(fileList{k});
-   [pathName, basename, ext] = fileparts(fileList{k});
-   fprintf(fid, '%s:\n', basename);
-   eventsFile = [pathName filesep basename(1:(end-3)) 'events.tsv'];
-   eventTable = getEventTable(eventsFile, columnTypes, renameColumns);
-   fprintf(fid, '\tCreate a table from the events file\n');
-   if convertLatency
-       eventTable.('latency') = eventTable.('latency')*EEG.srate + 1;
-       fprintf(fid, '\tConvert the latency column to samples\n');
-   end
-   fprintf('%s: EEG.event has %d events and BIDS event file has %d events\n', ...
-           basename, length(EEG.event), size(eventTable,1));
-   EEG.urevent = table2struct(eventTable)';
-   fprintf(fid, '\tSet the EEG.urevent\n');
-   eventTable.('urevent') = transpose(1:size(eventTable));
-   EEG.event = table2struct(eventTable)';
-   fprintf(fid, '\tSet the EEG.event\n');
-   if ~isempty(setname)    
-      EEG.setname = [setname basename];
-      fprintf(fid, '\tSet the EEG.setname\n');
-   end
-   fprintf(fid, '\tResave the EEG.set file\n');
-   EEG = pop_saveset(EEG, 'savemode', 'resave', 'version', '7.3');
-end
-fclose(fid);
+% % rootPath = 'G:/AttentionShift/AttentionShiftWorkingPhaseTwo';
+% % setname = 'Auditory Visual Attention Shift';
+% % log_name = 'attention_shift_18_import_events_log.txt';
+% 
+% % rootPath = 's:/bcit/AdvancedGuardDutyWorkingPhaseTwo';
+% % setname = 'BCIT Advanced Guard Duty';
+% % log_name = 'bcit_advanced_guard_duty_10_import_events_log.txt';
+% % 
+% % rootPath = 's:/bcit/AuditoryCueingWorkingPhaseTwo';
+% % setname = 'BCIT Auditory Cueing';
+% % log_name = 'bcit_auditory_cueing_10_import_events_log.txt';
+% 
+% % rootPath = 's:/bcit/BaselineDrivingWorkingPhaseTwo';
+% % setname = 'BCIT Baseline Driving';
+% % log_name = 'bcit_baseline_driving_10_import_events_log.txt';
+% 
+% % rootPath = 's:/bcit/BasicGuardDutyWorkingPhaseTwo';
+% % setname = 'BCIT Basic Guard Duty';
+% % log_name = 'bcit_basic_guard_duty_10_import_events_log.txt';
+% 
+% % rootPath = 's:/bcit/CalibrationDrivingWorkingPhaseTwo';
+% % setname = 'BCIT Calibration Driving';
+% % log_name = 'bcit_calibration_driving_10_import_events_log.txt';
+% 
+% % rootPath = 's:/bcit/MindWanderingWorkingPhaseTwo';
+% % setname = 'BCIT Mind Wandering';
+% % log_name = 'bcit_mind_wandering_10_import_events_log.txt';
+% 
+% % rootPath = 's:/bcit/SpeedControlWorkingPhaseTwo';
+% % setname = 'BCIT Speed Control';
+% % log_name = 'bcit_speed_control_10_import_events_log.txt';
+% 
+% % rootPath = 's:/bcit/TrafficComplexityWorkingPhaseTwo';
+% % setname = 'BCIT Traffic Complexity';
+% % log_name = 'bcit_traffic_complexity_10_import_events_log.txt';
+% 
+% excludeDirs = {'sourcedata', 'code', 'stimuli', 'derivatives'};
+% namePrefix = '';
+% nameSuffix = '_eeg';
+% extensions = {'.set'};
+% 
+% % Designate the columns that are numeric (rest are char)
+% columnTypes = {'onset', 'double'; 'duration', 'double'; 'sample', 'int32'};
+% 
+% convertLatency = true; 
+% 
+% %% Open the log
+% fid = fopen([rootPath filesep 'code/curation_logs', filesep log_name], 'w');
+% fprintf(fid, 'Log of runEeglabEventsImport.m on %s\n', datetime('now'));
+% 
+% %% Generate json file.
+% fileList = getFileList(rootPath, namePrefix, nameSuffix, ...
+%                            extensions, excludeDirs);
+% for k = 1:length(fileList)
+%    EEG = pop_loadset(fileList{k});
+%    [pathName, basename, ext] = fileparts(fileList{k});
+%    fprintf(fid, '%s:\n', basename);
+%    eventsFile = [pathName filesep basename(1:(end-3)) 'events.tsv'];
+%    eventTable = getEventTable(eventsFile, columnTypes, renameColumns);
+%    fprintf(fid, '\tCreate a table from the events file\n');
+%    if convertLatency
+%        eventTable.('latency') = eventTable.('latency')*EEG.srate + 1;
+%        fprintf(fid, '\tConvert the latency column to samples\n');
+%    end
+%    fprintf('%s: EEG.event has %d events and BIDS event file has %d events\n', ...
+%            basename, length(EEG.event), size(eventTable,1));
+%    EEG.urevent = table2struct(eventTable)';
+%    fprintf(fid, '\tSet the EEG.urevent\n');
+%    eventTable.('urevent') = transpose(1:size(eventTable));
+%    EEG.event = table2struct(eventTable)';
+%    fprintf(fid, '\tSet the EEG.event\n');
+%    if ~isempty(setname)    
+%       EEG.setname = [setname basename];
+%       fprintf(fid, '\tSet the EEG.setname\n');
+%    end
+%    fprintf(fid, '\tResave the EEG.set file\n');
+%    EEG = pop_saveset(EEG, 'savemode', 'resave', 'version', '7.3');
+% end
+% fclose(fid);
diff --git a/hedcode/matlab_scripts/data_cleaning/runEeglabImportEventsOld.m b/hedcode/matlab_scripts/data_cleaning/runEeglabImportEventsOld.m
@@ -0,0 +1,88 @@
+%% This imports the _events.tsv into the corresponding EEG.set file
+
+%% Set up the specifics for your dataset
+
+rootPath = '/XXX/SternbergWorkingPhaseTwo';
+setname = '';
+log_name = 'sternberg_12_import_events_log.txt';
+renameColumns = {'event_type', 'type'; 'onset', 'latency'};
+
+% rootPath = 'G:/AttentionShift/AttentionShiftWorkingPhaseTwo';
+% setname = 'Auditory Visual Attention Shift';
+% log_name = 'attention_shift_18_import_events_log.txt';
+
+% rootPath = 's:/bcit/AdvancedGuardDutyWorkingPhaseTwo';
+% setname = 'BCIT Advanced Guard Duty';
+% log_name = 'bcit_advanced_guard_duty_10_import_events_log.txt';
+% 
+% rootPath = 's:/bcit/AuditoryCueingWorkingPhaseTwo';
+% setname = 'BCIT Auditory Cueing';
+% log_name = 'bcit_auditory_cueing_10_import_events_log.txt';
+
+% rootPath = 's:/bcit/BaselineDrivingWorkingPhaseTwo';
+% setname = 'BCIT Baseline Driving';
+% log_name = 'bcit_baseline_driving_10_import_events_log.txt';
+
+% rootPath = 's:/bcit/BasicGuardDutyWorkingPhaseTwo';
+% setname = 'BCIT Basic Guard Duty';
+% log_name = 'bcit_basic_guard_duty_10_import_events_log.txt';
+
+% rootPath = 's:/bcit/CalibrationDrivingWorkingPhaseTwo';
+% setname = 'BCIT Calibration Driving';
+% log_name = 'bcit_calibration_driving_10_import_events_log.txt';
+
+% rootPath = 's:/bcit/MindWanderingWorkingPhaseTwo';
+% setname = 'BCIT Mind Wandering';
+% log_name = 'bcit_mind_wandering_10_import_events_log.txt';
+
+% rootPath = 's:/bcit/SpeedControlWorkingPhaseTwo';
+% setname = 'BCIT Speed Control';
+% log_name = 'bcit_speed_control_10_import_events_log.txt';
+
+% rootPath = 's:/bcit/TrafficComplexityWorkingPhaseTwo';
+% setname = 'BCIT Traffic Complexity';
+% log_name = 'bcit_traffic_complexity_10_import_events_log.txt';
+
+excludeDirs = {'sourcedata', 'code', 'stimuli', 'derivatives'};
+namePrefix = '';
+nameSuffix = '_eeg';
+extensions = {'.set'};
+
+% Designate the columns that are numeric (rest are char)
+columnTypes = {'onset', 'double'; 'duration', 'double'; 'sample', 'int32'};
+
+convertLatency = true; 
+
+%% Open the log
+fid = fopen([rootPath filesep 'code/curation_logs', filesep log_name], 'w');
+fprintf(fid, 'Log of runEeglabEventsImport.m on %s\n', datetime('now'));
+
+%% Generate json file.
+fileList = getFileList(rootPath, namePrefix, nameSuffix, ...
+                           extensions, excludeDirs);
+for k = 1:length(fileList)
+   EEG = pop_loadset(fileList{k});
+   [pathName, basename, ext] = fileparts(fileList{k});
+   fprintf(fid, '%s:\n', basename);
+   eventsFile = [pathName filesep basename(1:(end-3)) 'events.tsv'];
+   eventTable = getEventTable(eventsFile, columnTypes, renameColumns);
+   fprintf(fid, '\tCreate a table from the events file\n');
+   if convertLatency
+       eventTable.('latency') = eventTable.('latency')*EEG.srate + 1;
+       fprintf(fid, '\tConvert the latency column to samples\n');
+   end
+   fprintf('%s: EEG.event has %d events and BIDS event file has %d events\n', ...
+           basename, length(EEG.event), size(eventTable,1));
+   EEG.urevent = table2struct(eventTable)';
+   fprintf(fid, '\tSet the EEG.urevent\n');
+   eventTable.('urevent') = transpose(1:size(eventTable));
+   EEG.event = table2struct(eventTable)';
+   fprintf(fid, '\tSet the EEG.event\n');
+   if ~isempty(setname)    
+      EEG.setname = [setname basename];
+      fprintf(fid, '\tSet the EEG.setname\n');
+   end
+   fprintf(fid, '\tResave the EEG.set file\n');
+   EEG = pop_saveset(EEG, 'savemode', 'resave', 'version', '7.3');
+end
+fclose(fid);
diff --git a/hedcode/matlab_scripts/web_services/runAllTests.m b/hedcode/matlab_scripts/web_services/runAllTests.m
@@ -1,6 +1,6 @@
 dochost = 'https://hedtools.org/hed';
 %host = 'https://hedtools.org/hed_dev';
-%host = 'http://127.0.0.1:5000';
+host = 'http://127.0.0.1:5000';
 
 
 errorMap = containers.Map('KeyType', 'char', 'ValueType', 'any');

diff --git a/hedcode/matlab_scripts/web_services/runTest.m b/hedcode/matlab_scripts/web_services/runTest.m
@@ -4,4 +4,4 @@
 %host = 'https://hedtools.ucsd.edu/hed_dev';
 %errors = testLibraryServices(host);
 %errors = testSpreadsheetServices(host);
-errors = testStringServices(host);
+errors = testEventSearchServices(host);
diff --git a/hedcode/matlab_scripts/web_services/testEventSearchServices.m b/hedcode/matlab_scripts/web_services/testEventSearchServices.m
@@ -25,19 +25,19 @@
    errors{end + 1} = 'Example 1 failed execute the search.';
 end
 
-%% Example 2: Search an events file for HED
-request2 = struct('service', 'events_search', ...
-                  'schema_version', '8.0.0', ...
-                  'sidecar_string', data.jsonText, ...
-                  'events_string', data.eventsText, ...
-                  'columns_included', '', ...
-                  'query', '[[Intended-effect, Cue]]');
-request2.columns_included = {'onset'};
-response2 = webwrite(servicesUrl, request2, options);
-response2 = jsondecode(response2);
-outputReport(response2, 'Example 2 Querying an events file with extra columns');
-if ~isempty(response2.error_type) || ...
-   ~strcmpi(response2.results.msg_category, 'success')
-   errors{end + 1} = 'Example 2 failed execute the search.';
-end
+% %% Example 2: Search an events file for HED
+% request2 = struct('service', 'events_search', ...
+%                   'schema_version', '8.0.0', ...
+%                   'sidecar_string', data.jsonText, ...
+%                   'events_string', data.eventsText, ...
+%                   'columns_included', '', ...
+%                   'query', '[[Intended-effect, Cue]]');
+% request2.columns_included = {'onset'};
+% response2 = webwrite(servicesUrl, request2, options);
+% response2 = jsondecode(response2);
+% outputReport(response2, 'Example 2 Querying an events file with extra columns');
+% if ~isempty(response2.error_type) || ...
+%    ~strcmpi(response2.results.msg_category, 'success')
+%    errors{end + 1} = 'Example 2 failed execute the search.';
+% end