diff --git a/DataTree/AcquiredData/DataFiles/DataFilesClass.m b/DataTree/AcquiredData/DataFiles/DataFilesClass.m index 8bd1bf6d..6b38e8d9 100644 --- a/DataTree/AcquiredData/DataFiles/DataFilesClass.m +++ b/DataTree/AcquiredData/DataFiles/DataFilesClass.m @@ -16,6 +16,8 @@ properties (Access = private) lookupTable excludedFolders + savedFilename + changed end methods @@ -30,6 +32,8 @@ obj.err = -1; obj.errmsg = {}; obj.dirFormats = struct('type',0, 'choices',{{}}); + obj.lookupTable = []; + obj.changed = false; global logger global cfg @@ -38,7 +42,6 @@ cfg = InitConfig(cfg); obj.logger = logger; - obj.lookupTable = []; obj.excludedFolders = {}; skipconfigfile = false; @@ -76,7 +79,7 @@ obj.rootdir = filesepStandard(obj.rootdir,'full'); % Configuration parameters - obj.config = struct('RegressionTestActive','', 'AskToFixNameConflicts',1, 'DerivedDataDir',''); + obj.config = struct('RegressionTestActive','', 'AskToFixNameConflicts',1, 'DerivedDataDir','', 'DerivedDataRootDir',''); if skipconfigfile==false str = cfg.GetValue('Regression Test Active'); if strcmp(str,'true') @@ -110,108 +113,287 @@ [obj.rootdir, 'imagerecon']; [obj.rootdir, 'anatomical']; }; + obj.config.DerivedDataRootDir = [obj.rootdir, obj.config.DerivedDataDir, '/', f, '/']; + obj.savedFilename = [obj.config.DerivedDataRootDir, 'DatasetFiles', '_', obj.filetype, '.mat']; if nargin==0 return; end obj.err = 0; - - obj.InitFolderFromats(); + obj.InitFolderFormats(); obj.GetDataSet(); + obj.SaveDataSet(); + end + + + + % ----------------------------------------------------------------------------------- + function Compare(obj) + if obj.IsEmpty() + return + end + obj2 = DataFilesClass(); + obj2.rootdir = obj.rootdir; + obj2.filetype = obj.filetype; + obj2.dirFormats = obj.dirFormats; + obj2.GetDataSet('skipsaveddata') + + %%%% Create list of all the saved names for sorting later + filesSaved = cell(length(obj.files),1); + filesSavedErr = cell(length(obj.filesErr),1); + for ii = 1:length(obj.files) + filesSaved{ii} = obj.files(ii).name; + end + for ii = 1:length(obj.filesErr) + filesSavedErr{ii} = obj.filesErr(ii).name; + end + + %%%% Move any files from saved error list that have been updated to nonerror list + N = length(obj.filesErr); + for ii = N:-1:1 + for jj = 1:length(obj2.files) + if strcmp(obj.filesErr(ii).name, obj2.files(jj).name) + if obj.filesErr(ii).datenum ~= obj2.files(jj).datenum + obj.filesErr(ii) = []; + filesSavedErr{ii} = []; + obj.files(end+1) = obj2.files(jj); + if ~obj2.files(jj).isdir + obj.nfiles = obj.nfiles+1; + end + filesSaved{end+1} = obj2.files(jj).name; %#ok + obj.logger.Write('%d. Mismatch in dates for saved data in %s. Will re-validate this file.\n', jj, obj.filesErr(ii).name); + obj.changed = true; + end + break + end + end + end + + + %%%% Now check saved object nonerror list + N = length(obj.files); + for ii = N:-1:1 + found = false; + for jj = 1:length(obj2.files) + if strcmp(obj.files(ii).name, obj2.files(jj).name) + found = true; + if obj.files(ii).datenum ~= obj2.files(jj).datenum + obj.files(ii) = obj2.files(jj).copy(); + obj.logger.Write('%d. Mismatch in dates for saved data in %s. Will re-validate this file.\n', jj, obj.files(ii).name); + obj.changed = true; + end + break + end + end + if ~found + obj.logger.Write('%d. Deleting file %s from saved data.\n', ii, obj.files(ii).name); + if ~obj.files(ii).isdir + obj.nfiles = obj.nfiles-1; + end + obj.files(ii) = []; + filesSaved(ii) = []; + obj.changed = true; + end + end + + % We don't need to check if files were added if there's no + % change and the number of current files found is same as saved + % number of files. Exist early. + if obj.changed == false + if (length(obj.files) + length(obj.filesErr)) == length(obj2.files) + return + end + end + + + %%%% Find which files were added to current dataset and add + %%%% them to saved object + for ii = 1:length(obj2.files) + found = false; + for jj = 1:length(obj.files) + if strcmp(obj2.files(ii).name, obj.files(jj).name) + found = true; + break + end + end + + % If the current file filesCurr(ii) is not found in saved dataset then it's a new file. + % Add new file from current dataset to saved dataset + if ~found + obj.files(end+1) = obj2.files(ii); + if ~obj2.files(ii).isdir + obj.nfiles = obj.nfiles+1; + end + filesSaved{end+1} = obj2.files(ii).name; %#ok + obj.logger.Write('%d. Adding new file %s to saved data.\n', jj, obj2.files(ii).name); + obj.changed = true; + end + end + + + % Sort non-error files + if obj.changed + [~, order] = sort(filesSaved); + obj.files = obj.files(order); + obj.logger.Write('\n'); + end + + end + + + + % ----------------------------------------------------------------------------------- + function SaveDataSet(obj) + if ~obj.changed + return + end + obj.changed = false; + obj.logger.Write('Saving DataFilesClass object in %s\n', obj.savedFilename); + if ~exist(obj.config.DerivedDataRootDir, 'dir') + mkdir(obj.config.DerivedDataRootDir) + end + save(obj.savedFilename, '-mat', 'obj'); + end + + + + % ----------------------------------------------------------------------------------- + function b = LoadDataSet(obj) + b = false; + if ~exist(obj.savedFilename, 'file') + return + end + dataset = load(obj.savedFilename); + dataset.obj.Compare(); + if isempty(dataset.obj.files) && isempty(dataset.obj.filesErr) + return + end + obj.logger.Write('Loaded saved DataFilesClass object: %s\n', obj.savedFilename); + obj.Copy(dataset.obj); + + obj.ErrorCheck() + obj.ErrorCheckName() + b = true; end % ----------------------------------------------------------------------------------- - function GetDataSet(obj) + function GetDataSet(obj, skipsaveddata) + if ~exist('skipsaveddata','var') + skipsaveddata = ''; + end + if ~strcmp(skipsaveddata, 'skipsaveddata') + if obj.LoadDataSet() + return + end + end + + obj.changed = true; + + mode = ''; + if obj.dirFormats.type == 0 + iFormats = 1:length(obj.dirFormats.choices); + else + iFormats = obj.dirFormats.type; + mode = 'noerrorcheck'; + end if exist(obj.rootdir, 'dir')~=7 error('Invalid subject folder: ''%s''', obj.rootdir); end - - for ii = 1:length(obj.dirFormats.choices) + for ii = iFormats obj.InitLookupTable(); obj.FindDataSet(ii); + if strcmp(mode,'noerrorcheck') + continue + end + if isempty(obj.files) + continue + end + % Remove any files that cannot pass the basic test of loading % its data obj.ErrorCheck(); if ~isempty(obj.files) break end + end + if strcmp(mode,'noerrorcheck') + return end - obj.ErrorCheckFinal(); end % ----------------------------------------------------------------------------------- - function InitFolderFromats(obj) + function InitFolderFormats(obj) obj.dirFormats.choices = { %%%% 1. Flat #1 { - ['*_run*.', obj.filetype]; + ['*_run*.', obj.filetype]; } %%%% 2. Flat #2 { - ['*.', obj.filetype]; + ['*.', obj.filetype]; } %%%% 3. Subjects { - '*'; - ['*.', obj.filetype]; + '*'; + ['*.', obj.filetype]; } %%%% 4. BIDS #1, sub-