From 0db2617065358537c086206d4faa49c28d978e17 Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Sat, 9 Mar 2024 17:25:46 -0500 Subject: [PATCH] =?UTF-8?q?fix:=20=F0=9F=90=9B=20Fixing=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dacapo/compute_context/compute_context.py | 5 + dacapo/validate.py | 404 +++++++++++----------- tests/components/test_options.py | 69 ++-- 3 files changed, 243 insertions(+), 235 deletions(-) diff --git a/dacapo/compute_context/compute_context.py b/dacapo/compute_context/compute_context.py index ab0081909..9121f0d39 100644 --- a/dacapo/compute_context/compute_context.py +++ b/dacapo/compute_context/compute_context.py @@ -1,5 +1,7 @@ from abc import ABC, abstractmethod +import os import subprocess +import sys from dacapo import Options, compute_context @@ -20,6 +22,9 @@ def wrap_command(self, command): def execute(self, command): # A helper method to run a command in the context specific way. + + # add pythonpath to the environment + os.environ["PYTHONPATH"] = sys.executable subprocess.run(self.wrap_command(command)) diff --git a/dacapo/validate.py b/dacapo/validate.py index 1dbad2684..6dd279e3f 100644 --- a/dacapo/validate.py +++ b/dacapo/validate.py @@ -67,98 +67,90 @@ def validate_run( load the weights of that iteration, it is assumed that the model is already loaded correctly. Returns the best parameters and scores for this iteration.""" - try: # we don't want this to hold up training - # set benchmark flag to True for performance - compute_context = create_compute_context() - torch.backends.cudnn.benchmark = True - run.model.to(compute_context.device) - run.model.eval() - - if ( - run.datasplit.validate is None - or len(run.datasplit.validate) == 0 - or run.datasplit.validate[0].gt is None - ): - logger.info("Cannot validate run %s. Continuing training!", run.name) - return None, None + # set benchmark flag to True for performance + compute_context = create_compute_context() + torch.backends.cudnn.benchmark = True + run.model.to(compute_context.device) + run.model.eval() + + if ( + run.datasplit.validate is None + or len(run.datasplit.validate) == 0 + or run.datasplit.validate[0].gt is None + ): + logger.info("Cannot validate run %s. Continuing training!", run.name) + return None, None + + # get array and weight store + weights_store = create_weights_store() + array_store = create_array_store() + iteration_scores = [] - # get array and weight store - weights_store = create_weights_store() - array_store = create_array_store() - iteration_scores = [] + # get post processor and evaluator + post_processor = run.task.post_processor + evaluator = run.task.evaluator - # get post processor and evaluator - post_processor = run.task.post_processor - evaluator = run.task.evaluator + # Initialize the evaluator with the best scores seen so far + evaluator.set_best(run.validation_scores) - # Initialize the evaluator with the best scores seen so far - evaluator.set_best(run.validation_scores) + for validation_dataset in run.datasplit.validate: + if validation_dataset.gt is None: + logger.error( + "We do not yet support validating on datasets without ground truth" + ) + raise NotImplementedError - for validation_dataset in run.datasplit.validate: - if validation_dataset.gt is None: - logger.error( - "We do not yet support validating on datasets without ground truth" - ) - raise NotImplementedError + logger.info( + "Validating run %s on dataset %s", run.name, validation_dataset.name + ) - logger.info( - "Validating run %s on dataset %s", run.name, validation_dataset.name + ( + input_raw_array_identifier, + input_gt_array_identifier, + ) = array_store.validation_input_arrays(run.name, validation_dataset.name) + if ( + not Path( + f"{input_raw_array_identifier.container}/{input_raw_array_identifier.dataset}" + ).exists() + or not Path( + f"{input_gt_array_identifier.container}/{input_gt_array_identifier.dataset}" + ).exists() + ): + logger.info("Copying validation inputs!") + input_voxel_size = validation_dataset.raw.voxel_size + output_voxel_size = run.model.scale(input_voxel_size) + input_shape = run.model.eval_input_shape + input_size = input_voxel_size * input_shape + output_shape = run.model.compute_output_shape(input_shape)[1] + output_size = output_voxel_size * output_shape + context = (input_size - output_size) / 2 + output_roi = validation_dataset.gt.roi + + input_roi = ( + output_roi.grow(context, context) + .snap_to_grid(validation_dataset.raw.voxel_size, mode="grow") + .intersect(validation_dataset.raw.roi) ) - - ( + input_raw = ZarrArray.create_from_array_identifier( input_raw_array_identifier, + validation_dataset.raw.axes, + input_roi, + validation_dataset.raw.num_channels, + validation_dataset.raw.voxel_size, + validation_dataset.raw.dtype, + name=f"{run.name}_validation_raw", + write_size=input_size, + ) + input_raw[input_roi] = validation_dataset.raw[input_roi] + input_gt = ZarrArray.create_from_array_identifier( input_gt_array_identifier, - ) = array_store.validation_input_arrays(run.name, validation_dataset.name) - if ( - not Path( - f"{input_raw_array_identifier.container}/{input_raw_array_identifier.dataset}" - ).exists() - or not Path( - f"{input_gt_array_identifier.container}/{input_gt_array_identifier.dataset}" - ).exists() - ): - logger.info("Copying validation inputs!") - input_voxel_size = validation_dataset.raw.voxel_size - output_voxel_size = run.model.scale(input_voxel_size) - input_shape = run.model.eval_input_shape - input_size = input_voxel_size * input_shape - output_shape = run.model.compute_output_shape(input_shape)[1] - output_size = output_voxel_size * output_shape - context = (input_size - output_size) / 2 - output_roi = validation_dataset.gt.roi - - input_roi = ( - output_roi.grow(context, context) - .snap_to_grid(validation_dataset.raw.voxel_size, mode="grow") - .intersect(validation_dataset.raw.roi) - ) - input_raw = ZarrArray.create_from_array_identifier( - input_raw_array_identifier, - validation_dataset.raw.axes, - input_roi, - validation_dataset.raw.num_channels, - validation_dataset.raw.voxel_size, - validation_dataset.raw.dtype, - name=f"{run.name}_validation_raw", - write_size=input_size, - ) - input_raw[input_roi] = validation_dataset.raw[input_roi] - input_gt = ZarrArray.create_from_array_identifier( - input_gt_array_identifier, - validation_dataset.gt.axes, - output_roi, - validation_dataset.gt.num_channels, - validation_dataset.gt.voxel_size, - validation_dataset.gt.dtype, - name=f"{run.name}_validation_gt", - write_size=output_size, - ) - input_gt[output_roi] = validation_dataset.gt[output_roi] - else: - logger.info("validation inputs already copied!") - - prediction_array_identifier = array_store.validation_prediction_array( - run.name, iteration, validation_dataset.name + validation_dataset.gt.axes, + output_roi, + validation_dataset.gt.num_channels, + validation_dataset.gt.voxel_size, + validation_dataset.gt.dtype, + name=f"{run.name}_validation_gt", + write_size=output_size, ) input_gt[output_roi] = validation_dataset.gt[output_roi] else: @@ -167,151 +159,151 @@ def validate_run( prediction_array_identifier = array_store.validation_prediction_array( run.name, iteration, validation_dataset.name ) - predict( - run.name, - iteration, - input_container=input_raw_array_identifier.container, - input_dataset=input_raw_array_identifier.dataset, - output_path=prediction_array_identifier, - output_roi=validation_dataset.gt.roi, # type: ignore - num_workers=num_workers, - output_dtype=output_dtype, - overwrite=overwrite, + input_gt[output_roi] = validation_dataset.gt[output_roi] + else: + logger.info("validation inputs already copied!") + + prediction_array_identifier = array_store.validation_prediction_array( + run.name, iteration, validation_dataset.name + ) + predict( + run.name, + iteration, + input_container=input_raw_array_identifier.container, + input_dataset=input_raw_array_identifier.dataset, + output_path=prediction_array_identifier, + output_roi=validation_dataset.gt.roi, # type: ignore + num_workers=num_workers, + output_dtype=output_dtype, + overwrite=overwrite, + ) + + logger.info("Predicted on dataset %s", validation_dataset.name) + + post_processor.set_prediction(prediction_array_identifier) + + dataset_iteration_scores = [] + + # set up dict for overall best scores + overall_best_scores = {} + for criterion in run.validation_scores.criteria: + overall_best_scores[criterion] = evaluator.get_overall_best( + validation_dataset, + criterion, ) - logger.info("Predicted on dataset %s", validation_dataset.name) + for parameters in post_processor.enumerate_parameters(): + output_array_identifier = array_store.validation_output_array( + run.name, iteration, str(parameters), validation_dataset.name + ) post_processor.set_prediction(prediction_array_identifier) - dataset_iteration_scores = [] # set up dict for overall best scores overall_best_scores = {} for criterion in run.validation_scores.criteria: overall_best_scores[criterion] = evaluator.get_overall_best( - validation_dataset, - criterion, + validation_dataset, criterion ) + any_overall_best = False + output_array_identifiers = [] for parameters in post_processor.enumerate_parameters(): output_array_identifier = array_store.validation_output_array( run.name, iteration, str(parameters), validation_dataset.name ) + output_array_identifiers.append(output_array_identifier) + post_processed_array = post_processor.process( + parameters, output_array_identifier + ) - post_processor.set_prediction(prediction_array_identifier) - dataset_iteration_scores = [] - - # set up dict for overall best scores - overall_best_scores = {} - for criterion in run.validation_scores.criteria: - overall_best_scores[criterion] = evaluator.get_overall_best( - validation_dataset, criterion - ) - - any_overall_best = False - output_array_identifiers = [] - for parameters in post_processor.enumerate_parameters(): - output_array_identifier = array_store.validation_output_array( - run.name, iteration, str(parameters), validation_dataset.name - ) - output_array_identifiers.append(output_array_identifier) - post_processed_array = post_processor.process( - parameters, output_array_identifier + try: + scores = evaluator.evaluate( + output_array_identifier, validation_dataset.gt # type: ignore ) - - try: - scores = evaluator.evaluate( - output_array_identifier, validation_dataset.gt # type: ignore - ) - for criterion in run.validation_scores.criteria: - # replace predictions in array with the new better predictions - if evaluator.is_best( - validation_dataset, - parameters, - criterion, - scores, + for criterion in run.validation_scores.criteria: + # replace predictions in array with the new better predictions + if evaluator.is_best( + validation_dataset, + parameters, + criterion, + scores, + ): + # then this is the current best score for this parameter, but not necessarily the overall best + higher_is_better = scores.higher_is_better(criterion) + # initial_best_score = overall_best_scores[criterion] + current_score = getattr(scores, criterion) + if not overall_best_scores[ + criterion + ] or ( # TODO: should be in evaluator + ( + higher_is_better + and current_score > overall_best_scores[criterion] + ) + or ( + not higher_is_better + and current_score < overall_best_scores[criterion] + ) ): - # then this is the current best score for this parameter, but not necessarily the overall best - higher_is_better = scores.higher_is_better(criterion) - # initial_best_score = overall_best_scores[criterion] - current_score = getattr(scores, criterion) - if not overall_best_scores[ - criterion - ] or ( # TODO: should be in evaluator - ( - higher_is_better - and current_score > overall_best_scores[criterion] - ) - or ( - not higher_is_better - and current_score < overall_best_scores[criterion] - ) - ): - any_overall_best = True - overall_best_scores[criterion] = current_score - - # For example, if parameter 2 did better this round than it did in other rounds, but it was still worse than parameter 1 - # the code would have overwritten it below since all parameters write to the same file. Now each parameter will be its own file - # Either we do that, or we only write out the overall best, regardless of parameters - best_array_identifier = ( - array_store.best_validation_array( - run.name, - criterion, - index=validation_dataset.name, - ) - ) - best_array = ZarrArray.create_from_array_identifier( - best_array_identifier, - post_processed_array.axes, - post_processed_array.roi, - post_processed_array.num_channels, - post_processed_array.voxel_size, - post_processed_array.dtype, - ) - best_array[best_array.roi] = post_processed_array[ - post_processed_array.roi - ] - best_array.add_metadata( - { - "iteration": iteration, - criterion: getattr(scores, criterion), - "parameters_id": parameters.id, - } - ) - weights_store.store_best( - run.name, - iteration, - validation_dataset.name, - criterion, - ) - except: - logger.error( - f"Could not evaluate run {run.name} on dataset {validation_dataset.name} with parameters {parameters}.", - exc_info=True, - ) - - dataset_iteration_scores.append( - [getattr(scores, criterion) for criterion in scores.criteria] + any_overall_best = True + overall_best_scores[criterion] = current_score + + # For example, if parameter 2 did better this round than it did in other rounds, but it was still worse than parameter 1 + # the code would have overwritten it below since all parameters write to the same file. Now each parameter will be its own file + # Either we do that, or we only write out the overall best, regardless of parameters + best_array_identifier = array_store.best_validation_array( + run.name, + criterion, + index=validation_dataset.name, + ) + best_array = ZarrArray.create_from_array_identifier( + best_array_identifier, + post_processed_array.axes, + post_processed_array.roi, + post_processed_array.num_channels, + post_processed_array.voxel_size, + post_processed_array.dtype, + ) + best_array[best_array.roi] = post_processed_array[ + post_processed_array.roi + ] + best_array.add_metadata( + { + "iteration": iteration, + criterion: getattr(scores, criterion), + "parameters_id": parameters.id, + } + ) + weights_store.store_best( + run.name, + iteration, + validation_dataset.name, + criterion, + ) + except: + logger.error( + f"Could not evaluate run {run.name} on dataset {validation_dataset.name} with parameters {parameters}.", + exc_info=True, ) - if not any_overall_best: - # We only keep the best outputs as determined by the evaluator - for output_array_identifier in output_array_identifiers: - array_store.remove(prediction_array_identifier) - array_store.remove(output_array_identifier) + dataset_iteration_scores.append( + [getattr(scores, criterion) for criterion in scores.criteria] + ) + + if not any_overall_best: + # We only keep the best outputs as determined by the evaluator + for output_array_identifier in output_array_identifiers: + array_store.remove(prediction_array_identifier) + array_store.remove(output_array_identifier) - iteration_scores.append(dataset_iteration_scores) + iteration_scores.append(dataset_iteration_scores) - run.validation_scores.add_iteration_scores( - ValidationIterationScores(iteration, iteration_scores) - ) - stats_store = create_stats_store() - stats_store.store_validation_iteration_scores(run.name, run.validation_scores) - except Exception as e: - logger.error( - f"Validation failed for run {run.name} at iteration " f"{iteration}.", - exc_info=e, - ) + run.validation_scores.add_iteration_scores( + ValidationIterationScores(iteration, iteration_scores) + ) + stats_store = create_stats_store() + stats_store.store_validation_iteration_scores(run.name, run.validation_scores) if __name__ == "__main__": diff --git a/tests/components/test_options.py b/tests/components/test_options.py index 4211188a7..e12d90483 100644 --- a/tests/components/test_options.py +++ b/tests/components/test_options.py @@ -1,5 +1,7 @@ import os +import tempfile from dacapo import Options +from os.path import expanduser from pathlib import Path @@ -7,10 +9,13 @@ def test_no_config(): - # Make sure the config file does not exist - config_file = Path("dacapo.yaml") - if config_file.exists(): - config_file.unlink() + # Temporarilly move any dacapo config file + original_files = {} + while Options.config_file() is not None: + original_files[Options.config_file()] = Options.config_file().with_suffix( + ".bak" + ) + os.rename(Options.config_file(), Options.config_file().with_suffix(".bak")) # Remove the environment variable env_dict = dict(os.environ) @@ -35,37 +40,43 @@ def test_no_config(): assert options.mongo_db_host == "localhost" assert options.mongo_db_name == "dacapo" + # Restore the original config files + for original, new in original_files.items(): + os.rename(new, original) + # we need to change the working directory because # dacapo looks for the config file in the working directory def test_local_config_file(): - # Create a config file - config_file = Path("dacapo.yaml") - config_file.write_text( - textwrap.dedent( - """ - runs_base_dir: /tmp - mongo_db_host: localhost - mongo_db_name: dacapo - """ + # get temporary directory + with tempfile.TemporaryDirectory() as tmpdir: + # Create a config file + config_file = Path(tmpdir, "dacapo.yaml") + config_file.write_text( + textwrap.dedent( + """ + runs_base_dir: /tmp + mongo_db_host: localhost + mongo_db_name: dacapo + """ + ) ) - ) - os.environ["OPTIONS_FILE"] = str(config_file) + os.environ["OPTIONS_FILE"] = str(config_file) - # Parse the options - options = Options.instance() + # Parse the options + options = Options.instance() - # Check the options - assert options.runs_base_dir == Path("/tmp") - assert options.mongo_db_host == "localhost" - assert options.mongo_db_name == "dacapo" - assert Options.config_file() == config_file + # Check the options + assert options.runs_base_dir == Path("/tmp") + assert options.mongo_db_host == "localhost" + assert options.mongo_db_name == "dacapo" + assert Options.config_file() == config_file - # Parse the options - options = Options.instance(runs_base_dir="/tmp2") + # Parse the options + options = Options.instance(runs_base_dir="/tmp2") - # Check the options - assert options.runs_base_dir == Path("/tmp2") - assert options.mongo_db_host == "localhost" - assert options.mongo_db_name == "dacapo" - assert Options.config_file() == config_file + # Check the options + assert options.runs_base_dir == Path("/tmp2") + assert options.mongo_db_host == "localhost" + assert options.mongo_db_name == "dacapo" + assert Options.config_file() == config_file