From 46c03c62f0c592efe28a211896e79b0798ac4370 Mon Sep 17 00:00:00 2001 From: James Fulton Date: Fri, 8 Mar 2024 16:40:21 +0000 Subject: [PATCH 1/6] port all weights when converting to mutimodal model object --- pvnet/models/multimodal/unimodal_teacher.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pvnet/models/multimodal/unimodal_teacher.py b/pvnet/models/multimodal/unimodal_teacher.py index 56750260..5171b6d2 100644 --- a/pvnet/models/multimodal/unimodal_teacher.py +++ b/pvnet/models/multimodal/unimodal_teacher.py @@ -416,4 +416,10 @@ def convert_to_multimodal_model(self, config): multimodal_model.output_network.load_state_dict(self.output_network.state_dict()) + if self.embedding_dim: + multimodal_model.embed.load_state_dict(self.embed.state_dict()) + + if self.include_sun: + multimodal_model.sun_fc1.load_state_dict(self.sun_fc1.state_dict()) + return multimodal_model, config From cff58178093c313e1ea8a1afa419058bc84dcfe4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 8 Mar 2024 16:42:49 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pvnet/models/multimodal/unimodal_teacher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvnet/models/multimodal/unimodal_teacher.py b/pvnet/models/multimodal/unimodal_teacher.py index 5171b6d2..db896f05 100644 --- a/pvnet/models/multimodal/unimodal_teacher.py +++ b/pvnet/models/multimodal/unimodal_teacher.py @@ -421,5 +421,5 @@ def convert_to_multimodal_model(self, config): if self.include_sun: multimodal_model.sun_fc1.load_state_dict(self.sun_fc1.state_dict()) - + return multimodal_model, config From 7a427f60756dd4f7ae98de598093ad36921f70ca Mon Sep 17 00:00:00 2001 From: James Fulton Date: Wed, 10 Apr 2024 12:05:43 +0000 Subject: [PATCH 3/6] add umt conversion test and minor fixes --- pvnet/models/base_model.py | 9 +-- pvnet/models/multimodal/unimodal_teacher.py | 5 +- pvnet/optimizers.py | 1 - scripts/save_concurrent_batches.py | 68 +++++++++++-------- .../multimodal/test_unimodal_teacher.py | 15 ++++ 5 files changed, 65 insertions(+), 33 deletions(-) diff --git a/pvnet/models/base_model.py b/pvnet/models/base_model.py index 9c83418c..cae760c3 100644 --- a/pvnet/models/base_model.py +++ b/pvnet/models/base_model.py @@ -165,7 +165,7 @@ def save_pretrained( self, save_directory: Union[str, Path], config: dict, - data_config: Union[str, Path], + data_config: Optional[Union[str, Path]], repo_id: Optional[str] = None, push_to_hub: bool = False, wandb_ids: Optional[Union[list[str], str]] = None, @@ -206,7 +206,8 @@ def save_pretrained( (save_directory / CONFIG_NAME).write_text(json.dumps(config, indent=4)) # Save cleaned datapipes configuration file - make_clean_data_config(data_config, save_directory / DATA_CONFIG_NAME) + if data_config is not None: + make_clean_data_config(data_config, save_directory / DATA_CONFIG_NAME) # Creating and saving model card. card_data = ModelCardData(language="en", license="mit", library_name="pytorch") @@ -220,7 +221,7 @@ def save_pretrained( wandb_links = "" for wandb_id in wandb_ids: - link = f"https://wandb.ai/openclimatefix/pvnet2.1/runs/{wandb_id}" + link = f"https://wandb.ai/openclimatefix/pvnet_summation/runs/{wandb_id}" wandb_links += f" - [{link}]({link})\n" card = ModelCard.from_template( @@ -506,7 +507,7 @@ def validation_step(self, batch: dict, batch_idx): y_values.append(logged_losses[f"MAE_horizon/step_{x:03}/val"]) per_step_losses = [[x, y] for (x, y) in zip(x_values, y_values)] # Check if WandBLogger is being used - if isinstance(self.logger, pl.loggers.WandbLogger): + if False: #isinstance(self.logger, pl.loggers.WandbLogger): table = wandb.Table(data=per_step_losses, columns=["timestep", "MAE"]) wandb.log( { diff --git a/pvnet/models/multimodal/unimodal_teacher.py b/pvnet/models/multimodal/unimodal_teacher.py index 5171b6d2..d6dec75b 100644 --- a/pvnet/models/multimodal/unimodal_teacher.py +++ b/pvnet/models/multimodal/unimodal_teacher.py @@ -369,7 +369,10 @@ def training_step(self, batch, batch_idx): def convert_to_multimodal_model(self, config): """Convert the model into a multimodal model class whilst preserving weights""" config = config.copy() - del config["cold_start"] + + if "cold_start" in config: + del config["cold_start"] + config["_target_"] = "pvnet.models.multimodal.multimodal.Model" sources = [] diff --git a/pvnet/optimizers.py b/pvnet/optimizers.py index bfa322e9..704f5889 100644 --- a/pvnet/optimizers.py +++ b/pvnet/optimizers.py @@ -185,7 +185,6 @@ def __call__(self, model): if not isinstance(self._lr, float): return self._call_multi(model) else: - assert False default_lr = self._lr if model.lr is None else model.lr opt = torch.optim.AdamW(model.parameters(), lr=default_lr, **self.opt_kwargs) sch = torch.optim.lr_scheduler.ReduceLROnPlateau( diff --git a/scripts/save_concurrent_batches.py b/scripts/save_concurrent_batches.py index 4d628f36..8e8f5980 100644 --- a/scripts/save_concurrent_batches.py +++ b/scripts/save_concurrent_batches.py @@ -13,6 +13,13 @@ ``` """ +# This is needed to get multiprocessing/multiple workers to behave +try: + import torch.multiprocessing as mp + + mp.set_start_method("spawn", force=True) +except RuntimeError: + pass import logging import os @@ -157,9 +164,7 @@ def main(config: DictConfig): f.write(OmegaConf.to_yaml(config.datamodule)) shutil.copyfile(config_dm.configuration, f"{config.batch_output_dir}/data_configuration.yaml") - - os.mkdir(f"{config.batch_output_dir}/train") - os.mkdir(f"{config.batch_output_dir}/val") + dataloader_kwargs = dict( shuffle=False, @@ -176,35 +181,44 @@ def main(config: DictConfig): persistent_workers=False, ) - print("----- Saving val batches -----") + + + if config.num_val_batches>0: + + print("----- Saving val batches -----") + + os.mkdir(f"{config.batch_output_dir}/val") + + val_batch_pipe = _get_datapipe( + config_dm.configuration, + *config_dm.val_period, + config.num_val_batches, + ) - val_batch_pipe = _get_datapipe( - config_dm.configuration, - *config_dm.val_period, - config.num_val_batches, - ) + _save_batches_with_dataloader( + batch_pipe=val_batch_pipe, + batch_dir=f"{config.batch_output_dir}/val", + num_batches=config.num_val_batches, + dataloader_kwargs=dataloader_kwargs, + ) - _save_batches_with_dataloader( - batch_pipe=val_batch_pipe, - batch_dir=f"{config.batch_output_dir}/val", - num_batches=config.num_val_batches, - dataloader_kwargs=dataloader_kwargs, - ) + if config.num_train_batches>0: + print("----- Saving train batches -----") - print("----- Saving train batches -----") + os.mkdir(f"{config.batch_output_dir}/train") - train_batch_pipe = _get_datapipe( - config_dm.configuration, - *config_dm.train_period, - config.num_train_batches, - ) + train_batch_pipe = _get_datapipe( + config_dm.configuration, + *config_dm.train_period, + config.num_train_batches, + ) - _save_batches_with_dataloader( - batch_pipe=train_batch_pipe, - batch_dir=f"{config.batch_output_dir}/train", - num_batches=config.num_train_batches, - dataloader_kwargs=dataloader_kwargs, - ) + _save_batches_with_dataloader( + batch_pipe=train_batch_pipe, + batch_dir=f"{config.batch_output_dir}/train", + num_batches=config.num_train_batches, + dataloader_kwargs=dataloader_kwargs, + ) print("done") diff --git a/tests/models/multimodal/test_unimodal_teacher.py b/tests/models/multimodal/test_unimodal_teacher.py index e27af0fd..178bd5f1 100644 --- a/tests/models/multimodal/test_unimodal_teacher.py +++ b/tests/models/multimodal/test_unimodal_teacher.py @@ -49,6 +49,7 @@ def unimodal_model_kwargs(teacher_dir, model_minutes_kwargs): res_block_layers=2, dropout_frac=0.0, ), + cold_start=True, ) # Get the teacher model save directories @@ -89,3 +90,17 @@ def test_model_backward(unimodal_teacher_model, sample_batch): # Backwards on sum drives sum to zero y.sum().backward() + + +def test_model_conversion(unimodal_model_kwargs, sample_batch): + + # Create the unimodal model + um_model = Model(**unimodal_model_kwargs) + # Convert to the equivalent multimodel model + mm_model, _ = um_model.convert_to_multimodal_model(unimodal_model_kwargs) + + # If the model has been successfully converted the predictions should be identical + y_um = um_model(sample_batch, return_modes=False) + y_mm = mm_model(sample_batch) + + assert (y_um == y_mm).all() \ No newline at end of file From 4a2ced5c229489330b072683b59404a15822d92d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 12:06:09 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pvnet/models/base_model.py | 2 +- pvnet/models/multimodal/unimodal_teacher.py | 4 ++-- scripts/save_concurrent_batches.py | 12 ++++-------- tests/models/multimodal/test_unimodal_teacher.py | 9 ++++----- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/pvnet/models/base_model.py b/pvnet/models/base_model.py index cae760c3..b40eb128 100644 --- a/pvnet/models/base_model.py +++ b/pvnet/models/base_model.py @@ -507,7 +507,7 @@ def validation_step(self, batch: dict, batch_idx): y_values.append(logged_losses[f"MAE_horizon/step_{x:03}/val"]) per_step_losses = [[x, y] for (x, y) in zip(x_values, y_values)] # Check if WandBLogger is being used - if False: #isinstance(self.logger, pl.loggers.WandbLogger): + if False: # isinstance(self.logger, pl.loggers.WandbLogger): table = wandb.Table(data=per_step_losses, columns=["timestep", "MAE"]) wandb.log( { diff --git a/pvnet/models/multimodal/unimodal_teacher.py b/pvnet/models/multimodal/unimodal_teacher.py index 1672db37..8720f973 100644 --- a/pvnet/models/multimodal/unimodal_teacher.py +++ b/pvnet/models/multimodal/unimodal_teacher.py @@ -369,10 +369,10 @@ def training_step(self, batch, batch_idx): def convert_to_multimodal_model(self, config): """Convert the model into a multimodal model class whilst preserving weights""" config = config.copy() - + if "cold_start" in config: del config["cold_start"] - + config["_target_"] = "pvnet.models.multimodal.multimodal.Model" sources = [] diff --git a/scripts/save_concurrent_batches.py b/scripts/save_concurrent_batches.py index 8e8f5980..fbba9acb 100644 --- a/scripts/save_concurrent_batches.py +++ b/scripts/save_concurrent_batches.py @@ -164,7 +164,6 @@ def main(config: DictConfig): f.write(OmegaConf.to_yaml(config.datamodule)) shutil.copyfile(config_dm.configuration, f"{config.batch_output_dir}/data_configuration.yaml") - dataloader_kwargs = dict( shuffle=False, @@ -181,14 +180,11 @@ def main(config: DictConfig): persistent_workers=False, ) - - - if config.num_val_batches>0: - + if config.num_val_batches > 0: print("----- Saving val batches -----") - + os.mkdir(f"{config.batch_output_dir}/val") - + val_batch_pipe = _get_datapipe( config_dm.configuration, *config_dm.val_period, @@ -202,7 +198,7 @@ def main(config: DictConfig): dataloader_kwargs=dataloader_kwargs, ) - if config.num_train_batches>0: + if config.num_train_batches > 0: print("----- Saving train batches -----") os.mkdir(f"{config.batch_output_dir}/train") diff --git a/tests/models/multimodal/test_unimodal_teacher.py b/tests/models/multimodal/test_unimodal_teacher.py index 178bd5f1..758b32a3 100644 --- a/tests/models/multimodal/test_unimodal_teacher.py +++ b/tests/models/multimodal/test_unimodal_teacher.py @@ -91,16 +91,15 @@ def test_model_backward(unimodal_teacher_model, sample_batch): # Backwards on sum drives sum to zero y.sum().backward() - + def test_model_conversion(unimodal_model_kwargs, sample_batch): - # Create the unimodal model um_model = Model(**unimodal_model_kwargs) # Convert to the equivalent multimodel model mm_model, _ = um_model.convert_to_multimodal_model(unimodal_model_kwargs) - + # If the model has been successfully converted the predictions should be identical y_um = um_model(sample_batch, return_modes=False) y_mm = mm_model(sample_batch) - - assert (y_um == y_mm).all() \ No newline at end of file + + assert (y_um == y_mm).all() From 6dde5bd090f6b6735b2c400460d67ec942d1ca2e Mon Sep 17 00:00:00 2001 From: James Fulton <41546094+dfulu@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:33:41 +0100 Subject: [PATCH 5/6] Update base_model.py --- pvnet/models/base_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvnet/models/base_model.py b/pvnet/models/base_model.py index b40eb128..5f2f299e 100644 --- a/pvnet/models/base_model.py +++ b/pvnet/models/base_model.py @@ -221,7 +221,7 @@ def save_pretrained( wandb_links = "" for wandb_id in wandb_ids: - link = f"https://wandb.ai/openclimatefix/pvnet_summation/runs/{wandb_id}" + link = f"https://wandb.ai/openclimatefix/pvnet_2.1/runs/{wandb_id}" wandb_links += f" - [{link}]({link})\n" card = ModelCard.from_template( @@ -507,7 +507,7 @@ def validation_step(self, batch: dict, batch_idx): y_values.append(logged_losses[f"MAE_horizon/step_{x:03}/val"]) per_step_losses = [[x, y] for (x, y) in zip(x_values, y_values)] # Check if WandBLogger is being used - if False: # isinstance(self.logger, pl.loggers.WandbLogger): + if isinstance(self.logger, pl.loggers.WandbLogger): table = wandb.Table(data=per_step_losses, columns=["timestep", "MAE"]) wandb.log( { From 226bf2630f681d235519a21bdebeb0aaa79bc4e8 Mon Sep 17 00:00:00 2001 From: James Fulton <41546094+dfulu@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:34:03 +0100 Subject: [PATCH 6/6] Update base_model.py --- pvnet/models/base_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvnet/models/base_model.py b/pvnet/models/base_model.py index 5f2f299e..6b8faa36 100644 --- a/pvnet/models/base_model.py +++ b/pvnet/models/base_model.py @@ -221,7 +221,7 @@ def save_pretrained( wandb_links = "" for wandb_id in wandb_ids: - link = f"https://wandb.ai/openclimatefix/pvnet_2.1/runs/{wandb_id}" + link = f"https://wandb.ai/openclimatefix/pvnet2.1/runs/{wandb_id}" wandb_links += f" - [{link}]({link})\n" card = ModelCard.from_template(