From 959ef75e51506842b4d0906753770b26b49ec42e Mon Sep 17 00:00:00 2001 From: James Fulton Date: Wed, 28 Feb 2024 14:57:11 +0000 Subject: [PATCH 01/11] update to new pvnet --- pvnet_summation/models/base_model.py | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pvnet_summation/models/base_model.py b/pvnet_summation/models/base_model.py index c58e4f6..34f1d77 100644 --- a/pvnet_summation/models/base_model.py +++ b/pvnet_summation/models/base_model.py @@ -96,9 +96,9 @@ def sum_of_gsps(self, x): def pvnet_output_shape(self): """Return the expected shape of the PVNet outputs""" if self.pvnet_model.use_quantile_regression: - return (317, self.pvnet_model.forecast_len_30, len(self.pvnet_model.output_quantiles)) + return (317, self.pvnet_model.forecast_len, len(self.pvnet_model.output_quantiles)) else: - return (317, self.pvnet_model.forecast_len_30) + return (317, self.pvnet_model.forecast_len) def _training_accumulate_log(self, batch_idx, losses, y_hat, y, y_sum, times): """Internal function to accumulate training batches and log results. diff --git a/requirements.txt b/requirements.txt index 3b4b122..7264d31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ocf_datapipes==3.2.* -pvnet==2.6.* +pvnet3.0.* ocf_ml_metrics==0.0.* numpy pandas From 10a7e8c52989951283c04d915055e04ee0ead594 Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 10:58:13 +0000 Subject: [PATCH 02/11] fix reqs --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7264d31..0a964e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ocf_datapipes==3.2.* -pvnet3.0.* +pvnet==3.0.* ocf_ml_metrics==0.0.* numpy pandas From 331f7b859bfe7956f6c92029ce03f0f0c1656192 Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 11:00:06 +0000 Subject: [PATCH 03/11] temporarily move properties --- pvnet_summation/models/base_model.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pvnet_summation/models/base_model.py b/pvnet_summation/models/base_model.py index 34f1d77..aa3b488 100644 --- a/pvnet_summation/models/base_model.py +++ b/pvnet_summation/models/base_model.py @@ -74,6 +74,19 @@ def __init__( self._accumulated_y_hat = PredAccumulator() self._accumulated_y_sum = PredAccumulator() self._accumulated_times = PredAccumulator() + + self.use_quantile_regression = self.output_quantiles is not None + + if self.use_quantile_regression: + self.num_output_features = self.forecast_len_30 * len(self.output_quantiles) + else: + self.num_output_features = self.forecast_len_30 + + + if self.pvnet_model.use_quantile_regression: + self.pvnet_output_shape= (317, self.pvnet_model.forecast_len, len(self.pvnet_model.output_quantiles)) + else: + self.pvnet_output_shape= (317, self.pvnet_model.forecast_len) def predict_pvnet_batch(self, batch): """Use PVNet model to create predictions for batch""" @@ -93,7 +106,7 @@ def sum_of_gsps(self, x): return (y_hat * x["effective_capacity"]).sum(dim=1) @property - def pvnet_output_shape(self): + def _pvnet_output_shape(self): """Return the expected shape of the PVNet outputs""" if self.pvnet_model.use_quantile_regression: return (317, self.pvnet_model.forecast_len, len(self.pvnet_model.output_quantiles)) From b650c2da29f6d2e7439811c37034928f25eb5e7d Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 11:35:31 +0000 Subject: [PATCH 04/11] remove unused reqs --- requirements.txt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0a964e9..46d04d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ ocf_datapipes==3.2.* pvnet==3.0.* -ocf_ml_metrics==0.0.* numpy pandas matplotlib @@ -9,20 +8,14 @@ ipykernel h5netcdf torch>=2.0.0 lightning>=2.0.1 -torchdata pytest pytest-cov typer sqlalchemy -jedi fsspec[s3] -tables -tilemapbase -testcontainers wandb tensorboard tqdm -rich omegaconf hydra-core -python-dotenv +python-dotenv \ No newline at end of file From 34befcf88311db218c067f2cc11aa8c6402d2a6b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Feb 2024 11:37:11 +0000 Subject: [PATCH 05/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pvnet_summation/models/base_model.py | 15 +++++++++------ requirements.txt | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pvnet_summation/models/base_model.py b/pvnet_summation/models/base_model.py index aa3b488..e226112 100644 --- a/pvnet_summation/models/base_model.py +++ b/pvnet_summation/models/base_model.py @@ -74,19 +74,22 @@ def __init__( self._accumulated_y_hat = PredAccumulator() self._accumulated_y_sum = PredAccumulator() self._accumulated_times = PredAccumulator() - + self.use_quantile_regression = self.output_quantiles is not None - + if self.use_quantile_regression: self.num_output_features = self.forecast_len_30 * len(self.output_quantiles) else: self.num_output_features = self.forecast_len_30 - - + if self.pvnet_model.use_quantile_regression: - self.pvnet_output_shape= (317, self.pvnet_model.forecast_len, len(self.pvnet_model.output_quantiles)) + self.pvnet_output_shape = ( + 317, + self.pvnet_model.forecast_len, + len(self.pvnet_model.output_quantiles), + ) else: - self.pvnet_output_shape= (317, self.pvnet_model.forecast_len) + self.pvnet_output_shape = (317, self.pvnet_model.forecast_len) def predict_pvnet_batch(self, batch): """Use PVNet model to create predictions for batch""" diff --git a/requirements.txt b/requirements.txt index 46d04d2..793db32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ tensorboard tqdm omegaconf hydra-core -python-dotenv \ No newline at end of file +python-dotenv From f3f2635d268bee5cb8aa7351926ea44a7bc139f8 Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 11:52:52 +0000 Subject: [PATCH 06/11] update test pvnet model --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index e756706..edabfc8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -138,7 +138,7 @@ def model_kwargs(): # These kwargs define the pvnet model which the summation model uses kwargs = dict( model_name="openclimatefix/pvnet_v2", - model_version="22e577100d55787eb2547d701275b9bb48f7bfa0", + model_version="4203e12e719efd93da641c43d2e38527648f4915", ) return kwargs From d905c14d2905e18012d1ba63ca341bc2153064ac Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 16:37:25 +0000 Subject: [PATCH 07/11] limit reqs to older huggingfacehub --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 793db32..3bf6e5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ tqdm omegaconf hydra-core python-dotenv +huggingface_hub==2.0.* \ No newline at end of file From d72f5fb2f4a45fe9d94fc8c280bb49f56db858f8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:37:44 +0000 Subject: [PATCH 08/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3bf6e5d..5b687fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,4 @@ tqdm omegaconf hydra-core python-dotenv -huggingface_hub==2.0.* \ No newline at end of file +huggingface_hub==2.0.* From 694711c6349475bf668d44d0efc4203b94557897 Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 16:41:47 +0000 Subject: [PATCH 09/11] limit reqs to older huggingfacehub --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3bf6e5d..3d445e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,4 @@ tqdm omegaconf hydra-core python-dotenv -huggingface_hub==2.0.* \ No newline at end of file +huggingface-hub==0.20.* \ No newline at end of file From 6631bacc3c4ccd266a97c3f97148677178d7b540 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:43:34 +0000 Subject: [PATCH 10/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d445e2..f394115 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,4 @@ tqdm omegaconf hydra-core python-dotenv -huggingface-hub==0.20.* \ No newline at end of file +huggingface-hub==0.20.* From 9eab67bb61977319e9c32d523e0aa6c868a7976a Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 16:53:13 +0000 Subject: [PATCH 11/11] remove property --- pvnet_summation/models/base_model.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pvnet_summation/models/base_model.py b/pvnet_summation/models/base_model.py index e226112..6e7d541 100644 --- a/pvnet_summation/models/base_model.py +++ b/pvnet_summation/models/base_model.py @@ -108,14 +108,6 @@ def sum_of_gsps(self, x): return (y_hat * x["effective_capacity"]).sum(dim=1) - @property - def _pvnet_output_shape(self): - """Return the expected shape of the PVNet outputs""" - if self.pvnet_model.use_quantile_regression: - return (317, self.pvnet_model.forecast_len, len(self.pvnet_model.output_quantiles)) - else: - return (317, self.pvnet_model.forecast_len) - def _training_accumulate_log(self, batch_idx, losses, y_hat, y, y_sum, times): """Internal function to accumulate training batches and log results.