diff --git a/experiments/analysis.py b/experiments/analysis.py index 6212a881..75b00cd9 100644 --- a/experiments/analysis.py +++ b/experiments/analysis.py @@ -6,6 +6,7 @@ import matplotlib.pyplot as plt import numpy as np +import pandas as pd import wandb @@ -53,6 +54,9 @@ def main(runs: list[str], run_names: list[str]) -> None: [720, 1440], [1440, 2880], ] + + groups_df = [] + grouping_starts = [grouping[0] for grouping in groupings] header = "| Timestep |" separator = "| --- |" for run_name in run_names: @@ -68,14 +72,21 @@ def main(runs: list[str], run_names: list[str]) -> None: for idx, timestep in enumerate(column_timesteps) if timestep >= grouping[0] and timestep <= grouping[1] ] + data_one_group = [] for df in dfs: - group_string += f" {df.iloc[group_idx].mean()*100.:0.3f} |" + mean_row = df.iloc[group_idx].mean() + group_string += f" {mean_row:0.3f} |" + data_one_group.append(mean_row) print(group_string) + groups_df.append(data_one_group) + + groups_df = pd.DataFrame(groups_df, columns=run_names, index=grouping_starts) + for idx, df in enumerate(dfs): print(f"{run_names[idx]}: {df.mean()*100:0.3f}") - # Plot the error on per timestep, and grouped timesteps + # Plot the error on per timestep, and all timesteps plt.figure() for idx, df in enumerate(dfs): plt.plot(column_timesteps, df, label=run_names[idx]) @@ -86,6 +97,17 @@ def main(runs: list[str], run_names: list[str]) -> None: plt.savefig("mae_per_timestep.png") plt.show() + # Plot the error on per timestep, and grouped timesteps + plt.figure() + for run_name in run_names: + plt.plot(groups_df[run_name], label=run_name) + plt.legend() + plt.xlabel("Timestep (minutes)") + plt.ylabel("MAE %") + plt.title("MAE % for each timestep") + plt.savefig("mae_per_timestep.png") + plt.show() + if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/experiments/india/005_extra_nwp_variables/mae_steps.png b/experiments/india/005_extra_nwp_variables/mae_steps.png new file mode 100644 index 00000000..8ba09293 Binary files /dev/null and b/experiments/india/005_extra_nwp_variables/mae_steps.png differ diff --git a/experiments/india/005_extra_nwp_variables/mae_steps_grouped.png b/experiments/india/005_extra_nwp_variables/mae_steps_grouped.png new file mode 100644 index 00000000..e052cded Binary files /dev/null and b/experiments/india/005_extra_nwp_variables/mae_steps_grouped.png differ diff --git a/experiments/india/005_extra_nwp_variables/readmd.md b/experiments/india/005_extra_nwp_variables/readmd.md new file mode 100644 index 00000000..7d40f1e2 --- /dev/null +++ b/experiments/india/005_extra_nwp_variables/readmd.md @@ -0,0 +1,55 @@ +# Adding extra nwp variables + +I wanted to run Windnet but testing some new nwp variables from ecmwf + +General conclusion, although more experiments could be done. +The current nwp variables are about right. +If you add lots it makes it worse. +If you take some away, it makes it worse. + +## Bugs + +Ran into a problem where found that some xamples have +`d.__getitem__('nwp-ecmwf__init_time_utc').values` had size 50, where it should be just one values. I removed these examples. This might + +## Experiments + +The number of samples were 8000 when training. + +### 15 variablles +Run windnet with `'hcc', 'lcc', 'mcc', 'prate', 'sde', 'sr', 't2m', 'tcc', 'u10', + 'v10', 'u100', 'v100', 'u200', 'v200', 'dlwrf', 'dswrf'`. + +The experiment on wandb is [here](https://wandb.ai/openclimatefix/india/runs/k91rdffo) + +### 7 variables +Run windnet with the original 7 variables. +`t2m, u10, u100, u200, v10, v100, v200 ` + +The experiment on wandb is [here](https://wandb.ai/openclimatefix/india/runs/miszfep5) + +### 3 variables +Run windnet with only `t, u10, v100` + +The experiment on wandb is [here](https://wandb.ai/openclimatefix/india/runs/22v3a39g) + +## Results + +| Timestep | 15 MAE % | 7 MAE % | 3 MAE % | +| --- | --- | --- | --- | +| 0-0 minutes | 7.450 | 6.623 | 7.529 | +| 15-15 minutes | 7.348 | 6.441 | 7.408 | +| 30-45 minutes | 7.242 | 6.544 | 7.294 | +| 45-60 minutes | 7.134 | 6.567 | 7.185 | +| 60-120 minutes | 7.058 | 6.295 | 7.009 | +| 120-240 minutes | 6.965 | 6.290 | 6.800 | +| 240-360 minutes | 6.807 | 6.374 | 6.580 | +| 360-480 minutes | 6.749 | 6.482 | 6.548 | +| 480-720 minutes | 6.892 | 6.686 | 6.685 | +| 720-1440 minutes | 7.020 | 6.756 | 6.780 | +| 1440-2880 minutes | 7.445 | 7.095 | 7.214 | + +![](mae_steps_grouped.png "mae_steps") + +The raw data is here +![](mae_steps.png "mae_steps")