add try and except around wandb (#182)

* add try and except around wandb

* update

pvnet/models/base_model.py

@@ -506,7 +506,11 @@ def _log_forecast_plot(self, batch, y_hat, accum_batch_num, timesteps_to_plot, p
 
         plot_name = f"val_forecast_samples/batch_idx_{accum_batch_num}_{plot_suffix}"
 
-        self.logger.experiment.log({plot_name: wandb.Image(fig)})
+        try:
+            self.logger.experiment.log({plot_name: wandb.Image(fig)})
+        except Exception as e:
+            print(f"Failed to log {plot_name} to wandb")
+            print(e)
         plt.close(fig)
 
     def validation_step(self, batch: dict, batch_idx):
@@ -580,15 +584,18 @@ def on_validation_epoch_end(self):
         # Create the horizon accuracy curve
         if isinstance(self.logger, pl.loggers.WandbLogger):
             per_step_losses = [[i, horizon_maes_dict[i]] for i in range(self.forecast_len)]
-
-            table = wandb.Table(data=per_step_losses, columns=["horizon_step", "MAE"])
-            wandb.log(
-                {
-                    "horizon_loss_curve": wandb.plot.line(
-                        table, "horizon_step", "MAE", title="Horizon loss curve"
-                    )
-                },
-            )
+            try:
+                table = wandb.Table(data=per_step_losses, columns=["horizon_step", "MAE"])
+                wandb.log(
+                    {
+                        "horizon_loss_curve": wandb.plot.line(
+                            table, "horizon_step", "MAE", title="Horizon loss curve"
+                        )
+                    },
+                )
+            except Exception as e:
+                print("Failed to log horizon_loss_curve to wandb")
+                print(e)
 
     def test_step(self, batch, batch_idx):
         """Run test step"""