From 124869b4c0decfe109a7e8cb6efdbe009a2cd3ab Mon Sep 17 00:00:00 2001 From: Sukhil Patel Date: Tue, 22 Oct 2024 11:06:06 +0100 Subject: [PATCH] Remove run app locally script --- scripts/run_app_locally.py | 104 ------------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 scripts/run_app_locally.py diff --git a/scripts/run_app_locally.py b/scripts/run_app_locally.py deleted file mode 100644 index a78efe5e..00000000 --- a/scripts/run_app_locally.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Script to run the production app locally""" - -import logging -import os -import time -from datetime import timedelta - -import numpy as np -import pandas as pd -import xarray as xr -from ocf_datapipes.load import OpenGSPFromDatabase - -from pvnet.app import app - -formatter = logging.Formatter(fmt="%(levelname)s:%(name)s:%(message)s") -stream_handler = logging.StreamHandler() -stream_handler.setFormatter(formatter) - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logger.addHandler(stream_handler) - - -def sleep_until(wake_time): - """Sleep until the given time""" - now = pd.Timestamp.now() - sleep_duration = (wake_time - now).total_seconds() - if sleep_duration < 0: - logger.warning("Sleep for negative duration requested") - else: - logger.info(f"Sleeping for {sleep_duration} seconds") - time.sleep(sleep_duration) - - -if __name__ == "__main__": - # ---------------------------------------------------- - # USER SETTINGS - - # When to start and stop predictions - start_time = pd.Timestamp("2023-05-31 00:00") - end_time = pd.Timestamp("2023-06-05 21:00") - - output_dir = "/mnt/disks/batches/local_production_forecasts" - save_inputs = True - - # ---------------------------------------------------- - # RUN - - # Make output dirs - os.makedirs(f"{output_dir}/predictions", exist_ok=True) - os.makedirs(f"{output_dir}/logs", exist_ok=True) - if save_inputs: - os.makedirs(f"{output_dir}/inputs", exist_ok=True) - - # Wait until start time - if pd.Timestamp.now() < start_time: - sleep_until(start_time) - - while pd.Timestamp.now() < end_time: - # Next prediction time - t0 = pd.Timestamp.now().ceil(timedelta(minutes=30)) - - # Sleep until next prediction time - sleep_until(t0) - - try: - # Make predictions - df = app(write_predictions=False) - - # Save - df.to_csv(f"{output_dir}/predictions/{t0}.csv") - except Exception: - logger.exception(f"Predictions for {t0=} failed") - - try: - # Log delays of data sources - log = dict( - now=t0, - gsp_times=next(iter(OpenGSPFromDatabase())).time_utc.values, - sat_times=xr.open_zarr("latest.zarr.zip").time.values, - nwp_times=xr.open_zarr(os.environ["NWP_ZARR_PATH"]).init_time.values, - ) - np.save(f"{output_dir}/logs/{t0}.npy", log) - except Exception: - logger.exception(f"Logs for {t0=} failed") - - if save_inputs: - try: - # Set up directory to save inputs - input_dir = f"{output_dir}/inputs/{t0}" - os.makedirs(input_dir, exist_ok=True) - - # Save inputs - os.system(f"cp latest.zarr.zip '{input_dir}/sat.zarr.zip'") - - ds = xr.open_zarr(os.environ["NWP_ZARR_PATH"]) - for v in ds.variables: - ds[v].encoding.clear() - ds.to_zarr(f"{input_dir}/nwp.zarr") - - next(iter(OpenGSPFromDatabase())).to_dataset().to_zarr(f"{input_dir}/gsp.zarr") - - except Exception: - logger.exception(f"Saving inputs for {t0=} failed")