From f6caa152e37f0b6e990822b81e0c119e64d4a66f Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 10:55:34 +0000 Subject: [PATCH 1/5] update to use ECMWF as well as UKV NWP data --- data/nwp_ecmwf_target_coords.nc | Bin 0 -> 8932 bytes ...get_coords.nc => nwp_ukv_target_coords.nc} | Bin pvnet_app/app.py | 28 ++-- pvnet_app/consts.py | 3 +- pvnet_app/data.py | 108 +++++++++--- pvnet_app/utils.py | 4 +- requirements.txt | 4 +- tests/conftest.py | 28 +++- tests/test_app.py | 21 ++- tests/test_data/nwp_ecmwf_shell.zarr/.zattrs | 9 + .../.zgroup | 0 .../test_data/nwp_ecmwf_shell.zarr/.zmetadata | 156 ++++++++++++++++++ .../init_time/.zarray | 0 .../init_time/.zattrs | 0 .../nwp_ecmwf_shell.zarr/init_time/0 | Bin 0 -> 24 bytes .../nwp_ecmwf_shell.zarr/latitude/.zarray | 20 +++ .../nwp_ecmwf_shell.zarr/latitude/.zattrs | 9 + .../test_data/nwp_ecmwf_shell.zarr/latitude/0 | Bin 0 -> 233 bytes .../nwp_ecmwf_shell.zarr/longitude/.zarray | 20 +++ .../nwp_ecmwf_shell.zarr/longitude/.zattrs | 8 + .../nwp_ecmwf_shell.zarr/longitude/0 | Bin 0 -> 754 bytes .../nwp_ecmwf_shell.zarr/step/.zarray | 20 +++ .../step/.zattrs | 0 tests/test_data/nwp_ecmwf_shell.zarr/step/0 | Bin 0 -> 131 bytes .../nwp_ecmwf_shell.zarr/variable/.zarray | 20 +++ .../variable/.zattrs | 0 .../test_data/nwp_ecmwf_shell.zarr/variable/0 | Bin 0 -> 147 bytes tests/test_data/nwp_shell.zarr/init_time/0 | Bin 24 -> 0 bytes tests/test_data/nwp_shell.zarr/variable/0 | Bin 136 -> 0 bytes .../.zattrs | 1 - tests/test_data/nwp_ukv_shell.zarr/.zgroup | 3 + .../.zmetadata | 5 +- .../nwp_ukv_shell.zarr/init_time/.zarray | 20 +++ .../nwp_ukv_shell.zarr/init_time/.zattrs | 9 + .../test_data/nwp_ukv_shell.zarr/init_time/0 | Bin 0 -> 24 bytes .../latitude/.zarray | 0 .../latitude/.zattrs | 0 .../latitude/0.0 | Bin .../latitude/0.1 | Bin .../latitude/0.2 | Bin .../latitude/1.0 | Bin .../latitude/1.1 | Bin .../latitude/1.2 | Bin .../latitude/2.0 | Bin .../latitude/2.1 | Bin .../latitude/2.2 | Bin .../longitude/.zarray | 0 .../longitude/.zattrs | 0 .../longitude/0.0 | Bin .../longitude/0.1 | Bin .../longitude/0.2 | Bin .../longitude/1.0 | Bin .../longitude/1.1 | Bin .../longitude/1.2 | Bin .../longitude/2.0 | Bin .../longitude/2.1 | Bin .../longitude/2.2 | Bin .../step/.zarray | 0 .../test_data/nwp_ukv_shell.zarr/step/.zattrs | 8 + .../step/0 | Bin .../variable/.zarray | 4 +- .../nwp_ukv_shell.zarr/variable/.zattrs | 5 + tests/test_data/nwp_ukv_shell.zarr/variable/0 | Bin 0 -> 131 bytes .../x/.zarray | 0 .../x/.zattrs | 0 .../x/0 | Bin .../y/.zarray | 0 .../y/.zattrs | 0 .../y/0 | Bin 69 files changed, 459 insertions(+), 54 deletions(-) create mode 100644 data/nwp_ecmwf_target_coords.nc rename data/{nwp_target_coords.nc => nwp_ukv_target_coords.nc} (100%) create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/.zattrs rename tests/test_data/{nwp_shell.zarr => nwp_ecmwf_shell.zarr}/.zgroup (100%) create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/.zmetadata rename tests/test_data/{nwp_shell.zarr => nwp_ecmwf_shell.zarr}/init_time/.zarray (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ecmwf_shell.zarr}/init_time/.zattrs (100%) create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/init_time/0 create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/latitude/.zarray create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/latitude/.zattrs create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/latitude/0 create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/longitude/.zarray create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/longitude/.zattrs create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/longitude/0 create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/step/.zarray rename tests/test_data/{nwp_shell.zarr => nwp_ecmwf_shell.zarr}/step/.zattrs (100%) create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/step/0 create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/variable/.zarray rename tests/test_data/{nwp_shell.zarr => nwp_ecmwf_shell.zarr}/variable/.zattrs (100%) create mode 100644 tests/test_data/nwp_ecmwf_shell.zarr/variable/0 delete mode 100644 tests/test_data/nwp_shell.zarr/init_time/0 delete mode 100644 tests/test_data/nwp_shell.zarr/variable/0 rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/.zattrs (52%) create mode 100644 tests/test_data/nwp_ukv_shell.zarr/.zgroup rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/.zmetadata (94%) create mode 100644 tests/test_data/nwp_ukv_shell.zarr/init_time/.zarray create mode 100644 tests/test_data/nwp_ukv_shell.zarr/init_time/.zattrs create mode 100644 tests/test_data/nwp_ukv_shell.zarr/init_time/0 rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/.zarray (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/.zattrs (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/0.0 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/0.1 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/0.2 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/1.0 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/1.1 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/1.2 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/2.0 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/2.1 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/latitude/2.2 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/.zarray (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/.zattrs (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/0.0 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/0.1 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/0.2 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/1.0 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/1.1 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/1.2 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/2.0 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/2.1 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/longitude/2.2 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/step/.zarray (100%) create mode 100644 tests/test_data/nwp_ukv_shell.zarr/step/.zattrs rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/step/0 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/variable/.zarray (92%) create mode 100644 tests/test_data/nwp_ukv_shell.zarr/variable/.zattrs create mode 100644 tests/test_data/nwp_ukv_shell.zarr/variable/0 rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/x/.zarray (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/x/.zattrs (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/x/0 (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/y/.zarray (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/y/.zattrs (100%) rename tests/test_data/{nwp_shell.zarr => nwp_ukv_shell.zarr}/y/0 (100%) diff --git a/data/nwp_ecmwf_target_coords.nc b/data/nwp_ecmwf_target_coords.nc new file mode 100644 index 0000000000000000000000000000000000000000..f913633a5aff73c5e54561f244639c11ac9c9da2 GIT binary patch literal 8932 zcmeI%eQ;IvnFsKb+z^s*fxHoN!`n@AbCc4Dz(u#X%6SMOJ+F}=3>8-;NOB=g8f+36 zU>Rf`H^Z_S>KaPD);iH;vzCq28pjdG4Ff zzV3FWe{g2FC-WEoJHMCb9R5YCt{IZPgvXH&gHK40QcvdEYBS*c12Nct3frljE$TSoJ$_NMmr4UNC?zAKZaDmds%QNA1^ z`8_)u^zlruK%KW!(x>8&l(f{u_sr&XEo&t5oyiMI)^YF99#yJ8er{??K749QO3J5+ zg2}xvA2rSis)|>hlzHQnc~{&ZV_A+7>{53{F`rqtvQdsU$azOBHbcYk=sW(@CK1m;qE79B@6y&TH(u8%d~2@M(&x>0UIjrOq9q}l4tozb!CO9OsMgmO7~)X zq1Mb?o?BDr#!`}3k?ftxee$en$`WOxTN8KC$xmkzCz1i~{*v79ePzQ9Rm-dzw|4oE z3Z+4R_tlmdu__)wJ9uOI$Mi9)!O8OCU6_Qp#M zOUwQf2d`~^r52F;pE>GAjJnmM?&_#JI;x9|>OiBq#;995st2$y{M29Rs;-6oQ(M4W zZ9^$vthU!-2)bKx|L3i?UJ5XAf)}GA5i7aYX8hjN!7sCbPsvyO?muN^ZDafDhUmOS z>(;e4G_|a1Z){Vt-D$&{JA;xIBw{BfId5k2%M9gnC#4M7Mb^T)IwjAo3>E`SOx3!* zxVBEewRUCQqJ`FC#UH}&K6m1i0<{b6#o{rA!(Yu++`pGvGd{Um0rxv9@ZN_vee2dm zn(1HcesSYW@ZzkjU9_Zi-Mx*i?M-gYQGpWo9|CS@Y;9{=*K*6;$`-lATjtH0KWk3q z+J@*2x6GL}XYQ=I-Tzg!x~8sfv65V+QrrpufL!fY8h384(W6=Xrqi-S3>TW(|oO{P{&uf~Ro0qR@Uf=k? z=61*gzkJKR2+VIzg?}Xx2LUrRZGLF)`mYsbG(Rc?dw__ z8&)ON-qFywy0vjtTT{!LtIfB*rK#P!<5LWGZgx7>Es>bnzZhnI_cDHan7QMzecR&^NiF}3j>7Pkonm2EaX0k+Aeo8j7QdJC@E zrhDO{Hp!E4y-j=IR@k%vuFj^n;Vhe4!D?*6AXdnx9=Q275nRNkci<{*YKIHkBza}f zrYGS7HdTS;*>n`nZ_@_2bep_qf1CEgjaXC-ciN(N;f5{R2p6|V-VF^|)C)IgQ4QP? zi-zC^EZPj$Z;`x$B(suwIL{!1Y)pvre}~Ps7D5GT^!_ItJHi(N?$) ziwdRvEP4iPlSLL>n?-TBW{bALMJ+0VtGDP`xD^)F!qr)H9L}=ney|#guL zim_{$fOtH22ENHcf_QVa04dofa^CY1ji;FfZK12|m-lg_|(nzR$H!z6dHar>W1FM(|`=?=Iylg`35oAe-D)TEhk^(GyJTVc{& zaCIh~gR@L}2&~2=cR6zVpGp02^G&LUiL)F2eau z+6|X(lKTq7-AsB7Zp5GlxYGt*h8s5MVYs+KbHRoT8h{%#C<=GPAmw#HgC2qFH)tLl z8}tU;euLJ+^%;~3*K5$Da6Jay2-j`UVYrw<_rP@-lm^#n&|`2N27L`~n?XMV+hkBP zT$@4ZaLopF!$l3c39jCtpTn&%s0FUhpbR+6peMj;47wSP40;Q0zCriGMGVS$DNBPp4eCUY&a3dUUFR>(*%qE~e9FxGtUY z;W~BN2iKvK2DeS855P9*v<0qBrvkWUot}n^>SVyx>vRlmg-%=H>U1iEvvhg}tVSmb zj&zE{&DUugTtufLxJsR#g$wIc3m4SsI9x!d`@!;bnhNLF={dM`otD5UI(-Z`qR|6z zr!|@mH>}a~aB+>6f(>aj3^%Az2iy^jf^Y*Gy#Uv*(K0yJ=mgw;jkd$}X;cE&tI>;a zJsK^C>(=NbTuh@Ka9tXO;5s!r0N0_>ZE)K(It8{#qfWRsjlyuv8Zlf{qub%?H97;g zLZh8$xcM4A2p7?4CS0XP2jRjR-31ra=p0-?qlduqG^&L2 zYt#>yt`V4`B=;3wb>ijhDt9aAxhU6H&JQC_teh|74fB2F+<@}Ya()fWD(Bg7k#gP* z7AWV7@+LaWcZ0>ld{#@KNJ}7P~|C_j}e3!VX{8e#N`JA|^{1b6g_#tsqc%`^0Tq$k}kBFPX{o;za zOI#7xi!0*m#1-**aYg)!xB%}G7vNRm0z6AxfG>y(@F8)9{2g(He7CqlzFu4*Uldo! zuZo+@yTwiB)#4`eY;lwMlDNtInz#b~uDAkj5Ldu+#1-&maRvNSag+FAag(@F+$5eW zZW3P+H;D(t<#SA2K1apnb3|M|D{`On`E_x5{D`RkgN#}qpT)%Nba{Pp|MAQ zi{gIml!%Kv+r=GoPKf)^Stjm%XZ=1.5.30 fsspec[s3] xarray diff --git a/tests/conftest.py b/tests/conftest.py index 1be9ae4..c2041a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,12 +81,9 @@ def db_session(db_connection, engine_url): s.rollback() -@pytest.fixture -def nwp_data(): +def make_nwp_data(shell_path, varname): # Load dataset which only contains coordinates, but no data - ds = xr.open_zarr( - f"{os.path.dirname(os.path.abspath(__file__))}/test_data/nwp_shell.zarr" - ) + ds = xr.open_zarr(shell_path) # Last init time was at least 2 hours ago and floor to 3-hour interval t0_datetime_utc = time_before_present(timedelta(hours=2)).floor(timedelta(hours=3)) @@ -106,15 +103,32 @@ def nwp_data(): ds[v].encoding.clear() # Add data to dataset - ds["UKV"] = xr.DataArray( + ds[varname] = xr.DataArray( np.zeros([len(ds[c]) for c in ds.xindexes]), coords=[ds[c] for c in ds.xindexes], ) # Add stored attributes to DataArray - ds.UKV.attrs = ds.attrs["_data_attrs"] + ds[varname].attrs = ds.attrs["_data_attrs"] del ds.attrs["_data_attrs"] + return ds + + +@pytest.fixture +def nwp_ukv_data(): + return make_nwp_data( + shell_path=f"{os.path.dirname(os.path.abspath(__file__))}/test_data/nwp_ukv_shell.zarr", + varname="UKV", + ) + + +@pytest.fixture +def nwp_ecmwf_data(): + return make_nwp_data( + shell_path=f"{os.path.dirname(os.path.abspath(__file__))}/test_data/nwp_ecmwf_shell.zarr", + varname="UKV", + ) @pytest.fixture() diff --git a/tests/test_app.py b/tests/test_app.py index 9bff233..56bcdfc 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -10,8 +10,12 @@ ForecastValueSQL, ) +from pvnet_app.consts import sat_path, nwp_ukv_path, nwp_ecmwf_path +from pvnet_app.data import sat_5_path, sat_15_path -def test_app(db_session, nwp_data, sat_5_data, gsp_yields_and_systems, me_latest): +def test_app( + db_session, nwp_ukv_data, nwp_ecmwf_data, sat_5_data, gsp_yields_and_systems, me_latest +): # Environment variable DB_URL is set in engine_url, which is called by db_session # set NWP_ZARR_PATH # save nwp_data to temporary file, and set NWP_ZARR_PATH @@ -22,9 +26,13 @@ def test_app(db_session, nwp_data, sat_5_data, gsp_yields_and_systems, me_latest with tempfile.TemporaryDirectory() as tmpdirname: # The app loads sat and NWP data from environment variable # Save out data, and set paths as environmental variables - temp_nwp_path = f"{tmpdirname}/nwp.zarr" - os.environ["NWP_ZARR_PATH"] = temp_nwp_path - nwp_data.to_zarr(temp_nwp_path) + temp_nwp_path = f"{tmpdirname}/nwp_ukv.zarr" + os.environ["NWP_UKV_ZARR_PATH"] = temp_nwp_path + nwp_ukv_data.to_zarr(temp_nwp_path) + + temp_nwp_path = f"{tmpdirname}/nwp_ecmwf.zarr" + os.environ["NWP_ECMWF_ZARR_PATH"] = temp_nwp_path + nwp_ecmwf_data.to_zarr(temp_nwp_path) # In production sat zarr is zipped temp_sat_path = f"{tmpdirname}/sat.zarr.zip" @@ -41,6 +49,11 @@ def test_app(db_session, nwp_data, sat_5_data, gsp_yields_and_systems, me_latest from pvnet_app.app import app app(gsp_ids=list(range(1, 318)), num_workers=2) + os.system(f"rm {sat_5_path}") + os.system(f"rm {sat_15_path}") + os.system(f"rm -r {sat_path}") + os.system(f"rm -r {nwp_ukv_path}") + os.system(f"rm -r {nwp_ecmwf_path}") # Check forecasts have been made # (317 GSPs + 1 National + GSP-sum) = 319 forecasts # Doubled for historic and forecast diff --git a/tests/test_data/nwp_ecmwf_shell.zarr/.zattrs b/tests/test_data/nwp_ecmwf_shell.zarr/.zattrs new file mode 100644 index 0000000..37e0c1e --- /dev/null +++ b/tests/test_data/nwp_ecmwf_shell.zarr/.zattrs @@ -0,0 +1,9 @@ +{ + "_data_attrs": { + "Conventions": "CF-1.7", + "GRIB_centre": "ecmf", + "GRIB_centreDescription": "European Centre for Medium-Range Weather Forecasts", + "GRIB_subCentre": 0, + "institution": "European Centre for Medium-Range Weather Forecasts" + } +} \ No newline at end of file diff --git a/tests/test_data/nwp_shell.zarr/.zgroup b/tests/test_data/nwp_ecmwf_shell.zarr/.zgroup similarity index 100% rename from tests/test_data/nwp_shell.zarr/.zgroup rename to tests/test_data/nwp_ecmwf_shell.zarr/.zgroup diff --git a/tests/test_data/nwp_ecmwf_shell.zarr/.zmetadata b/tests/test_data/nwp_ecmwf_shell.zarr/.zmetadata new file mode 100644 index 0000000..d6c2040 --- /dev/null +++ b/tests/test_data/nwp_ecmwf_shell.zarr/.zmetadata @@ -0,0 +1,156 @@ +{ + "metadata": { + ".zattrs": { + "_data_attrs": { + "Conventions": "CF-1.7", + "GRIB_centre": "ecmf", + "GRIB_centreDescription": "European Centre for Medium-Range Weather Forecasts", + "GRIB_subCentre": 0, + "institution": "European Centre for Medium-Range Weather Forecasts" + } + }, + ".zgroup": { + "zarr_format": 2 + }, + "init_time/.zarray": { + "chunks": [ + 1 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "LH#Ymhwc7+035$;L$trUNm<*;(l(jOS)&m@=K8lWE3$VQ!W=%f)%w7Oa=% z=UB2`UXW|Weq~{vHODHKdlf=!+#Xbktn+wOBeucoNu9(dpJxqHTl`)$$!rUF)gree z=uMkK@v@sd_C-r?aN89wxz1%~3^Re`f1rhD{)23X5FlBQdB(%fJPrK*od}E%1r2pZ1_3o7Rs~`eAXWxq zB_LJ=Vg*Kq5ApKya&mIAvNAF<($Z2=Qj(Gq5)$I#Vq#*Vq9P(9!oosALV|(<0s{Pe zygWSIT$~&n>};$oEX+)d3=IGN{Qmjl+n3KDKfHPQ?D2z}mrw4L-Ffoz&HIm^y?pcj vV+f0~`PVvMnNQ literal 0 HcmV?d00001 diff --git a/tests/test_data/nwp_ecmwf_shell.zarr/step/.zarray b/tests/test_data/nwp_ecmwf_shell.zarr/step/.zarray new file mode 100644 index 0000000..7c72480 --- /dev/null +++ b/tests/test_data/nwp_ecmwf_shell.zarr/step/.zarray @@ -0,0 +1,20 @@ +{ + "chunks": [ + 89 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "5-KLF0GiIgz{>Fde@FnB2>``+7Rmqs literal 0 HcmV?d00001 diff --git a/tests/test_data/nwp_ecmwf_shell.zarr/variable/.zarray b/tests/test_data/nwp_ecmwf_shell.zarr/variable/.zarray new file mode 100644 index 0000000..cb17acb --- /dev/null +++ b/tests/test_data/nwp_ecmwf_shell.zarr/variable/.zarray @@ -0,0 +1,20 @@ +{ + "chunks": [ + 18 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "<_+%~$YQJKJB6`<6S7lfqP^ z01h?)74v2Gi*?S8{u5$MIp>+hF;Jq}3-4pL){-L_)3T=*8X@s3(z49lbUmO?#Tqw> cIuH@B(*r?ku7K;2tNSplQ2i6Clz5*Hf7bsWbN~PV literal 0 HcmV?d00001 diff --git a/tests/test_data/nwp_shell.zarr/init_time/0 b/tests/test_data/nwp_shell.zarr/init_time/0 deleted file mode 100644 index c70f33673b1f16cf6489be06d4de39ed3354e455..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 bcmZQ#H0I!7U|;}Y2_R-zV7g(=oQdK97IXvL diff --git a/tests/test_data/nwp_shell.zarr/variable/0 b/tests/test_data/nwp_shell.zarr/variable/0 deleted file mode 100644 index 119d688dfe8a40119fe655c5b6567390054dd7d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 136 zcmZQ#G?u7fU|;~@4j@he;$k2!2jU_iP6J{P$N}PHCM+cg-OLAtqA1;Ed%{imH*xlhcWV)k;M34F-_C8s}Nn Date: Thu, 29 Feb 2024 11:39:30 +0000 Subject: [PATCH 2/5] update app --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e5af306..faaa1a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ pydantic pytorch-lightning==2.1.3 torch[cpu]==2.2.0 -PVNet-summation==0.1.2 -pvnet==3.0.8 +PVNet-summation==0.1.3 +pvnet==3.0.9 ocf_datapipes==3.2.11 nowcasting_datamodel>=1.5.30 fsspec[s3] From abbe4f8cb3cab9302e876b9896f66518aeb23f67 Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 17:04:13 +0000 Subject: [PATCH 3/5] update reqs --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index faaa1a3..83c559a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ pydantic pytorch-lightning==2.1.3 torch[cpu]==2.2.0 PVNet-summation==0.1.3 -pvnet==3.0.9 +pvnet==3.0.11 ocf_datapipes==3.2.11 nowcasting_datamodel>=1.5.30 fsspec[s3] From 2112ebba54f73f481092a959e9a96c46f93a1bb7 Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 17:15:19 +0000 Subject: [PATCH 4/5] update func name --- tests/test_data.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_data.py b/tests/test_data.py index b634644..ddc0938 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -11,7 +11,7 @@ Note that I'm not sure these tests will work in parallel, due to files being saved in the same places """ -from pvnet_app.data import download_sat_data, preprocess_sat_data, sat_path, sat_5_path, sat_15_path +from pvnet_app.data import download_all_sat_data, preprocess_sat_data, sat_path, sat_5_path, sat_15_path import zarr import os import pandas as pd @@ -36,7 +36,7 @@ def test_download_sat_data(sat_5_data): save_to_zarr_zip(sat_5_data, filename=filename) os.environ["SATELLITE_ZARR_PATH"] = filename - download_sat_data() + download_all_sat_data() # assert that the file 'sat_5_path' exists assert os.path.exists(sat_5_path) @@ -54,7 +54,7 @@ def test_download_sat_15_data(sat_5_data): save_to_zarr_zip(sat_5_data, filename=filename) os.environ["SATELLITE_ZARR_PATH"] = os.path.join(tmpdirname, "latest.zarr.zip") - download_sat_data() + download_all_sat_data() assert not os.path.exists(sat_5_path) assert os.path.exists(sat_15_path) @@ -72,7 +72,7 @@ def test_download_sat_both_data(sat_5_data): save_to_zarr_zip(sat_5_data, filename=filename) os.environ["SATELLITE_ZARR_PATH"] = os.path.join(tmpdirname, "latest.zarr.zip") - download_sat_data() + download_all_sat_data() assert os.path.exists(sat_5_path) assert os.path.exists(sat_15_path) @@ -90,7 +90,7 @@ def test_preprocess_sat_data(sat_5_data): save_to_zarr_zip(sat_5_data, filename=filename) os.environ["SATELLITE_ZARR_PATH"] = filename - download_sat_data() + download_all_sat_data() use_15_minute = preprocess_sat_data(pd.Timestamp.now(tz=None)) assert use_15_minute == False @@ -106,7 +106,7 @@ def test_preprocess_sat_15_data(sat_5_data): save_to_zarr_zip(sat_5_data, filename=filename) os.environ["SATELLITE_ZARR_PATH"] = os.path.join(tmpdirname, "latest.zarr.zip") - download_sat_data() + download_all_sat_data() assert not os.path.exists(sat_5_path) assert os.path.exists(sat_15_path) @@ -132,7 +132,7 @@ def test_preprocess_old_sat_5_data_(sat_5_data): save_to_zarr_zip(sat_5_data, filename=filename) os.environ["SATELLITE_ZARR_PATH"] = os.path.join(tmpdirname, "latest.zarr.zip") - download_sat_data() + download_all_sat_data() assert os.path.exists(sat_5_path) assert os.path.exists(sat_15_path) From 83dc26cc6e91241f1d9cbca4547dd1b5be67c4b8 Mon Sep 17 00:00:00 2001 From: James Fulton Date: Thu, 29 Feb 2024 17:55:59 +0000 Subject: [PATCH 5/5] extend ECMWF data to cover shetlands with NaNs --- pvnet_app/data.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pvnet_app/data.py b/pvnet_app/data.py index 63c8415..6632dd5 100644 --- a/pvnet_app/data.py +++ b/pvnet_app/data.py @@ -150,7 +150,7 @@ def regrid_nwp_data(nwp_zarr, target_coords_path, method): ).to_zarr(nwp_zarr) -def rename_ecmwf_variables(): +def fix_ecmwf_data(): ds = xr.open_zarr(nwp_ecmwf_path).compute() ds["variable"] = ds["variable"].astype(str) @@ -160,12 +160,16 @@ def rename_ecmwf_variables(): "clt": "tcc" } - if not any(v in name_sub for v in ds["variable"].values): + if any(v in name_sub for v in ds["variable"].values): + logger.info(f"Renaming the ECMWF variables") + ds["variable"] = np.array([name_sub[v] if v in name_sub else v for v in ds["variable"].values]) + else: logger.info(f"No ECMWF renaming required - skipping this step") - return - logger.info(f"Renaming the ECMWF variables") - ds["variable"] = np.array([name_sub[v] if v in name_sub else v for v in ds["variable"].values]) + logger.info(f"Extending the ECMWF data to reach the shetlands") + # Thw data must be extended to reach the shetlands. This will fill missing lats with NaNs + # and reflects what the model saw in training + ds = ds.reindex(latitude=np.concatenate([np.arange(62, 60, -0.05), ds.latitude.values])) # Re-save inplace os.system(f"rm -rf {nwp_ecmwf_path}") @@ -188,5 +192,5 @@ def preprocess_nwp_data(): method="conservative" # this is needed to avoid zeros around edges of ECMWF data ) - # Names need to be aligned between training and prod - rename_ecmwf_variables() \ No newline at end of file + # Names need to be aligned between training and prod, and we need to infill the shetlands + fix_ecmwf_data() \ No newline at end of file