From 0d7dfa71c52b49bd93bc59132b2c121255d47628 Mon Sep 17 00:00:00 2001 From: Michael Maier-Gerber Date: Sun, 16 Jun 2024 19:06:17 +0200 Subject: [PATCH 01/12] Connected to CLINT data repo & added plotting --- shearwater/processes/wps_cyclone.py | 147 ++++++++++++++++++---------- 1 file changed, 96 insertions(+), 51 deletions(-) diff --git a/shearwater/processes/wps_cyclone.py b/shearwater/processes/wps_cyclone.py index 808c114..84dffb5 100644 --- a/shearwater/processes/wps_cyclone.py +++ b/shearwater/processes/wps_cyclone.py @@ -1,10 +1,11 @@ -from pywps import Process, LiteralInput, ComplexOutput +from pywps import Process, LiteralInput, ComplexOutput, LiteralOutput from pywps.app.Common import Metadata # from tensorflow.keras import models # import pickle import numpy as np import numpy import pandas as pd +import metview as mv # from datetime import datetime import os from pywps import FORMATS @@ -18,22 +19,22 @@ class Cyclone(Process): - """A process to forecast tropical cyclone activities.""" + """A process to forecast tropical cyclone activity.""" def __init__(self): inputs = [ LiteralInput( "start_day", "Start Day", data_type="string", - abstract="Enter the start date, like 2021-01-01", - default="2022-01-01" + abstract="Enter the start date, like 2024-05-01", + default="2024-05-01" ), LiteralInput( "end_day", "End Day", data_type="string", - abstract="Enter the end date, like 2023-10-12", - default="2022-01-31" + abstract="Enter the end date, like 2024-05-28", + default="2024-05-28" ), LiteralInput( "area", @@ -52,18 +53,21 @@ def __init__(self): # abstract='netcdf', # # keywords=['output', 'result', 'response'], # data_type='string'), - ComplexOutput('output_csv', 'Cyclone activity forecast', - abstract='csv file', - as_reference=True, - # keywords=['output', 'result', 'response'], - supported_formats=[FORMATS.CSV],) + LiteralOutput('output_png', 'Cyclone activity forecast', + abstract='png file', + data_type='string'), + #ComplexOutput('output_csv', 'Cyclone activity forecast', + # abstract='csv file', + # as_reference=True, + # # keywords=['output', 'result', 'response'], + # supported_formats=[FORMATS.CSV],) ] super(Cyclone, self).__init__( self._handler, identifier='cyclone', title='Cyclone', - abstract='A process to forecast tropical cyclone activities.', + abstract='A process to forecast tropical cyclone activity.', # keywords=['hello', 'demo'], metadata=[ Metadata('PyWPS', 'https://pywps.org/'), @@ -80,45 +84,34 @@ def _handler(self, request, response): # TODO: lazy load tensorflow ... issues with sphinx doc build from tensorflow.keras import models -# master_catalog = intake.open_catalog(["https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_catalog.yaml"]) # noqa -# # master_catalog = intake.open_catalog('/pool/data/Catalogs/dkrz_catalog.yaml') -# era5_catalog = master_catalog['dkrz_era5_disk'] - -# query = { -# 'era_id': 'ET', -# 'level_type': 'surface', -# 'table_id': 128, -# # 'frequency':'hourly', -# 'code': 34, -# 'dataType': 'an', -# 'validation_date': '2023-06-27', -# } - -# my_catalog = era5_catalog.search(**query) -# # my_catalog.df - -# era_path = my_catalog.df['uri'].iloc[0] -# response.outputs['output'].data = f'netcdf {era_path}' -# response.outputs['output_csv'].data = 'csv ' + request.inputs['model'][0].data -# return response - start_date = request.inputs['start_day'][0].data end_date = request.inputs['end_day'][0].data # area = request.inputs['area'][0].data - # to be updated with data repository - data1 = pd.read_csv( - "https://github.com/climateintelligence/shearwater/raw/main/data/test_dailymeans_Sindian_1.zip") - # ("../shearwater/data/test_dailymeans_Sindian_1.zip") - data2 = pd.read_csv( - "https://github.com/climateintelligence/shearwater/raw/main/data/test_dailymeans_Sindian_2.zip") - # ("../shearwater/data/test_dailymeans_Sindian_2.zip") - data = pd.concat((data1, data2), ignore_index=True) - data = data.loc[(data.time >= start_date) & (data.time <= end_date)] - - variables = ['vo', 'r', 'u_200', 'u_850', 'v_200', 'v_850', 'tcwv', 'sst', 'shear'] - # with open("https://github.com/climateintelligence/shearwater/raw/main/data/full_statistics.pkl", 'rb') as f: - # means, stds = pickle.load(f) + parameters = [[138, 'vo', [850]], + [157, 'r', [700]], + [131, 'u', [200,850]], + [132, 'v', [200,850]], + [34, 'sst', [0]], + [137, 'tcwv', [0]]] + reso=2.5 + area_bbox = [0,20,-30,90] + + data=pd.DataFrame() + for param1 in parameters: + path = f'/pool/data/ERA5/ET/{"sf" if param1[2]==[0] else "pl"}/an/1D/{str(param1[0]).zfill(3)}' + fs1_param = mv.read(f"{path}/ET{'sf' if param1[2]==[0] else 'pl'}00_1D_{start_date[:7]}_{str(param1[0]).zfill(3)}.grb") + fs1_param = fs1_param.select(date=start_date.replace("-",""),level=param1[2]) + fs1_param_interp = mv.read(data=fs1_param,grid=[reso,reso],area=area_bbox,interpolation = '"--interpolation=grid-box-average"') + for level in param1[2]: + data.loc[:,f"{param1[1]}{'_'+str(level) if (param1[1]=='u' or param1[1]=='v') else ''}"] = fs1_param_interp.select(level=level).to_dataset().to_dataframe().reset_index(drop=True)[param1[1]] + + data.loc[:,['latitude','longitude']]= fs1_param_interp.select(level=level).to_dataset().to_dataframe().reset_index()[['latitude','longitude']] + data.loc[:,'time'] = start_date + data.loc[:,'shear'] = ((data.u_200-data.u_850)**2+(data.v_200-data.v_850)**2)**0.5 + data.loc[:,'sst'] = data.sst.fillna(0) + + variables = ['vo','r','u_200','u_850','v_200','v_850','tcwv','sst','shear'] means, stds = pd.read_pickle( "https://github.com/climateintelligence/shearwater/raw/main/data/full_statistics.zip") @@ -162,8 +155,60 @@ def _handler(self, request, response): data = data[["latitude", "longitude", "time"]] data['predictions_lag0'] = prediction.reshape(-1, 1) - prediction_path = os.path.join(workdir, "prediction_Sindian.csv") - data.to_csv(prediction_path) - - response.outputs['output_csv'].file = prediction_path + lag=0 + workdir = Path(self.workdir) + outfilename = os.path.join(workdir, f'tcactivity_48_17_{start_date.replace("-","")}_lag{lag}_Sindian') + + if True: + predscol=f"predictions_lag{lag}" + gpt = mv.create_geo(latitudes=data["latitude"].values, longitudes=data["longitude"].values, values=data[predscol].values).set_dates([pd.Timestamp(start_date)]*data.shape[0]) + fs = mv.geo_to_grib(geopoints = gpt, grid = [2.5,2.5], tolerance = 1.5)*1e2 + + cont_gen = mv.mcont( + legend = "on", + contour_line_colour = "avocado", + contour_shade = "on", + contour_shade_technique = "grid_shading", + contour_shade_max_level_colour = "red", + contour_shade_min_level_colour = "blue", + contour_shade_colour_direction = "clockwise" + ) + cont_tc = mv.mcont( + legend = "on", + contour_line_colour = "avocado", + contour_shade = "on", + contour_max_level = 105, + contour_min_level = 0, + contour_shade_technique = "grid_shading", + contour_shade_max_level_colour = "red", + contour_shade_min_level_colour = "blue", + contour_shade_colour_direction = "clockwise" + ) + + cont_oper = mv.mcont( + contour_automatic_setting = "style_name", + contour_style_name = "prob_green2yellow", + legend = "on" + ) + coastlines = mv.mcoast( + map_coastline_land_shade = "on", + map_coastline_land_shade_colour = "grey" + ) + + gview = mv.geoview( + map_area_definition = "corners", + area = area_bbox, + coastlines = coastlines + ) + legend = mv.mlegend( + legend_text_font_size = 0.5, + ) + + mv.setoutput(mv.png_output(output_name = outfilename+".png")) + mv.plot(gview,fs,cont_oper,legend) + response.outputs['output_png'].data = outfilename+".png" + else: + data.to_csv(outfilename+".csv") + response.outputs['output_csv'].data = outfilename+".csv" + return response From e6412cd3b4ed117a0935660bc01440752c23e1c0 Mon Sep 17 00:00:00 2001 From: PaoloBonettiPolimi <94172434+PaoloBonettiPolimi@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:05:52 +0200 Subject: [PATCH 02/12] adding metview --- environment.yml | 1 + requirements.txt | 3 ++- requirements_dev.txt | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d236cd8..589783b 100644 --- a/environment.yml +++ b/environment.yml @@ -19,6 +19,7 @@ dependencies: - birdy - tensorflow=2.15.0 - numpy=1.26.0 +- metview # tests - pytest - pandoc diff --git a/requirements.txt b/requirements.txt index 71da141..e991b49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ birdy tensorflow scipy ipykernel -numpy \ No newline at end of file +numpy +metview \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 765d88b..7ab917d 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -16,6 +16,7 @@ cruft # Changing dependencies above this comment will create merge conflicts when updating the cookiecutter template with cruft. Add extra requirements below this line. birdy tensorflow +metview scipy ipykernel numpy \ No newline at end of file From 1217291cc30d674878ad6763b481bb5912ec1112 Mon Sep 17 00:00:00 2001 From: PaoloBonettiPolimi <94172434+PaoloBonettiPolimi@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:13:26 +0200 Subject: [PATCH 03/12] adding eccodes to environment --- environment.yml | 1 + requirements.txt | 3 ++- requirements_dev.txt | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 589783b..0d86d9a 100644 --- a/environment.yml +++ b/environment.yml @@ -20,6 +20,7 @@ dependencies: - tensorflow=2.15.0 - numpy=1.26.0 - metview +- eccodes # tests - pytest - pandoc diff --git a/requirements.txt b/requirements.txt index e991b49..ac0d939 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,5 @@ tensorflow scipy ipykernel numpy -metview \ No newline at end of file +metview +eccodes \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 7ab917d..79ad144 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -17,6 +17,7 @@ cruft birdy tensorflow metview +eccodes scipy ipykernel numpy \ No newline at end of file From 3f4b5717b1e30de4da663de13764591196d04b8c Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 14:25:14 +0200 Subject: [PATCH 04/12] keep only additial dev packages --- requirements_dev.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 79ad144..ac37539 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -14,10 +14,4 @@ bump2version Click cruft # Changing dependencies above this comment will create merge conflicts when updating the cookiecutter template with cruft. Add extra requirements below this line. -birdy -tensorflow -metview -eccodes -scipy -ipykernel -numpy \ No newline at end of file +birdy \ No newline at end of file From 115492e6b6424e508db419161d4649b51624ca5e Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 14:25:34 +0200 Subject: [PATCH 05/12] updated conda spec list --- spec-list.txt | 81 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 9 deletions(-) diff --git a/spec-list.txt b/spec-list.txt index 88e3fc4..483d9d2 100644 --- a/spec-list.txt +++ b/spec-list.txt @@ -9,56 +9,67 @@ https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_2.conda -https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-hf3520f5_3.conda +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-hf3520f5_6.conda https://conda.anaconda.org/conda-forge/linux-64/libboost-headers-1.85.0-ha770c72_1.conda https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-4_cp311.conda https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h77fa898_8.conda +https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h77fa898_10.conda https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h77fa898_8.conda +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h77fa898_10.conda +https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.12-h4ab18f5_0.conda +https://conda.anaconda.org/conda-forge/linux-64/attr-2.5.1-h166bdaf_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.13-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.28.1-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/dav1d-1.2.1-hd590300_0.conda +https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.10-h36c2ea0_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/gettext-tools-0.22.5-h59595ed_2.conda https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.2-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/json-c-0.17-h7ab15ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/jxrlib-1.1-hd590300_3.conda https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/lame-3.100-h166bdaf_1003.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hd590300_1.conda https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.19-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-hd590300_2.conda https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-h3d2ce59_8.conda +https://conda.anaconda.org/conda-forge/linux-64/libgettextpo-0.22.5-h59595ed_2.conda +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-h3d2ce59_10.conda https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-hd590300_2.conda https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.0.0-hd590300_1.conda https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libogg-1.3.4-h7f98852_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libopus-1.3.1-h7f98852_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.18-h36c2ea0_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-hc0a3c3a_8.conda +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-hc0a3c3a_10.conda https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.4.0-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-h4ab18f5_1.conda https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-hd590300_1001.conda +https://conda.anaconda.org/conda-forge/linux-64/m4-1.4.18-h516909a_1001.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h59595ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.1-h4ab18f5_0.conda https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/rav1e-0.6.6-he8a937b_2.conda https://conda.anaconda.org/conda-forge/linux-64/tzcode-2024a-h3f72095_0.conda +https://conda.anaconda.org/conda-forge/linux-64/xorg-inputproto-2.3.2-h7f98852_1002.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-kbproto-1.0.7-h7f98852_1002.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.1-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.11-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.3-h7f98852_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-renderproto-0.11.1-h7f98852_1002.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-h0b41bf4_1003.conda +https://conda.anaconda.org/conda-forge/linux-64/xorg-xf86vidmodeproto-2.3.1-h7f98852_1002.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.0.7-h0b41bf4_0.conda -https://conda.anaconda.org/conda-forge/linux-64/aom-3.9.0-hac33072_0.conda +https://conda.anaconda.org/conda-forge/linux-64/aom-3.9.1-hac33072_0.conda https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.10-hb29e0c7_1.conda https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.18-hecc5fa9_1.conda https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.15-hecc5fa9_1.conda @@ -66,26 +77,35 @@ https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.18-hecc5fa9_1. https://conda.anaconda.org/conda-forge/linux-64/charls-2.4.2-h59595ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/expat-2.6.2-h59595ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/flatbuffers-23.5.26-h59595ed_1.conda +https://conda.anaconda.org/conda-forge/linux-64/flex-2.6.4-h58526e2_1004.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/geos-3.12.1-h59595ed_0.conda +https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h59595ed_1003.conda https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h2a13503_7.conda https://conda.anaconda.org/conda-forge/linux-64/icu-73.2-h59595ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libabseil-20230802.1-cxx17_h59595ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/libaec-1.1.3-h59595ed_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libasprintf-0.22.5-h661eb56_2.conda https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hd590300_1.conda https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hd590300_1.conda +https://conda.anaconda.org/conda-forge/linux-64/libcap-2.69-h0f662aa_0.conda https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_8.conda +https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda +https://conda.anaconda.org/conda-forge/linux-64/libgettextpo-devel-0.22.5-h59595ed_2.conda +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_10.conda https://conda.anaconda.org/conda-forge/linux-64/libhwy-1.1.0-h00ab1b0_0.conda https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.58.0-h47da74e_1.conda https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.43-h2797004_0.conda https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.46.0-hde9e2c9_0.conda https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.0-h0841786_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libvorbis-1.3.7-h9c3ff4c_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.15-h0b41bf4_0.conda https://conda.anaconda.org/conda-forge/linux-64/libzip-1.10.1-h2629f0a_3.conda https://conda.anaconda.org/conda-forge/linux-64/libzopfli-1.0.3-h9c3ff4c_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda +https://conda.anaconda.org/conda-forge/linux-64/mpg123-1.32.6-h59595ed_0.conda +https://conda.anaconda.org/conda-forge/linux-64/mysql-common-8.3.0-hf1915f5_4.conda https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.43-hcad00b1_0.conda https://conda.anaconda.org/conda-forge/linux-64/pixman-0.43.2-h59595ed_0.conda @@ -95,16 +115,21 @@ https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.10-hdb0a2a9_1.conda https://conda.anaconda.org/conda-forge/linux-64/svt-av1-2.1.0-hac33072_0.conda https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda https://conda.anaconda.org/conda-forge/linux-64/uriparser-0.9.8-hac33072_0.conda +https://conda.anaconda.org/conda-forge/linux-64/xorg-fixesproto-5.0-h7f98852_1002.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.4-h7391055_0.conda https://conda.anaconda.org/conda-forge/linux-64/zfp-1.0.1-h59595ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.1-h4ab18f5_1.conda https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.6-ha6fb4c9_0.conda https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.14.5-h50678d4_1.conda +https://conda.anaconda.org/conda-forge/linux-64/bison-3.8.2-h59595ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.5-h0f2a231_0.conda https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hd590300_1.conda https://conda.anaconda.org/conda-forge/linux-64/c-blosc2-2.14.3-hb4ffafa_0.conda +https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.10-nompi_hf1063bd_110.conda https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-h267a509_2.conda +https://conda.anaconda.org/conda-forge/linux-64/gdbm-1.18-h0a1914f_2.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.2-h659d440_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libasprintf-devel-0.22.5-h661eb56_2.conda https://conda.anaconda.org/conda-forge/linux-64/libavif16-1.0.4-hd2f8ffe_4.conda https://conda.anaconda.org/conda-forge/linux-64/libglib-2.80.2-hf974151_0.conda https://conda.anaconda.org/conda-forge/linux-64/libjxl-0.10.2-hcae5a98_0.conda @@ -115,10 +140,15 @@ https://conda.anaconda.org/conda-forge/linux-64/libre2-11-2023.09.01-h7a70373_1. https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-h8917695_15.conda https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.6.0-ha9c0a0a_2.conda https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.12.7-hc051c1a_1.conda -https://conda.anaconda.org/conda-forge/linux-64/minizip-4.0.6-h9d307f2_0.conda -https://conda.anaconda.org/conda-forge/linux-64/nss-3.100-hca3bf56_0.conda +https://conda.anaconda.org/conda-forge/linux-64/minizip-4.0.7-h401b404_0.conda +https://conda.anaconda.org/conda-forge/linux-64/mysql-libs-8.3.0-hca2cd23_4.conda +https://conda.anaconda.org/conda-forge/linux-64/nss-3.101-h593d115_0.conda https://conda.anaconda.org/conda-forge/linux-64/python-3.11.9-hb806964_0_cpython.conda https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.46.0-h6d4b2fc_0.conda +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-0.4.0-hd590300_1.conda +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-keysyms-0.4.0-h8ee46fc_1.conda +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.9-hd590300_1.conda +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.1-h8ee46fc_1.conda https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.9-h8ee46fc_0.conda https://conda.anaconda.org/conda-forge/noarch/absl-py-2.1.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.16-pyhd8ed1ab_0.conda @@ -139,6 +169,7 @@ https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.cond https://conda.anaconda.org/conda-forge/noarch/cloudpickle-3.0.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/dataclasses-0.8-pyhc8e2a94_3.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/dbus-1.13.6-h5008d03_3.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.1-py311hb755f60_0.conda https://conda.anaconda.org/conda-forge/noarch/decorator-5.1.1-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/docutils-0.21.2-pyhd8ed1ab_0.conda @@ -152,6 +183,8 @@ https://conda.anaconda.org/conda-forge/linux-64/frozenlist-1.4.1-py311h459d7ec_0 https://conda.anaconda.org/conda-forge/noarch/fsspec-2024.6.0-pyhff2d567_0.conda https://conda.anaconda.org/conda-forge/noarch/funcsigs-1.0.2-py_3.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/gast-0.5.4-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/gettext-0.22.5-h59595ed_2.conda +https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.80.2-hb6ce0ca_0.conda https://conda.anaconda.org/conda-forge/linux-64/greenlet-3.0.3-py311hb755f60_0.conda https://conda.anaconda.org/conda-forge/noarch/humanize-4.9.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/idna-3.7-pyhd8ed1ab_0.conda @@ -161,7 +194,10 @@ https://conda.anaconda.org/conda-forge/noarch/keras-2.15.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.16-hb7c19ff_0.conda https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.7.4-hfca40fe_0.conda https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-22_linux64_openblas.conda +https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h4637d8d_4.conda https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.8.0-hca28451_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libllvm15-15.0.7-hb3ce162_4.conda +https://conda.anaconda.org/conda-forge/linux-64/libllvm18-18.1.7-hc9dba70_1.conda https://conda.anaconda.org/conda-forge/linux-64/libpq-16.3-ha72fbe1_0.conda https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.39-h76b75d6_0.conda https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2 @@ -170,6 +206,7 @@ https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.5-py311h459d7ec_0 https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.8-py311h52f7536_0.conda https://conda.anaconda.org/conda-forge/linux-64/multidict-6.0.5-py311h459d7ec_0.conda https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.6.0-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/openblas-0.3.27-pthreads_h7a3da1a_0.conda https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.2-h488ebb8_0.conda https://conda.anaconda.org/conda-forge/noarch/packaging-24.1-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/parso-0.8.4-pyhd8ed1ab_0.conda @@ -192,6 +229,7 @@ https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py311h459d7ec_1.con https://conda.anaconda.org/conda-forge/linux-64/re2-2023.09.01-h7f4b329_1.conda https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.18.1-py311h5ecf98a_0.conda https://conda.anaconda.org/conda-forge/noarch/setuptools-70.0.0-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/simplejson-3.19.2-py311h459d7ec_0.conda https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-2.2.0-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.4.0-pyhd8ed1ab_0.tar.bz2 @@ -207,7 +245,10 @@ https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.12.2-pyha770c7 https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.13-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.14.1-py311hd4cff14_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-h8ee46fc_1.conda +https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.42-h4ab18f5_0.conda https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.conda +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-5.0.3-h7f98852_1004.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.11-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h75354e8_4.conda https://conda.anaconda.org/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_0.conda @@ -228,6 +269,7 @@ https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/comm-0.2.2-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.3-py311h459d7ec_0.conda +https://conda.anaconda.org/conda-forge/linux-64/glib-2.80.2-hf974151_0.conda https://conda.anaconda.org/conda-forge/noarch/google-pasta-0.2.0-pyh8c360ce_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.3-nompi_hdf9ad27_105.conda https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda @@ -237,8 +279,14 @@ https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.1-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.4-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/jupyter_core-5.7.2-py311h38be061_0.conda https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-22_linux64_openblas.conda +https://conda.anaconda.org/conda-forge/linux-64/libclang-cpp15-15.0.7-default_h127d8a8_5.conda +https://conda.anaconda.org/conda-forge/linux-64/libclang13-18.1.7-default_h087397f_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libflac-1.4.3-h59595ed_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libglu-9.0.0-hac7e632_1003.conda +https://conda.anaconda.org/conda-forge/linux-64/libgpg-error-1.49-h4f305b6_0.conda https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.59.3-hd6c4280_0.conda https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-22_linux64_openblas.conda +https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.7.0-h662e7e4_0.conda https://conda.anaconda.org/conda-forge/linux-64/lxml-5.2.2-py311hc0a218f_0.conda https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.7-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/partd-1.4.2-pyhd8ed1ab_0.conda @@ -260,19 +308,25 @@ https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.12.2-hd8ed1ab_ https://conda.anaconda.org/conda-forge/noarch/urllib3-2.2.1-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/werkzeug-3.0.3-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.5-hac6953d_0.conda +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.7.10-h7f98852_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/yarl-1.9.4-py311h459d7ec_0.conda https://conda.anaconda.org/conda-forge/linux-64/aiohttp-3.9.5-py311h459d7ec_0.conda https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.5.2-h4ad9680_0.conda https://conda.anaconda.org/conda-forge/linux-64/azure-storage-common-cpp-12.5.0-hb858b4b_2.conda https://conda.anaconda.org/conda-forge/linux-64/cryptography-42.0.8-py311h4a61cc7_0.conda https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.3.0-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/freeglut-3.2.2-hac7e632_2.conda https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h6b2125f_15.conda https://conda.anaconda.org/conda-forge/linux-64/grpcio-1.59.3-py311ha6695c7_0.conda +https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.24.4-haf2f30d_0.conda +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-8.5.0-hfac3d4d_0.conda https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-7.1.0-hd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.12.1-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.3-hee9dde6_1.conda +https://conda.anaconda.org/conda-forge/linux-64/libgcrypt-1.10.3-hd590300_0.conda https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.17.0-h884d01b_1.conda https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h135f659_114.conda +https://conda.anaconda.org/conda-forge/linux-64/libsndfile-1.2.2-hc60ed4a_1.conda https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.1.0-h7bd4643_4.conda https://conda.anaconda.org/conda-forge/noarch/markdown-3.6-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.0-py311h64a7726_0.conda @@ -287,27 +341,33 @@ https://conda.anaconda.org/conda-forge/linux-64/azure-storage-blobs-cpp-12.10.0- https://conda.anaconda.org/conda-forge/noarch/bokeh-2.4.3-pyhd8ed1ab_3.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.4-py311h18e1886_0.conda https://conda.anaconda.org/conda-forge/noarch/distributed-2023.3.0-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/gst-plugins-base-1.24.4-h9ad1361_0.conda https://conda.anaconda.org/conda-forge/linux-64/h5py-3.11.0-nompi_py311h439e445_102.conda https://conda.anaconda.org/conda-forge/linux-64/imagecodecs-2024.1.1-py311hd0e15ba_2.conda https://conda.anaconda.org/conda-forge/noarch/ipython-8.25.0-pyh707e725_0.conda +https://conda.anaconda.org/conda-forge/linux-64/jasper-4.2.4-h536e39c_0.conda https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.22.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.2-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-storage-2.17.0-hc7a4891_1.conda +https://conda.anaconda.org/conda-forge/linux-64/libsystemd0-255-h3516f8a_1.conda https://conda.anaconda.org/conda-forge/linux-64/ml_dtypes-0.2.0-py311h320fe9a_2.conda https://conda.anaconda.org/conda-forge/linux-64/numcodecs-0.12.1-py311h4332511_1.conda https://conda.anaconda.org/conda-forge/noarch/oauthlib-3.2.2-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/opt_einsum-3.3.0-pyhc1e730c_2.conda https://conda.anaconda.org/conda-forge/noarch/owslib-0.31.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/pandas-2.2.2-py311h14de704_1.conda +https://conda.anaconda.org/conda-forge/linux-64/pango-1.54.0-h84a9a3c_0.conda https://conda.anaconda.org/conda-forge/noarch/pyopenssl-24.0.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/scipy-1.13.1-py311h517d4fd_0.conda https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.4-py311h0bed3d6_1.conda https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.267-h5606698_1.conda https://conda.anaconda.org/conda-forge/noarch/birdy-0.8.7-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/dask-2023.3.0-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/eccodes-2.35.0-he84ddb8_0.conda https://conda.anaconda.org/conda-forge/noarch/google-auth-2.30.0-pyhff2d567_0.conda https://conda.anaconda.org/conda-forge/noarch/ipykernel-6.29.4-pyh3099207_0.conda https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.5-nompi_py311h74118c1_102.conda +https://conda.anaconda.org/conda-forge/linux-64/pulseaudio-client-17.0-hb77b528_0.conda https://conda.anaconda.org/conda-forge/noarch/requests-oauthlib-2.0.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/tifffile-2024.5.22-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/xarray-2024.3.0-pyhd8ed1ab_0.conda @@ -315,10 +375,13 @@ https://conda.anaconda.org/conda-forge/noarch/zarr-2.12.0-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/google-auth-oauthlib-1.2.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/intake-esm-2023.6.14-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/python-geotiff-0.2.10-pyh6eeeb26_0.conda +https://conda.anaconda.org/conda-forge/linux-64/qt-main-5.15.8-hc9dc06e_21.conda https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.20.1-hd75ad12_1.conda https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.8.4-hab4ef92_3.conda +https://conda.anaconda.org/conda-forge/linux-64/magics-metview-4.15.3-he53f960_0.conda https://conda.anaconda.org/conda-forge/noarch/tensorboard-2.15.2-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py311h8be719e_3.conda +https://conda.anaconda.org/conda-forge/linux-64/metview-5.22.0-h3a65212_0.conda https://conda.anaconda.org/conda-forge/linux-64/tensorflow-base-2.15.0-cpu_py311h6aa969b_2.conda https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.6-py311hf8e0aa6_0.conda https://conda.anaconda.org/conda-forge/linux-64/tensorflow-estimator-2.15.0-cpu_py311ha26c8b9_2.conda From ff4e9ca1e8453c2941620841f07bc4be6e7b6c24 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 14:34:09 +0200 Subject: [PATCH 06/12] use conda build --- .github/workflows/main.yml | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0b8b031..d9a3ae6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,3 @@ - name: build ⚙️ on: [ push, pull_request ] @@ -6,22 +5,32 @@ on: [ push, pull_request ] jobs: build: + name: Conda Build with Python${{ matrix.python-version }} runs-on: ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11"] + defaults: + run: + shell: bash -l {0} steps: - - name: Checkout repository and submodules - uses: actions/checkout@v2 - with: - submodules: recursive + - uses: actions/checkout@v3 - name: Install packages run: | - sudo apt-get -y install pandoc - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + sudo apt-get -y install pandoc graphviz + - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@v1 with: - python-version: ${{ matrix.python-version }} + cache-downloads: true + cache-environment: true + environment-file: environment.yml + create-args: >- + conda + python=${{ matrix.python-version }} + - name: Conda and Mamba versions + run: | + conda --version + echo "micromamba: $(micromamba --version)" - name: Install requirements 📦 run: | python -m pip install --upgrade pip @@ -33,6 +42,9 @@ jobs: - name: Lint with flake8 ⚙️ run: make lint if: matrix.python-version == 3.9 + # - name: Check formatting with black ⚙️ + # run: black --check --target-version py39 rook tests + # if: matrix.python-version == 3.9 - name: Build docs 🏗️ run: make docs if: matrix.python-version == 3.9 From 3333460242fbcd6c5432459fbe421d68ea9e8591 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 14:42:59 +0200 Subject: [PATCH 07/12] linting --- shearwater/processes/wps_cyclone.py | 238 +++++++++++++++++----------- 1 file changed, 145 insertions(+), 93 deletions(-) diff --git a/shearwater/processes/wps_cyclone.py b/shearwater/processes/wps_cyclone.py index 84dffb5..e53c0e2 100644 --- a/shearwater/processes/wps_cyclone.py +++ b/shearwater/processes/wps_cyclone.py @@ -1,11 +1,13 @@ from pywps import Process, LiteralInput, ComplexOutput, LiteralOutput from pywps.app.Common import Metadata + # from tensorflow.keras import models # import pickle import numpy as np import numpy import pandas as pd import metview as mv + # from datetime import datetime import os from pywps import FORMATS @@ -15,11 +17,13 @@ # import intake import logging + LOGGER = logging.getLogger("PYWPS") class Cyclone(Process): """A process to forecast tropical cyclone activity.""" + def __init__(self): inputs = [ LiteralInput( @@ -27,36 +31,36 @@ def __init__(self): "Start Day", data_type="string", abstract="Enter the start date, like 2024-05-01", - default="2024-05-01" + default="2024-05-01", ), LiteralInput( "end_day", "End Day", data_type="string", abstract="Enter the end date, like 2024-05-28", - default="2024-05-28" + default="2024-05-28", ), LiteralInput( "area", "Area", data_type="string", abstract="Choose the region of your interest", - allowed_values=[ - "Sindian", - "TBD" - ], + allowed_values=["Sindian", "TBD"], default="Sindian", - ) + ), ] outputs = [ # LiteralOutput('output', 'Cyclone activity forecast', # abstract='netcdf', # # keywords=['output', 'result', 'response'], # data_type='string'), - LiteralOutput('output_png', 'Cyclone activity forecast', - abstract='png file', - data_type='string'), - #ComplexOutput('output_csv', 'Cyclone activity forecast', + LiteralOutput( + "output_png", + "Cyclone activity forecast", + abstract="png file", + data_type="string", + ), + # ComplexOutput('output_csv', 'Cyclone activity forecast', # abstract='csv file', # as_reference=True, # # keywords=['output', 'result', 'response'], @@ -65,18 +69,18 @@ def __init__(self): super(Cyclone, self).__init__( self._handler, - identifier='cyclone', - title='Cyclone', - abstract='A process to forecast tropical cyclone activity.', + identifier="cyclone", + title="Cyclone", + abstract="A process to forecast tropical cyclone activity.", # keywords=['hello', 'demo'], metadata=[ - Metadata('PyWPS', 'https://pywps.org/'), + Metadata("PyWPS", "https://pywps.org/"), ], - version='0.1', + version="0.1", inputs=inputs, outputs=outputs, store_supported=True, - status_supported=True + status_supported=True, ) def _handler(self, request, response): @@ -84,64 +88,109 @@ def _handler(self, request, response): # TODO: lazy load tensorflow ... issues with sphinx doc build from tensorflow.keras import models - start_date = request.inputs['start_day'][0].data - end_date = request.inputs['end_day'][0].data + start_date = request.inputs["start_day"][0].data + end_date = request.inputs["end_day"][0].data # area = request.inputs['area'][0].data - parameters = [[138, 'vo', [850]], - [157, 'r', [700]], - [131, 'u', [200,850]], - [132, 'v', [200,850]], - [34, 'sst', [0]], - [137, 'tcwv', [0]]] - reso=2.5 - area_bbox = [0,20,-30,90] - - data=pd.DataFrame() + parameters = [ + [138, "vo", [850]], + [157, "r", [700]], + [131, "u", [200, 850]], + [132, "v", [200, 850]], + [34, "sst", [0]], + [137, "tcwv", [0]], + ] + reso = 2.5 + area_bbox = [0, 20, -30, 90] + + data = pd.DataFrame() for param1 in parameters: path = f'/pool/data/ERA5/ET/{"sf" if param1[2]==[0] else "pl"}/an/1D/{str(param1[0]).zfill(3)}' - fs1_param = mv.read(f"{path}/ET{'sf' if param1[2]==[0] else 'pl'}00_1D_{start_date[:7]}_{str(param1[0]).zfill(3)}.grb") - fs1_param = fs1_param.select(date=start_date.replace("-",""),level=param1[2]) - fs1_param_interp = mv.read(data=fs1_param,grid=[reso,reso],area=area_bbox,interpolation = '"--interpolation=grid-box-average"') + fs1_param = mv.read( + f"{path}/ET{'sf' if param1[2]==[0] else 'pl'}00_1D_{start_date[:7]}_{str(param1[0]).zfill(3)}.grb" + ) + fs1_param = fs1_param.select( + date=start_date.replace("-", ""), level=param1[2] + ) + fs1_param_interp = mv.read( + data=fs1_param, + grid=[reso, reso], + area=area_bbox, + interpolation='"--interpolation=grid-box-average"', + ) for level in param1[2]: - data.loc[:,f"{param1[1]}{'_'+str(level) if (param1[1]=='u' or param1[1]=='v') else ''}"] = fs1_param_interp.select(level=level).to_dataset().to_dataframe().reset_index(drop=True)[param1[1]] + data.loc[ + :, + f"{param1[1]}{'_'+str(level) if (param1[1]=='u' or param1[1]=='v') else ''}", + ] = ( + fs1_param_interp.select(level=level) + .to_dataset() + .to_dataframe() + .reset_index(drop=True)[param1[1]] + ) - data.loc[:,['latitude','longitude']]= fs1_param_interp.select(level=level).to_dataset().to_dataframe().reset_index()[['latitude','longitude']] - data.loc[:,'time'] = start_date - data.loc[:,'shear'] = ((data.u_200-data.u_850)**2+(data.v_200-data.v_850)**2)**0.5 - data.loc[:,'sst'] = data.sst.fillna(0) + data.loc[:, ["latitude", "longitude"]] = ( + fs1_param_interp.select(level=level) + .to_dataset() + .to_dataframe() + .reset_index()[["latitude", "longitude"]] + ) + data.loc[:, "time"] = start_date + data.loc[:, "shear"] = ( + (data.u_200 - data.u_850) ** 2 + (data.v_200 - data.v_850) ** 2 + ) ** 0.5 + data.loc[:, "sst"] = data.sst.fillna(0) - variables = ['vo','r','u_200','u_850','v_200','v_850','tcwv','sst','shear'] + variables = [ + "vo", + "r", + "u_200", + "u_850", + "v_200", + "v_850", + "tcwv", + "sst", + "shear", + ] means, stds = pd.read_pickle( - "https://github.com/climateintelligence/shearwater/raw/main/data/full_statistics.zip") + "https://github.com/climateintelligence/shearwater/raw/main/data/full_statistics.zip" + ) - data[variables] = (data[variables]-means[variables])/stds[variables] + data[variables] = (data[variables] - means[variables]) / stds[variables] - number_of_img, rows, cols = len(data.time.unique()), len(data.latitude.unique()), len(data.longitude.unique()) + number_of_img, rows, cols = ( + len(data.time.unique()), + len(data.latitude.unique()), + len(data.longitude.unique()), + ) images = np.zeros((number_of_img, rows, cols, len(variables))) - df = data.sort_values(by=['time', 'latitude', 'longitude']) + df = data.sort_values(by=["time", "latitude", "longitude"]) verbose = False k = 0 for day in range(0, number_of_img): - a = df.iloc[377*day:377*(day+1)] + a = df.iloc[377 * day : 377 * (day + 1)] i = 0 for var in variables: - images[day, :, :, i] = a.pivot(index='latitude', columns='longitude').sort_index(ascending=False)[var] + images[day, :, :, i] = a.pivot( + index="latitude", columns="longitude" + ).sort_index(ascending=False)[var] i += 1 k += 1 - if ((k % 100 == 0) & (verbose is True)): + if (k % 100 == 0) & (verbose is True): print(k) test_img_std = images - test_img_std = np.pad(test_img_std, ((0, 0), (1, 2), (1, 2), (0, 0)), 'constant') + test_img_std = np.pad( + test_img_std, ((0, 0), (1, 2), (1, 2), (0, 0)), "constant" + ) workdir = Path(self.workdir) model_path = os.path.join(workdir, "Unet_sevenAreas_fullStd_0lag_model.keras") urllib.request.urlretrieve( "https://github.com/climateintelligence/shearwater/raw/main/data/Unet_sevenAreas_fullStd_0lag_model.keras", - model_path # "Unet_sevenAreas_fullStd_0lag_model.keras" + model_path, # "Unet_sevenAreas_fullStd_0lag_model.keras" ) # model_trained = models.load_model( @@ -153,62 +202,65 @@ def _handler(self, request, response): prediction = model_trained.predict(test_img_std) data = data[["latitude", "longitude", "time"]] - data['predictions_lag0'] = prediction.reshape(-1, 1) + data["predictions_lag0"] = prediction.reshape(-1, 1) - lag=0 + lag = 0 workdir = Path(self.workdir) - outfilename = os.path.join(workdir, f'tcactivity_48_17_{start_date.replace("-","")}_lag{lag}_Sindian') - + outfilename = os.path.join( + workdir, f'tcactivity_48_17_{start_date.replace("-","")}_lag{lag}_Sindian' + ) + if True: - predscol=f"predictions_lag{lag}" - gpt = mv.create_geo(latitudes=data["latitude"].values, longitudes=data["longitude"].values, values=data[predscol].values).set_dates([pd.Timestamp(start_date)]*data.shape[0]) - fs = mv.geo_to_grib(geopoints = gpt, grid = [2.5,2.5], tolerance = 1.5)*1e2 - + predscol = f"predictions_lag{lag}" + gpt = mv.create_geo( + latitudes=data["latitude"].values, + longitudes=data["longitude"].values, + values=data[predscol].values, + ).set_dates([pd.Timestamp(start_date)] * data.shape[0]) + fs = mv.geo_to_grib(geopoints=gpt, grid=[2.5, 2.5], tolerance=1.5) * 1e2 + cont_gen = mv.mcont( - legend = "on", - contour_line_colour = "avocado", - contour_shade = "on", - contour_shade_technique = "grid_shading", - contour_shade_max_level_colour = "red", - contour_shade_min_level_colour = "blue", - contour_shade_colour_direction = "clockwise" - ) + legend="on", + contour_line_colour="avocado", + contour_shade="on", + contour_shade_technique="grid_shading", + contour_shade_max_level_colour="red", + contour_shade_min_level_colour="blue", + contour_shade_colour_direction="clockwise", + ) cont_tc = mv.mcont( - legend = "on", - contour_line_colour = "avocado", - contour_shade = "on", - contour_max_level = 105, - contour_min_level = 0, - contour_shade_technique = "grid_shading", - contour_shade_max_level_colour = "red", - contour_shade_min_level_colour = "blue", - contour_shade_colour_direction = "clockwise" - ) + legend="on", + contour_line_colour="avocado", + contour_shade="on", + contour_max_level=105, + contour_min_level=0, + contour_shade_technique="grid_shading", + contour_shade_max_level_colour="red", + contour_shade_min_level_colour="blue", + contour_shade_colour_direction="clockwise", + ) cont_oper = mv.mcont( - contour_automatic_setting = "style_name", - contour_style_name = "prob_green2yellow", - legend = "on" - ) + contour_automatic_setting="style_name", + contour_style_name="prob_green2yellow", + legend="on", + ) coastlines = mv.mcoast( - map_coastline_land_shade = "on", - map_coastline_land_shade_colour = "grey" - ) + map_coastline_land_shade="on", map_coastline_land_shade_colour="grey" + ) gview = mv.geoview( - map_area_definition = "corners", - area = area_bbox, - coastlines = coastlines - ) + map_area_definition="corners", area=area_bbox, coastlines=coastlines + ) legend = mv.mlegend( - legend_text_font_size = 0.5, - ) - - mv.setoutput(mv.png_output(output_name = outfilename+".png")) - mv.plot(gview,fs,cont_oper,legend) - response.outputs['output_png'].data = outfilename+".png" + legend_text_font_size=0.5, + ) + + mv.setoutput(mv.png_output(output_name=outfilename + ".png")) + mv.plot(gview, fs, cont_oper, legend) + response.outputs["output_png"].data = outfilename + ".png" else: - data.to_csv(outfilename+".csv") - response.outputs['output_csv'].data = outfilename+".csv" - + data.to_csv(outfilename + ".csv") + response.outputs["output_csv"].data = outfilename + ".csv" + return response From 78aaeeb5c0a36b0e336f5ef7f556f401d9d486e3 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 14:53:09 +0200 Subject: [PATCH 08/12] fix pep8 --- shearwater/processes/wps_cyclone.py | 49 +++++++++++++++-------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/shearwater/processes/wps_cyclone.py b/shearwater/processes/wps_cyclone.py index e53c0e2..a576906 100644 --- a/shearwater/processes/wps_cyclone.py +++ b/shearwater/processes/wps_cyclone.py @@ -1,4 +1,4 @@ -from pywps import Process, LiteralInput, ComplexOutput, LiteralOutput +from pywps import Process, LiteralInput, LiteralOutput from pywps.app.Common import Metadata # from tensorflow.keras import models @@ -10,7 +10,8 @@ # from datetime import datetime import os -from pywps import FORMATS + +# from pywps import FORMATS from pathlib import Path import urllib.request @@ -89,7 +90,7 @@ def _handler(self, request, response): from tensorflow.keras import models start_date = request.inputs["start_day"][0].data - end_date = request.inputs["end_day"][0].data + # end_date = request.inputs["end_day"][0].data # area = request.inputs['area'][0].data parameters = [ @@ -169,7 +170,7 @@ def _handler(self, request, response): k = 0 for day in range(0, number_of_img): - a = df.iloc[377 * day : 377 * (day + 1)] + a = df.iloc[377 * day : 377 * (day + 1)] # nopep8: E203 i = 0 for var in variables: images[day, :, :, i] = a.pivot( @@ -219,26 +220,26 @@ def _handler(self, request, response): ).set_dates([pd.Timestamp(start_date)] * data.shape[0]) fs = mv.geo_to_grib(geopoints=gpt, grid=[2.5, 2.5], tolerance=1.5) * 1e2 - cont_gen = mv.mcont( - legend="on", - contour_line_colour="avocado", - contour_shade="on", - contour_shade_technique="grid_shading", - contour_shade_max_level_colour="red", - contour_shade_min_level_colour="blue", - contour_shade_colour_direction="clockwise", - ) - cont_tc = mv.mcont( - legend="on", - contour_line_colour="avocado", - contour_shade="on", - contour_max_level=105, - contour_min_level=0, - contour_shade_technique="grid_shading", - contour_shade_max_level_colour="red", - contour_shade_min_level_colour="blue", - contour_shade_colour_direction="clockwise", - ) + # cont_gen = mv.mcont( + # legend="on", + # contour_line_colour="avocado", + # contour_shade="on", + # contour_shade_technique="grid_shading", + # contour_shade_max_level_colour="red", + # contour_shade_min_level_colour="blue", + # contour_shade_colour_direction="clockwise", + # ) + # cont_tc = mv.mcont( + # legend="on", + # contour_line_colour="avocado", + # contour_shade="on", + # contour_max_level=105, + # contour_min_level=0, + # contour_shade_technique="grid_shading", + # contour_shade_max_level_colour="red", + # contour_shade_min_level_colour="blue", + # contour_shade_colour_direction="clockwise", + # ) cont_oper = mv.mcont( contour_automatic_setting="style_name", From 92306c19373940211cb26d60541a8c6104b8452c Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 14:55:41 +0200 Subject: [PATCH 09/12] fix pep8 --- shearwater/processes/wps_cyclone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shearwater/processes/wps_cyclone.py b/shearwater/processes/wps_cyclone.py index a576906..9fae8d1 100644 --- a/shearwater/processes/wps_cyclone.py +++ b/shearwater/processes/wps_cyclone.py @@ -170,7 +170,7 @@ def _handler(self, request, response): k = 0 for day in range(0, number_of_img): - a = df.iloc[377 * day : 377 * (day + 1)] # nopep8: E203 + a = df.iloc[377 * day: 377 * (day + 1)] i = 0 for var in variables: images[day, :, :, i] = a.pivot( From 8f06a3254a3d724b0a1261d79ae41cad3889d052 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 15:01:54 +0200 Subject: [PATCH 10/12] fix docs build .. lazy load metview --- shearwater/processes/wps_cyclone.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shearwater/processes/wps_cyclone.py b/shearwater/processes/wps_cyclone.py index 9fae8d1..7f8e25f 100644 --- a/shearwater/processes/wps_cyclone.py +++ b/shearwater/processes/wps_cyclone.py @@ -6,7 +6,8 @@ import numpy as np import numpy import pandas as pd -import metview as mv + +# import metview as mv # from datetime import datetime import os @@ -86,8 +87,9 @@ def __init__(self): def _handler(self, request, response): LOGGER.info("running cyclone ...") - # TODO: lazy load tensorflow ... issues with sphinx doc build + # TODO: lazy load for tensorflow and metview ... issues with sphinx doc build from tensorflow.keras import models + import metview as mv start_date = request.inputs["start_day"][0].data # end_date = request.inputs["end_day"][0].data From 3c5c1e7fa6ec8ac8df0bb3651cc212ae4a93a6da Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 15:05:21 +0200 Subject: [PATCH 11/12] fix linting --- tests/test_wps_caps.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_wps_caps.py b/tests/test_wps_caps.py index 0e1c131..72e547b 100644 --- a/tests/test_wps_caps.py +++ b/tests/test_wps_caps.py @@ -6,12 +6,11 @@ def test_wps_caps(): client = client_for(Service(processes=processes)) - resp = client.get(service='wps', request='getcapabilities', version='1.0.0') - names = resp.xpath_text('/wps:Capabilities' - '/wps:ProcessOfferings' - '/wps:Process' - '/ows:Identifier') + resp = client.get(service="wps", request="getcapabilities", version="1.0.0") + names = resp.xpath_text( + "/wps:Capabilities" "/wps:ProcessOfferings" "/wps:Process" "/ows:Identifier" + ) assert sorted(names.split()) == [ - 'cyclone', - 'hello', + "cyclone", + "hello", ] From e0640b21700953009b9955803471bf9db70f2f47 Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Mon, 17 Jun 2024 15:11:56 +0200 Subject: [PATCH 12/12] fix linting --- shearwater/processes/__init__.py | 2 -- tests/test_wps_caps.py | 1 - 2 files changed, 3 deletions(-) diff --git a/shearwater/processes/__init__.py b/shearwater/processes/__init__.py index d880e28..c5aa82c 100644 --- a/shearwater/processes/__init__.py +++ b/shearwater/processes/__init__.py @@ -1,7 +1,5 @@ -# from .wps_say_hello import SayHello from .wps_cyclone import Cyclone processes = [ -# SayHello(), Cyclone(), ] diff --git a/tests/test_wps_caps.py b/tests/test_wps_caps.py index c2e3ebf..8ac5612 100644 --- a/tests/test_wps_caps.py +++ b/tests/test_wps_caps.py @@ -12,5 +12,4 @@ def test_wps_caps(): ) assert sorted(names.split()) == [ "cyclone", - # "hello", ]