Refine Animations (#3)

* Fixes value range and add rain colormap * fix rain scale and min max stabilize all the animation pipes * add iberia heatwave animations * rollback cmap for temperature to inferno * Makes videos better formated for web * Fix temperature divergent colormap scaling * Setup for running global low resolution of all the zoomed in variables * fix global vars pipes * Remvoe unneceary intermediate parts * Add post process re encoding script * Add capacity factor and fix video scaling * Separate pipelines * add basemap to rain * Adds new colormap for capacity factor * Apply power norm to precipitation values * Move node to proper module and other minor stuff * Remove diff scenarios * Add image plotting for scenario comparison * Add image plotting for scenario comparison * moar updates * Cleanup and update README.md
Vizzuality · Dec 9, 2024 · 6e61fc8 · 6e61fc8
1 parent 676c04f
commit 6e61fc8
Show file tree

Hide file tree

Showing 23 changed files with 1,707 additions and 476 deletions.
diff --git a/science/.gitignore b/science/.gitignore
@@ -149,3 +149,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+session_store.db
+.viz
diff --git a/science/README.md b/science/README.md
@@ -10,15 +10,20 @@ Take a look at the [Kedro documentation](https://docs.kedro.org) to get started.
 
 The project contains one pipeline for now: `globe`
 
-### `lowvshigh`
+### `global`
 
-Pipeline to generate the comparisson between low and high resolution simulations. Currently it has:
+Generates global videos of wind and so
 
-- splits nextgems global datasets into a set of tiffs (one per timestep) to use in blender to render a rotating globe.
-- video generation pipeline for a regions defined in `conf/parameters.yml`
 
+### `scenarios`
 
-## How to install dependencies
+Scenario comparison, image and videos
+
+### `zooms`
+
+Zoomed in videos for the globe visualization
+
+##  Dependencies
 
 Declare any dependencies in `requirements.txt` for `pip` installation.
 
@@ -35,19 +40,14 @@ You can run your Kedro project with:
 ```
 kedro run
 ```
-I recomend use the `ParallelRunner` to run the nodes in parallel 
-
-```
-kedro run --runner=ParallelRunner
-```
 
-### Run a subset of the pipeline
+### Run a single pipeline
 
 Kedro allows run subsets by selecting only nodes, pipelines or tags. Check the tags in the pipeline code or in kedro viz.
 For example to run only the detailed videos pipelines use
 
 ```
-kedro run --runner=ParallelRunner --tags zoomin
+kedro run --pipeline zoomms --tags precipitation
 ```
 
 
@@ -67,4 +67,4 @@ In order to get the best out of the template:
 * Don't remove any lines from the `.gitignore` file we provide
 * Make sure your results can be reproduced by following a [data engineering convention](https://docs.kedro.org/en/stable/faq/faq.html#what-is-data-engineering-convention)
 * Don't commit data to your repository
-* Don't commit any credentials or your local configuration to your repository. Keep all your credentials and local configuration in `conf/local/`
+* Don't commit any credentials or your local configuration to your repository. Keep all your credentials and local configuration in `conf/local/`
diff --git a/science/conf/base/catalog.yml b/science/conf/base/catalog.yml
@@ -1,3 +1,7 @@
+test_video:
+  type: video.VideoDataset
+  filepath: data/03_primary/test.mp4
+
 # ------------- WINDSPEED --------------
 
 wind_speed_global_100km.raw:
@@ -8,33 +12,15 @@ wind_speed_global_10km.raw:
   type: kedro_datasets_experimental.netcdf.NetCDFDataset
   filepath: data/01_raw/nextgems/ws_global_10km.nc
 
-wind_speed_global_100km.parts:
-  type: partitions.PartitionedDataset
-  path: data/03_primary/ws-100-parts
-  dataset: 
-    type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-    save_args:
-      compress: zstd
-  filename_suffix: ".tif"
-
-wind_speed_global_10km.parts:
-  type: partitions.PartitionedDataset
-  path: data/03_primary/ws-10-parts
-  dataset: 
-    type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-    save_args:
-      compress: zstd
-  filename_suffix: ".tif"
-
 wind_speed_global_10km.video:
   type: video.VideoDataset
   filepath: data/03_primary/wind_speed_global_10km.mp4
-  fourcc: "h264"
+  fourcc: avc1
 
 wind_speed_global_100km.video:
   type: video.VideoDataset
   filepath: data/03_primary/wind_speed_global_100km.mp4
-  fourcc: "h264"
+  fourcc: avc1
 
 
 # -------------- CLOUD COVER --------------
@@ -48,36 +34,26 @@ cloud_cover_100km.raw:
   type: kedro_datasets_experimental.netcdf.NetCDFDataset
   filepath: data/01_raw/nextgems/lcc_global_100km.nc
 
-
-cloud_cover_10km.parts:
-  type: partitions.PartitionedDataset
-  path: data/02_intermediate/amazonia-10-parts
-  dataset: 
-    type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-    save_args:
-      compress: zstd
-  filename_suffix: ".tif"
-
-cloud_cover_100km.parts:
-  type: partitions.PartitionedDataset
-  path: data/02_intermediate/amazonia-100-parts
-  dataset: 
-    type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-    save_args:
-      compress: zstd
-  filename_suffix: ".tif"
-
 cloud_cover_10km.video:
   type: video.VideoDataset
   filepath: data/03_primary/cloud_cover_10km.mp4
+  fourcc: avc1
 
 cloud_cover_100km.video:
   type: video.VideoDataset
-  filepath: data/03_primary/cloud_cover_100km.mp4
+  filepath: data/03_primary/cloud_cover_global_100km.mp4
+  fourcc: avc1
 
 
 # -------------- PRECIPITATION --------------
 
+total_precipitation_10km.mapbox_credentials:
+  type: science.datasets.credentials.CredentialsDataset
+  credentials: mapbox
+
+total_precipitation_10km.basemap:
+  type: kedro_datasets.pillow.ImageDataset
+  filepath: data/01_raw/amazonia_basemap.png
 
 total_precipitation_10km.raw:
   type: kedro_datasets_experimental.netcdf.NetCDFDataset
@@ -87,55 +63,36 @@ total_precipitation_100km.raw:
   type: kedro_datasets_experimental.netcdf.NetCDFDataset
   filepath: data/01_raw/nextgems/tp_global_100km.nc
 
-total_precipitation_10km.parts:
-  type: partitions.PartitionedDataset
-  path: data/02_intermediate/hurricane-10-parts
-  dataset: 
-    type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-    save_args:
-      compress: zstd
-  filename_suffix: ".tif"
-
-total_precipitation_100km.parts:
-  type: partitions.PartitionedDataset
-  path: data/02_intermediate/hurricane-100-parts
-  dataset: 
-    type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-    save_args:
-      compress: zstd
-  filename_suffix: ".tif"
-
-
 total_precipitation_10km.video:
   type: video.VideoDataset
-  filepath: data/03_primary/tp_global_10km.mp4
+  filepath: data/03_primary/tp_amazonia_10km.mp4
+  fourcc: avc1
 
 total_precipitation_100km.video:
   type: video.VideoDataset
   filepath: data/03_primary/tp_global_100km.mp4
-
+  fourcc: avc1
 
 # -------------- TEMP --------------
 
 
-temp_10km.raw:
+temperature_10km.raw:
   type: kedro_datasets_experimental.netcdf.NetCDFDataset
   filepath: data/01_raw/nextgems/tas_global_10km.nc
 
-temp_10km.parts:
-  type: partitions.PartitionedDataset
-  path: data/02_intermediate/tas-10-parts
-  dataset: 
-    type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-    save_args:
-      compress: zstd
-  filename_suffix: ".tif"
-
+temperature_100km.raw:
+  type: kedro_datasets_experimental.netcdf.NetCDFDataset
+  filepath: data/01_raw/nextgems/tas_global_100km.nc
 
-temp_10km.video:
+temperature_10km.video:
   type: video.VideoDataset
   filepath: data/03_primary/tas_10km.mp4
+  fourcc: avc1
 
+temperature_100km.video:
+  type: video.VideoDataset
+  filepath: data/03_primary/tas_global_100km.mp4
+  fourcc: avc1
 
 # -------------- SEA SURFACE TEMP --------------
 
@@ -144,60 +101,84 @@ sst_10km.raw:
   type: kedro_datasets_experimental.netcdf.NetCDFDataset
   filepath: data/01_raw/nextgems/sst_global_10km.nc
 
-sst_10km.parts:
-  type: partitions.PartitionedDataset
-  path: data/02_intermediate/sst-10-parts
-  dataset: 
-    type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-    save_args:
-      compress: zstd
-  filename_suffix: ".tif"
+
+sst_100km.raw:
+  type: kedro_datasets_experimental.netcdf.NetCDFDataset
+  filepath: data/01_raw/nextgems/sst_global_100km.nc
 
 
 sst_10km.video:
   type: video.VideoDataset
   filepath: data/03_primary/sst_10km.mp4
+  fourcc: avc1
+
 
+sst_100km.video:
+  type: video.VideoDataset
+  filepath: data/03_primary/sst_global_100km.mp4
+  fourcc: avc1
 
 # ============= SCENARIOS ==============
 
+europe_plus2k.raw:
+  type: kedro_datasets_experimental.netcdf.NetCDFDataset
+  filepath: data/01_raw/destine/2K/2018_07_29_T00_00_to_2018_08_09_T23_00_2t_europe.nc
+
+europe_plus2k.image:
+  type: kedro_datasets.pillow.ImageDataset
+  filepath: data/03_primary/europe_plus_2k_scenario.png
 
-plus2k.raw:
+europe_hist.raw:
   type: kedro_datasets_experimental.netcdf.NetCDFDataset
-  filepath: data/01_raw/destine/IFS_FESOM_storyline/latlon_new/2K/2018_07_29_T00_00_to_2018_08_09_T23_00_2t_europe.nc
+  filepath: data/01_raw/destine/hist/2018_07_29_T00_00_to_2018_08_09_T23_00_2t_europe.nc
 
-# plus2k.parts:
-#   type: partitions.PartitionedDataset
-#   path: data/02_intermediate/plus2k
-#   dataset: 
-#     type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-#     save_args:
-#       compress: zstd
-#   filename_suffix: ".tif"
 
-plus2k.video:
-  type: video.VideoDataset
-  filepath: data/03_primary/plus_2k_scenario.mp4
+europe_hist.image:
+  type: pillow.ImageDataset
+  filepath: data/03_primary/europe_hist_scenario.png
 
 
-hist.raw:
+iberia_plus2k.raw:
   type: kedro_datasets_experimental.netcdf.NetCDFDataset
-  filepath: data/01_raw/destine/IFS_FESOM_storyline/latlon_new/hist/2018_07_29_T00_00_to_2018_08_09_T23_00_2t_europe.nc
+  filepath: data/01_raw/destine/2K/2018_07_29_T00_00_to_2018_08_09_T23_00_2t_iberia.nc
 
-# hist.parts:
-#   type: partitions.PartitionedDataset
-#   path: data/02_intermediate/hist
-#   dataset: 
-#     type: kedro_datasets_experimental.rioxarray.GeoTIFFDataset
-#     save_args:
-#       compress: zstd
-#   filename_suffix: ".tif"
+iberia_plus2k.image:
+  type: pillow.ImageDataset
+  filepath: data/03_primary/iberia_plus_2k_scenario.png
 
+iberia_hist.raw:
+  type: kedro_datasets_experimental.netcdf.NetCDFDataset
+  filepath: data/01_raw/destine/hist/2018_07_29_T00_00_to_2018_08_09_T23_00_2t_iberia.nc
+
+iberia_hist.image:
+  type: pillow.ImageDataset
+  filepath: data/03_primary/iberia_hist_scenario.png
+
+observations.raw:
+  type: kedro_datasets_experimental.netcdf.NetCDFDataset
+  filepath: data/01_raw/destine/2K/2018_07_29_T00_00_to_2018_08_09_T23_00_2t_europe.nc
 
-hist.video:
+observations.video:
   type: video.VideoDataset
-  filepath: data/03_primary/hist_scenario.mp4
+  filepath: data/03_primary/observations.mp4
+  fourcc: avc1
+
+# ============= CAPACITY FACTOR ==============
+
+capacity_factor_100.raw:
+  type: kedro_datasets_experimental.netcdf.NetCDFDataset
+  filepath: data/01_raw/nextgems/cf_global_100km.nc
+
+capacity_factor_100.video:
+  type: video.VideoDataset
+  filepath: data/03_primary/capacity_factor_100km.mp4
+  fourcc: avc1
+
+capacity_factor_10.raw:
+  type: kedro_datasets_experimental.netcdf.NetCDFDataset
+  filepath: data/01_raw/nextgems/cf_global_10km.nc
 
-diff.video:
+capacity_factor_10.video:
   type: video.VideoDataset
-  filepath: data/03_primary/diff_video.mp4
+  filepath: data/03_primary/capacity_factor_10km.mp4
+  fourcc: avc1
-Original file line number
+Diff line change
@@ Expand Up / @@ -149,3 +149,6 @@ venv.bak/ @@
     # mypy
     .mypy_cache/
+    session_store.db
+    .viz