"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "dset.sum(dim=\"time\")[\"tx90pETCCDI\"].plot()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8a9be89c-ac03-4e51-9cb0-65bb4af02b81",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "## What happens if the data get's lost?\n",
+ "\n",
+ "Let's delete the data:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "id": "d8731d29-47af-48d5-9f23-ded8d31ad269",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "-"
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "!rm -fr /scratch/b/b380001/futures/6def5135a687932d27f419a3e993b5bd68aa03425ff0378cfb7745c0aef497a5"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3147eb51",
+ "metadata": {},
+ "source": [
+ "The data is still in the databrowser:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "id": "34ee1fbe-9a67-4342-93c9-06fcabf56573",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "-"
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['/scratch/b/b380001/futures/6def5135a687932d27f419a3e993b5bd68aa03425ff0378cfb7745c0aef497a5/cmip5/output1/mpi-m/mpi-esm-lr/historical/yr/atmos/yr/r1i1p1/v20230911/tx90pETCCDI/tx90pETCCDI_yr_mpi-esm-lr_historical_r1i1p1_199007020000-199207011200.nc']"
+ ]
+ },
+ "execution_count": 37,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "list(freva.databrowser(variable=\"tx90petccdi\"))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5d8604c6-a208-4595-b70b-289fd010a4e6",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "Because of that the data can be re-created:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "id": "666b5193-a70f-4e8d-a063-05097c6e5e04",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "-"
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "dd49267695d047eab2d5432fa1ed7041",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Output()"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n"
+ ],
+ "text/plain": []
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "Dimensions: (time: 3, bnds: 2, lon: 192, lat: 96)\n",
+ "Coordinates:\n",
+ " * time (time) datetime64[ns] 1990-07-02 1991-07-02 1992-07-01T12:00:00\n",
+ " * lon (lon) float64 -179.1 -177.2 -175.3 -173.4 ... 175.3 177.2 179.1\n",
+ " * lat (lat) float64 -89.06 -87.19 -85.31 -83.44 ... 85.31 87.19 89.06\n",
+ " height float64 ...\n",
+ "Dimensions without coordinates: bnds\n",
+ "Data variables:\n",
+ " time_bnds (time, bnds) datetime64[ns] dask.array\n",
+ " tx90pETCCDI (time, lat, lon) float32 dask.array\n",
+ "Attributes: (12/36)\n",
+ " CDI: Climate Data Interface version 2.0.5 (https://m...\n",
+ " Conventions: CF-1.4\n",
+ " source: MPI-ESM-LR 2011; URL: http://svn.zmaw.de/svn/co...\n",
+ " institution: Max Planck Institute for Meteorology\n",
+ " institute_id: MPI-M\n",
+ " experiment_id: historical\n",
+ " ... ...\n",
+ " ETCCDI_software: climdex.pcic\n",
+ " ETCCDI_software_version: 1.1.11\n",
+ " frequency: yr\n",
+ " creation_date: 2023-09-11T20:05:11Z\n",
+ " title: ETCCDI indices computed on MPI-ESM-LR model out...\n",
+ " CDO: Climate Data Operators version 2.0.5 (https://m..."
+ ]
+ },
+ "execution_count": 35,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "dset = xr.open_mfdataset(\n",
+ " freva.databrowser(variable=\"tx90petccdi\",\n",
+ " execute_future=True\n",
+ " )\n",
+ ")\n",
+ "dset"
+ ]
+ }
+ ],
+ "metadata": {
+ "celltoolbar": "Slideshow",
+ "kernelspec": {
+ "display_name": "freva-dev",
+ "language": "python",
+ "name": "frev-dev"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.4"
+ },
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "state": {
+ "2286aa25721f441b8f22cbdcce4c6bab": {
+ "model_module": "@jupyter-widgets/output",
+ "model_module_version": "1.0.0",
+ "model_name": "OutputModel",
+ "state": {
+ "layout": "IPY_MODEL_fb04cb54a3204412a958a0055e186784",
+ "outputs": [
+ {
+ "data": {
+ "text/html": "
+
+
+
+
+
+
+
+
+
+## What happens if the data get's lost?
+
+Let's delete the data:
+
+
+```python
+!rm -fr /scratch/b/b380001/futures/6def5135a687932d27f419a3e993b5bd68aa03425ff0378cfb7745c0aef497a5
+```
+
+The data is still in the databrowser:
+
+
+```python
+list(freva.databrowser(variable="tx90petccdi"))
+```
+
+
+
+
+ ['/scratch/b/b380001/futures/6def5135a687932d27f419a3e993b5bd68aa03425ff0378cfb7745c0aef497a5/cmip5/output1/mpi-m/mpi-esm-lr/historical/yr/atmos/yr/r1i1p1/v20230911/tx90pETCCDI/tx90pETCCDI_yr_mpi-esm-lr_historical_r1i1p1_199007020000-199207011200.nc']
+
+
+
+Because of that the data can be re-created:
+
+
+```python
+dset = xr.open_mfdataset(
+ freva.databrowser(variable="tx90petccdi",
+ execute_future=True
+ )
+)
+dset
+```
+
+
+ Output()
+
+
+
+
+
+
+
+
+
+
+FuturesExample slides
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\n",
+ "
"
+ ],
+ "text/plain": [
+ "<xarray.Dataset>\n", + "Dimensions: (time: 3, bnds: 2, lon: 192, lat: 96)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 1990-07-02 1991-07-02 1992-07-01T12:00:00\n", + " * lon (lon) float64 -179.1 -177.2 -175.3 -173.4 ... 175.3 177.2 179.1\n", + " * lat (lat) float64 -89.06 -87.19 -85.31 -83.44 ... 85.31 87.19 89.06\n", + " height float64 ...\n", + "Dimensions without coordinates: bnds\n", + "Data variables:\n", + " time_bnds (time, bnds) datetime64[ns] dask.array<chunksize=(3, 2), meta=np.ndarray>\n", + " tx90pETCCDI (time, lat, lon) float32 dask.array<chunksize=(3, 96, 192), meta=np.ndarray>\n", + "Attributes: (12/36)\n", + " CDI: Climate Data Interface version 2.0.5 (https://m...\n", + " Conventions: CF-1.4\n", + " source: MPI-ESM-LR 2011; URL: http://svn.zmaw.de/svn/co...\n", + " institution: Max Planck Institute for Meteorology\n", + " institute_id: MPI-M\n", + " experiment_id: historical\n", + " ... ...\n", + " ETCCDI_software: climdex.pcic\n", + " ETCCDI_software_version: 1.1.11\n", + " frequency: yr\n", + " creation_date: 2023-09-11T20:05:11Z\n", + " title: ETCCDI indices computed on MPI-ESM-LR model out...\n", + " CDO: Climate Data Operators version 2.0.5 (https://m...
⛈ Executing futures ...\n\n", + "text/plain": "⛈ Executing futures ...\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ] + } + }, + "61acf91180514d5e803f091435c1612e": { + "model_module": "@jupyter-widgets/output", + "model_module_version": "1.0.0", + "model_name": "OutputModel", + "state": { + "layout": "IPY_MODEL_9c18d70ab886442f9112929e378411d7", + "outputs": [ + { + "data": { + "text/html": "
⛈ Executing futures ...\n\n", + "text/plain": "⛈ Executing futures ...\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ] + } + }, + "9c18d70ab886442f9112929e378411d7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "c7825b1332ed4b48a784b88ef87270fd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + }, + "dd49267695d047eab2d5432fa1ed7041": { + "model_module": "@jupyter-widgets/output", + "model_module_version": "1.0.0", + "model_name": "OutputModel", + "state": { + "layout": "IPY_MODEL_c7825b1332ed4b48a784b88ef87270fd", + "outputs": [ + { + "data": { + "text/html": "
🌧 Executing futures ...\n\n", + "text/plain": "🌧 Executing futures ...\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ] + } + }, + "fb04cb54a3204412a958a0055e186784": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": {} + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/talks/FrevaFutures/FuturesExample.md b/talks/FrevaFutures/FuturesExample.md new file mode 100644 index 0000000..f1bcc7f --- /dev/null +++ b/talks/FrevaFutures/FuturesExample.md @@ -0,0 +1,1305 @@ +# Registering a dataset that will exist in the future + +## Here we use a freva plugin run that has been applied + + +```python +import freva +import xarray as xr +from freva._futures import Futures +hist_id = 3085 # We can get this ID using the freva.history command +_ = Futures.register_future_from_history_id(hist_id) +``` + +## Let's search for the data + + +```python +list(freva.databrowser(variable="tx90petccdi")) +``` + + + + + ['future:///scratch/b/b380001/futures/6def5135a687932d27f419a3e993b5bd68aa03425ff0378cfb7745c0aef497a5/cmip5/output1/mpi-m/mpi-esm-lr/historical/yr/atmos/1day/r1i1p1/tx90pETCCDI/tx90pETCCDI_1day_mpi-esm-lr_historical_r1i1p1_199007020000-199207011200'] + + + +## The data doesn't exist yet, but can be created on demand: + + +```python +dset = xr.open_mfdataset( + freva.databrowser(variable="tx90petccdi", + execute_future=True + ) +) +dset +``` + + + Output() + + + + + + + + + + +
+
+
+
+
+### The data has bee loaded, we can work with it (plot it)
+
+
+```python
+dset.sum(dim="time")["tx90pETCCDI"].plot()
+```
+
+
+
+
+ <xarray.Dataset> +Dimensions: (time: 3, bnds: 2, lon: 192, lat: 96) +Coordinates: + * time (time) datetime64[ns] 1990-07-02 1991-07-02 1992-07-01T12:00:00 + * lon (lon) float64 -179.1 -177.2 -175.3 -173.4 ... 175.3 177.2 179.1 + * lat (lat) float64 -89.06 -87.19 -85.31 -83.44 ... 85.31 87.19 89.06 + height float64 ... +Dimensions without coordinates: bnds +Data variables: + time_bnds (time, bnds) datetime64[ns] dask.array<chunksize=(3, 2), meta=np.ndarray> + tx90pETCCDI (time, lat, lon) float32 dask.array<chunksize=(3, 96, 192), meta=np.ndarray> +Attributes: (12/36) + CDI: Climate Data Interface version 2.0.5 (https://m... + Conventions: CF-1.4 + source: MPI-ESM-LR 2011; URL: http://svn.zmaw.de/svn/co... + institution: Max Planck Institute for Meteorology + institute_id: MPI-M + experiment_id: historical + ... ... + ETCCDI_software: climdex.pcic + ETCCDI_software_version: 1.1.11 + frequency: yr + creation_date: 2023-09-11T19:57:50Z + title: ETCCDI indices computed on MPI-ESM-LR model out... + CDO: Climate Data Operators version 2.0.5 (https://m...
+
+
+
diff --git a/talks/FrevaFutures/index.slides.html b/talks/FrevaFutures/index.slides.html
new file mode 100644
index 0000000..dfaa199
--- /dev/null
+++ b/talks/FrevaFutures/index.slides.html
@@ -0,0 +1,16336 @@
+
+
+
+
+
+
+
+
+
+<xarray.Dataset> +Dimensions: (time: 3, bnds: 2, lon: 192, lat: 96) +Coordinates: + * time (time) datetime64[ns] 1990-07-02 1991-07-02 1992-07-01T12:00:00 + * lon (lon) float64 -179.1 -177.2 -175.3 -173.4 ... 175.3 177.2 179.1 + * lat (lat) float64 -89.06 -87.19 -85.31 -83.44 ... 85.31 87.19 89.06 + height float64 ... +Dimensions without coordinates: bnds +Data variables: + time_bnds (time, bnds) datetime64[ns] dask.array<chunksize=(3, 2), meta=np.ndarray> + tx90pETCCDI (time, lat, lon) float32 dask.array<chunksize=(3, 96, 192), meta=np.ndarray> +Attributes: (12/36) + CDI: Climate Data Interface version 2.0.5 (https://m... + Conventions: CF-1.4 + source: MPI-ESM-LR 2011; URL: http://svn.zmaw.de/svn/co... + institution: Max Planck Institute for Meteorology + institute_id: MPI-M + experiment_id: historical + ... ... + ETCCDI_software: climdex.pcic + ETCCDI_software_version: 1.1.11 + frequency: yr + creation_date: 2023-09-11T20:05:11Z + title: ETCCDI indices computed on MPI-ESM-LR model out... + CDO: Climate Data Operators version 2.0.5 (https://m...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/talks/FrevaFutures/output_8_1.png b/talks/FrevaFutures/output_8_1.png
new file mode 100644
index 0000000..a7867e4
Binary files /dev/null and b/talks/FrevaFutures/output_8_1.png differ
+
+
+
+
+
+
+
+
+Registering a dataset that will exist in the future¶
+
+
+
+
+
+
+
+
+
+Here we use a freva plugin run that has been applied¶
+
+
+
+
+
+
+
+
+In [24]:
+
+
+
+
+
+
+import freva
+import xarray as xr
+from freva._futures import Futures
+hist_id = 3085 # We can get this ID using the freva.history command
+_ = Futures.register_future_from_history_id(hist_id)
+
+
+
+
+
+
+
+
+
+Let's search for the data¶
+
+
+
+
+
+
+
+
+In [23]:
+
+
+
+
+
+
+list(freva.databrowser(variable="tx90petccdi"))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Out[23]:
+
+
+
+
+
+
+
+['future:///scratch/b/b380001/futures/6def5135a687932d27f419a3e993b5bd68aa03425ff0378cfb7745c0aef497a5/cmip5/output1/mpi-m/mpi-esm-lr/historical/yr/atmos/1day/r1i1p1/tx90pETCCDI/tx90pETCCDI_1day_mpi-esm-lr_historical_r1i1p1_199007020000-199207011200']+
+
+
+
+
+
+
+
+
+The data doesn't exist yet, but can be created on demand:¶
+
+
+
+
+
+
+
+
+In [25]:
+
+
+
+
+
+
+dset = xr.open_mfdataset(
+ freva.databrowser(variable="tx90petccdi",
+ execute_future=True
+ )
+)
+dset
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Out[25]:
+
+
+
+
+
+
+
+
+<xarray.Dataset> +Dimensions: (time: 3, bnds: 2, lon: 192, lat: 96) +Coordinates: + * time (time) datetime64[ns] 1990-07-02 1991-07-02 1992-07-01T12:00:00 + * lon (lon) float64 -179.1 -177.2 -175.3 -173.4 ... 175.3 177.2 179.1 + * lat (lat) float64 -89.06 -87.19 -85.31 -83.44 ... 85.31 87.19 89.06 + height float64 ... +Dimensions without coordinates: bnds +Data variables: + time_bnds (time, bnds) datetime64[ns] dask.array<chunksize=(3, 2), meta=np.ndarray> + tx90pETCCDI (time, lat, lon) float32 dask.array<chunksize=(3, 96, 192), meta=np.ndarray> +Attributes: (12/36) + CDI: Climate Data Interface version 2.0.5 (https://m... + Conventions: CF-1.4 + source: MPI-ESM-LR 2011; URL: http://svn.zmaw.de/svn/co... + institution: Max Planck Institute for Meteorology + institute_id: MPI-M + experiment_id: historical + ... ... + ETCCDI_software: climdex.pcic + ETCCDI_software_version: 1.1.11 + frequency: yr + creation_date: 2023-09-11T19:57:50Z + title: ETCCDI indices computed on MPI-ESM-LR model out... + CDO: Climate Data Operators version 2.0.5 (https://m...
+
+
+
+
+
+
+
+
+The data has bee loaded, we can work with it (plot it)¶
+
+
+
+
+
+
+
+
+In [36]:
+
+
+
+
+
+
+dset.sum(dim="time")["tx90pETCCDI"].plot()
+
+
+
+
+
+
+
+
+
+
+
+
+
+Out[36]:
+
+
+
+
+
+
+
+<matplotlib.collections.QuadMesh at 0x7fff430ff450>+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+What happens if the data get's lost?¶
Let's delete the data:
+ +
+
+
+
+
+
+
+
+
+In [32]:
+
+
+
+
+
+
+!rm -fr /scratch/b/b380001/futures/6def5135a687932d27f419a3e993b5bd68aa03425ff0378cfb7745c0aef497a5
+
+
+
+
+
+
+
+
+
+The data is still in the databrowser:
+ +
+
+
+
+
+
+
+
+In [37]:
+
+
+
+
+
+
+list(freva.databrowser(variable="tx90petccdi"))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Out[37]:
+
+
+
+
+
+
+
+['/scratch/b/b380001/futures/6def5135a687932d27f419a3e993b5bd68aa03425ff0378cfb7745c0aef497a5/cmip5/output1/mpi-m/mpi-esm-lr/historical/yr/atmos/yr/r1i1p1/v20230911/tx90pETCCDI/tx90pETCCDI_yr_mpi-esm-lr_historical_r1i1p1_199007020000-199207011200.nc']+
+
+
+
+
+
+
+
+
+Because of that the data can be re-created:
+ +
+
+
+
+
+
+
+
+In [35]:
+
+
+
+
+
+
+dset = xr.open_mfdataset(
+ freva.databrowser(variable="tx90petccdi",
+ execute_future=True
+ )
+)
+dset
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Out[35]:
+
+
+
+
+
+
+
+
+<xarray.Dataset> +Dimensions: (time: 3, bnds: 2, lon: 192, lat: 96) +Coordinates: + * time (time) datetime64[ns] 1990-07-02 1991-07-02 1992-07-01T12:00:00 + * lon (lon) float64 -179.1 -177.2 -175.3 -173.4 ... 175.3 177.2 179.1 + * lat (lat) float64 -89.06 -87.19 -85.31 -83.44 ... 85.31 87.19 89.06 + height float64 ... +Dimensions without coordinates: bnds +Data variables: + time_bnds (time, bnds) datetime64[ns] dask.array<chunksize=(3, 2), meta=np.ndarray> + tx90pETCCDI (time, lat, lon) float32 dask.array<chunksize=(3, 96, 192), meta=np.ndarray> +Attributes: (12/36) + CDI: Climate Data Interface version 2.0.5 (https://m... + Conventions: CF-1.4 + source: MPI-ESM-LR 2011; URL: http://svn.zmaw.de/svn/co... + institution: Max Planck Institute for Meteorology + institute_id: MPI-M + experiment_id: historical + ... ... + ETCCDI_software: climdex.pcic + ETCCDI_software_version: 1.1.11 + frequency: yr + creation_date: 2023-09-11T20:05:11Z + title: ETCCDI indices computed on MPI-ESM-LR model out... + CDO: Climate Data Operators version 2.0.5 (https://m...