From 42ee9359e9e0e77d686a98a64311dfa48d76167e Mon Sep 17 00:00:00 2001 From: Eli Holmes Date: Wed, 21 Aug 2024 01:25:15 +0000 Subject: [PATCH] working to get the CNN tutorial working --- book/notebooks/CHL_prediction_CNN.ipynb | 1677 ++++++++++++++----- book/notebooks/IO_Zarr_visualizations.ipynb | 2 +- 2 files changed, 1252 insertions(+), 427 deletions(-) diff --git a/book/notebooks/CHL_prediction_CNN.ipynb b/book/notebooks/CHL_prediction_CNN.ipynb index 7242ee6..00915dd 100644 --- a/book/notebooks/CHL_prediction_CNN.ipynb +++ b/book/notebooks/CHL_prediction_CNN.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -26,14 +26,14 @@ "import matplotlib.pyplot as plt\n", "\n", "import tensorflow as tf\n", - "from tensorflow.keras.callbacks import EarlyStopping\n", - "from tensorflow.keras.models import Sequential\n", - "from tensorflow.keras.layers import ConvLSTM2D, BatchNormalization, Conv2D, Dropout" + "from keras.callbacks import EarlyStopping\n", + "from keras.models import Sequential\n", + "from keras.layers import ConvLSTM2D, BatchNormalization, Conv2D, Dropout" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -42,7 +42,17 @@ "text": [ "All Physical Devices: [PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n", "Available GPUs: [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n", - "GPU Details: {'device_name': 'NVIDIA GeForce RTX 4070 Laptop GPU', 'compute_capability': (8, 9)}\n" + "GPU Details: {'compute_capability': (7, 5), 'device_name': 'Tesla T4'}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-08-21 01:07:44.661600: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2024-08-21 01:07:44.720010: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2024-08-21 01:07:44.720279: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2024-08-21 01:07:44.721706: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" ] } ], @@ -66,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -103,6 +113,7 @@ "}\n", "\n", "html[theme=dark],\n", + "html[data-theme=dark],\n", "body[data-theme=dark],\n", "body.vscode-dark {\n", " --xr-font-color0: rgba(255, 255, 255, 1);\n", @@ -384,127 +395,784 @@ " padding-left: 25px !important;\n", "}\n", "\n", - ".xr-attrs,\n", - ".xr-var-attrs,\n", - ".xr-var-data,\n", - ".xr-index-data {\n", - " grid-column: 1 / -1;\n", - "}\n", + ".xr-attrs,\n", + ".xr-var-attrs,\n", + ".xr-var-data,\n", + ".xr-index-data {\n", + " grid-column: 1 / -1;\n", + "}\n", + "\n", + "dl.xr-attrs {\n", + " padding: 0;\n", + " margin: 0;\n", + " display: grid;\n", + " grid-template-columns: 125px auto;\n", + "}\n", + "\n", + ".xr-attrs dt,\n", + ".xr-attrs dd {\n", + " padding: 0;\n", + " margin: 0;\n", + " float: left;\n", + " padding-right: 10px;\n", + " width: auto;\n", + "}\n", + "\n", + ".xr-attrs dt {\n", + " font-weight: normal;\n", + " grid-column: 1;\n", + "}\n", + "\n", + ".xr-attrs dt:hover span {\n", + " display: inline-block;\n", + " background: var(--xr-background-color);\n", + " padding-right: 10px;\n", + "}\n", + "\n", + ".xr-attrs dd {\n", + " grid-column: 2;\n", + " white-space: pre-wrap;\n", + " word-break: break-all;\n", + "}\n", + "\n", + ".xr-icon-database,\n", + ".xr-icon-file-text2,\n", + ".xr-no-icon {\n", + " display: inline-block;\n", + " vertical-align: middle;\n", + " width: 1em;\n", + " height: 1.5em !important;\n", + " stroke-width: 0;\n", + " stroke: currentColor;\n", + " fill: currentColor;\n", + "}\n", + "
<xarray.Dataset> Size: 958MB\n",
+       "Dimensions:                       (time: 366, lat: 149, lon: 181)\n",
+       "Coordinates:\n",
+       "  * lat                           (lat) float32 596B 32.0 31.75 ... -4.75 -5.0\n",
+       "  * lon                           (lon) float32 724B 45.0 45.25 ... 89.75 90.0\n",
+       "  * time                          (time) datetime64[ns] 3kB 2020-01-01 ... 20...\n",
+       "Data variables: (12/27)\n",
+       "    CHL                           (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    CHL_cmes-cloud                (time, lat, lon) uint8 10MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    CHL_cmes-gapfree              (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    CHL_cmes-land                 (lat, lon) uint8 27kB dask.array<chunksize=(149, 181), meta=np.ndarray>\n",
+       "    CHL_cmes-level3               (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    CHL_cmes_flags-gapfree        (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    ...                            ...\n",
+       "    ug_curr                       (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    v_curr                        (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    v_wind                        (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    vg_curr                       (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    wind_dir                      (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "    wind_speed                    (time, lat, lon) float32 39MB dask.array<chunksize=(23, 149, 181), meta=np.ndarray>\n",
+       "Attributes: (12/92)\n",
+       "    Conventions:                     CF-1.8, ACDD-1.3\n",
+       "    DPM_reference:                   GC-UD-ACRI-PUG\n",
+       "    IODD_reference:                  GC-UD-ACRI-PUG\n",
+       "    acknowledgement:                 The Licensees will ensure that original ...\n",
+       "    citation:                        The Licensees will ensure that original ...\n",
+       "    cmems_product_id:                OCEANCOLOUR_GLO_BGC_L3_MY_009_103\n",
+       "    ...                              ...\n",
+       "    time_coverage_end:               2024-04-18T02:58:23Z\n",
+       "    time_coverage_resolution:        P1D\n",
+       "    time_coverage_start:             2024-04-16T21:12:05Z\n",
+       "    title:                           cmems_obs-oc_glo_bgc-plankton_my_l3-mult...\n",
+       "    westernmost_longitude:           -180.0\n",
+       "    westernmost_valid_longitude:     -180.0