From c20cd25cacdb648abd7da2561f94e8b035b2d928 Mon Sep 17 00:00:00 2001 From: Rob Fatland Date: Mon, 27 May 2024 09:10:46 -0700 Subject: [PATCH] added zarr --- .../chapters/loadData-Shallow_profilers.ipynb | 3509 +++++++++++++++++ book/chapters/rob/documentation.ipynb | 72 +- 2 files changed, 3545 insertions(+), 36 deletions(-) create mode 100755 book/chapters/loadData-Shallow_profilers.ipynb diff --git a/book/chapters/loadData-Shallow_profilers.ipynb b/book/chapters/loadData-Shallow_profilers.ipynb new file mode 100755 index 0000000..a6edb8a --- /dev/null +++ b/book/chapters/loadData-Shallow_profilers.ipynb @@ -0,0 +1,3509 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "2228ec14-bbdb-429b-b19a-49d273c67dbd", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import xarray as xr\n", + "import s3fs\n", + "\n", + "def loadData(zarrDir):\n", + " fs = s3fs.S3FileSystem(anon=True)\n", + " zarr_store = fs.get_mapper(zarrDir)\n", + " ds = xr.open_zarr(zarr_store, consolidated=True)\n", + " return ds\n", + "\n", + "#zarrFile = 'ooi-data/CE04OSPS-SF01B-2A-CTDPFA107-streamed-ctdpf_sbe43_sample'\n", + "#zarrFile = 'ooi-data/RS01SBPS-SF01A-2A-CTDPFA102-streamed-ctdpf_sbe43_sample'\n", + "zarrFile = 'ooi-data/RS03AXPS-SF03A-2A-CTDPFA302-streamed-ctdpf_sbe43_sample'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a56bc2c6-e16d-49ee-b3cc-8166935f0c42", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:                                            (time: 245304383)\n",
+       "Coordinates:\n",
+       "  * time                                               (time) datetime64[ns] ...\n",
+       "Data variables: (12/39)\n",
+       "    conductivity                                       (time) float64 dask.array<chunksize=(12000000,), meta=np.ndarray>\n",
+       "    corrected_dissolved_oxygen                         (time) float64 dask.array<chunksize=(12000000,), meta=np.ndarray>\n",
+       "    corrected_dissolved_oxygen_qartod_executed         (time) <U2 dask.array<chunksize=(12000000,), meta=np.ndarray>\n",
+       "    corrected_dissolved_oxygen_qartod_results          (time) uint8 dask.array<chunksize=(100000000,), meta=np.ndarray>\n",
+       "    corrected_dissolved_oxygen_qc_executed             (time) uint8 dask.array<chunksize=(100000000,), meta=np.ndarray>\n",
+       "    corrected_dissolved_oxygen_qc_results              (time) uint8 dask.array<chunksize=(100000000,), meta=np.ndarray>\n",
+       "    ...                                                 ...\n",
+       "    sea_water_temperature                              (time) float64 dask.array<chunksize=(12000000,), meta=np.ndarray>\n",
+       "    sea_water_temperature_qartod_executed              (time) <U2 dask.array<chunksize=(12000000,), meta=np.ndarray>\n",
+       "    sea_water_temperature_qartod_results               (time) uint8 dask.array<chunksize=(100000000,), meta=np.ndarray>\n",
+       "    sea_water_temperature_qc_executed                  (time) uint8 dask.array<chunksize=(100000000,), meta=np.ndarray>\n",
+       "    sea_water_temperature_qc_results                   (time) uint8 dask.array<chunksize=(100000000,), meta=np.ndarray>\n",
+       "    temperature                                        (time) float64 dask.array<chunksize=(12000000,), meta=np.ndarray>\n",
+       "Attributes: (12/62)\n",
+       "    AssetManagementRecordLastModified:  2024-05-10T17:57:50.594000\n",
+       "    AssetUniqueID:                      ATAPL-66662-00010\n",
+       "    Conventions:                        CF-1.6\n",
+       "    Description:                        CTD Profiler: CTDPF Series A\n",
+       "    FirmwareVersion:                    Not specified.\n",
+       "    Manufacturer:                       Sea-Bird Electronics\n",
+       "    ...                                 ...\n",
+       "    stream:                             ctdpf_sbe43_sample\n",
+       "    subsite:                            RS03AXPS\n",
+       "    summary:                            Dataset Generated by Stream Engine fr...\n",
+       "    time_coverage_end:                  2024-05-26T11:27:20.932029952\n",
+       "    time_coverage_start:                2014-10-07T21:32:53.601809920\n",
+       "    title:                              Data produced by Stream Engine versio...
" + ], + "text/plain": [ + "\n", + "Dimensions: (time: 245304383)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] ...\n", + "Data variables: (12/39)\n", + " conductivity (time) float64 dask.array\n", + " corrected_dissolved_oxygen (time) float64 dask.array\n", + " corrected_dissolved_oxygen_qartod_executed (time) \n", + " corrected_dissolved_oxygen_qartod_results (time) uint8 dask.array\n", + " corrected_dissolved_oxygen_qc_executed (time) uint8 dask.array\n", + " corrected_dissolved_oxygen_qc_results (time) uint8 dask.array\n", + " ... ...\n", + " sea_water_temperature (time) float64 dask.array\n", + " sea_water_temperature_qartod_executed (time) \n", + " sea_water_temperature_qartod_results (time) uint8 dask.array\n", + " sea_water_temperature_qc_executed (time) uint8 dask.array\n", + " sea_water_temperature_qc_results (time) uint8 dask.array\n", + " temperature (time) float64 dask.array\n", + "Attributes: (12/62)\n", + " AssetManagementRecordLastModified: 2024-05-10T17:57:50.594000\n", + " AssetUniqueID: ATAPL-66662-00010\n", + " Conventions: CF-1.6\n", + " Description: CTD Profiler: CTDPF Series A\n", + " FirmwareVersion: Not specified.\n", + " Manufacturer: Sea-Bird Electronics\n", + " ... ...\n", + " stream: ctdpf_sbe43_sample\n", + " subsite: RS03AXPS\n", + " summary: Dataset Generated by Stream Engine fr...\n", + " time_coverage_end: 2024-05-26T11:27:20.932029952\n", + " time_coverage_start: 2014-10-07T21:32:53.601809920\n", + " title: Data produced by Stream Engine versio..." + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "siteData = loadData(zarrFile)\n", + "siteData" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "6d82a0a7-c1a0-48dc-ae14-f32a41356e99", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.DataArray 'time' ()>\n",
+       "array('2024-05-26T11:27:20.932029952', dtype='datetime64[ns]')\n",
+       "Coordinates:\n",
+       "    time     datetime64[ns] 2024-05-26T11:27:20.932029952\n",
+       "Attributes:\n",
+       "    axis:           T\n",
+       "    long_name:      time\n",
+       "    standard_name:  time
" + ], + "text/plain": [ + "\n", + "array('2024-05-26T11:27:20.932029952', dtype='datetime64[ns]')\n", + "Coordinates:\n", + " time datetime64[ns] 2024-05-26T11:27:20.932029952\n", + "Attributes:\n", + " axis: T\n", + " long_name: time\n", + " standard_name: time" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "siteData.time[-1]" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e08e785a-5741-45e5-9f4c-cb497fa83eed", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'siteData' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28mtype\u001b[39m(siteData)\n", + "\u001b[0;31mNameError\u001b[0m: name 'siteData' is not defined" + ] + } + ], + "source": [ + "type(siteData)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5646cad3-6c2a-4898-9092-59cfdc88eb60", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "siteData.to_netcdf(\"~/RS03AXPS-SF03A-2A-CTDPFA302-streamed-ctdpf_sbe43_sample.nc\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/book/chapters/rob/documentation.ipynb b/book/chapters/rob/documentation.ipynb index 69b7627..dc15cc1 100644 --- a/book/chapters/rob/documentation.ipynb +++ b/book/chapters/rob/documentation.ipynb @@ -2001,42 +2001,6 @@ "HTML(im_ani.to_html5_video())" ] }, - { - "cell_type": "markdown", - "id": "responsible-client", - "metadata": {}, - "source": [ - "## Binder\n", - "\n", - "[Top](#Introduction)\n", - "\n", - "* Create a binder badge in the home page `README.md` of the repository. \n", - "\n", - "```\n", - "[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh///HEAD)\n", - "\n", - "```\n", - "\n", - "* In `/binder` create `environment.yml` to match the working environment\n", - " * For this repo as of 10/23/2021 `binder/environment.yml` was: \n", - "\n", - "\n", - "```\n", - "channels:\n", - " - conda-forge\n", - "dependencies:\n", - " - python=3\n", - " - numpy\n", - " - pandas\n", - " - matplotlib\n", - " - netcdf4\n", - " - xarray\n", - " - ffmpeg\n", - "```\n", - "\n", - "\n" - ] - }, { "cell_type": "markdown", "id": "89a502fd", @@ -3138,6 +3102,42 @@ " - ...noting that August through October 2018 appears to be good data\n" ] }, + { + "cell_type": "markdown", + "id": "responsible-client", + "metadata": {}, + "source": [ + "## Binder\n", + "\n", + "[Top](#Introduction)\n", + "\n", + "* Create a binder badge in the home page `README.md` of the repository. \n", + "\n", + "```\n", + "[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh///HEAD)\n", + "\n", + "```\n", + "\n", + "* In `/binder` create `environment.yml` to match the working environment\n", + " * For this repo as of 10/23/2021 `binder/environment.yml` was: \n", + "\n", + "\n", + "```\n", + "channels:\n", + " - conda-forge\n", + "dependencies:\n", + " - python=3\n", + " - numpy\n", + " - pandas\n", + " - matplotlib\n", + " - netcdf4\n", + " - xarray\n", + " - ffmpeg\n", + "```\n", + "\n", + "\n" + ] + }, { "cell_type": "code", "execution_count": null,