From 3f2fc33d87d7fc76a40454068134ed750517c46e Mon Sep 17 00:00:00 2001 From: spestana Date: Wed, 25 Sep 2024 22:49:17 +0000 Subject: [PATCH 1/3] Add goes2go downloader functionality --- docs/examples/download.ipynb | 217 ++++ docs/examples/example.json | 19 - docs/examples/quick_start.ipynb | 2038 +------------------------------ src/goes_ortho/clip.py | 13 +- src/goes_ortho/get_data.py | 143 ++- src/goes_ortho/orthorectify.py | 71 +- 6 files changed, 390 insertions(+), 2111 deletions(-) create mode 100644 docs/examples/download.ipynb delete mode 100644 docs/examples/example.json diff --git a/docs/examples/download.ipynb b/docs/examples/download.ipynb new file mode 100644 index 0000000..3e4fddf --- /dev/null +++ b/docs/examples/download.ipynb @@ -0,0 +1,217 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3aea64b8-4c4c-478f-85a3-a5c36321c4c9", + "metadata": {}, + "source": [ + "# Download GOES\n", + "\n", + "This notebook demonstrates downloading a short time series of GOES-R ABI imagery." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "5c0d77eb-457b-4b4d-a669-73778a54637e", + "metadata": {}, + "outputs": [], + "source": [ + "import goes_ortho as go\n", + "import xarray as xr" + ] + }, + { + "cell_type": "markdown", + "id": "0e01f57b-bbbe-4ebf-9c25-fff635326c2d", + "metadata": {}, + "source": [ + "First, specify the time range, location bounds, satellite, product (and if applicable, band and variable) that we'd like to access.\n", + "\n", + "We will also need to provide an API key for [OpenTopography.org](https://portal.opentopography.org/requestService?service=api) which you can create with a free account. This allows goes_ortho to access digital elevation models to perform the orthorectification step.\n", + "\n", + "The workflow below was developed to read a json file containing information about what we'd like to download. This was done to 1) allow these functions to run through github actions (still an experimental feature) and 2) keep a record of datasets we've downloaded. This is something that may change in the near future since it adds an unnecessary step for most use cases." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "6fc6c19c-f21f-4319-a45e-47434b933600", + "metadata": {}, + "outputs": [], + "source": [ + "lon = -119.31212\n", + "lat = 37.88175\n", + "z = 2811 # elevation in meters" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "28208c5f-e93c-4ce9-b4f7-c4c7ea14e12e", + "metadata": {}, + "outputs": [], + "source": [ + "startDatetime = \"2024-04-19T00:00:00Z\"\n", + "endDatetime = \"2024-04-19T00:59:00Z\"\n", + "[min_lon, min_lat, max_lon, max_lat] = [lon-1, lat-1, lon+1, lat+1] # set bounds to just a little area around the location we want\n", + "satellite = \"goes18\"\n", + "product = \"ABI-L1b-RadC\"\n", + "band = 2\n", + "variable = \"Rad\"\n", + "OPENTOPO_API_KEY = \"585b1d1639bc5ef8a4a5bdea7e45a8d1\"" + ] + }, + { + "cell_type": "markdown", + "id": "8a4c97cb-b630-4771-8e80-4c7f70c0f1bf", + "metadata": {}, + "source": [ + "Using the data above, we make the required json file." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "cac7ae71-01db-4fa3-9e9f-856b82ff738b", + "metadata": {}, + "outputs": [], + "source": [ + "# Make request file from user input\n", + "go.get_data.make_request_json(\n", + " f\"{satellite}-example-ACMC\",\n", + " startDatetime,\n", + " endDatetime,\n", + " [min_lon, min_lat, max_lon, max_lat],\n", + " satellite,\n", + " product,\n", + " band,\n", + " variable,\n", + " OPENTOPO_API_KEY,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9fbfaf9c-938a-475a-a1f7-f2a36b062505", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Estimated 1 batches to download\n", + "Batch number 1\n", + "Download batch of imagery from 2024-04-19 00:00:00+00:00 to 2024-04-19 03:00:00+00:00\n", + "📦 Finished downloading [576] files to [/home/spestana/data/noaa-goes18/ABI-L1b-RadC].\n", + "Cropping image batch to [-120.31212, 36.88175, -118.31212, 38.88175]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 576/576 [01:44<00:00, 5.52it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done\n", + "CPU times: user 1min 58s, sys: 22 s, total: 2min 20s\n", + "Wall time: 2min 50s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "%%time\n", + "filepaths = go.get_data.download_abi_goes2go(f\"{satellite}-example-ACMC.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "738b548c-8807-4fb8-82c2-eb380a5284c8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Files will be downloaded and then cropped to these bounds:\n", + "\t(-120.31212,38.88175).\t.(-118.31212,38.88175)\n", + "\n", + "\n", + "\n", + "\t(-120.31212,36.88175).\t.(-118.31212,36.88175)\n", + "\n", + "For each S3 bucket, download the corresponding observations\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02\n", + "Downloading... the product ABI-L1b-RadC \n", + "Downloading dataset to... 00 UTC\n", + "OR_ABI-L1b-RadC-M6C02_G18_s20241100056172_e20241100058545_c20241100058573.nc: [##########] 100% 67.930455 MB/67.930455 MB\n", + "Subsetting files in...goes18/2024/4/19/ABI-L1b-RadC/00/C02\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100036172_e20241100038545_c20241100038570.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100006172_e20241100008545_c20241100008570.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100046172_e20241100048545_c20241100048571.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100021172_e20241100023545_c20241100023570.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100011172_e20241100013545_c20241100013573.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100016172_e20241100018545_c20241100018577.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100026172_e20241100028545_c20241100028580.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100031172_e20241100033545_c20241100033572.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100056172_e20241100058545_c20241100058573.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100001172_e20241100003545_c20241100003572.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100041172_e20241100043545_c20241100043570.nc\n", + "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100051172_e20241100053545_c20241100053568.nc\n", + "Done\n", + "CPU times: user 15.7 s, sys: 4.8 s, total: 20.5 s\n", + "Wall time: 51 s\n" + ] + } + ], + "source": [ + "%%time\n", + "filepaths = go.get_data.download_abi_goespy(f\"{satellite}-example-ACMC.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd893536-d241-4966-a49c-dd93729d6c43", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "goes-test-env", + "language": "python", + "name": "goes-test-env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/examples/example.json b/docs/examples/example.json deleted file mode 100644 index 575b351..0000000 --- a/docs/examples/example.json +++ /dev/null @@ -1,19 +0,0 @@ -{ -"dateRange" : { - "startDatetime" : "2024-03-10T00:00:00Z", - "endDatetime" : "2024-03-17T00:00:00Z" - }, -"bounds" : { - "min_lon" : -122.12, - "min_lat" : 46.6, - "max_lon" : -121.35, - "max_lat" : 47.1 - }, -"satellite" : "goes18", -"product" : "ABI-L1b-RadC", -"bands" : [7], -"variables" : ["Rad"], -"downloadDirectory" : "/storage/spestana/scratchspace/", -"outputFilepath" : "/storage/spestana/scratchspace/example.zarr", -"apiKey" : "585b1d1639bc5ef8a4a5bdea7e45a8d1" -} diff --git a/docs/examples/quick_start.ipynb b/docs/examples/quick_start.ipynb index edfa730..84c0e05 100644 --- a/docs/examples/quick_start.ipynb +++ b/docs/examples/quick_start.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "5c0d77eb-457b-4b4d-a669-73778a54637e", "metadata": {}, "outputs": [], @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "28208c5f-e93c-4ce9-b4f7-c4c7ea14e12e", "metadata": {}, "outputs": [], @@ -52,7 +52,7 @@ "product = \"ABI-L1b-RadC\"\n", "band = 2\n", "variable = \"Rad\"\n", - "OPENTOPO_API_KEY = \"585b1d1639bc5ef8a4a5bdea7e45a8d1\"" + "OPENTOPO_API_KEY = \"YOUR_API_KEY_HERE\"" ] }, { @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "cac7ae71-01db-4fa3-9e9f-856b82ff738b", "metadata": {}, "outputs": [], @@ -94,2034 +94,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "faa60edb-76c0-4315-b1df-387819140c24", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download requested imagery\n", - "\n", - "Files will be downloaded and then cropped to these bounds:\n", - "\t(-123,48).\t.(-121,48)\n", - "\n", - "\n", - "\n", - "\t(-123,46).\t.(-121,46)\n", - "\n", - "For each S3 bucket, download the corresponding observations\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02\n", - "Downloading... the product ABI-L1b-RadC \n", - "Downloading dataset to... 00 UTC\n", - "OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566.nc: [##########] 100% 77.23365 MB/77.23365 MBMB\n", - "Subsetting files in...goes18/2022/8/10/ABI-L1b-RadC/00/C02\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:03:56.7Z\n", - " time_coverage_start: 2022-08-10T00:01:17.0Z\n", - " time_coverage_end: 2022-08-10T00:03:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: d2f9e9d4-f486-40cd-b7ad-3384164e4c0e\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:33:56.7Z\n", - " time_coverage_start: 2022-08-10T00:31:17.0Z\n", - " time_coverage_end: 2022-08-10T00:33:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: 16281c04-b4cf-47cb-b0b1-5563401991d1\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:53:56.8Z\n", - " time_coverage_start: 2022-08-10T00:51:17.0Z\n", - " time_coverage_end: 2022-08-10T00:53:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: ee43da04-0d6d-4edc-b82d-94a15fcbb148\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:08:56.7Z\n", - " time_coverage_start: 2022-08-10T00:06:17.0Z\n", - " time_coverage_end: 2022-08-10T00:08:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: 715373cd-4e92-4eb8-8326-652ae65bd935\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:43:56.7Z\n", - " time_coverage_start: 2022-08-10T00:41:17.0Z\n", - " time_coverage_end: 2022-08-10T00:43:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: 7a565460-7431-4c7c-97a1-03a2a50dfa24\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:13:56.7Z\n", - " time_coverage_start: 2022-08-10T00:11:17.0Z\n", - " time_coverage_end: 2022-08-10T00:13:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: 7914dcf9-f36c-4bdf-8aeb-be777f1b3507\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:58:56.6Z\n", - " time_coverage_start: 2022-08-10T00:56:17.0Z\n", - " time_coverage_end: 2022-08-10T00:58:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: 820a89a7-a84e-42f5-839b-b1a4dd922e63\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:18:56.9Z\n", - " time_coverage_start: 2022-08-10T00:16:17.0Z\n", - " time_coverage_end: 2022-08-10T00:18:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: 26e22b19-19b9-4e78-9523-18b6d31de50d\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:28:56.7Z\n", - " time_coverage_start: 2022-08-10T00:26:17.0Z\n", - " time_coverage_end: 2022-08-10T00:28:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: 6a1b3476-feb7-41b7-bff0-1e32160483c7\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:38:56.5Z\n", - " time_coverage_start: 2022-08-10T00:36:17.0Z\n", - " time_coverage_end: 2022-08-10T00:38:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: c94bc4e6-8213-4f7a-8f08-284ff76d69e7\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:23:56.8Z\n", - " time_coverage_start: 2022-08-10T00:21:17.0Z\n", - " time_coverage_end: 2022-08-10T00:23:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: e6095181-c235-442b-aa78-07612d1a4f4e\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569.nc\n", - " Size: 480MB\n", - "Dimensions: (y: 6000, x: 10000,\n", - " number_of_time_bounds: 2,\n", - " number_of_image_bounds: 2,\n", - " band: 1,\n", - " num_star_looks: 24)\n", - "Coordinates:\n", - " t datetime64[ns] 8B ...\n", - " * y (y) float32 24kB 0.1282...\n", - " * x (x) float32 40kB -0.069...\n", - " y_image float32 4B ...\n", - " x_image float32 4B ...\n", - " band_id (band) int8 1B ...\n", - " band_wavelength (band) float32 4B ...\n", - " t_star_look (num_star_looks) datetime64[ns] 192B ...\n", - " band_wavelength_star_look (num_star_looks) float32 96B ...\n", - "Dimensions without coordinates: number_of_time_bounds, number_of_image_bounds,\n", - " band, num_star_looks\n", - "Data variables: (12/37)\n", - " Rad (y, x) float32 240MB ...\n", - " DQF (y, x) float32 240MB ...\n", - " time_bounds (number_of_time_bounds) datetime64[ns] 16B ...\n", - " goes_imager_projection int32 4B ...\n", - " y_image_bounds (number_of_image_bounds) float32 8B ...\n", - " x_image_bounds (number_of_image_bounds) float32 8B ...\n", - " ... ...\n", - " algorithm_dynamic_input_data_container int32 4B ...\n", - " processing_parm_version_container int32 4B ...\n", - " algorithm_product_version_container int32 4B ...\n", - " star_id (num_star_looks) float32 96B ...\n", - " channel_integration_time float64 8B ...\n", - " channel_gain_field float64 8B ...\n", - "Attributes: (12/30)\n", - " naming_authority: gov.nesdis.noaa\n", - " Conventions: CF-1.7\n", - " standard_name_vocabulary: CF Standard Name Table (v35, 20 July 2016)\n", - " institution: DOC/NOAA/NESDIS > U.S. Department of Commerce,...\n", - " project: GOES\n", - " production_site: WCDAS\n", - " ... ...\n", - " timeline_id: ABI Mode 6\n", - " date_created: 2022-08-10T00:48:56.9Z\n", - " time_coverage_start: 2022-08-10T00:46:17.0Z\n", - " time_coverage_end: 2022-08-10T00:48:54.3Z\n", - " LUT_Filenames: SpaceLookParams(FM3A_CDRL79RevC_PR_09_08_18)-7...\n", - " id: 32d8d8d8-6c28-454c-b737-0044fa933079\n", - "SW Corner: 0.028150758694010212, 0.120010256764695\n", - "SE Corner: 0.032038042426967735, 0.11988449939379014\n", - "NW Corner: 0.026996097229990735, 0.1234469575576232\n", - "NE Corner: 0.0307254555843864, 0.12332291940734816\n", - "Corner coords chosen: N: 0.1234469575576232, S: 0.11988449939379014; E: 0.032038042426967735, W: 0.026996097229990735\n", - "Done\n", - "parse json request file\n", - "orthorectify all images\n", - "[PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569.nc')]\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567_o.nc\n", - "https://portal.opentopography.org/API/globaldem?demtype=SRTMGL3&west=-123&south=46&east=-121&north=48&outputFormat=GTiff&API_Key=585b1d1639bc5ef8a4a5bdea7e45a8d1\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n", - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568_o.nc\n", - "...done\n", - "filename: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569.nc\n", - "renamed to: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569_o.nc\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "/bin/gdalwarp -r cubic -co COMPRESS=LZW -co TILED=YES -co BIGTIFF=IF_SAFER -tr 30 30 -t_srs '+proj=lonlat +datum=GRS80' temp_SRTMGL3_DEM.tif temp_SRTMGL3_DEM_proj.tif\n", - "Usage: gdalwarp [--help-general] [--formats]\n", - " [-s_srs srs_def] [-t_srs srs_def] [-to \"NAME=VALUE\"]* [-novshiftgrid]\n", - " [-order n | -tps | -rpc | -geoloc] [-et err_threshold]\n", - " [-refine_gcps tolerance [minimum_gcps]]\n", - " [-te xmin ymin xmax ymax] [-tr xres yres] [-tap] [-ts width height]\n", - " [-ovr level|AUTO|AUTO-n|NONE] [-wo \"NAME=VALUE\"] [-ot Byte/Int16/...] [-wt Byte/Int16]\n", - " [-srcnodata \"value [value...]\"] [-dstnodata \"value [value...]\"] -dstalpha\n", - " [-r resampling_method] [-wm memory_in_mb] [-multi] [-q]\n", - " [-cutline datasource] [-cl layer] [-cwhere expression]\n", - " [-csql statement] [-cblend dist_in_pixels] [-crop_to_cutline]\n", - " [-if format]* [-of format] [-co \"NAME=VALUE\"]* [-overwrite]\n", - " [-nomd] [-cvmd meta_conflict_value] [-setci] [-oo NAME=VALUE]*\n", - " [-doo NAME=VALUE]*\n", - " srcfile* dstfile\n", - "\n", - "Available resampling methods:\n", - " near (default), bilinear, cubic, cubicspline, lanczos, average, rms,\n", - " mode, max, min, med, Q1, Q3, sum.\n", - "\n", - "RUNNING: make_ortho_map()\n", - "\n", - "Opening GOES ABI image...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR 1: PROJ: proj_create: Error -9 (unknown elliptical parameter name)\n", - "ERROR 1: Translating source or target SRS failed:\n", - "+proj=lonlat +datum=GRS80\n", - "Child returned 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Get inputs: projection information from the ABI radiance product\n", - "...done\n", - "\n", - "Opening DEM file...\n", - "\n", - "Create 2D arrays of longitude and latitude from the DEM\n", - "...done\n", - "\n", - "For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)\n", - "...done\n", - "\n", - "Create metadata dictionary about this map\n", - "...done\n", - "\n", - "Create pixel map dataset\n", - " Size: 115MB\n", - "Dimensions: (latitude: 2400, longitude: 2400)\n", - "Coordinates:\n", - " * longitude (longitude) float64 19kB -123.0 -123.0 ... -121.0 -121.0\n", - " * latitude (latitude) float64 19kB 48.0 48.0 48.0 ... 46.0 46.0 46.0\n", - " dem_px_angle_x (latitude, longitude) float64 46MB 0.027 0.027 ... 0.03204\n", - " dem_px_angle_y (latitude, longitude) float64 46MB 0.1235 0.1235 ... 0.1199\n", - "Data variables:\n", - " elevation (latitude, longitude) float32 23MB 277.0 289.0 ... 1.107e+03\n", - "Attributes: (12/21)\n", - " longitude_of_projection_origin: -137.0\n", - " semi_major_axis: 6378137.0\n", - " semi_minor_axis: 6356752.31414\n", - " satellite_height: 42164160.0\n", - " grs80_eccentricity: 0.0818191910435\n", - " longitude_of_projection_origin_info: longitude of geostationary satellit...\n", - " ... ...\n", - " dem_ifov_info: instantaneous field of view (angula...\n", - " dem_px_angle_x_info: DEM grid cell X coordinate (east/we...\n", - " dem_px_angle_y_info: DEM grid cell Y coordinate (north/s...\n", - " longitude_info: longitude from DEM file\n", - " latitude_info: latitude from DEM file\n", - " elevation_info: elevation from DEM file\n", - "...done\n", - "\n", - "Return the pixel map dataset.\n", - "\n", - "RUNNING: orthorectify_abi_rad()\n", - "\n", - "Does the projection info in the image match our mapping?\n", - "\n", - "Opening GOES ABI image...\t\t\tABI image value\tPixel map value\n", - "perspective_point_height + semi_major_axis:\t42164160.0\t42164160.0\n", - "semi_major_axis:\t\t\t\t6378137.0\t6378137.0\n", - "semi_minor_axis:\t\t\t\t6356752.31414\t6356752.31414\n", - "longitude_of_projection_origin:\t\t\t-137.0\t\t-137.0\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for Rad\n", - "...done\n", - "\n", - "Map (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates\n", - "...done\n", - "\n", - "Create zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)\n", - "...done\n", - "\n", - "Output this result to a new NetCDF file\n", - "Saving file as: goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569_o.nc\n", - "...done\n", - "[PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569_o.nc')]\n", - "add time dimension, fix CRS, build zarr file\n", - "add_datetime_crs\n", - "[PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568_o.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569_o.nc')]\n", - "Rad\n", - "EPSG:4326\n", - "[datetime.datetime(2022, 8, 10, 0, 1, 17), datetime.datetime(2022, 8, 10, 0, 31, 17), datetime.datetime(2022, 8, 10, 0, 51, 17), datetime.datetime(2022, 8, 10, 0, 6, 17), datetime.datetime(2022, 8, 10, 0, 41, 17), datetime.datetime(2022, 8, 10, 0, 11, 17), datetime.datetime(2022, 8, 10, 0, 56, 17), datetime.datetime(2022, 8, 10, 0, 16, 17), datetime.datetime(2022, 8, 10, 0, 26, 17), datetime.datetime(2022, 8, 10, 0, 36, 17), datetime.datetime(2022, 8, 10, 0, 21, 17), datetime.datetime(2022, 8, 10, 0, 46, 17)]\n", - "Processing 0 of 12...\n", - "2022-08-10 00:01:17\n", - "Processing 1 of 12...\n", - "2022-08-10 00:31:17\n", - "Processing 2 of 12...\n", - "2022-08-10 00:51:17\n", - "Processing 3 of 12...\n", - "2022-08-10 00:06:17\n", - "Processing 4 of 12...\n", - "2022-08-10 00:41:17\n", - "Processing 5 of 12...\n", - "2022-08-10 00:11:17\n", - "Processing 6 of 12...\n", - "2022-08-10 00:56:17\n", - "Processing 7 of 12...\n", - "2022-08-10 00:16:17\n", - "Processing 8 of 12...\n", - "2022-08-10 00:26:17\n", - "Processing 9 of 12...\n", - "2022-08-10 00:36:17\n", - "Processing 10 of 12...\n", - "2022-08-10 00:21:17\n", - "Processing 11 of 12...\n", - "2022-08-10 00:46:17\n", - "start Dask cluster\n", - "\n", - "Dask dashboard at: http://127.0.0.1:8786/status\n", - "Workers: 6\n", - "Threads per worker: 2 \n", - "\n", - "[PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569_o_Rad.nc')]\n", - "[PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220001170_e20222220003543_c20222220003567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220006170_e20222220008543_c20222220008567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220011170_e20222220013543_c20222220013567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220016170_e20222220018543_c20222220018569_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220021170_e20222220023543_c20222220023568_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220026170_e20222220028543_c20222220028567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220031170_e20222220033543_c20222220033567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220036170_e20222220038543_c20222220038565_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220041170_e20222220043543_c20222220043567_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220046170_e20222220048543_c20222220048569_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220051170_e20222220053543_c20222220053568_o_Rad.nc'), PosixPath('goes18/2022/8/10/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20222220056170_e20222220058543_c20222220058566_o_Rad.nc')]\n", - "open all rasters\n", - "rechunk\n", - "saving zarr file\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/spestana/git/goes-ortho/src/goes_ortho/get_data.py:112: UserWarning: Times can't be serialized faithfully to int64 with requested units 'days since 2022-08-10T00:01:17'. Serializing with units 'minutes since 2022-08-10T00:01:17' instead. Set encoding['dtype'] to floating point dtype to serialize with units 'days since 2022-08-10T00:01:17'. Set encoding['units'] to 'minutes since 2022-08-10T00:01:17' to silence this warning .\n", - " ds.to_zarr(outputFilepath)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/\n", - " ├── Rad (12, 2400, 2400) float32\n", - " ├── latitude (2400,) float64\n", - " ├── longitude (2400,) float64\n", - " ├── spatial_ref (12,) int64\n", - " └── time (12,) int64\n", - "Name : /Rad\n", - "Type : zarr.core.Array\n", - "Data type : float32\n", - "Shape : (12, 2400, 2400)\n", - "Chunk shape : (1, 2400, 2400)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : zarr.storage.DirectoryStore\n", - "No. bytes : 276480000 (263.7M)\n", - "No. bytes stored : 65062172 (62.0M)\n", - "Storage ratio : 4.2\n", - "Chunks initialized : 12/12\n", - "\n", - "Done.\n" - ] - } - ], + "outputs": [], "source": [ "# Download GOES imagery and build zarr file\n", "go.get_data.build_zarr(f\"{satellite}-build-zarr-b{band}.json\")" @@ -2137,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "fdb54d4d-1675-4106-becc-2cef6c3715f0", "metadata": {}, "outputs": [], diff --git a/src/goes_ortho/clip.py b/src/goes_ortho/clip.py index 5b6e86a..b6b0e6c 100644 --- a/src/goes_ortho/clip.py +++ b/src/goes_ortho/clip.py @@ -2,6 +2,8 @@ Functions for clipping GOES ABI imagery to smaller areas """ +import logging + import xarray as xr from goes_ortho.geometry import LonLat2ABIangle @@ -40,7 +42,6 @@ def subsetNetCDF(filepath, bounds, newfilepath=None): lat_north = bounds[3] with xr.open_dataset(filepath) as file: - print(file) f = file.load() # Values needed for geometry calculations req = f.goes_imager_projection.semi_major_axis @@ -56,19 +57,19 @@ def subsetNetCDF(filepath, bounds, newfilepath=None): x_rad_sw, y_rad_sw = LonLat2ABIangle( lon_west, lat_south, 0, H, req, rpol, e, lon_0 ) - print("SW Corner: {}, {}".format(x_rad_sw, y_rad_sw)) + logging.info("SW Corner: {}, {}".format(x_rad_sw, y_rad_sw)) x_rad_se, y_rad_se = LonLat2ABIangle( lon_east, lat_south, 0, H, req, rpol, e, lon_0 ) - print("SE Corner: {}, {}".format(x_rad_se, y_rad_se)) + logging.info("SE Corner: {}, {}".format(x_rad_se, y_rad_se)) x_rad_nw, y_rad_nw = LonLat2ABIangle( lon_west, lat_north, 0, H, req, rpol, e, lon_0 ) - print("NW Corner: {}, {}".format(x_rad_nw, y_rad_nw)) + logging.info("NW Corner: {}, {}".format(x_rad_nw, y_rad_nw)) x_rad_ne, y_rad_ne = LonLat2ABIangle( lon_east, lat_north, 0, H, req, rpol, e, lon_0 ) - print("NE Corner: {}, {}".format(x_rad_ne, y_rad_ne)) + logging.info("NE Corner: {}, {}".format(x_rad_ne, y_rad_ne)) # choose the bounds that cover the largest extent y_rad_s = min( y_rad_sw, y_rad_se @@ -76,7 +77,7 @@ def subsetNetCDF(filepath, bounds, newfilepath=None): y_rad_n = max(y_rad_nw, y_rad_ne) # northern-most x_rad_e = max(x_rad_se, x_rad_ne) # eastern-most (scan angle in radians) x_rad_w = min(x_rad_sw, x_rad_nw) # western-most - print( + logging.info( "Corner coords chosen: N: {}, S: {}; E: {}, W: {}".format( y_rad_n, y_rad_s, x_rad_e, x_rad_w ) diff --git a/src/goes_ortho/get_data.py b/src/goes_ortho/get_data.py index 15075a2..5dc11b1 100644 --- a/src/goes_ortho/get_data.py +++ b/src/goes_ortho/get_data.py @@ -4,7 +4,9 @@ import datetime as dt import json +import logging import os +import re import shutil import subprocess import sys @@ -12,50 +14,56 @@ import urllib.request from pathlib import Path +import goes2go as g2g +import numpy as np import xarray as xr import zarr from dateutil import parser, rrule +from tqdm import tqdm import goes_ortho as go -def build_zarr(downloadRequest_filepath): +def build_zarr(downloadRequest_filepath, downloader="goes2go"): # download requested imagery - print("download requested imagery") - image_path_list = download_abi(downloadRequest_filepath) + logging.info("download requested imagery") + if downloader == "goespy": + image_path_list = download_abi_goespy(downloadRequest_filepath) + if downloader == "goes2go": + image_path_list = download_abi_goes2go(downloadRequest_filepath) # parse json request file - print("parse json request file") + logging.info("parse json request file") _, _, bounds, _, _, _, _, variables, apiKey, _, outputFilepath = parse_json( downloadRequest_filepath ) # orthorectify all images - print("orthorectify all images") + logging.info("orthorectify all images") new_image_path_list = [] - print(image_path_list) + logging.info(image_path_list) for goes_image_path in image_path_list: - print("filename: ", goes_image_path) + logging.info("filename: ", goes_image_path) new_goes_filename = goes_image_path.with_name( goes_image_path.stem + "_o" ).with_suffix(".nc") # new_goes_filename = goes_image_path.split('.')[:-1][0] + '_o.nc' - print("renamed to: ", new_goes_filename) + logging.info("renamed to: ", new_goes_filename) new_image_path_list.append(new_goes_filename) go.orthorectify.ortho( goes_image_path, variables, bounds, apiKey, new_goes_filename, keep_dem=True ) - print(new_image_path_list) + logging.info(new_image_path_list) # add time dimension, fix CRS, build zarr file - print("add time dimension, fix CRS, build zarr file") + logging.info("add time dimension, fix CRS, build zarr file") for variable in variables: - print("add_datetime_crs") + logging.info("add_datetime_crs") new_image_path_list, datetimes_list = add_datetime_crs( new_image_path_list, variable ) # start Dask cluster - print("start Dask cluster") + logging.info("start Dask cluster") _ = go.io.dask_start_cluster( workers=6, threads=2, @@ -63,7 +71,7 @@ def build_zarr(downloadRequest_filepath): verbose=True, ) - print(new_image_path_list) + logging.info(new_image_path_list) # nc_files = sorted( # new_image_path_list, # key=datetimes_list @@ -72,11 +80,11 @@ def build_zarr(downloadRequest_filepath): img_path for img_dt, img_path in sorted(zip(datetimes_list, new_image_path_list)) ] - print(nc_files) + logging.info(nc_files) # Open all the raster files as a single dataset (combining them together) # Why did we choose chunks = 500? 100MB? # https://docs.xarray.dev/en/stable/user-guide/dask.html#optimization-tips - print("open all rasters") + logging.info("open all rasters") ds = xr.open_mfdataset(nc_files, chunks={"time": 500}) # ## if 'Rad' is our variable, check if we should add reflectance 'ref', or brightness temperature 'tb' to the list too @@ -92,7 +100,7 @@ def build_zarr(downloadRequest_filepath): # Dask's rechunk documentation: https://docs.dask.org/en/stable/generated/dask.array.rechunk.html # 0:-1 specifies that we want the dataset to be chunked along the 0th dimension -- the time dimension, which means that each chunk will have all 40 thousand values in time dimension # 1:'auto', 2:'auto' and balance=True specifies that dask can freely rechunk along the latitude and longitude dimensions to attain blocks that have a uniform size - print("rechunk") + logging.info("rechunk") ds[variable].data.rechunk( {0: -1, 1: "auto", 2: "auto"}, block_size_limit=1e8, balance=True ) @@ -108,13 +116,13 @@ def build_zarr(downloadRequest_filepath): # chunk ds[variable].encoding = {"chunks": (t, y, x)} # output zarr file - print("saving zarr file") + logging.info("saving zarr file") ds.to_zarr(outputFilepath) # Display source_group = zarr.open(outputFilepath) source_array = source_group[variable] - print(source_group.tree()) - print(source_array.info) + logging.info(source_group.tree()) + logging.info(source_array.info) del source_group del source_array print("Done.") @@ -235,7 +243,7 @@ def parse_json(downloadRequest_filepath): ) -def download_abi(downloadRequest_filepath): +def download_abi_goespy(downloadRequest_filepath): """Download GOES ABI imagery as specified by an input JSON file. (this function wraps around goespy.ABIDownloader())""" ( @@ -297,7 +305,7 @@ def download_abi(downloadRequest_filepath): / product / "{:02}".format(this_datetime.hour) ) - print(this_filepath) + # print(this_filepath) download_filepaths.append( this_filepath ) #'{}/{}/{}/{}/{}/{}/{}/{}/'.format(outDir,satellite,dt.year,dt.month,dt.day,product,'{:02}'.format(dt.hour),channel) @@ -328,6 +336,99 @@ def download_abi(downloadRequest_filepath): return output_filepaths +def download_abi_goes2go(downloadRequest_filepath): + """Download GOES ABI imagery as specified by an input JSON file. (this function wraps around goes2go functions)""" + + ( + start_datetime, + end_datetime, + bounds, + satellite, + bucket, + product_and_domain, + channels, + _, + _, + outDir, + _, + ) = parse_json(downloadRequest_filepath) + + output_filepaths = [] + + # get satellite integer (e.g. the 16, 17, or 18 part of goes16, goes17, goes18 + satellite_int = int(re.search(r"\d+", satellite).group()) + # separate product and domain (C, F, M) + product = product_and_domain[:-1] + domain = product_and_domain[-1] + # check domain + valid_domains = ["M", "C", "F"] + assert any( + domain in d for d in valid_domains + ), "Invalid GOES-R ABI product name. Product name should end with a domain identifier: M, C, or F. (e.g. ABI-L2-LSTC)" + + this_start_datetime = start_datetime + + # step through and download n hours at a time + n = 3 # try 3 hours at a time + n_steps = int( + np.ceil((end_datetime - start_datetime).total_seconds() / (3600 * n)) + ) # how many steps we'll have to take + print(f"Estimated {n_steps} batches to download") + batch_number = 1 + while this_start_datetime < end_datetime: + print(f"Batch number {batch_number}") + this_end_datetime = this_start_datetime + dt.timedelta(hours=n) + print( + f"Download batch of imagery from {this_start_datetime} to {this_end_datetime}" + ) + + # set up goes2go object for this satellite, product, and domain + G = g2g.GOES(satellite=satellite_int, product=product, domain=domain) + + # see what is available to download + # df = G.df(start=startDatetime.strftime("%Y-%m-%d %H:%M"), end=endDatetime.strftime("%Y-%m-%d %H:%M")) + + # download this batch + df = G.timerange( + start=this_start_datetime.strftime("%Y-%m-%d %H:%M"), + end=this_end_datetime.strftime("%Y-%m-%d %H:%M"), + ) + + # get the filepaths of this batch + batch_filepaths = [ + Path(g2g.config["default"]["save_dir"], filepath) for filepath in df["file"] + ] + + # now try and crop these so they don't take up so much space - this is very inefficient but oh well it's what I have right now + print(f"Cropping image batch to {bounds}") + pbar = tqdm(total=len(batch_filepaths)) + for _, this_filepath in enumerate(batch_filepaths): + if Path.exists( + this_filepath + ): # we have to make sure the path exists (meaning we downloaded something) before running the subsetNetCDF function + logging.info("\nSubsetting file {}".format(this_filepath)) + go.clip.subsetNetCDF(this_filepath, bounds) + pbar.update(1) + + pbar.close() + + # append the batch filepaths to our main filepaths list + output_filepaths.append(batch_filepaths) + + # set the next start datetime to the end of this batch + this_start_datetime = this_end_datetime + + batch_number += 1 + + # flatten the list of lists that we've compiled + output_filepaths = [ + filepath for batch_filepaths in output_filepaths for filepath in batch_filepaths + ] + + print("Done") + return output_filepaths + + def get_dem(demtype, bounds, api_key, out_fn=None, proj="EPSG:4326"): """ download a DEM of choice from OpenTopography World DEM (modified by Shashank Bhushan, first written by David Shean) diff --git a/src/goes_ortho/orthorectify.py b/src/goes_ortho/orthorectify.py index 971b3a3..51ffca7 100644 --- a/src/goes_ortho/orthorectify.py +++ b/src/goes_ortho/orthorectify.py @@ -2,6 +2,7 @@ Functions to orthorectify GOES-R ABI images using a DEM """ +import logging import os import numpy as np @@ -77,14 +78,14 @@ def make_ortho_map(goes_filepath, dem_filepath, out_filepath=None): """ - print("\nRUNNING: make_ortho_map()") + logging.info("\nRUNNING: make_ortho_map()") # Open the GOES ABI image - print("\nOpening GOES ABI image...") + logging.info("\nOpening GOES ABI image...") abi_image = xr.open_dataset(goes_filepath, decode_times=False) # NOTE: for some reason (?) I sometimes get an error "ValueError: unable to decode time units 'seconds since 2000-01-01 12:00:00' with the default calendar. Try opening your dataset with decode_times=False." so I've added decode_times=False here. # Get inputs: projection information from the ABI radiance product (values needed for geometry calculations) - print("\nGet inputs: projection information from the ABI radiance product") + logging.info("\nGet inputs: projection information from the ABI radiance product") req = abi_image.goes_imager_projection.semi_major_axis rpol = abi_image.goes_imager_projection.semi_minor_axis H = ( @@ -93,10 +94,10 @@ def make_ortho_map(goes_filepath, dem_filepath, out_filepath=None): ) lon_0 = abi_image.goes_imager_projection.longitude_of_projection_origin e = 0.0818191910435 # GRS-80 eccentricity - print("...done") + logging.info("...done") # Load DEM - print("\nOpening DEM file...") + logging.info("\nOpening DEM file...") dem = rioxarray.open_rasterio(dem_filepath) dem = dem.where(dem != dem.attrs["_FillValue"])[0, :, :] # replace nodata with nans dem = dem.fillna( @@ -104,20 +105,20 @@ def make_ortho_map(goes_filepath, dem_filepath, out_filepath=None): ) # fill nans with zeros for the ocean (temporary fix for fog project) # dem = dem.where(dem!=0) # replace zeros with nans # Create 2D arrays of longitude and latitude from the DEM - print("\nCreate 2D arrays of longitude and latitude from the DEM") + logging.info("\nCreate 2D arrays of longitude and latitude from the DEM") X, Y = np.meshgrid(dem.x, dem.y) # Lon and Lat of each DEM grid cell Z = dem.values # elevation of each DEM grid cell - print("...done") + logging.info("...done") # For each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians) - print( + logging.info( "\nFor each grid cell in the DEM, compute the corresponding ABI scan angle (x and y, radians)" ) abi_grid_x, abi_grid_y = LonLat2ABIangle(X, Y, Z, H, req, rpol, e, lon_0) - print("...done") + logging.info("...done") # Create metadata dictionary about this map (should probably clean up metadata, adhere to some set of standards) - print("\nCreate metadata dictionary about this map") + logging.info("\nCreate metadata dictionary about this map") metadata = { # Information about the projection geometry: "longitude_of_projection_origin": lon_0, @@ -148,10 +149,10 @@ def make_ortho_map(goes_filepath, dem_filepath, out_filepath=None): "latitude_info": "latitude from DEM file", "elevation_info": "elevation from DEM file", } - print("...done") + logging.info("...done") # Create pixel map dataset - print("\nCreate pixel map dataset") + logging.info("\nCreate pixel map dataset") ds = xr.Dataset( {"elevation": (["latitude", "longitude"], dem.values)}, coords={ @@ -162,17 +163,19 @@ def make_ortho_map(goes_filepath, dem_filepath, out_filepath=None): }, attrs=metadata, ) - print(ds) - print("...done") + # print(ds) + logging.info("...done") if out_filepath is not None: - print("\nExport this pixel map along with the metadata (NetCDF with xarray)") + logging.info( + "\nExport this pixel map along with the metadata (NetCDF with xarray)" + ) # Export this pixel map along with the metadata (NetCDF with xarray) ds.to_netcdf(out_filepath, mode="w") - print("...done") + logging.info("...done") # Return the pixel map dataset - print("\nReturn the pixel map dataset.") + logging.info("\nReturn the pixel map dataset.") return ds @@ -202,41 +205,41 @@ def orthorectify_abi(goes_filepath, pixel_map, data_vars, out_filename=None): ------------ """ - print("\nRUNNING: orthorectify_abi_rad()") + logging.info("\nRUNNING: orthorectify_abi_rad()") # First check, Does the projection info in the image match our mapping? - print("\nDoes the projection info in the image match our mapping?") + logging.info("\nDoes the projection info in the image match our mapping?") # Open the GOES ABI image - print("\nOpening GOES ABI image...\t\t\tABI image value\tPixel map value") + logging.info("\nOpening GOES ABI image...\t\t\tABI image value\tPixel map value") abi_image = xr.open_dataset(goes_filepath, decode_times=False) - print( + logging.info( "perspective_point_height + semi_major_axis:\t{}\t{}".format( abi_image.goes_imager_projection.perspective_point_height + abi_image.goes_imager_projection.semi_major_axis, pixel_map.satellite_height, ) ) - print( + logging.info( "semi_major_axis:\t\t\t\t{}\t{}".format( abi_image.goes_imager_projection.semi_major_axis, pixel_map.semi_major_axis ) ) - print( + logging.info( "semi_minor_axis:\t\t\t\t{}\t{}".format( abi_image.goes_imager_projection.semi_minor_axis, pixel_map.semi_minor_axis ) ) - print( + logging.info( "longitude_of_projection_origin:\t\t\t{}\t\t{}".format( abi_image.goes_imager_projection.longitude_of_projection_origin, pixel_map.longitude_of_projection_origin, ) ) - print("...done") + logging.info("...done") # Map (orthorectify) and clip the image to the pixel map for each data variable we want for var in data_vars: - print( + logging.info( "\nMap (orthorectify) and clip the image to the pixel map for {}".format( var ) @@ -244,7 +247,7 @@ def orthorectify_abi(goes_filepath, pixel_map, data_vars, out_filename=None): abi_var_values = abi_image.sel( x=pixel_map.dem_px_angle_x, y=pixel_map.dem_px_angle_y, method="nearest" )[var].values - print("...done") + logging.info("...done") # Create a new xarray dataset with the orthorectified ABI radiance values, # Lat, Lon, Elevation, and metadata from the pixel map. @@ -267,7 +270,7 @@ def orthorectify_abi(goes_filepath, pixel_map, data_vars, out_filename=None): ) # Map (orthorectify) the original ABI Fixed Grid coordinate values to the new pixels for reference - print( + logging.info( "\nMap (orthorectify) and clip the image to the pixel map for ABI Fixed Grid coordinates" ) abi_fixed_grid_x_values = abi_image.sel( @@ -290,12 +293,12 @@ def orthorectify_abi(goes_filepath, pixel_map, data_vars, out_filename=None): ("latitude", "longitude"), abi_fixed_grid_y_values_reshaped, ) - print("...done") + logging.info("...done") # drop DEM from dataset # pixel_map = pixel_map.drop(['elevation']) - print( + logging.info( "\nCreate zone labels for each unique pair of ABI Fixed Grid coordinates (for each orthorectified pixel footprint)" ) # Found this clever solution here: https://stackoverflow.com/a/32326297/11699349 @@ -311,16 +314,16 @@ def orthorectify_abi(goes_filepath, pixel_map, data_vars, out_filename=None): zone_labels = idx.reshape(pixel_map.abi_fixed_grid_y.values.shape) # Add the zone_labels to the dataset pixel_map["zone_labels"] = (("latitude", "longitude"), zone_labels) - print("...done") + logging.info("...done") # Output this result to a new NetCDF file - print("\nOutput this result to a new NetCDF file") + logging.info("\nOutput this result to a new NetCDF file") if out_filename is None: out_filename = abi_image.dataset_name + "_ortho.nc" - print("Saving file as: {}".format(out_filename)) + logging.info("Saving file as: {}".format(out_filename)) pixel_map.to_netcdf(out_filename) - print("...done") + logging.info("...done") return pixel_map From 19dcac0b1fb2e2c81f0ad7345def2f3c7e626d5b Mon Sep 17 00:00:00 2001 From: spestana Date: Wed, 25 Sep 2024 22:50:00 +0000 Subject: [PATCH 2/3] Add goes2go dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ff752f2..0b9513b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ "flake8", "scikit-learn", "geogif", + "goes2go", ] requires-python = ">=3.8" authors = [ From a560e19deb65a2ffd85e15306e5a8e99d7042831 Mon Sep 17 00:00:00 2001 From: spestana Date: Thu, 26 Sep 2024 17:10:39 +0000 Subject: [PATCH 3/3] Update download functions and add example notebook --- docs/examples/download.ipynb | 621 ++++++++++++++++++++++++++++++----- docs/index.rst | 1 + pyproject.toml | 1 + src/goes_ortho/get_data.py | 206 +++++++++--- 4 files changed, 705 insertions(+), 124 deletions(-) diff --git a/docs/examples/download.ipynb b/docs/examples/download.ipynb index 3e4fddf..ddc9aac 100644 --- a/docs/examples/download.ipynb +++ b/docs/examples/download.ipynb @@ -12,10 +12,21 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 1, "id": "5c0d77eb-457b-4b4d-a669-73778a54637e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/spestana/.conda/envs/goes-test-env/lib/python3.12/site-packages/goes2go/data.py:665: FutureWarning: 'H' is deprecated and will be removed in a future version. Please use 'h' instead of 'H'.\n", + " within=pd.to_timedelta(config[\"nearesttime\"].get(\"within\", \"1h\")),\n", + "/home/spestana/.conda/envs/goes-test-env/lib/python3.12/site-packages/goes2go/NEW.py:188: FutureWarning: 'H' is deprecated and will be removed in a future version. Please use 'h' instead of 'H'.\n", + " within=pd.to_timedelta(config[\"nearesttime\"].get(\"within\", \"1h\")),\n" + ] + } + ], "source": [ "import goes_ortho as go\n", "import xarray as xr" @@ -35,65 +46,35 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "6fc6c19c-f21f-4319-a45e-47434b933600", - "metadata": {}, - "outputs": [], - "source": [ - "lon = -119.31212\n", - "lat = 37.88175\n", - "z = 2811 # elevation in meters" - ] - }, - { - "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "id": "28208c5f-e93c-4ce9-b4f7-c4c7ea14e12e", "metadata": {}, "outputs": [], "source": [ - "startDatetime = \"2024-04-19T00:00:00Z\"\n", - "endDatetime = \"2024-04-19T00:59:00Z\"\n", - "[min_lon, min_lat, max_lon, max_lat] = [lon-1, lat-1, lon+1, lat+1] # set bounds to just a little area around the location we want\n", - "satellite = \"goes18\"\n", - "product = \"ABI-L1b-RadC\"\n", - "band = 2\n", - "variable = \"Rad\"\n", - "OPENTOPO_API_KEY = \"585b1d1639bc5ef8a4a5bdea7e45a8d1\"" + "# Make request file from user input\n", + "request_filepath = go.get_data.make_request_json(workflowName = \"example\",\n", + " startDatetime = \"2020-04-19T00:00:00Z\",\n", + " endDatetime = \"2020-04-21T00:59:00Z\",\n", + " bounds = go.get_data.bounds_from_geojson(\"grand_mesa.geojson\"),\n", + " satellite = \"goes16\",\n", + " product = \"ABI-L2-LSTC\",\n", + " band = 2,\n", + " variable = \"LST\",\n", + " apiKey = None, #585b1d1639bc5ef8a4a5bdea7e45a8d1\n", + " )" ] }, { "cell_type": "markdown", - "id": "8a4c97cb-b630-4771-8e80-4c7f70c0f1bf", + "id": "8eebc03e-2e78-44b8-beeb-e6914e454b2e", "metadata": {}, "source": [ - "Using the data above, we make the required json file." + "The functions below demonstrate downloading GOES imagery using two different downloader packages: [goes2go](https://goes2go.readthedocs.io/en/latest/) and [goespy](https://github.com/spestana/goes-py) (the goespy functions are now integrated directly within the goes-ortho package). I have found goes2go is typically faster." ] }, { "cell_type": "code", - "execution_count": 9, - "id": "cac7ae71-01db-4fa3-9e9f-856b82ff738b", - "metadata": {}, - "outputs": [], - "source": [ - "# Make request file from user input\n", - "go.get_data.make_request_json(\n", - " f\"{satellite}-example-ACMC\",\n", - " startDatetime,\n", - " endDatetime,\n", - " [min_lon, min_lat, max_lon, max_lat],\n", - " satellite,\n", - " product,\n", - " band,\n", - " variable,\n", - " OPENTOPO_API_KEY,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, + "execution_count": 3, "id": "9fbfaf9c-938a-475a-a1f7-f2a36b062505", "metadata": {}, "outputs": [ @@ -101,18 +82,277 @@ "name": "stdout", "output_type": "stream", "text": [ - "Estimated 1 batches to download\n", + "Estimated 17 batches to download\n", "Batch number 1\n", - "Download batch of imagery from 2024-04-19 00:00:00+00:00 to 2024-04-19 03:00:00+00:00\n", - "📦 Finished downloading [576] files to [/home/spestana/data/noaa-goes18/ABI-L1b-RadC].\n", - "Cropping image batch to [-120.31212, 36.88175, -118.31212, 38.88175]\n" + "Download batch of imagery from 2020-04-19 00:00:00+00:00 to 2020-04-19 03:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 576/576 [01:44<00:00, 5.52it/s]" + "100%|██████████| 3/3 [00:01<00:00, 2.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 2\n", + "Download batch of imagery from 2020-04-19 03:00:00+00:00 to 2020-04-19 06:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 10.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 3\n", + "Download batch of imagery from 2020-04-19 06:00:00+00:00 to 2020-04-19 09:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 10.80it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 4\n", + "Download batch of imagery from 2020-04-19 09:00:00+00:00 to 2020-04-19 12:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 10.74it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 5\n", + "Download batch of imagery from 2020-04-19 12:00:00+00:00 to 2020-04-19 15:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 10.90it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 6\n", + "Download batch of imagery from 2020-04-19 15:00:00+00:00 to 2020-04-19 18:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.84it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 7\n", + "Download batch of imagery from 2020-04-19 18:00:00+00:00 to 2020-04-19 21:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 8\n", + "Download batch of imagery from 2020-04-19 21:00:00+00:00 to 2020-04-20 00:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.92it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 9\n", + "Download batch of imagery from 2020-04-20 00:00:00+00:00 to 2020-04-20 03:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.67it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 10\n", + "Download batch of imagery from 2020-04-20 03:00:00+00:00 to 2020-04-20 06:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.58it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 11\n", + "Download batch of imagery from 2020-04-20 06:00:00+00:00 to 2020-04-20 09:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.43it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 12\n", + "Download batch of imagery from 2020-04-20 09:00:00+00:00 to 2020-04-20 12:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.46it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 13\n", + "Download batch of imagery from 2020-04-20 12:00:00+00:00 to 2020-04-20 15:00:00+00:00\n", + "FileNotFoundError encountered. The requested image may not exist. Because this searched a time window of 3 hours, there may be some valid imagery within the time window. Try a smaller time window to search for valid imagery.\n", + "noaa-goes16/ABI-L2-LSTC/2020/111/13\n", + "Batch number 14\n", + "Download batch of imagery from 2020-04-20 15:00:00+00:00 to 2020-04-20 18:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 15\n", + "Download batch of imagery from 2020-04-20 18:00:00+00:00 to 2020-04-20 21:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.70it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 16\n", + "Download batch of imagery from 2020-04-20 21:00:00+00:00 to 2020-04-21 00:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.65it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 17\n", + "Download batch of imagery from 2020-04-21 00:00:00+00:00 to 2020-04-21 03:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.43it/s]" ] }, { @@ -120,8 +360,8 @@ "output_type": "stream", "text": [ "Done\n", - "CPU times: user 1min 58s, sys: 22 s, total: 2min 20s\n", - "Wall time: 2min 50s\n" + "CPU times: user 5.94 s, sys: 863 ms, total: 6.8 s\n", + "Wall time: 22.3 s\n" ] }, { @@ -134,12 +374,12 @@ ], "source": [ "%%time\n", - "filepaths = go.get_data.download_abi_goes2go(f\"{satellite}-example-ACMC.json\")" + "filepaths = go.get_data.download_abi_goes2go(request_filepath)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 4, "id": "738b548c-8807-4fb8-82c2-eb380a5284c8", "metadata": {}, "outputs": [ @@ -149,45 +389,270 @@ "text": [ "\n", "Files will be downloaded and then cropped to these bounds:\n", - "\t(-120.31212,38.88175).\t.(-118.31212,38.88175)\n", + "\t(-108.368202,39.211234).\t.(-107.627676,39.211234)\n", "\n", "\n", "\n", - "\t(-120.31212,36.88175).\t.(-118.31212,36.88175)\n", + "\t(-108.368202,38.80429).\t.(-107.627676,38.80429)\n", "\n", "For each S3 bucket, download the corresponding observations\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02\n", - "Downloading... the product ABI-L1b-RadC \n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 00 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100001131_e20201100003504_c20201100004343.nc: [##########] 100% 1.701659 MB/1.701659 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/00\n", + "goes16/2020/4/19/ABI-L2-LSTC/00/OR_ABI-L2-LSTC-M6_G16_s20201100001131_e20201100003504_c20201100004343.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 01 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100101131_e20201100103504_c20201100104361.nc: [##########] 100% 1.762344 MB/1.762344 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/01\n", + "goes16/2020/4/19/ABI-L2-LSTC/01/OR_ABI-L2-LSTC-M6_G16_s20201100101131_e20201100103504_c20201100104361.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 02 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100201132_e20201100203504_c20201100204346.nc: [##########] 100% 1.827398 MB/1.827398 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/02\n", + "goes16/2020/4/19/ABI-L2-LSTC/02/OR_ABI-L2-LSTC-M6_G16_s20201100201132_e20201100203504_c20201100204346.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 03 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100301132_e20201100303504_c20201100304348.nc: [##########] 100% 1.893841 MB/1.893841 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/03\n", + "goes16/2020/4/19/ABI-L2-LSTC/03/OR_ABI-L2-LSTC-M6_G16_s20201100301132_e20201100303504_c20201100304348.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 04 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100401132_e20201100403504_c20201100404347.nc: [##########] 100% 1.912388 MB/1.912388 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/04\n", + "goes16/2020/4/19/ABI-L2-LSTC/04/OR_ABI-L2-LSTC-M6_G16_s20201100401132_e20201100403504_c20201100404347.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 05 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100501132_e20201100503504_c20201100504340.nc: [##########] 100% 1.948882 MB/1.948882 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/05\n", + "goes16/2020/4/19/ABI-L2-LSTC/05/OR_ABI-L2-LSTC-M6_G16_s20201100501132_e20201100503504_c20201100504340.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 06 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100601132_e20201100603505_c20201100604316.nc: [##########] 100% 1.973455 MB/1.973455 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/06\n", + "goes16/2020/4/19/ABI-L2-LSTC/06/OR_ABI-L2-LSTC-M6_G16_s20201100601132_e20201100603505_c20201100604316.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 07 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100701132_e20201100703505_c20201100704354.nc: [##########] 100% 1.979452 MB/1.979452 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/07\n", + "goes16/2020/4/19/ABI-L2-LSTC/07/OR_ABI-L2-LSTC-M6_G16_s20201100701132_e20201100703505_c20201100704354.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 08 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100801132_e20201100803505_c20201100804369.nc: [##########] 100% 2.035866 MB/2.035866 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/08\n", + "goes16/2020/4/19/ABI-L2-LSTC/08/OR_ABI-L2-LSTC-M6_G16_s20201100801132_e20201100803505_c20201100804369.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 09 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100901132_e20201100903505_c20201100904378.nc: [##########] 100% 2.069201 MB/2.069201 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/09\n", + "goes16/2020/4/19/ABI-L2-LSTC/09/OR_ABI-L2-LSTC-M6_G16_s20201100901132_e20201100903505_c20201100904378.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 10 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101001132_e20201101003505_c20201101004356.nc: [##########] 100% 2.08223 MB/2.08223 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/10\n", + "goes16/2020/4/19/ABI-L2-LSTC/10/OR_ABI-L2-LSTC-M6_G16_s20201101001132_e20201101003505_c20201101004356.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 11 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101101132_e20201101103505_c20201101104373.nc: [##########] 100% 2.057548 MB/2.057548 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/11\n", + "goes16/2020/4/19/ABI-L2-LSTC/11/OR_ABI-L2-LSTC-M6_G16_s20201101101132_e20201101103505_c20201101104373.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 12 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101201132_e20201101203505_c20201101204347.nc: [##########] 100% 1.966647 MB/1.966647 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/12\n", + "goes16/2020/4/19/ABI-L2-LSTC/12/OR_ABI-L2-LSTC-M6_G16_s20201101201132_e20201101203505_c20201101204347.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 13 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101301132_e20201101303505_c20201101304353.nc: [##########] 100% 1.739812 MB/1.739812 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/13\n", + "goes16/2020/4/19/ABI-L2-LSTC/13/OR_ABI-L2-LSTC-M6_G16_s20201101301132_e20201101303505_c20201101304353.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 14 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101401132_e20201101403505_c20201101404378.nc: [##########] 100% 1.568007 MB/1.568007 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/14\n", + "goes16/2020/4/19/ABI-L2-LSTC/14/OR_ABI-L2-LSTC-M6_G16_s20201101401132_e20201101403505_c20201101404378.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 15 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101501132_e20201101503505_c20201101504372.nc: [##########] 100% 1.53152 MB/1.53152 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/15\n", + "goes16/2020/4/19/ABI-L2-LSTC/15/OR_ABI-L2-LSTC-M6_G16_s20201101501132_e20201101503505_c20201101504372.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 16 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101601132_e20201101603505_c20201101604343.nc: [##########] 100% 1.536471 MB/1.536471 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/16\n", + "goes16/2020/4/19/ABI-L2-LSTC/16/OR_ABI-L2-LSTC-M6_G16_s20201101601132_e20201101603505_c20201101604343.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 17 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101701132_e20201101703505_c20201101704397.nc: [##########] 100% 1.530618 MB/1.530618 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/17\n", + "goes16/2020/4/19/ABI-L2-LSTC/17/OR_ABI-L2-LSTC-M6_G16_s20201101701132_e20201101703505_c20201101704397.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 18 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101801132_e20201101803505_c20201101804367.nc: [##########] 100% 1.507082 MB/1.507082 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/18\n", + "goes16/2020/4/19/ABI-L2-LSTC/18/OR_ABI-L2-LSTC-M6_G16_s20201101801132_e20201101803505_c20201101804367.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 19 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101901132_e20201101903505_c20201101904334.nc: [##########] 100% 1.496716 MB/1.496716 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/19\n", + "goes16/2020/4/19/ABI-L2-LSTC/19/OR_ABI-L2-LSTC-M6_G16_s20201101901132_e20201101903505_c20201101904334.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 20 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201102001132_e20201102003505_c20201102004371.nc: [##########] 100% 1.467092 MB/1.467092 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/20\n", + "goes16/2020/4/19/ABI-L2-LSTC/20/OR_ABI-L2-LSTC-M6_G16_s20201102001132_e20201102003505_c20201102004371.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 21 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201102101132_e20201102103505_c20201102104445.nc: [##########] 100% 1.447728 MB/1.447728 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/21\n", + "goes16/2020/4/19/ABI-L2-LSTC/21/OR_ABI-L2-LSTC-M6_G16_s20201102101132_e20201102103505_c20201102104445.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 22 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201102201132_e20201102203505_c20201102204366.nc: [##########] 100% 1.461739 MB/1.461739 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/22\n", + "goes16/2020/4/19/ABI-L2-LSTC/22/OR_ABI-L2-LSTC-M6_G16_s20201102201132_e20201102203505_c20201102204366.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 23 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201102301132_e20201102303505_c20201102304375.nc: [##########] 100% 1.526796 MB/1.526796 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/23\n", + "goes16/2020/4/19/ABI-L2-LSTC/23/OR_ABI-L2-LSTC-M6_G16_s20201102301132_e20201102303505_c20201102304375.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 00 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110001132_e20201110003505_c20201110004359.nc: [##########] 100% 1.601177 MB/1.601177 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/00\n", + "goes16/2020/4/20/ABI-L2-LSTC/00/OR_ABI-L2-LSTC-M6_G16_s20201110001132_e20201110003505_c20201110004359.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 01 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110101132_e20201110103505_c20201110104364.nc: [##########] 100% 1.73089 MB/1.73089 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/01\n", + "goes16/2020/4/20/ABI-L2-LSTC/01/OR_ABI-L2-LSTC-M6_G16_s20201110101132_e20201110103505_c20201110104364.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 02 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110201132_e20201110203505_c20201110204344.nc: [##########] 100% 1.784311 MB/1.784311 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/02\n", + "goes16/2020/4/20/ABI-L2-LSTC/02/OR_ABI-L2-LSTC-M6_G16_s20201110201132_e20201110203505_c20201110204344.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 03 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110301132_e20201110303505_c20201110304351.nc: [##########] 100% 1.836626 MB/1.836626 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/03\n", + "goes16/2020/4/20/ABI-L2-LSTC/03/OR_ABI-L2-LSTC-M6_G16_s20201110301132_e20201110303505_c20201110304351.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 04 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110401132_e20201110403504_c20201110404358.nc: [##########] 100% 1.858458 MB/1.858458 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/04\n", + "goes16/2020/4/20/ABI-L2-LSTC/04/OR_ABI-L2-LSTC-M6_G16_s20201110401132_e20201110403504_c20201110404358.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 05 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110501131_e20201110503504_c20201110504389.nc: [##########] 100% 1.872962 MB/1.872962 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/05\n", + "goes16/2020/4/20/ABI-L2-LSTC/05/OR_ABI-L2-LSTC-M6_G16_s20201110501131_e20201110503504_c20201110504389.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 06 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110601131_e20201110603504_c20201110604343.nc: [##########] 100% 1.882345 MB/1.882345 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/06\n", + "goes16/2020/4/20/ABI-L2-LSTC/06/OR_ABI-L2-LSTC-M6_G16_s20201110601131_e20201110603504_c20201110604343.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 07 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110701131_e20201110703504_c20201110704364.nc: [##########] 100% 1.906022 MB/1.906022 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/07\n", + "goes16/2020/4/20/ABI-L2-LSTC/07/OR_ABI-L2-LSTC-M6_G16_s20201110701131_e20201110703504_c20201110704364.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 08 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110801131_e20201110803504_c20201110804329.nc: [##########] 100% 1.921032 MB/1.921032 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/08\n", + "goes16/2020/4/20/ABI-L2-LSTC/08/OR_ABI-L2-LSTC-M6_G16_s20201110801131_e20201110803504_c20201110804329.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 09 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110901131_e20201110903504_c20201110904360.nc: [##########] 100% 1.954707 MB/1.954707 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/09\n", + "goes16/2020/4/20/ABI-L2-LSTC/09/OR_ABI-L2-LSTC-M6_G16_s20201110901131_e20201110903504_c20201110904360.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 10 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111001131_e20201111003504_c20201111004351.nc: [##########] 100% 1.975989 MB/1.975989 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/10\n", + "goes16/2020/4/20/ABI-L2-LSTC/10/OR_ABI-L2-LSTC-M6_G16_s20201111001131_e20201111003504_c20201111004351.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 11 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111101131_e20201111103504_c20201111104359.nc: [##########] 100% 1.984507 MB/1.984507 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/11\n", + "goes16/2020/4/20/ABI-L2-LSTC/11/OR_ABI-L2-LSTC-M6_G16_s20201111101131_e20201111103504_c20201111104359.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 12 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111201131_e20201111203503_c20201111204339.nc: [##########] 100% 1.967923 MB/1.967923 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/12\n", + "goes16/2020/4/20/ABI-L2-LSTC/12/OR_ABI-L2-LSTC-M6_G16_s20201111201131_e20201111203503_c20201111204339.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 13 UTC\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 14 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111401130_e20201111403503_c20201111404374.nc: [##########] 100% 1.810978 MB/1.810978 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/14\n", + "goes16/2020/4/20/ABI-L2-LSTC/14/OR_ABI-L2-LSTC-M6_G16_s20201111401130_e20201111403503_c20201111404374.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 15 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111501130_e20201111503503_c20201111504393.nc: [##########] 100% 1.880033 MB/1.880033 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/15\n", + "goes16/2020/4/20/ABI-L2-LSTC/15/OR_ABI-L2-LSTC-M6_G16_s20201111501130_e20201111503503_c20201111504393.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 16 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111601130_e20201111603503_c20201111604388.nc: [##########] 100% 1.912107 MB/1.912107 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/16\n", + "goes16/2020/4/20/ABI-L2-LSTC/16/OR_ABI-L2-LSTC-M6_G16_s20201111601130_e20201111603503_c20201111604388.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 17 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111701130_e20201111703503_c20201111704362.nc: [##########] 100% 1.920667 MB/1.920667 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/17\n", + "goes16/2020/4/20/ABI-L2-LSTC/17/OR_ABI-L2-LSTC-M6_G16_s20201111701130_e20201111703503_c20201111704362.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 18 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111801130_e20201111803503_c20201111804416.nc: [##########] 100% 1.845728 MB/1.845728 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/18\n", + "goes16/2020/4/20/ABI-L2-LSTC/18/OR_ABI-L2-LSTC-M6_G16_s20201111801130_e20201111803503_c20201111804416.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 19 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111901130_e20201111903503_c20201111904383.nc: [##########] 100% 1.763694 MB/1.763694 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/19\n", + "goes16/2020/4/20/ABI-L2-LSTC/19/OR_ABI-L2-LSTC-M6_G16_s20201111901130_e20201111903503_c20201111904383.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 20 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201112001130_e20201112003503_c20201112004371.nc: [##########] 100% 1.727079 MB/1.727079 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/20\n", + "goes16/2020/4/20/ABI-L2-LSTC/20/OR_ABI-L2-LSTC-M6_G16_s20201112001130_e20201112003503_c20201112004371.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 21 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201112101130_e20201112103503_c20201112104424.nc: [##########] 100% 1.687915 MB/1.687915 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/21\n", + "goes16/2020/4/20/ABI-L2-LSTC/21/OR_ABI-L2-LSTC-M6_G16_s20201112101130_e20201112103503_c20201112104424.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 22 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201112201130_e20201112203503_c20201112204424.nc: [##########] 100% 1.691709 MB/1.691709 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/22\n", + "goes16/2020/4/20/ABI-L2-LSTC/22/OR_ABI-L2-LSTC-M6_G16_s20201112201130_e20201112203503_c20201112204424.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 23 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201112301130_e20201112303503_c20201112304429.nc: [##########] 100% 1.729744 MB/1.729744 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/23\n", + "goes16/2020/4/20/ABI-L2-LSTC/23/OR_ABI-L2-LSTC-M6_G16_s20201112301130_e20201112303503_c20201112304429.nc\n", + "Downloading... the product ABI-L2-LSTC \n", "Downloading dataset to... 00 UTC\n", - "OR_ABI-L1b-RadC-M6C02_G18_s20241100056172_e20241100058545_c20241100058573.nc: [##########] 100% 67.930455 MB/67.930455 MB\n", - "Subsetting files in...goes18/2024/4/19/ABI-L1b-RadC/00/C02\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100036172_e20241100038545_c20241100038570.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100006172_e20241100008545_c20241100008570.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100046172_e20241100048545_c20241100048571.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100021172_e20241100023545_c20241100023570.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100011172_e20241100013545_c20241100013573.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100016172_e20241100018545_c20241100018577.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100026172_e20241100028545_c20241100028580.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100031172_e20241100033545_c20241100033572.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100056172_e20241100058545_c20241100058573.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100001172_e20241100003545_c20241100003572.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100041172_e20241100043545_c20241100043570.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100051172_e20241100053545_c20241100053568.nc\n", + "OR_ABI-L2-LSTC-M6_G16_s20201120001130_e20201120003503_c20201120004384.nc: [##########] 100% 1.842908 MB/1.842908 MB\n", + "Subsetting files in...goes16/2020/4/21/ABI-L2-LSTC/00\n", + "goes16/2020/4/21/ABI-L2-LSTC/00/OR_ABI-L2-LSTC-M6_G16_s20201120001130_e20201120003503_c20201120004384.nc\n", "Done\n", - "CPU times: user 15.7 s, sys: 4.8 s, total: 20.5 s\n", - "Wall time: 51 s\n" + "CPU times: user 6.49 s, sys: 870 ms, total: 7.36 s\n", + "Wall time: 55.2 s\n" ] } ], "source": [ "%%time\n", - "filepaths = go.get_data.download_abi_goespy(f\"{satellite}-example-ACMC.json\")" + "filepaths = go.get_data.download_abi_goespy(request_filepath)" ] }, { "cell_type": "code", "execution_count": null, - "id": "bd893536-d241-4966-a49c-dd93729d6c43", + "id": "374725bb-d6fb-4efa-b194-dfe9bab317d0", "metadata": {}, "outputs": [], "source": [] diff --git a/docs/index.rst b/docs/index.rst index dc86a6c..b0f5e7e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,6 +7,7 @@ installation examples/quick_start + examples/download examples/subset_abi_netcdf_example examples/orthorectify_abi_example examples/orthorectify_abi_example2 diff --git a/pyproject.toml b/pyproject.toml index 0b9513b..c4b5294 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "aiohttp", "scipy", "geopandas", + "geojson", "folium", "distributed", "tqdm", diff --git a/src/goes_ortho/get_data.py b/src/goes_ortho/get_data.py index 5dc11b1..b28a330 100644 --- a/src/goes_ortho/get_data.py +++ b/src/goes_ortho/get_data.py @@ -1,6 +1,4 @@ -"""Get test data for tests and/or examples""" - -# based on https://github.com/GlacioHack/xdem/blob/d91bf1cc9b3f36d77f3729649bc8e9edc6b42f9f/xdem/examples.py#L33 +"""Functions for downloading data""" import datetime as dt import json @@ -13,29 +11,35 @@ import tarfile import urllib.request from pathlib import Path +from typing import List, Tuple, Union +import geojson import goes2go as g2g import numpy as np import xarray as xr import zarr from dateutil import parser, rrule +from shapely.geometry import shape from tqdm import tqdm import goes_ortho as go -def build_zarr(downloadRequest_filepath, downloader="goes2go"): +def build_zarr(download_request_filepath, downloader="goes2go"): + """ + For running through github actions. Download and build a zarr file from a download request json file + """ # download requested imagery logging.info("download requested imagery") if downloader == "goespy": - image_path_list = download_abi_goespy(downloadRequest_filepath) + image_path_list = download_abi_goespy(download_request_filepath) if downloader == "goes2go": - image_path_list = download_abi_goes2go(downloadRequest_filepath) + image_path_list = download_abi_goes2go(download_request_filepath) # parse json request file logging.info("parse json request file") _, _, bounds, _, _, _, _, variables, apiKey, _, outputFilepath = parse_json( - downloadRequest_filepath + download_request_filepath ) # orthorectify all images @@ -139,8 +143,10 @@ def make_request_json( band, variable, apiKey, -): - """For running through github actions, make a request json file from github user input to be read by the build_zarr function""" +) -> str: + """ + For running through github actions, make a request json file from github user input to be read by the build_zarr function + """ request_dict = { "dateRange": {"startDatetime": startDatetime, "endDatetime": endDatetime}, "bounds": { @@ -160,13 +166,33 @@ def make_request_json( filename = workflowName + ".json" with open(filename, "w") as f: json.dump(request_dict, f) + return filename -def get_start_date_from_abi_filename(s): - return s.split("_s")[1].split("_")[0] +def get_start_date_from_abi_filename(s: str) -> str: + """ + Read the start time from an ABI filename string. + + Parameters + ------------ + s : str + filename for an ABI image product + + Returns + ----------- + time_str : str + start time from the filename + """ + time_str = s.split("_s")[1].split("_")[0] + return time_str -def add_datetime_crs(files, variable, crs="EPSG:4326"): +def add_datetime_crs( + files: List[str], variable: str, crs: str = "EPSG:4326" +) -> Tuple[List[str], List[dt.datetime]]: + """ + Add datetime and CRS information to a GOES ABI image product file + """ print(files) print(variable) print(crs) @@ -206,29 +232,46 @@ def add_datetime_crs(files, variable, crs="EPSG:4326"): return new_files, datetimes -def parse_json(downloadRequest_filepath): +def parse_json(download_request_filepath: str) -> Tuple: + """ + Parses a json file that specifies what should be downloaded, created with make_request_json() + + Parameters + ------------ + download_request_filepath : str + file path to a download request json file, created with make_request_json() + + Returns + ----------- + parsed : Tuple + Tuple of parsed information (startDatetime, endDatetime, bounds, satellite, bucket, product, channels, variables, apiKey, outDir, outputFilepath) + + Examples + ------------ + (startDatetime, endDatetime, bounds, satellite, bucket, product, channels, variables, apiKey, outDir, outputFilepath) = parse_json("my_download_request.json") + """ # load json file that specifies what we'd like to download and parse its contents - with open(downloadRequest_filepath, "r") as f: - downloadRequest = json.load(f) + with open(download_request_filepath, "r") as f: + download_request = json.load(f) - startDatetime = parser.parse(downloadRequest["dateRange"]["startDatetime"]) - endDatetime = parser.parse(downloadRequest["dateRange"]["endDatetime"]) + startDatetime = parser.parse(download_request["dateRange"]["startDatetime"]) + endDatetime = parser.parse(download_request["dateRange"]["endDatetime"]) bounds = [ - downloadRequest["bounds"]["min_lon"], - downloadRequest["bounds"]["min_lat"], - downloadRequest["bounds"]["max_lon"], - downloadRequest["bounds"]["max_lat"], + download_request["bounds"]["min_lon"], + download_request["bounds"]["min_lat"], + download_request["bounds"]["max_lon"], + download_request["bounds"]["max_lat"], ] # bounds = [min_lon, min_lat, max_lon, max_lat] - satellite = downloadRequest["satellite"] + satellite = download_request["satellite"] bucket = "noaa-" + satellite - product = downloadRequest["product"] - channels = downloadRequest["bands"] - variables = downloadRequest["variables"] - apiKey = downloadRequest["apiKey"] - outDir = downloadRequest["downloadDirectory"] - outputFilepath = downloadRequest["outputFilepath"] - - return ( + product = download_request["product"] + channels = download_request["bands"] + variables = download_request["variables"] + apiKey = download_request["apiKey"] + outDir = download_request["downloadDirectory"] + outputFilepath = download_request["outputFilepath"] + + parsed = ( startDatetime, endDatetime, bounds, @@ -242,9 +285,27 @@ def parse_json(downloadRequest_filepath): outputFilepath, ) + return parsed + -def download_abi_goespy(downloadRequest_filepath): - """Download GOES ABI imagery as specified by an input JSON file. (this function wraps around goespy.ABIDownloader())""" +def download_abi_goespy(download_request_filepath: str) -> List[Path]: + """ + Download GOES ABI imagery as specified by an input JSON file using the goespy package for downloading. + + Parameters + ------------ + download_request_filepath : str + file path to a download request json file, created with make_request_json() + + Returns + ----------- + output_filepaths : List[Path] + path to output DEM (useful if the downloaded DEM is reprojected to custom proj) + + Examples + ------------ + filepaths = download_abi_goespy("my_download_request.json") + """ ( startDatetime, @@ -258,7 +319,7 @@ def download_abi_goespy(downloadRequest_filepath): _, outDir, _, - ) = parse_json(downloadRequest_filepath) + ) = parse_json(download_request_filepath) output_filepaths = [] @@ -336,8 +397,24 @@ def download_abi_goespy(downloadRequest_filepath): return output_filepaths -def download_abi_goes2go(downloadRequest_filepath): - """Download GOES ABI imagery as specified by an input JSON file. (this function wraps around goes2go functions)""" +def download_abi_goes2go(download_request_filepath: str) -> List[Path]: + """ + Download GOES ABI imagery as specified by an input JSON file using the goes2go package for downloading. + + Parameters + ------------ + download_request_filepath : str + file path to a download request json file, created with make_request_json() + + Returns + ----------- + output_filepaths : List[Path] + path to output DEM (useful if the downloaded DEM is reprojected to custom proj) + + Examples + ------------ + filepaths = download_abi_goes2go("my_download_request.json") + """ ( start_datetime, @@ -351,7 +428,7 @@ def download_abi_goes2go(downloadRequest_filepath): _, outDir, _, - ) = parse_json(downloadRequest_filepath) + ) = parse_json(download_request_filepath) output_filepaths = [] @@ -388,11 +465,20 @@ def download_abi_goes2go(downloadRequest_filepath): # see what is available to download # df = G.df(start=startDatetime.strftime("%Y-%m-%d %H:%M"), end=endDatetime.strftime("%Y-%m-%d %H:%M")) - # download this batch - df = G.timerange( - start=this_start_datetime.strftime("%Y-%m-%d %H:%M"), - end=this_end_datetime.strftime("%Y-%m-%d %H:%M"), - ) + try: + # download this batch + df = G.timerange( + start=this_start_datetime.strftime("%Y-%m-%d %H:%M"), + end=this_end_datetime.strftime("%Y-%m-%d %H:%M"), + ) + except FileNotFoundError as e: + print( + f"FileNotFoundError encountered. The requested image may not exist. Because this searched a time window of {n} hours, there may be some valid imagery within the time window. Try a smaller time window to search for valid imagery.\n{e}" + ) + # set the next start datetime to the end of this batch + this_start_datetime = this_end_datetime + batch_number += 1 + continue # get the filepaths of this batch batch_filepaths = [ @@ -431,7 +517,7 @@ def download_abi_goes2go(downloadRequest_filepath): def get_dem(demtype, bounds, api_key, out_fn=None, proj="EPSG:4326"): """ - download a DEM of choice from OpenTopography World DEM (modified by Shashank Bhushan, first written by David Shean) + download a DEM of choice from OpenTopography World DEM Parameters ------------ @@ -457,7 +543,8 @@ def get_dem(demtype, bounds, api_key, out_fn=None, proj="EPSG:4326"): import requests - ### From David Shean + # written by Shashank Bhushan, David Shean + base_url = "https://portal.opentopography.org/API/globaldem?demtype={}&west={}&south={}&east={}&north={}&outputFormat=GTiff&API_Key={}" if out_fn is None: out_fn = "{}.tif".format(demtype) @@ -488,8 +575,9 @@ def get_dem(demtype, bounds, api_key, out_fn=None, proj="EPSG:4326"): def run_bash_command(cmd): # written by Scott Henderson - # move to asp_binder_utils - """Call a system command through the subprocess python module.""" + """ + Call a system command through the subprocess python module. + """ print(cmd) try: retcode = subprocess.call(cmd, shell=True) @@ -505,7 +593,7 @@ def download_example_data() -> None: """ Fetch the GOES ABI example files. """ - + # based on https://github.com/GlacioHack/xdem/blob/d91bf1cc9b3f36d77f3729649bc8e9edc6b42f9f/xdem/examples.py#L33 # Static commit hash (to be updated as needed) # commit = "16756d3aff6ca41ebb0be999a82d2f66930e7851" # The URL from which to download the tarball @@ -535,5 +623,31 @@ def download_example_data() -> None: def remove_example_data() -> None: + """ + Remove example files downloaded by download_example_data(). + """ tmp_dir = "./tests/resources/" shutil.rmtree(tmp_dir) + + +def bounds_from_geojson(geojson_filepath: str) -> Union[List[List[float]], List[float]]: + """ + Get a bounding box around a polygon from a geojson file + """ + + with open(geojson_filepath) as f: + geojson_data = geojson.load(f) + bounds = [] + for feature in geojson_data["features"]: + bounds.append(shape(feature["geometry"]).bounds) # [minx, miny, maxx, maxy] + + if len(bounds) > 1: + print( + "geojson file contains more than one feature, returning a list of bounds, one set of bounds per feature" + ) + if len(bounds) == 1: + bounds = bounds[ + 0 + ] # if there is only one set of bounds, remove it from the list + + return bounds