Skip to content

Commit

Permalink
use fixed output file path
Browse files Browse the repository at this point in the history
  • Loading branch information
@ZihengSun
ZihengSun committed Jun 21, 2023
1 parent 6a69429 commit ac2c6a2
Show file tree
Hide file tree
Showing 51 changed files with 1,690 additions and 4 deletions.
2 changes: 2 additions & 0 deletions code/earth_data_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

# Harmony
import geopandas as gpd
import geoviews as gv
gv.extension('bokeh', 'matplotlib')
from harmony import BBox, Client, Collection, Request, LinkType
import datetime as dt
import s3fs
Expand Down
3 changes: 3 additions & 0 deletions code/install_packages.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash
pip3 install geopandas earthaccess xarray harmony-py s3fs

4 changes: 2 additions & 2 deletions code/process.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@
"id" : "nlsntt",
"name" : "retrieve_sst",
"description" : null,
"code" : "from earth_data_utils import *\n\ndef search_and_get_sst():\n auth_earthaccess()\n results = earthaccess.search_data(\n short_name=sst_short_name,\n cloud_hosted=True,\n temporal=(\"2021-07-01\", \"2021-07-02\"),\n )\n print(\"found results: \", results)\n return results\n\ndef plot_subset(ds_subset):\n ds_subset.plot(figsize=(12,6), x='lon', y='lat')\n home_dir = os.path.expanduser('~')\n \n file_name = f\"geoweaver_plot_{str(uuid.uuid4())}.png\"\n save_file_path = os.path.join(home_dir, file_name)\n plt.savefig(save_file_path)\n print(f\"new figure is saved to {save_file_path}\")\n\nwith get_dataset(search_and_get_sst()) as ds:\n subset = ds[\"analysed_sst\"].sel(lat=slice(27, 37), \n lon=slice(-70,-50)).std('time')\n plot_subset(subset)\n\n",
"code" : "from earth_data_utils import *\n\ndef search_and_get_sst():\n auth_earthaccess()\n results = earthaccess.search_data(\n short_name=sst_short_name,\n cloud_hosted=True,\n temporal=(\"2021-07-01\", \"2021-07-02\"),\n )\n print(\"found results: \", results)\n return results\n\ndef plot_subset(ds_subset):\n ds_subset.plot(figsize=(12,6), x='lon', y='lat')\n home_dir = os.path.expanduser('~')\n \n file_name = f\"geoweaver_plot_sst.png\"\n save_file_path = os.path.join(home_dir, file_name)\n plt.savefig(save_file_path)\n print(f\"new figure is saved to {save_file_path}\")\n\nwith get_dataset(search_and_get_sst()) as ds:\n subset = ds[\"analysed_sst\"].sel(lat=slice(27, 37), \n lon=slice(-70,-50)).std('time')\n plot_subset(subset)\n\n",
"lang" : "python",
"owner" : "111111",
"confidential" : "FALSE"
},{
"id" : "e57twj",
"name" : "retrieve_ssh",
"description" : null,
"code" : "from earth_data_utils import *\n\ndef search_ssh():\n auth_earthaccess()\n results = earthaccess.search_data(\n short_name=ssh_short_name,\n cloud_hosted=True,\n temporal=(\"2021-07-01\", \"2021-09-30\"),\n )\n return results\n\ndef plot_subset(ds_subset):\n ds_subset.plot(figsize=(12,6), x='Longitude', y='Latitude')\n home_dir = os.path.expanduser('~')\n \n file_name = f\"geoweaver_plot_{str(uuid.uuid4())}.png\"\n save_file_path = os.path.join(home_dir, file_name)\n plt.savefig(save_file_path)\n print(f\"new figure is saved to {save_file_path}\")\n\nwith get_dataset(search_ssh()) as ds:\n subset = ds[\"SLA\"].sel(Latitude=slice(-79, 79), \n Longitude=slice(100,300)).std('Time')\n plot_subset(subset)\n",
"code" : "from earth_data_utils import *\n\ndef search_ssh():\n auth_earthaccess()\n results = earthaccess.search_data(\n short_name=ssh_short_name,\n cloud_hosted=True,\n temporal=(\"2021-07-01\", \"2021-09-30\"),\n )\n return results\n\ndef plot_subset(ds_subset):\n ds_subset.plot(figsize=(12,6), x='Longitude', y='Latitude')\n home_dir = os.path.expanduser('~')\n \n file_name = f\"geoweaver_plot_ssh.png\"\n save_file_path = os.path.join(home_dir, file_name)\n plt.savefig(save_file_path)\n print(f\"new figure is saved to {save_file_path}\")\n\nwith get_dataset(search_ssh()) as ds:\n subset = ds[\"SLA\"].sel(Latitude=slice(-79, 79), \n Longitude=slice(100,300)).std('Time')\n plot_subset(subset)\n",
"lang" : "python",
"owner" : "111111",
"confidential" : "FALSE"
Expand Down
2 changes: 1 addition & 1 deletion code/retrieve_ssh.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def plot_subset(ds_subset):
ds_subset.plot(figsize=(12,6), x='Longitude', y='Latitude')
home_dir = os.path.expanduser('~')

file_name = f"geoweaver_plot_{str(uuid.uuid4())}.png"
file_name = f"geoweaver_plot_ssh.png"
save_file_path = os.path.join(home_dir, file_name)
plt.savefig(save_file_path)
print(f"new figure is saved to {save_file_path}")
Expand Down
2 changes: 1 addition & 1 deletion code/retrieve_sst.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def plot_subset(ds_subset):
ds_subset.plot(figsize=(12,6), x='lon', y='lat')
home_dir = os.path.expanduser('~')

file_name = f"geoweaver_plot_{str(uuid.uuid4())}.png"
file_name = f"geoweaver_plot_sst.png"
save_file_path = os.path.join(home_dir, file_name)
plt.savefig(save_file_path)
print(f"new figure is saved to {save_file_path}")
Expand Down
41 changes: 41 additions & 0 deletions history/3ZlO4QZpqjhGyKmyXO.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
[{
"history_id" : "qjzve8oimqm",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686629569918,
"history_end_time" : 1686629570051,
"history_notes" : null,
"history_process" : "wttkvt",
"host_id" : "100001",
"indicator" : "Failed"
},{
"history_id" : "htpbdvjc8no",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686629570919,
"history_end_time" : 1686629570977,
"history_notes" : null,
"history_process" : "yvo1ds",
"host_id" : "100001",
"indicator" : "Failed"
},{
"history_id" : "j2rg3ef6rul",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686629571059,
"history_end_time" : 1686629571096,
"history_notes" : null,
"history_process" : "cgvhiy",
"host_id" : "100001",
"indicator" : "Failed"
},{
"history_id" : "zpwvlwn3rb8",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686629571983,
"history_end_time" : 1686629572047,
"history_notes" : null,
"history_process" : "1b4ko0",
"host_id" : "100001",
"indicator" : "Failed"
}]
31 changes: 31 additions & 0 deletions history/54jcaiqoso3iree437.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
[{
"history_id" : "u51ro7imrzb",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686719429276,
"history_end_time" : 1686719429431,
"history_notes" : null,
"history_process" : "wttkvt",
"host_id" : "100001",
"indicator" : "Failed"
},{
"history_id" : "gwrbbwkvoyc",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686719430276,
"history_end_time" : 1686719430322,
"history_notes" : null,
"history_process" : "cgvhiy",
"host_id" : "100001",
"indicator" : "Failed"
},{
"history_id" : "bp9bgjy10zr",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686719430446,
"history_end_time" : 1686719430487,
"history_notes" : null,
"history_process" : "1b4ko0",
"host_id" : "100001",
"indicator" : "Failed"
}]
21 changes: 21 additions & 0 deletions history/5dixki8g2scz0sjj56.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[{
"history_id" : "y8vpu7ag2f3",
"history_input" : "# Write first python in Geoweaver\nx = 1",
"history_output" : "",
"history_begin_time" : 1686363724744,
"history_end_time" : 1686363725573,
"history_notes" : null,
"history_process" : "qmc7xi",
"host_id" : "100001",
"indicator" : "Done"
},{
"history_id" : "xfgwx8dm9v7",
"history_input" : "# Write first python in Geoweaver\nx = 1",
"history_output" : "",
"history_begin_time" : 1686363726747,
"history_end_time" : 1686363727401,
"history_notes" : null,
"history_process" : "qmc7xi",
"host_id" : "100001",
"indicator" : "Done"
}]
41 changes: 41 additions & 0 deletions history/5ou0GVSM1NQnlv7FOB.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
[{
"history_id" : "s5e7087vr1q",
"history_input" : "\n\n\n# Suppress warnings\nimport warnings\nwarnings.simplefilter('ignore')\nwarnings.filterwarnings('ignore')\n\n# Direct access\nimport earthaccess \nfrom pprint import pprint\nimport xarray as xr\n\n# Harmony\nimport geopandas as gpd\nimport geoviews as gv\ngv.extension('bokeh', 'matplotlib')\nfrom harmony import BBox, Client, Collection, Request, LinkType\nimport datetime as dt\nimport s3fs",
"history_output" : "/srv/conda/envs/notebook/lib/python3.9/site-packages/geoviews/operation/__init__.py:14: HoloviewsDeprecationWarning: 'ResamplingOperation' is deprecated and will be removed in version 1.17, use 'ResampleOperation2D' instead.\n from holoviews.operation.datashader import (\n",
"history_begin_time" : 1686632672962,
"history_end_time" : 1686632681141,
"history_notes" : null,
"history_process" : "wttkvt",
"host_id" : "100001",
"indicator" : "Done"
},{
"history_id" : "zqhu0tomq42",
"history_input" : "from earth_data_utils import *\n\nssh_short_name = \"SEA_SURFACE_HEIGHT_ALT_GRIDS_L4_2SATS_5DAY_6THDEG_V_JPL2205\"\n\nresults = earthaccess.search_data(\n short_name=ssh_short_name,\n cloud_hosted=True,\n temporal=(\"2021-07-01\", \"2021-09-30\"),\n)",
"history_output" : "/srv/conda/envs/notebook/lib/python3.9/site-packages/geoviews/operation/__init__.py:14: HoloviewsDeprecationWarning: 'ResamplingOperation' is deprecated and will be removed in version 1.17, use 'ResampleOperation2D' instead.\n from holoviews.operation.datashader import (\nGranules found: 18\n",
"history_begin_time" : 1686632682995,
"history_end_time" : 1686632692171,
"history_notes" : null,
"history_process" : "yvo1ds",
"host_id" : "100001",
"indicator" : "Done"
},{
"history_id" : "1llchcspi3l",
"history_input" : "No code saved",
"history_output" : "Skipped",
"history_begin_time" : 1686632672823,
"history_end_time" : 1686632672823,
"history_notes" : null,
"history_process" : "cgvhiy",
"host_id" : "100001",
"indicator" : "Skipped"
},{
"history_id" : "joj0lfoon7u",
"history_input" : "# Suppress warnings\nimport os\nimport warnings\nwarnings.simplefilter('ignore')\nwarnings.filterwarnings('ignore')\n\n# Direct access\nimport earthaccess \nfrom pprint import pprint\nimport xarray as xr\n\n# Harmony\nimport geopandas as gpd\nimport geoviews as gv\ngv.extension('bokeh', 'matplotlib')\nfrom harmony import BBox, Client, Collection, Request, LinkType\nimport datetime as dt\nimport s3fs\nimport matplotlib.pyplot as plt\nfrom earth_access_authenticate import *\n\ncreds = harmony_client.aws_credentials()\ns3_fs = s3fs.S3FileSystem(\n key=creds['aws_access_key_id'],\n secret=creds['aws_secret_access_key'],\n token=creds['aws_session_token'],\n client_kwargs={'region_name':'us-west-2'},\n)\nf = s3_fs.open(url, mode='rb')\nds = xr.open_dataset(f)\nds.analysed_sst.plot()\nhome_dir = os.path.expanduser('~')\nplt.savefig(os.path.join(home_dir, 'plot2.png'))",
"history_output" : "/srv/conda/envs/notebook/lib/python3.9/site-packages/geoviews/operation/__init__.py:14: HoloviewsDeprecationWarning: 'ResamplingOperation' is deprecated and will be removed in version 1.17, use 'ResampleOperation2D' instead.\n from holoviews.operation.datashader import (\nGranules found: 18\nGranules found: 18\nYou're now authenticated with NASA Earthdata Login\nUsing token with expiration date: 08/07/2023\nUsing .netrc file for EDL\n Opening 18 granules, approx size: 0.0 GB\nSUBMITTING | : 0%| | 0/18 [00:00<?, ?it/s]\nSUBMITTING | : 100%|██████████| 18/18 [00:00<00:00, 1538.38it/s]\nPROCESSING | : 0%| | 0/18 [00:00<?, ?it/s]\nPROCESSING | : 6%|▌ | 1/18 [00:00<00:01, 9.71it/s]\nPROCESSING | : 100%|██████████| 18/18 [00:00<00:00, 126.26it/s]\nCOLLECTING | : 0%| | 0/18 [00:00<?, ?it/s]\nCOLLECTING | : 100%|██████████| 18/18 [00:00<00:00, 300786.74it/s]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 100% ] |###################################################| [|]\n [ Processing: 100% ] |###################################################| [|]\nException ignored in: <function CachingFileManager.__del__ at 0x7f136a3e5e50>\nTraceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/xarray/backends/file_manager.py\", line 249, in __del__\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/xarray/backends/file_manager.py\", line 233, in close\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/h5netcdf/core.py\", line 1209, in close\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/h5py/_hl/files.py\", line 585, in close\nTypeError: bad operand type for unary ~: 'NoneType'\nException ignored in: <function File.close at 0x7f150674d3a0>\nTraceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/h5netcdf/core.py\", line 1209, in close\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/h5py/_hl/files.py\", line 585, in close\nTypeError: bad operand type for unary ~: 'NoneType'\n",
"history_begin_time" : 1686632672898,
"history_end_time" : 1686632761172,
"history_notes" : null,
"history_process" : "1b4ko0",
"host_id" : "100001",
"indicator" : "Done"
}]
Loading

0 comments on commit ac2c6a2

Please sign in to comment.