Skip to content

Commit

Permalink
added history files
Browse files Browse the repository at this point in the history
  • Loading branch information
@gokulprathin8
gokulprathin8 committed Jun 13, 2023
1 parent 6646c10 commit 3c6cf75
Show file tree
Hide file tree
Showing 18 changed files with 694 additions and 19 deletions.
2 changes: 1 addition & 1 deletion code/earth_access_authenticate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
},
)
job_id = harmony_client.submit(request)
harmony_client.wait_for_processing(job_id, show_progress=True)
harmony_client.wait_for_processing(job_id, show_progress=False)
data = harmony_client.result_json(job_id)
results = harmony_client.result_urls(job_id, link_type=LinkType.s3)
urls = list(results)
Expand Down
4 changes: 2 additions & 2 deletions code/process.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@
"id" : "cgvhiy",
"name" : "earth_access_authenticate",
"description" : null,
"code" : "from search_data import *\n\nsst_short_name=\"MUR-JPL-L4-GLOB-v4.1\"\n\nresults = earthaccess.search_data(\n short_name=ssh_short_name,\n cloud_hosted=True,\n temporal=(\"2021-07-01\", \"2021-09-30\"),\n)\n\nauth = earthaccess.login(strategy=\"netrc\", persist=True)\nds = xr.open_mfdataset(earthaccess.open(results))\ngeojson_path = '/home/jovyan/gulf.json'\ngdf = gpd.read_file(geojson_path) #Return a GeoDataFrame object\n\n\nharmony_client = Client()\nsst_short_name=\"MUR-JPL-L4-GLOB-v4.1\"\n\nrequest = Request(\n collection=Collection(id=sst_short_name),\n shape=geojson_path,\n temporal={\n 'start': dt.datetime(2021, 8, 1, 1),\n 'stop': dt.datetime(2021, 8, 1, 2) \n },\n)\njob_id = harmony_client.submit(request)\nharmony_client.wait_for_processing(job_id, show_progress=True)\ndata = harmony_client.result_json(job_id)\nresults = harmony_client.result_urls(job_id, link_type=LinkType.s3)\nurls = list(results)\nurl = urls[0]\n",
"code" : "from search_data import *\n\nsst_short_name=\"MUR-JPL-L4-GLOB-v4.1\"\n\nresults = earthaccess.search_data(\n short_name=ssh_short_name,\n cloud_hosted=True,\n temporal=(\"2021-07-01\", \"2021-09-30\"),\n)\n\nauth = earthaccess.login(strategy=\"netrc\", persist=True)\nds = xr.open_mfdataset(earthaccess.open(results))\ngeojson_path = '/home/jovyan/gulf.json'\ngdf = gpd.read_file(geojson_path) #Return a GeoDataFrame object\n\n\nharmony_client = Client()\nsst_short_name=\"MUR-JPL-L4-GLOB-v4.1\"\n\nrequest = Request(\n collection=Collection(id=sst_short_name),\n shape=geojson_path,\n temporal={\n 'start': dt.datetime(2021, 8, 1, 1),\n 'stop': dt.datetime(2021, 8, 1, 2) \n },\n)\njob_id = harmony_client.submit(request)\nharmony_client.wait_for_processing(job_id, show_progress=False)\ndata = harmony_client.result_json(job_id)\nresults = harmony_client.result_urls(job_id, link_type=LinkType.s3)\nurls = list(results)\nurl = urls[0]\n",
"lang" : "python",
"owner" : "111111",
"confidential" : "FALSE"
},{
"id" : "1b4ko0",
"name" : "s3_file_storage",
"description" : null,
"code" : "import matplotlib.pyplot as plt\nfrom earth_access_authenticate import *\n\ntry:\n creds = harmony_client.aws_credentials()\n s3_fs = s3fs.S3FileSystem(\n key=creds['aws_access_key_id'],\n secret=creds['aws_secret_access_key'],\n token=creds['aws_session_token'],\n client_kwargs={'region_name':'us-west-2'},\n )\n f = s3_fs.open(url, mode='rb')\n ds = xr.open_dataset(f)\n ds.analysed_sst.plot()\n plt.savefig('plot.png')\nexcept Exception as e:\n print('please check the write permissions to directory', e)",
"code" : "# Suppress warnings\nimport os\nimport warnings\nwarnings.simplefilter('ignore')\nwarnings.filterwarnings('ignore')\n\n# Direct access\nimport earthaccess \nfrom pprint import pprint\nimport xarray as xr\n\n# Harmony\nimport geopandas as gpd\nimport geoviews as gv\ngv.extension('bokeh', 'matplotlib')\nfrom harmony import BBox, Client, Collection, Request, LinkType\nimport datetime as dt\nimport s3fs\nimport matplotlib.pyplot as plt\nfrom earth_access_authenticate import *\n\ncreds = harmony_client.aws_credentials()\ns3_fs = s3fs.S3FileSystem(\n key=creds['aws_access_key_id'],\n secret=creds['aws_secret_access_key'],\n token=creds['aws_session_token'],\n client_kwargs={'region_name':'us-west-2'},\n)\nf = s3_fs.open(url, mode='rb')\nds = xr.open_dataset(f)\nds.analysed_sst.plot()\nhome_dir = os.path.expanduser('~')\nplt.savefig(os.path.join(home_dir, 'plot2.png'))",
"lang" : "python",
"owner" : "111111",
"confidential" : "FALSE"
Expand Down
44 changes: 30 additions & 14 deletions code/s3_file_storage.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,33 @@
# Suppress warnings
import os
import warnings
warnings.simplefilter('ignore')
warnings.filterwarnings('ignore')

# Direct access
import earthaccess
from pprint import pprint
import xarray as xr

# Harmony
import geopandas as gpd
import geoviews as gv
gv.extension('bokeh', 'matplotlib')
from harmony import BBox, Client, Collection, Request, LinkType
import datetime as dt
import s3fs
import matplotlib.pyplot as plt
from earth_access_authenticate import *

try:
creds = harmony_client.aws_credentials()
s3_fs = s3fs.S3FileSystem(
key=creds['aws_access_key_id'],
secret=creds['aws_secret_access_key'],
token=creds['aws_session_token'],
client_kwargs={'region_name':'us-west-2'},
)
f = s3_fs.open(url, mode='rb')
ds = xr.open_dataset(f)
ds.analysed_sst.plot()
plt.savefig('plot.png')
except Exception as e:
print('please check the write permissions to directory', e)
creds = harmony_client.aws_credentials()
s3_fs = s3fs.S3FileSystem(
key=creds['aws_access_key_id'],
secret=creds['aws_secret_access_key'],
token=creds['aws_session_token'],
client_kwargs={'region_name':'us-west-2'},
)
f = s3_fs.open(url, mode='rb')
ds = xr.open_dataset(f)
ds.analysed_sst.plot()
home_dir = os.path.expanduser('~')
plt.savefig(os.path.join(home_dir, 'plot2.png'))
41 changes: 41 additions & 0 deletions history/3ZlO4QZpqjhGyKmyXO.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
[{
"history_id" : "qjzve8oimqm",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686629569918,
"history_end_time" : 1686629570051,
"history_notes" : null,
"history_process" : "wttkvt",
"host_id" : "100001",
"indicator" : "Failed"
},{
"history_id" : "htpbdvjc8no",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686629570919,
"history_end_time" : 1686629570977,
"history_notes" : null,
"history_process" : "yvo1ds",
"host_id" : "100001",
"indicator" : "Failed"
},{
"history_id" : "j2rg3ef6rul",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686629571059,
"history_end_time" : 1686629571096,
"history_notes" : null,
"history_process" : "cgvhiy",
"host_id" : "100001",
"indicator" : "Failed"
},{
"history_id" : "zpwvlwn3rb8",
"history_input" : null,
"history_output" : "Authentication Failed. Wrong Password.",
"history_begin_time" : 1686629571983,
"history_end_time" : 1686629572047,
"history_notes" : null,
"history_process" : "1b4ko0",
"host_id" : "100001",
"indicator" : "Failed"
}]
21 changes: 21 additions & 0 deletions history/5dixki8g2scz0sjj56.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[{
"history_id" : "y8vpu7ag2f3",
"history_input" : "# Write first python in Geoweaver\nx = 1",
"history_output" : "",
"history_begin_time" : 1686363724744,
"history_end_time" : 1686363725573,
"history_notes" : null,
"history_process" : "qmc7xi",
"host_id" : "100001",
"indicator" : "Done"
},{
"history_id" : "xfgwx8dm9v7",
"history_input" : "# Write first python in Geoweaver\nx = 1",
"history_output" : "",
"history_begin_time" : 1686363726747,
"history_end_time" : 1686363727401,
"history_notes" : null,
"history_process" : "qmc7xi",
"host_id" : "100001",
"indicator" : "Done"
}]
41 changes: 41 additions & 0 deletions history/5ou0GVSM1NQnlv7FOB.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
[{
"history_id" : "s5e7087vr1q",
"history_input" : "\n\n\n# Suppress warnings\nimport warnings\nwarnings.simplefilter('ignore')\nwarnings.filterwarnings('ignore')\n\n# Direct access\nimport earthaccess \nfrom pprint import pprint\nimport xarray as xr\n\n# Harmony\nimport geopandas as gpd\nimport geoviews as gv\ngv.extension('bokeh', 'matplotlib')\nfrom harmony import BBox, Client, Collection, Request, LinkType\nimport datetime as dt\nimport s3fs",
"history_output" : "/srv/conda/envs/notebook/lib/python3.9/site-packages/geoviews/operation/__init__.py:14: HoloviewsDeprecationWarning: 'ResamplingOperation' is deprecated and will be removed in version 1.17, use 'ResampleOperation2D' instead.\n from holoviews.operation.datashader import (\n",
"history_begin_time" : 1686632672962,
"history_end_time" : 1686632681141,
"history_notes" : null,
"history_process" : "wttkvt",
"host_id" : "100001",
"indicator" : "Done"
},{
"history_id" : "zqhu0tomq42",
"history_input" : "from earth_data_utils import *\n\nssh_short_name = \"SEA_SURFACE_HEIGHT_ALT_GRIDS_L4_2SATS_5DAY_6THDEG_V_JPL2205\"\n\nresults = earthaccess.search_data(\n short_name=ssh_short_name,\n cloud_hosted=True,\n temporal=(\"2021-07-01\", \"2021-09-30\"),\n)",
"history_output" : "/srv/conda/envs/notebook/lib/python3.9/site-packages/geoviews/operation/__init__.py:14: HoloviewsDeprecationWarning: 'ResamplingOperation' is deprecated and will be removed in version 1.17, use 'ResampleOperation2D' instead.\n from holoviews.operation.datashader import (\nGranules found: 18\n",
"history_begin_time" : 1686632682995,
"history_end_time" : 1686632692171,
"history_notes" : null,
"history_process" : "yvo1ds",
"host_id" : "100001",
"indicator" : "Done"
},{
"history_id" : "1llchcspi3l",
"history_input" : "No code saved",
"history_output" : "Skipped",
"history_begin_time" : 1686632672823,
"history_end_time" : 1686632672823,
"history_notes" : null,
"history_process" : "cgvhiy",
"host_id" : "100001",
"indicator" : "Skipped"
},{
"history_id" : "joj0lfoon7u",
"history_input" : "# Suppress warnings\nimport os\nimport warnings\nwarnings.simplefilter('ignore')\nwarnings.filterwarnings('ignore')\n\n# Direct access\nimport earthaccess \nfrom pprint import pprint\nimport xarray as xr\n\n# Harmony\nimport geopandas as gpd\nimport geoviews as gv\ngv.extension('bokeh', 'matplotlib')\nfrom harmony import BBox, Client, Collection, Request, LinkType\nimport datetime as dt\nimport s3fs\nimport matplotlib.pyplot as plt\nfrom earth_access_authenticate import *\n\ncreds = harmony_client.aws_credentials()\ns3_fs = s3fs.S3FileSystem(\n key=creds['aws_access_key_id'],\n secret=creds['aws_secret_access_key'],\n token=creds['aws_session_token'],\n client_kwargs={'region_name':'us-west-2'},\n)\nf = s3_fs.open(url, mode='rb')\nds = xr.open_dataset(f)\nds.analysed_sst.plot()\nhome_dir = os.path.expanduser('~')\nplt.savefig(os.path.join(home_dir, 'plot2.png'))",
"history_output" : "/srv/conda/envs/notebook/lib/python3.9/site-packages/geoviews/operation/__init__.py:14: HoloviewsDeprecationWarning: 'ResamplingOperation' is deprecated and will be removed in version 1.17, use 'ResampleOperation2D' instead.\n from holoviews.operation.datashader import (\nGranules found: 18\nGranules found: 18\nYou're now authenticated with NASA Earthdata Login\nUsing token with expiration date: 08/07/2023\nUsing .netrc file for EDL\n Opening 18 granules, approx size: 0.0 GB\nSUBMITTING | : 0%| | 0/18 [00:00<?, ?it/s]\nSUBMITTING | : 100%|██████████| 18/18 [00:00<00:00, 1538.38it/s]\nPROCESSING | : 0%| | 0/18 [00:00<?, ?it/s]\nPROCESSING | : 6%|▌ | 1/18 [00:00<00:01, 9.71it/s]\nPROCESSING | : 100%|██████████| 18/18 [00:00<00:00, 126.26it/s]\nCOLLECTING | : 0%| | 0/18 [00:00<?, ?it/s]\nCOLLECTING | : 100%|██████████| 18/18 [00:00<00:00, 300786.74it/s]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 0% ] | | [|]\n [ Processing: 0% ] | | [/]\n [ Processing: 0% ] | | [-]\n [ Processing: 0% ] | | [\\]\n [ Processing: 100% ] |###################################################| [|]\n [ Processing: 100% ] |###################################################| [|]\nException ignored in: <function CachingFileManager.__del__ at 0x7f136a3e5e50>\nTraceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/xarray/backends/file_manager.py\", line 249, in __del__\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/xarray/backends/file_manager.py\", line 233, in close\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/h5netcdf/core.py\", line 1209, in close\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/h5py/_hl/files.py\", line 585, in close\nTypeError: bad operand type for unary ~: 'NoneType'\nException ignored in: <function File.close at 0x7f150674d3a0>\nTraceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/h5netcdf/core.py\", line 1209, in close\n File \"/srv/conda/envs/notebook/lib/python3.9/site-packages/h5py/_hl/files.py\", line 585, in close\nTypeError: bad operand type for unary ~: 'NoneType'\n",
"history_begin_time" : 1686632672898,
"history_end_time" : 1686632761172,
"history_notes" : null,
"history_process" : "1b4ko0",
"host_id" : "100001",
"indicator" : "Done"
}]
Loading

0 comments on commit 3c6cf75

Please sign in to comment.