Skip to content

Commit

Permalink
Issue #12
Browse files Browse the repository at this point in the history
  • Loading branch information
@ckrogers
ckrogers committed Apr 20, 2018
1 parent aa07253 commit d146709
Showing 1 changed file with 19 additions and 213 deletions.
232 changes: 19 additions & 213 deletions notebooks/NSO2EE-MultiSpectral.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"import exifread\n",
"import zipfile\n",
"import gdal\n",
"import math\n",
"import pandas as pd\n",
"\n",
"import ee\n",
Expand Down Expand Up @@ -62,7 +63,7 @@
},
{
"cell_type": "code",
"execution_count": 31,
"execution_count": 4,
"metadata": {
"collapsed": false
},
Expand All @@ -71,8 +72,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"['Number of files to upload as assets = 3']\n",
"['Number of zipped files = 7']\n"
"['Number of files to upload as assets = 12']\n",
"['Number of zipped files = 0']\n"
]
}
],
Expand Down Expand Up @@ -106,24 +107,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"print(zippedfiles[0])\n",
"filename = os.path.basename(zippedfiles[0])\n",
"print(filename)\n",
"filename_no_ext = os.path.splitext(filename)[0]\n",
"print(filename_no_ext)\n",
"filename_unzipped = os.path.join(input_dir,dst,filename_no_ext+'.jp2000')\n",
"print(filename_unzipped)"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {
"collapsed": false
},
Expand All @@ -132,8 +116,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"['Unzipped .tif file from 20180302_093358_Tri_80cm_RD_12bit_RGBI_Schiermonnikoog.zip']\n",
"['Unzipped .tif file from 20180306_094211_Tri_80cm_RD_12bit_RGBI_Westkapelle.zip']\n"
"['Unzipped .tif file from 20180224_093625_Tri_80cm_RD_12bit_RGBI_Zierikzee.zip']\n"
]
}
],
Expand All @@ -157,7 +140,7 @@
"elif triplesat:\n",
" for j in zippedfiles:\n",
" filename = os.path.basename(j)\n",
" filename_no_ext = os.path.splitext(j)[0]\n",
" filename_no_ext = os.path.splitext(filename)[0]\n",
" filename_unzipped = os.path.join(input_dir,filename_no_ext+'.tif')\n",
" is_unzipped = os.path.lexists(filename_unzipped)\n",
" if is_unzipped: \n",
Expand Down Expand Up @@ -193,46 +176,16 @@
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"105\n"
]
}
],
"source": [
"# files = glob.glob('*.jp2')\n",
"# print(len(files))\n",
"files = glob.glob('*.tif')\n",
"print(len(files))"
]
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['RE3_20170420_3163017_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162917_RD_12bit_RGBREI_5m.jp2', 'RE5_20170707_3162917_RD_12bit_RGBREI_5m.jp2', 'RE1_20170829_3163721_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162817_RD_12bit_RGBREI_5m.jp2', 'RE5_20170707_3162817_RD_12bit_RGBREI_5m.jp2', 'RE2_20170331_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170602_3163722_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162818_RD_12bit_RGBREI_5m.jp2', 'RE3_20170601_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162817_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3163721_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162917_RD_12bit_RGBREI_5m.jp2', 'RE2_20170522_3163721_RD_12bit_RGBREI_5m.jp2', 'RE4_20170425_3163721_RD_12bit_RGBREI_5m.jp2', 'RE5_20170902_3163017_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162916_RD_12bit_RGBREI_5m.jp2', 'RE5_20170809_3162916_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3162817_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3162917_RD_12bit_RGBREI_5m.jp2', 'RE1_20170330_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162816_RD_12bit_RGBREI_5m.jp2', 'RE5_20170707_3162816_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162816_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3163721_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3163722_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162916_RD_12bit_RGBREI_5m.jp2', 'RE4_20170602_3163721_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162916_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3162817_RD_12bit_RGBREI_5m.jp2', 'RE3_20170327_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3163721_RD_12bit_RGBREI_5m.jp2', 'RE1_20170325_3263706_RD_12bit_RGBREI_5m.jp2', 'RE2_20170331_3163721_RD_12bit_RGBREI_5m.jp2', 'RE1_20170829_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162816_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3162917_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170425_3163722_RD_12bit_RGBREI_5m.jp2', 'RE2_20170522_3163722_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3163721_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3163017_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3162916_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162817_RD_12bit_RGBREI_5m.jp2', 'RE2_20170409_3263706_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162816_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3162917_RD_12bit_RGBREI_5m.jp2', 'RE3_20170904_3263706_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162916_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3162817_RD_12bit_RGBREI_5m.jp2', 'RE1_20170511_3163722_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3163721_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3162916_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162817_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3162816_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170426_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3162816_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3162818_RD_12bit_RGBREI_5m.jp2', 'RE2_20170409_3163722_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3163017_RD_12bit_RGBREI_5m.jp2', 'RE3_20170904_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3162916_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3163017_RD_12bit_RGBREI_5m.jp2', 'RE1_20170511_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162818_RD_12bit_RGBREI_5m.jp2', 'RE2_20170331_3162818_RD_12bit_RGBREI_5m.jp2', 'RE3_20170807_3263706_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3163017_RD_12bit_RGBREI_5m.jp2', 'RE1_20170325_3163722_RD_12bit_RGBREI_5m.jp2', 'RE3_20170327_3163722_RD_12bit_RGBREI_5m.jp2', 'RE1_20170829_3263706_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162818_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170425_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3162817_RD_12bit_RGBREI_5m.jp2', 'RE3_20170709_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170426_3162916_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3163017_RD_12bit_RGBREI_5m.jp2', 'RE3_20170904_3163721_RD_12bit_RGBREI_5m.jp2', 'RE5_20170902_3162816_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3163017_RD_12bit_RGBREI_5m.jp2', 'RE2_20170409_3163721_RD_12bit_RGBREI_5m.jp2', 'RE5_20170902_3162916_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3162818_RD_12bit_RGBREI_5m.jp2', 'RE3_20170831_3162817_RD_12bit_RGBREI_5m.jp2', 'RE1_20170330_3163722_RD_12bit_RGBREI_5m.jp2', 'RE5_20170315_3162916_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3163721_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162916_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162816_RD_12bit_RGBREI_5m.jp2', 'RE2_20170825_3163722_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162817_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162818_RD_12bit_RGBREI_5m.jp2', 'RE5_20170707_3162818_RD_12bit_RGBREI_5m.jp2', 'RE3_20170327_3163721_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3163017_RD_12bit_RGBREI_5m.jp2', 'RE2_20170331_3263706_RD_12bit_RGBREI_5m.jp2', 'RE1_20170619_3163722_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162917_RD_12bit_RGBREI_5m.jp2', 'RE5_20170809_3163017_RD_12bit_RGBREI_5m.jp2', 'RE5_20170902_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3163017_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162818_RD_12bit_RGBREI_5m.jp2']\n"
]
}
],
"outputs": [],
"source": [
"if rapideye:\n",
"# files = glob.glob('*.jp2000')\n",
" files = glob.glob('*.jp2')\n",
"# print(files)\n",
" print(len(files))\n",
" for i, f in enumerate(files):\n",
" print(str(i)+' of '+str(len(files)))\n",
" filename = os.path.splitext(os.path.basename(f))[0]\n",
Expand Down Expand Up @@ -260,7 +213,7 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": null,
"metadata": {
"collapsed": false
},
Expand All @@ -269,80 +222,11 @@
"name": "stdout",
"output_type": "stream",
"text": [
"['Number of files to upload as assets = 105']\n"
"['Number of files to upload as assets = 12']\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>system:time_start</th>\n",
" </tr>\n",
" <tr>\n",
" <th>filename</th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>RE3_20170904_3163721_RD_12bit_RGBREI_5m</th>\n",
" <td>1.504523e+12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>RE5_20170902_3162816_RD_12bit_RGBREI_5m</th>\n",
" <td>1.504351e+12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>RE5_20170525_3163017_RD_12bit_RGBREI_5m</th>\n",
" <td>1.495711e+12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>RE5_20170902_3162916_RD_12bit_RGBREI_5m</th>\n",
" <td>1.504351e+12</td>\n",
" </tr>\n",
" <tr>\n",
" <th>RE2_20170409_3163721_RD_12bit_RGBREI_5m</th>\n",
" <td>1.491736e+12</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" system:time_start\n",
"filename \n",
"RE3_20170904_3163721_RD_12bit_RGBREI_5m 1.504523e+12\n",
"RE5_20170902_3162816_RD_12bit_RGBREI_5m 1.504351e+12\n",
"RE5_20170525_3163017_RD_12bit_RGBREI_5m 1.495711e+12\n",
"RE5_20170902_3162916_RD_12bit_RGBREI_5m 1.504351e+12\n",
"RE2_20170409_3163721_RD_12bit_RGBREI_5m 1.491736e+12"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"files = glob.glob('*.tif')\n",
"print(['Number of files to upload as assets = '+str(len(files))])\n",
"df = pd.DataFrame(columns=['filename','system:time_start'])\n",
"\n",
"if rapideye:\n",
Expand Down Expand Up @@ -376,10 +260,12 @@
"\n",
" for i, f in enumerate(files):\n",
" filesize = os.path.getsize(f)\n",
" filename = os.path.splitext(os.path.basename(f))[0]\n",
" filename_no_ext = os.path.splitext(os.path.basename(f))[0]\n",
" dst = 'TripleSat/Tri_RD_12bit_RGBI_80cm_Split/'\n",
" \n",
" if (filesize > 10e9):\n",
" in_path = os.path.join(input_dir,f)\n",
" out_path = os.path.join(input_dir,filename_no_ext)\n",
" in_path = os.path.join(input_dir,dst,f)\n",
" out_path = os.path.join(input_dir,dst,filename_no_ext)\n",
" ds = gdal.Open(in_path)\n",
" band = ds.GetRasterBand(1)\n",
" xsize = band.XSize\n",
Expand Down Expand Up @@ -414,19 +300,11 @@
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"geebam upload --source P:/11202200.005-kpp-cip2018/data/Satellitedataportaal\\RapidEye/RE_RD_12bit_RGBEI_5m_jp2000/ --dest users/rogersckw9/eo-bathymetry/rapideye -m P:/11202200.005-kpp-cip2018/data/Satellitedataportaal\\RapidEye/RE_RD_12bit_RGBEI_5m_jp2000/metadata.csv -u [email protected]\n"
]
}
],
"outputs": [],
"source": [
"# Use Google Earth Engine Batch Asset Manager\n",
"# Details at https://github.com/tracek/gee_asset_manager\n",
Expand All @@ -443,78 +321,6 @@
" print(\"No satellite selected.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# if uploading to Google Cloud first, follow this protocol (this is for RapidEye)\n",
"\n",
"for i, f in enumerate(files):\n",
" print('Processing file ' + f + ', file index: ' + str(i))\n",
"\n",
" # extract time in UTC\n",
" fh = open(f, 'rb')\n",
" tags = exifread.process_file(fh, stop_tag=\"Image DateTime\")\n",
" datestring = str(tags[\"Image DateTime\"].values)\n",
" t = datetime.datetime.strptime(datestring, '%Y:%m:%d %H:%M:%S')\n",
" local_t = local.localize(t, is_dst=None)\n",
" utc_t = local_t.astimezone(pytz.utc)\n",
" time_start = utc_t.strftime('%Y-%m-%dT%H:%M:%S')\n",
"\n",
" # parse file names\n",
" filename = os.path.basename(f)\n",
" filename_no_ext = os.path.splitext(filename)[0]\n",
"\n",
" # get nodata value ... UGLY, UGLY code!\n",
" nodata_value = -99999999\n",
" with open(f, encoding=\"latin_1\") as asc:\n",
" for line in asc:\n",
" if \"nodata_value\" in line.lower():\n",
" nodata_value = line.split()[1]\n",
" break\n",
" \n",
" run(r\"C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp {0} gs://eo-bathymetry/rapideye/{1}\".format(os.path.join(input_dir,f),filename))\n",
" \n",
" # upload to GEE\n",
" retry_count = 0\n",
"\n",
" while True:\n",
" run(\"earthengine upload image --wait --asset_id=users/rogersckw9/rapideye/{1} --nodata_value={0} gs://eo-bathymetry/rapideye/{1}\".format(nodata_value, filename))\n",
" \n",
" check last task status\n",
" tasks = ee.data.getTaskList()\n",
" task_state = None\n",
" for task in tasks:\n",
" task_status = ee.data.getTaskStatus([task['id']])\n",
" task_state = task_status[0]['state']\n",
" print(task_status)\n",
" break\n",
"\n",
" if task_state != 'FAILED':\n",
" break # done\n",
" else:\n",
" retry_count += 1\n",
" print('Retrying upload ' + str(retry_count) + ' ...')\n",
"\n",
" if retry_count > 10:\n",
" print('Maximum number of retry reached, exiting ...')\n",
" sys.exit(0)\n",
" \n",
" # set time\n",
" run(\"earthengine asset set --time_start {0} users/rogersckw9/rapideye/{1}\".format(time_start, filename_no_ext))\n",
"\n",
" if rapideye:\n",
" \n",
"elif triplesat:\n",
"\n",
"else:\n",
" print(\"No satellite selected.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down

0 comments on commit d146709

Please sign in to comment.