Skip to content

Commit

Permalink
Issue #12
Browse files Browse the repository at this point in the history 
 Prepare and upload RapidEye+TripleSat images
  • Loading branch information
@ckrogers
ckrogers authored Apr 3, 2018
1 parent e5fdd14 commit c99e4e2
Showing 1 changed file with 284 additions and 0 deletions.
284 changes: 284 additions & 0 deletions notebooks/NSO2EE.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,284 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 35,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"import re\n",
"import glob\n",
"import subprocess\n",
"import datetime\n",
"import time\n",
"import pytz\n",
"import exifread\n",
"import zipfile\n",
"# from osgeo import gdal\n",
"\n",
"import ee\n",
"import ee.cli\n",
"import ee.cli.commands\n",
"import ee.cli.utils"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def run(cmd):\n",
" print(cmd)\n",
" subprocess.call(cmd)\n",
" \n",
"def extractAll(zipName):\n",
" z = zipfile.PyZipFile(zipName)\n",
" for f in z.namelist():\n",
" if f.endswith('/'):\n",
" os.makedirs(f)\n",
" else:\n",
" z.extract(f)"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"ee.Initialize()\n",
"ee_config = ee.cli.utils.CommandLineConfig()"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"input_dir = r'P:/11202200.005-kpp-cip2018/data/Satellitedataportaal'\n",
"os.chdir(input_dir)\n",
"\n",
"rapideye = True\n",
"triplesat = False\n",
"\n",
"local = pytz.timezone(\"Europe/Amsterdam\")\n"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Number of files to upload as assets = 200']\n"
]
}
],
"source": [
"# unzip all .zip files\n",
"\n",
"# find all .tif files\n",
"if rapideye:\n",
" files = glob.glob('RapidEye/**/**/*.tif', recursive=True)\n",
" zippedfiles = glob.glob('RapidEye/**/**/*.zip', recursive=True)\n",
"elif triplesat:\n",
" files = glob.glob('TripleSat/**/**/*.tif', recursive=True)\n",
" zippedfiles = glob.glob('TripleSat/**/**/*.zip', recursive=True)\n",
"else:\n",
" files = None\n",
" zippedfiles = None\n",
" print('Can only upload one satellite at a time. Please set rapideye or triplesat only.')\n",
"\n",
"# files = glob.glob('TripleSat/**/*.tif')\n",
"\n",
"print(['Number of files to upload as assets = '+str(len(files))])\n"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"False\n",
"['tif file DOES NOT exist for 20170331_3163722_RE_5m_8bit_RGB_RD.zip']\n"
]
}
],
"source": [
"filename = os.path.basename(zippedfiles[1])\n",
"filename_no_ext = os.path.splitext(zippedfiles[1])[0]\n",
"dir_path = os.path.dirname(os.path.abspath(zippedfiles[1]))\n",
"filename_unzipped = os.path.join(dir_path,filename_no_ext+'.tif')\n",
"test_unzipped = os.path.exists(filename_unzipped)\n",
"print(test_unzipped)\n",
"if test_unzipped: \n",
" print(['tif file exists for ' + filename])\n",
"else:\n",
" print(['tif file DOES NOT exist for ' + filename])"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"\n",
"# z = zipfile.ZipFile(zippedfiles)\n",
"\n",
"for j in zippedfiles:\n",
" filename = os.path.basename(j)\n",
" filename_no_ext = os.path.splitext(j)[0]\n",
" dir_path = os.path.dirname(os.path.abspath(files[1]))\n",
" unzipped = \n",
" if (dir_path,filename,'.tif'):\n",
" else:\n",
" extractAll(j)\n",
"\n",
"\n",
"# if __name__ == '__main__':\n",
"# zipList = ['one.zip', 'two.zip', 'three.zip']\n",
"# for zip in zipList:\n",
"# extractAll(zipName)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing file RapidEye\\2017_03\\20170325\\20170325_3163722_RE_5m_8bit_RGB_RD.tif, file index: 0\n",
"C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp P:/11202200.005-kpp-cip2018/data/Satellitedataportaal\\RapidEye\\2017_03\\20170325\\20170325_3163722_RE_5m_8bit_RGB_RD.tif gs://eo-bathymetry/rapideye/20170325_3163722_RE_5m_8bit_RGB_RD.tif\n",
"earthengine upload image --wait --asset_id=users/rogersckw9/rapideye/20170325_3163722_RE_5m_8bit_RGB_RD.tif --nodata_value=-99999999 gs://eo-bathymetry/rapideye/20170325_3163722_RE_5m_8bit_RGB_RD.tif\n",
"[{'id': 'IHJ2XPQJZSPO2GZHRAFA27PQ', 'state': 'COMPLETED', 'creation_timestamp_ms': 1522415523002, 'update_timestamp_ms': 1522415627887, 'description': 'Asset ingestion: users/rogersckw9/20170327_3163722_RE_5m_8bit_RGB_RD', 'task_type': 'INGEST', 'output_url': ['https://code.earthengine.google.com/?asset=users/rogersckw9/20170327_3163722_RE_5m_8bit_RGB_RD'], 'start_timestamp_ms': 1522415526221}]\n",
"earthengine asset set --time_start 2017-03-25T10:14:57 users/rogersckw9/rapideye/20170325_3163722_RE_5m_8bit_RGB_RD\n",
"Processing file RapidEye\\2017_03\\20170321\\20170321_3162818_RE_5m_8bit_RGB_RD.tif, file index: 1\n",
"C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp P:/11202200.005-kpp-cip2018/data/Satellitedataportaal\\RapidEye\\2017_03\\20170321\\20170321_3162818_RE_5m_8bit_RGB_RD.tif gs://eo-bathymetry/rapideye/20170321_3162818_RE_5m_8bit_RGB_RD.tif\n",
"earthengine upload image --wait --asset_id=users/rogersckw9/rapideye/20170321_3162818_RE_5m_8bit_RGB_RD.tif --nodata_value=-99999999 gs://eo-bathymetry/rapideye/20170321_3162818_RE_5m_8bit_RGB_RD.tif\n",
"[{'id': 'IHJ2XPQJZSPO2GZHRAFA27PQ', 'state': 'COMPLETED', 'creation_timestamp_ms': 1522415523002, 'update_timestamp_ms': 1522415627887, 'description': 'Asset ingestion: users/rogersckw9/20170327_3163722_RE_5m_8bit_RGB_RD', 'task_type': 'INGEST', 'output_url': ['https://code.earthengine.google.com/?asset=users/rogersckw9/20170327_3163722_RE_5m_8bit_RGB_RD'], 'start_timestamp_ms': 1522415526221}]\n",
"earthengine asset set --time_start 2017-03-21T10:14:42 users/rogersckw9/rapideye/20170321_3162818_RE_5m_8bit_RGB_RD\n"
]
}
],
"source": [
"for i, f in enumerate(files[1:3]):\n",
" print('Processing file ' + f + ', file index: ' + str(i))\n",
"\n",
" # extract time in UTC\n",
" fh = open(f, 'rb')\n",
" tags = exifread.process_file(fh, stop_tag=\"Image DateTime\")\n",
" datestring = str(tags[\"Image DateTime\"].values)\n",
" t = datetime.datetime.strptime(datestring, '%Y:%m:%d %H:%M:%S')\n",
" local_t = local.localize(t, is_dst=None)\n",
" utc_t = local_t.astimezone(pytz.utc)\n",
" time_start = utc_t.strftime('%Y-%m-%dT%H:%M:%S')\n",
"\n",
" # parse file names\n",
" filename = os.path.basename(f)\n",
" filename_no_ext = os.path.splitext(filename)[0]\n",
"\n",
" # get nodata value ... UGLY, UGLY code!\n",
" nodata_value = -99999999\n",
" with open(f, encoding=\"latin_1\") as asc:\n",
" for line in asc:\n",
" if \"nodata_value\" in line.lower():\n",
" nodata_value = line.split()[1]\n",
" break\n",
" if rapideye:\n",
" # copy to GCS \n",
" run(r\"C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp {0} gs://eo-bathymetry/rapideye/{1}\".format(os.path.join(input_dir,f),filename))\n",
" elif triplesat:\n",
" run(r\"C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp {0} gs://eo-bathymetry/triplesat/{1}\".format(os.path.join(input_dir,f),filename))\n",
" else:\n",
" print('Can only upload one satellite at a time. Please set rapideye or triplesat only.')\n",
" break\n",
" \n",
" # upload to GEE\n",
" retry_count = 0\n",
"\n",
"# while True:\n",
"# if rapideye:\n",
"# run(\"earthengine upload image --wait --asset_id=users/rogersckw9/rapideye/{1} --nodata_value={0} gs://eo-bathymetry/rapideye/{1}\".format(nodata_value, filename))\n",
"# else:\n",
"# run(\"earthengine upload image --wait --asset_id=users/rogersckw9/triplesat/{1} --nodata_value={0} gs://eo-bathymetry/triplesat/{1}\".format(nodata_value, filename))\n",
" \n",
" # check last task status\n",
" tasks = ee.data.getTaskList()\n",
" task_state = None\n",
" for task in tasks:\n",
" task_status = ee.data.getTaskStatus([task['id']])\n",
" task_state = task_status[0]['state']\n",
" print(task_status)\n",
" break\n",
"\n",
" if task_state != 'FAILED':\n",
" break # done\n",
" else:\n",
" retry_count += 1\n",
" print('Retrying upload ' + str(retry_count) + ' ...')\n",
"\n",
" if retry_count > 10:\n",
" print('Maximum number of retry reached, exiting ...')\n",
" sys.exit(0)\n",
"\n",
" # set time\n",
" if rapideye:\n",
" run(\"earthengine asset set --time_start {0} users/rogersckw9/rapideye/{1}\".format(time_start, filename_no_ext))\n",
" else:\n",
" run(\"earthengine asset set --time_start {0} users/rogersckw9/triplesat/{1}\".format(time_start, filename_no_ext))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:Anaconda]",
"language": "python",
"name": "conda-env-Anaconda-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.0"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

0 comments on commit c99e4e2

Please sign in to comment.