From c99e4e2f149173ba32b727596fc211b2713fd6ae Mon Sep 17 00:00:00 2001 From: ckrogers <35336768+ckrogers@users.noreply.github.com> Date: Tue, 3 Apr 2018 16:53:19 +0200 Subject: [PATCH] Issue #12 Prepare and upload RapidEye+TripleSat images --- notebooks/NSO2EE.ipynb | 284 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 notebooks/NSO2EE.ipynb diff --git a/notebooks/NSO2EE.ipynb b/notebooks/NSO2EE.ipynb new file mode 100644 index 0000000..21910ba --- /dev/null +++ b/notebooks/NSO2EE.ipynb @@ -0,0 +1,284 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import re\n", + "import glob\n", + "import subprocess\n", + "import datetime\n", + "import time\n", + "import pytz\n", + "import exifread\n", + "import zipfile\n", + "# from osgeo import gdal\n", + "\n", + "import ee\n", + "import ee.cli\n", + "import ee.cli.commands\n", + "import ee.cli.utils" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def run(cmd):\n", + " print(cmd)\n", + " subprocess.call(cmd)\n", + " \n", + "def extractAll(zipName):\n", + " z = zipfile.PyZipFile(zipName)\n", + " for f in z.namelist():\n", + " if f.endswith('/'):\n", + " os.makedirs(f)\n", + " else:\n", + " z.extract(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "ee.Initialize()\n", + "ee_config = ee.cli.utils.CommandLineConfig()" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "input_dir = r'P:/11202200.005-kpp-cip2018/data/Satellitedataportaal'\n", + "os.chdir(input_dir)\n", + "\n", + "rapideye = True\n", + "triplesat = False\n", + "\n", + "local = pytz.timezone(\"Europe/Amsterdam\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Number of files to upload as assets = 200']\n" + ] + } + ], + "source": [ + "# unzip all .zip files\n", + "\n", + "# find all .tif files\n", + "if rapideye:\n", + " files = glob.glob('RapidEye/**/**/*.tif', recursive=True)\n", + " zippedfiles = glob.glob('RapidEye/**/**/*.zip', recursive=True)\n", + "elif triplesat:\n", + " files = glob.glob('TripleSat/**/**/*.tif', recursive=True)\n", + " zippedfiles = glob.glob('TripleSat/**/**/*.zip', recursive=True)\n", + "else:\n", + " files = None\n", + " zippedfiles = None\n", + " print('Can only upload one satellite at a time. Please set rapideye or triplesat only.')\n", + "\n", + "# files = glob.glob('TripleSat/**/*.tif')\n", + "\n", + "print(['Number of files to upload as assets = '+str(len(files))])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n", + "['tif file DOES NOT exist for 20170331_3163722_RE_5m_8bit_RGB_RD.zip']\n" + ] + } + ], + "source": [ + "filename = os.path.basename(zippedfiles[1])\n", + "filename_no_ext = os.path.splitext(zippedfiles[1])[0]\n", + "dir_path = os.path.dirname(os.path.abspath(zippedfiles[1]))\n", + "filename_unzipped = os.path.join(dir_path,filename_no_ext+'.tif')\n", + "test_unzipped = os.path.exists(filename_unzipped)\n", + "print(test_unzipped)\n", + "if test_unzipped: \n", + " print(['tif file exists for ' + filename])\n", + "else:\n", + " print(['tif file DOES NOT exist for ' + filename])" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "\n", + "# z = zipfile.ZipFile(zippedfiles)\n", + "\n", + "for j in zippedfiles:\n", + " filename = os.path.basename(j)\n", + " filename_no_ext = os.path.splitext(j)[0]\n", + " dir_path = os.path.dirname(os.path.abspath(files[1]))\n", + " unzipped = \n", + " if (dir_path,filename,'.tif'):\n", + " else:\n", + " extractAll(j)\n", + "\n", + "\n", + "# if __name__ == '__main__':\n", + "# zipList = ['one.zip', 'two.zip', 'three.zip']\n", + "# for zip in zipList:\n", + "# extractAll(zipName)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing file RapidEye\\2017_03\\20170325\\20170325_3163722_RE_5m_8bit_RGB_RD.tif, file index: 0\n", + "C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp P:/11202200.005-kpp-cip2018/data/Satellitedataportaal\\RapidEye\\2017_03\\20170325\\20170325_3163722_RE_5m_8bit_RGB_RD.tif gs://eo-bathymetry/rapideye/20170325_3163722_RE_5m_8bit_RGB_RD.tif\n", + "earthengine upload image --wait --asset_id=users/rogersckw9/rapideye/20170325_3163722_RE_5m_8bit_RGB_RD.tif --nodata_value=-99999999 gs://eo-bathymetry/rapideye/20170325_3163722_RE_5m_8bit_RGB_RD.tif\n", + "[{'id': 'IHJ2XPQJZSPO2GZHRAFA27PQ', 'state': 'COMPLETED', 'creation_timestamp_ms': 1522415523002, 'update_timestamp_ms': 1522415627887, 'description': 'Asset ingestion: users/rogersckw9/20170327_3163722_RE_5m_8bit_RGB_RD', 'task_type': 'INGEST', 'output_url': ['https://code.earthengine.google.com/?asset=users/rogersckw9/20170327_3163722_RE_5m_8bit_RGB_RD'], 'start_timestamp_ms': 1522415526221}]\n", + "earthengine asset set --time_start 2017-03-25T10:14:57 users/rogersckw9/rapideye/20170325_3163722_RE_5m_8bit_RGB_RD\n", + "Processing file RapidEye\\2017_03\\20170321\\20170321_3162818_RE_5m_8bit_RGB_RD.tif, file index: 1\n", + "C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp P:/11202200.005-kpp-cip2018/data/Satellitedataportaal\\RapidEye\\2017_03\\20170321\\20170321_3162818_RE_5m_8bit_RGB_RD.tif gs://eo-bathymetry/rapideye/20170321_3162818_RE_5m_8bit_RGB_RD.tif\n", + "earthengine upload image --wait --asset_id=users/rogersckw9/rapideye/20170321_3162818_RE_5m_8bit_RGB_RD.tif --nodata_value=-99999999 gs://eo-bathymetry/rapideye/20170321_3162818_RE_5m_8bit_RGB_RD.tif\n", + "[{'id': 'IHJ2XPQJZSPO2GZHRAFA27PQ', 'state': 'COMPLETED', 'creation_timestamp_ms': 1522415523002, 'update_timestamp_ms': 1522415627887, 'description': 'Asset ingestion: users/rogersckw9/20170327_3163722_RE_5m_8bit_RGB_RD', 'task_type': 'INGEST', 'output_url': ['https://code.earthengine.google.com/?asset=users/rogersckw9/20170327_3163722_RE_5m_8bit_RGB_RD'], 'start_timestamp_ms': 1522415526221}]\n", + "earthengine asset set --time_start 2017-03-21T10:14:42 users/rogersckw9/rapideye/20170321_3162818_RE_5m_8bit_RGB_RD\n" + ] + } + ], + "source": [ + "for i, f in enumerate(files[1:3]):\n", + " print('Processing file ' + f + ', file index: ' + str(i))\n", + "\n", + " # extract time in UTC\n", + " fh = open(f, 'rb')\n", + " tags = exifread.process_file(fh, stop_tag=\"Image DateTime\")\n", + " datestring = str(tags[\"Image DateTime\"].values)\n", + " t = datetime.datetime.strptime(datestring, '%Y:%m:%d %H:%M:%S')\n", + " local_t = local.localize(t, is_dst=None)\n", + " utc_t = local_t.astimezone(pytz.utc)\n", + " time_start = utc_t.strftime('%Y-%m-%dT%H:%M:%S')\n", + "\n", + " # parse file names\n", + " filename = os.path.basename(f)\n", + " filename_no_ext = os.path.splitext(filename)[0]\n", + "\n", + " # get nodata value ... UGLY, UGLY code!\n", + " nodata_value = -99999999\n", + " with open(f, encoding=\"latin_1\") as asc:\n", + " for line in asc:\n", + " if \"nodata_value\" in line.lower():\n", + " nodata_value = line.split()[1]\n", + " break\n", + " if rapideye:\n", + " # copy to GCS \n", + " run(r\"C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp {0} gs://eo-bathymetry/rapideye/{1}\".format(os.path.join(input_dir,f),filename))\n", + " elif triplesat:\n", + " run(r\"C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp {0} gs://eo-bathymetry/triplesat/{1}\".format(os.path.join(input_dir,f),filename))\n", + " else:\n", + " print('Can only upload one satellite at a time. Please set rapideye or triplesat only.')\n", + " break\n", + " \n", + " # upload to GEE\n", + " retry_count = 0\n", + "\n", + "# while True:\n", + "# if rapideye:\n", + "# run(\"earthengine upload image --wait --asset_id=users/rogersckw9/rapideye/{1} --nodata_value={0} gs://eo-bathymetry/rapideye/{1}\".format(nodata_value, filename))\n", + "# else:\n", + "# run(\"earthengine upload image --wait --asset_id=users/rogersckw9/triplesat/{1} --nodata_value={0} gs://eo-bathymetry/triplesat/{1}\".format(nodata_value, filename))\n", + " \n", + " # check last task status\n", + " tasks = ee.data.getTaskList()\n", + " task_state = None\n", + " for task in tasks:\n", + " task_status = ee.data.getTaskStatus([task['id']])\n", + " task_state = task_status[0]['state']\n", + " print(task_status)\n", + " break\n", + "\n", + " if task_state != 'FAILED':\n", + " break # done\n", + " else:\n", + " retry_count += 1\n", + " print('Retrying upload ' + str(retry_count) + ' ...')\n", + "\n", + " if retry_count > 10:\n", + " print('Maximum number of retry reached, exiting ...')\n", + " sys.exit(0)\n", + "\n", + " # set time\n", + " if rapideye:\n", + " run(\"earthengine asset set --time_start {0} users/rogersckw9/rapideye/{1}\".format(time_start, filename_no_ext))\n", + " else:\n", + " run(\"earthengine asset set --time_start {0} users/rogersckw9/triplesat/{1}\".format(time_start, filename_no_ext))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:Anaconda]", + "language": "python", + "name": "conda-env-Anaconda-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}