From d14670954012f6e42ae60b0374e056f3549b3c4e Mon Sep 17 00:00:00 2001 From: ckrogers Date: Fri, 20 Apr 2018 09:21:11 +0200 Subject: [PATCH] Issue #12 --- notebooks/NSO2EE-MultiSpectral.ipynb | 232 +++------------------------ 1 file changed, 19 insertions(+), 213 deletions(-) diff --git a/notebooks/NSO2EE-MultiSpectral.ipynb b/notebooks/NSO2EE-MultiSpectral.ipynb index 8918f76..528f836 100644 --- a/notebooks/NSO2EE-MultiSpectral.ipynb +++ b/notebooks/NSO2EE-MultiSpectral.ipynb @@ -19,6 +19,7 @@ "import exifread\n", "import zipfile\n", "import gdal\n", + "import math\n", "import pandas as pd\n", "\n", "import ee\n", @@ -62,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 4, "metadata": { "collapsed": false }, @@ -71,8 +72,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "['Number of files to upload as assets = 3']\n", - "['Number of zipped files = 7']\n" + "['Number of files to upload as assets = 12']\n", + "['Number of zipped files = 0']\n" ] } ], @@ -106,24 +107,7 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "print(zippedfiles[0])\n", - "filename = os.path.basename(zippedfiles[0])\n", - "print(filename)\n", - "filename_no_ext = os.path.splitext(filename)[0]\n", - "print(filename_no_ext)\n", - "filename_unzipped = os.path.join(input_dir,dst,filename_no_ext+'.jp2000')\n", - "print(filename_unzipped)" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "collapsed": false }, @@ -132,8 +116,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "['Unzipped .tif file from 20180302_093358_Tri_80cm_RD_12bit_RGBI_Schiermonnikoog.zip']\n", - "['Unzipped .tif file from 20180306_094211_Tri_80cm_RD_12bit_RGBI_Westkapelle.zip']\n" + "['Unzipped .tif file from 20180224_093625_Tri_80cm_RD_12bit_RGBI_Zierikzee.zip']\n" ] } ], @@ -157,7 +140,7 @@ "elif triplesat:\n", " for j in zippedfiles:\n", " filename = os.path.basename(j)\n", - " filename_no_ext = os.path.splitext(j)[0]\n", + " filename_no_ext = os.path.splitext(filename)[0]\n", " filename_unzipped = os.path.join(input_dir,filename_no_ext+'.tif')\n", " is_unzipped = os.path.lexists(filename_unzipped)\n", " if is_unzipped: \n", @@ -193,46 +176,16 @@ }, { "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "105\n" - ] - } - ], - "source": [ - "# files = glob.glob('*.jp2')\n", - "# print(len(files))\n", - "files = glob.glob('*.tif')\n", - "print(len(files))" - ] - }, - { - "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['RE3_20170420_3163017_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162917_RD_12bit_RGBREI_5m.jp2', 'RE5_20170707_3162917_RD_12bit_RGBREI_5m.jp2', 'RE1_20170829_3163721_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162817_RD_12bit_RGBREI_5m.jp2', 'RE5_20170707_3162817_RD_12bit_RGBREI_5m.jp2', 'RE2_20170331_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170602_3163722_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162818_RD_12bit_RGBREI_5m.jp2', 'RE3_20170601_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162817_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3163721_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162917_RD_12bit_RGBREI_5m.jp2', 'RE2_20170522_3163721_RD_12bit_RGBREI_5m.jp2', 'RE4_20170425_3163721_RD_12bit_RGBREI_5m.jp2', 'RE5_20170902_3163017_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162916_RD_12bit_RGBREI_5m.jp2', 'RE5_20170809_3162916_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3162817_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3162917_RD_12bit_RGBREI_5m.jp2', 'RE1_20170330_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162816_RD_12bit_RGBREI_5m.jp2', 'RE5_20170707_3162816_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162816_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3163721_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3163722_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162916_RD_12bit_RGBREI_5m.jp2', 'RE4_20170602_3163721_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162916_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3162817_RD_12bit_RGBREI_5m.jp2', 'RE3_20170327_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3163721_RD_12bit_RGBREI_5m.jp2', 'RE1_20170325_3263706_RD_12bit_RGBREI_5m.jp2', 'RE2_20170331_3163721_RD_12bit_RGBREI_5m.jp2', 'RE1_20170829_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162816_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3162917_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170425_3163722_RD_12bit_RGBREI_5m.jp2', 'RE2_20170522_3163722_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3163721_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3163017_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3162916_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162817_RD_12bit_RGBREI_5m.jp2', 'RE2_20170409_3263706_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162816_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3162917_RD_12bit_RGBREI_5m.jp2', 'RE3_20170904_3263706_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162916_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3162817_RD_12bit_RGBREI_5m.jp2', 'RE1_20170511_3163722_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3163721_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3162916_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162817_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3162816_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170426_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3162816_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3162818_RD_12bit_RGBREI_5m.jp2', 'RE2_20170409_3163722_RD_12bit_RGBREI_5m.jp2', 'RE2_20170321_3163017_RD_12bit_RGBREI_5m.jp2', 'RE3_20170904_3163722_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3162916_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3163017_RD_12bit_RGBREI_5m.jp2', 'RE1_20170511_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170323_3162818_RD_12bit_RGBREI_5m.jp2', 'RE2_20170331_3162818_RD_12bit_RGBREI_5m.jp2', 'RE3_20170807_3263706_RD_12bit_RGBREI_5m.jp2', 'RE2_20170709_3163017_RD_12bit_RGBREI_5m.jp2', 'RE1_20170325_3163722_RD_12bit_RGBREI_5m.jp2', 'RE3_20170327_3163722_RD_12bit_RGBREI_5m.jp2', 'RE1_20170829_3263706_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3162818_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170425_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170827_3162817_RD_12bit_RGBREI_5m.jp2', 'RE3_20170709_3263706_RD_12bit_RGBREI_5m.jp2', 'RE4_20170426_3162916_RD_12bit_RGBREI_5m.jp2', 'RE1_20170409_3163017_RD_12bit_RGBREI_5m.jp2', 'RE3_20170904_3163721_RD_12bit_RGBREI_5m.jp2', 'RE5_20170902_3162816_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3163017_RD_12bit_RGBREI_5m.jp2', 'RE2_20170409_3163721_RD_12bit_RGBREI_5m.jp2', 'RE5_20170902_3162916_RD_12bit_RGBREI_5m.jp2', 'RE5_20170525_3162818_RD_12bit_RGBREI_5m.jp2', 'RE3_20170831_3162817_RD_12bit_RGBREI_5m.jp2', 'RE1_20170330_3163722_RD_12bit_RGBREI_5m.jp2', 'RE5_20170315_3162916_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3163721_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162916_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162816_RD_12bit_RGBREI_5m.jp2', 'RE2_20170825_3163722_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162817_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3162818_RD_12bit_RGBREI_5m.jp2', 'RE5_20170707_3162818_RD_12bit_RGBREI_5m.jp2', 'RE3_20170327_3163721_RD_12bit_RGBREI_5m.jp2', 'RE1_20170404_3163017_RD_12bit_RGBREI_5m.jp2', 'RE2_20170331_3263706_RD_12bit_RGBREI_5m.jp2', 'RE1_20170619_3163722_RD_12bit_RGBREI_5m.jp2', 'RE3_20170420_3162917_RD_12bit_RGBREI_5m.jp2', 'RE5_20170809_3163017_RD_12bit_RGBREI_5m.jp2', 'RE5_20170902_3162917_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3163017_RD_12bit_RGBREI_5m.jp2', 'RE4_20170510_3162818_RD_12bit_RGBREI_5m.jp2']\n" - ] - } - ], + "outputs": [], "source": [ "if rapideye:\n", "# files = glob.glob('*.jp2000')\n", " files = glob.glob('*.jp2')\n", - "# print(files)\n", + " print(len(files))\n", " for i, f in enumerate(files):\n", " print(str(i)+' of '+str(len(files)))\n", " filename = os.path.splitext(os.path.basename(f))[0]\n", @@ -260,7 +213,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": { "collapsed": false }, @@ -269,80 +222,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "['Number of files to upload as assets = 105']\n" + "['Number of files to upload as assets = 12']\n" ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
system:time_start
filename
RE3_20170904_3163721_RD_12bit_RGBREI_5m1.504523e+12
RE5_20170902_3162816_RD_12bit_RGBREI_5m1.504351e+12
RE5_20170525_3163017_RD_12bit_RGBREI_5m1.495711e+12
RE5_20170902_3162916_RD_12bit_RGBREI_5m1.504351e+12
RE2_20170409_3163721_RD_12bit_RGBREI_5m1.491736e+12
\n", - "
" - ], - "text/plain": [ - " system:time_start\n", - "filename \n", - "RE3_20170904_3163721_RD_12bit_RGBREI_5m 1.504523e+12\n", - "RE5_20170902_3162816_RD_12bit_RGBREI_5m 1.504351e+12\n", - "RE5_20170525_3163017_RD_12bit_RGBREI_5m 1.495711e+12\n", - "RE5_20170902_3162916_RD_12bit_RGBREI_5m 1.504351e+12\n", - "RE2_20170409_3163721_RD_12bit_RGBREI_5m 1.491736e+12" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ - "files = glob.glob('*.tif')\n", - "print(['Number of files to upload as assets = '+str(len(files))])\n", "df = pd.DataFrame(columns=['filename','system:time_start'])\n", "\n", "if rapideye:\n", @@ -376,10 +260,12 @@ "\n", " for i, f in enumerate(files):\n", " filesize = os.path.getsize(f)\n", - " filename = os.path.splitext(os.path.basename(f))[0]\n", + " filename_no_ext = os.path.splitext(os.path.basename(f))[0]\n", + " dst = 'TripleSat/Tri_RD_12bit_RGBI_80cm_Split/'\n", + " \n", " if (filesize > 10e9):\n", - " in_path = os.path.join(input_dir,f)\n", - " out_path = os.path.join(input_dir,filename_no_ext)\n", + " in_path = os.path.join(input_dir,dst,f)\n", + " out_path = os.path.join(input_dir,dst,filename_no_ext)\n", " ds = gdal.Open(in_path)\n", " band = ds.GetRasterBand(1)\n", " xsize = band.XSize\n", @@ -414,19 +300,11 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "geebam upload --source P:/11202200.005-kpp-cip2018/data/Satellitedataportaal\\RapidEye/RE_RD_12bit_RGBEI_5m_jp2000/ --dest users/rogersckw9/eo-bathymetry/rapideye -m P:/11202200.005-kpp-cip2018/data/Satellitedataportaal\\RapidEye/RE_RD_12bit_RGBEI_5m_jp2000/metadata.csv -u rogersckw9@gmail.com\n" - ] - } - ], + "outputs": [], "source": [ "# Use Google Earth Engine Batch Asset Manager\n", "# Details at https://github.com/tracek/gee_asset_manager\n", @@ -443,78 +321,6 @@ " print(\"No satellite selected.\")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# if uploading to Google Cloud first, follow this protocol (this is for RapidEye)\n", - "\n", - "for i, f in enumerate(files):\n", - " print('Processing file ' + f + ', file index: ' + str(i))\n", - "\n", - " # extract time in UTC\n", - " fh = open(f, 'rb')\n", - " tags = exifread.process_file(fh, stop_tag=\"Image DateTime\")\n", - " datestring = str(tags[\"Image DateTime\"].values)\n", - " t = datetime.datetime.strptime(datestring, '%Y:%m:%d %H:%M:%S')\n", - " local_t = local.localize(t, is_dst=None)\n", - " utc_t = local_t.astimezone(pytz.utc)\n", - " time_start = utc_t.strftime('%Y-%m-%dT%H:%M:%S')\n", - "\n", - " # parse file names\n", - " filename = os.path.basename(f)\n", - " filename_no_ext = os.path.splitext(filename)[0]\n", - "\n", - " # get nodata value ... UGLY, UGLY code!\n", - " nodata_value = -99999999\n", - " with open(f, encoding=\"latin_1\") as asc:\n", - " for line in asc:\n", - " if \"nodata_value\" in line.lower():\n", - " nodata_value = line.split()[1]\n", - " break\n", - " \n", - " run(r\"C:\\Users\\Wilson\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gsutil.cmd cp {0} gs://eo-bathymetry/rapideye/{1}\".format(os.path.join(input_dir,f),filename))\n", - " \n", - " # upload to GEE\n", - " retry_count = 0\n", - "\n", - " while True:\n", - " run(\"earthengine upload image --wait --asset_id=users/rogersckw9/rapideye/{1} --nodata_value={0} gs://eo-bathymetry/rapideye/{1}\".format(nodata_value, filename))\n", - " \n", - " check last task status\n", - " tasks = ee.data.getTaskList()\n", - " task_state = None\n", - " for task in tasks:\n", - " task_status = ee.data.getTaskStatus([task['id']])\n", - " task_state = task_status[0]['state']\n", - " print(task_status)\n", - " break\n", - "\n", - " if task_state != 'FAILED':\n", - " break # done\n", - " else:\n", - " retry_count += 1\n", - " print('Retrying upload ' + str(retry_count) + ' ...')\n", - "\n", - " if retry_count > 10:\n", - " print('Maximum number of retry reached, exiting ...')\n", - " sys.exit(0)\n", - " \n", - " # set time\n", - " run(\"earthengine asset set --time_start {0} users/rogersckw9/rapideye/{1}\".format(time_start, filename_no_ext))\n", - "\n", - " if rapideye:\n", - " \n", - "elif triplesat:\n", - "\n", - "else:\n", - " print(\"No satellite selected.\")" - ] - }, { "cell_type": "code", "execution_count": null,