From c6059c0193643600e845dd186cd89865873d1a06 Mon Sep 17 00:00:00 2001 From: ckrogers <35336768+ckrogers@users.noreply.github.com> Date: Wed, 4 Apr 2018 16:15:00 +0200 Subject: [PATCH] Issue #12 GEE image collection upload using https://github.com/tracek/gee_asset_manager --- notebooks/NSO2EE.ipynb | 337 ++++++++++++++++++++++++++++------------- 1 file changed, 234 insertions(+), 103 deletions(-) diff --git a/notebooks/NSO2EE.ipynb b/notebooks/NSO2EE.ipynb index 21910ba..b77c9fe 100644 --- a/notebooks/NSO2EE.ipynb +++ b/notebooks/NSO2EE.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 35, + "execution_count": 1, "metadata": { "collapsed": false }, @@ -13,12 +13,12 @@ "import re\n", "import glob\n", "import subprocess\n", - "import datetime\n", "import time\n", + "import datetime\n", "import pytz\n", "import exifread\n", "import zipfile\n", - "# from osgeo import gdal\n", + "import pandas as pd\n", "\n", "import ee\n", "import ee.cli\n", @@ -28,7 +28,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 2, "metadata": { "collapsed": true }, @@ -49,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 3, "metadata": { "collapsed": true }, @@ -61,59 +61,77 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 4, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Number of files to upload as assets = 69']\n", + "['Number of zipped files = 0']\n" + ] + } + ], "source": [ "input_dir = r'P:/11202200.005-kpp-cip2018/data/Satellitedataportaal'\n", "os.chdir(input_dir)\n", "\n", - "rapideye = True\n", - "triplesat = False\n", + "# find all .tif and .zip files\n", + "files = glob.glob('RapidEye/**/**/*.tif', recursive=True)\n", + "zippedfiles = glob.glob('RapidEye/**/**/*.zip', recursive=True)\n", + "print(['Number of files to upload as assets = '+str(len(files))])\n", + "print(['Number of zipped files = '+str(len(zippedfiles))])\n", "\n", - "local = pytz.timezone(\"Europe/Amsterdam\")\n" + "local = pytz.timezone(\"Europe/Amsterdam\")" ] }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 6, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['Number of files to upload as assets = 200']\n" - ] - } - ], + "outputs": [], "source": [ - "# unzip all .zip files\n", + "# Files must all be in the same file for Google Earth Engine Batch Asset Manager,\n", + "# but ftp.satellietdataportaal.nl transfer for RapidEye was categorized in monthly/daily folders\n", "\n", - "# find all .tif files\n", - "if rapideye:\n", - " files = glob.glob('RapidEye/**/**/*.tif', recursive=True)\n", - " zippedfiles = glob.glob('RapidEye/**/**/*.zip', recursive=True)\n", - "elif triplesat:\n", - " files = glob.glob('TripleSat/**/**/*.tif', recursive=True)\n", - " zippedfiles = glob.glob('TripleSat/**/**/*.zip', recursive=True)\n", - "else:\n", - " files = None\n", - " zippedfiles = None\n", - " print('Can only upload one satellite at a time. Please set rapideye or triplesat only.')\n", + "# Unzip all\n", + "for j in zippedfiles:\n", + " filename = os.path.basename(j)\n", + " filename_no_ext = os.path.splitext(j)[0]\n", + " filename_unzipped = os.path.join(input_dir,filename_no_ext+'.tif')\n", + " is_unzipped = os.path.lexists(filename_unzipped)\n", + " if is_unzipped: \n", + " os.remove(os.path.join(input_dir,j))\n", + " else:\n", + " extractAll(j)\n", + " print(['Unzipped .tif file from ' + filename])\n", + " os.remove(os.path.join(input_dir,j))\n", + "\n", + "dst = 'RapidEye/'\n", + "# Move all files to destination path\n", + "for k in files:\n", + " filename = os.path.basename(k)\n", + " path_move_file = os.path.join(input_dir,dst,filename)\n", + " if os.path.lexists(path_move_file):\n", + " pass\n", + " else:\n", + " os.rename(os.path.join(input_dir,k), path_move_file)\n", "\n", - "# files = glob.glob('TripleSat/**/*.tif')\n", + "# delete empty directories\n", + "empty_dir = glob.glob('RapidEye/**/**')\n", "\n", - "print(['Number of files to upload as assets = '+str(len(files))])\n" + "for e in empty_dir:\n", + " os.rmdir(os.path.join(input_dir,e))" ] }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 7, "metadata": { "collapsed": false }, @@ -122,54 +140,168 @@ "name": "stdout", "output_type": "stream", "text": [ - "False\n", - "['tif file DOES NOT exist for 20170331_3163722_RE_5m_8bit_RGB_RD.zip']\n" + "['Number of files to upload as assets = 69']\n", + "Processing file RapidEye\\20170511_3163722_RE_5m_8bit_RGB_RD.tif, file index: 0\n", + "Processing file RapidEye\\20170409_3162817_RE_5m_8bit_RGB_RD.tif, file index: 1\n", + "Processing file RapidEye\\20170409_3163721_RE_5m_8bit_RGB_RD.tif, file index: 2\n", + "Processing file RapidEye\\20170331_3263706_RE_5m_8bit_RGB_RD.tif, file index: 3\n", + "Processing file RapidEye\\20170425_3163722_RE_5m_8bit_RGB_RD.tif, file index: 4\n", + "Processing file RapidEye\\20170321_3163721_RE_5m_8bit_RGB_RD.tif, file index: 5\n", + "Processing file RapidEye\\20170321_3162817_RE_5m_8bit_RGB_RD.tif, file index: 6\n", + "Processing file RapidEye\\20170404_3162816_RE_5m_8bit_RGB_RD.tif, file index: 7\n", + "Processing file RapidEye\\20170829_3263706_RE_5m_8bit_RGB_RD.tif, file index: 8\n", + "Processing file RapidEye\\20170327_3163721_RE_5m_8bit_RGB_RD.tif, file index: 9\n", + "Processing file RapidEye\\20170420_3163721_RE_5m_8bit_RGB_RD.tif, file index: 10\n", + "Processing file RapidEye\\20170420_3162817_RE_5m_8bit_RGB_RD.tif, file index: 11\n", + "Processing file RapidEye\\20170525_3163722_RE_5m_8bit_RGB_RD.tif, file index: 12\n", + "Processing file RapidEye\\20170709_3163721_RE_5m_8bit_RGB_RD.tif, file index: 13\n", + "Processing file RapidEye\\20170709_3162817_RE_5m_8bit_RGB_RD.tif, file index: 14\n", + "Processing file RapidEye\\20170904_3163722_RE_5m_8bit_RGB_RD.tif, file index: 15\n", + "Processing file RapidEye\\20170601_3163722_RE_5m_8bit_RGB_RD.tif, file index: 16\n", + "Processing file RapidEye\\20170807_3263706_RE_5m_8bit_RGB_RD.tif, file index: 17\n", + "Processing file RapidEye\\20170331_3163721_RE_5m_8bit_RGB_RD.tif, file index: 18\n", + "Processing file RapidEye\\20170602_3163721_RE_5m_8bit_RGB_RD.tif, file index: 19\n", + "Processing file RapidEye\\20170315_3162816_RE_5m_8bit_RGB_RD.tif, file index: 20\n", + "Processing file RapidEye\\20170709_3263706_RE_5m_8bit_RGB_RD.tif, file index: 21\n", + "Processing file RapidEye\\20170327_3263706_RE_5m_8bit_RGB_RD.tif, file index: 22\n", + "Processing file RapidEye\\20170829_3163721_RE_5m_8bit_RGB_RD.tif, file index: 23\n", + "Processing file RapidEye\\20170409_3162816_RE_5m_8bit_RGB_RD.tif, file index: 24\n", + "Processing file RapidEye\\20170522_3163722_RE_5m_8bit_RGB_RD.tif, file index: 25\n", + "Processing file RapidEye\\20170330_3263706_RE_5m_8bit_RGB_RD.tif, file index: 26\n", + "Processing file RapidEye\\20170404_3162817_RE_5m_8bit_RGB_RD.tif, file index: 27\n", + "Processing file RapidEye\\20170404_3163721_RE_5m_8bit_RGB_RD.tif, file index: 28\n", + "Processing file RapidEye\\20170321_3162816_RE_5m_8bit_RGB_RD.tif, file index: 29\n", + "Processing file RapidEye\\20170420_3162816_RE_5m_8bit_RGB_RD.tif, file index: 30\n", + "Processing file RapidEye\\20170409_3263706_RE_5m_8bit_RGB_RD.tif, file index: 31\n", + "Processing file RapidEye\\20170323_3162818_RE_5m_8bit_RGB_RD.tif, file index: 32\n", + "Processing file RapidEye\\20170325_3163722_RE_5m_8bit_RGB_RD.tif, file index: 33\n", + "Processing file RapidEye\\20170709_3162816_RE_5m_8bit_RGB_RD.tif, file index: 34\n", + "Processing file RapidEye\\20170602_3163722_RE_5m_8bit_RGB_RD.tif, file index: 35\n", + "Processing file RapidEye\\20170425_3263706_RE_5m_8bit_RGB_RD.tif, file index: 36\n", + "Processing file RapidEye\\20170331_3163722_RE_5m_8bit_RGB_RD.tif, file index: 37\n", + "Processing file RapidEye\\20170510_3162817_RE_5m_8bit_RGB_RD.tif, file index: 38\n", + "Processing file RapidEye\\20170619_3163722_RE_5m_8bit_RGB_RD.tif, file index: 39\n", + "Processing file RapidEye\\20170707_3162817_RE_5m_8bit_RGB_RD.tif, file index: 40\n", + "Processing file RapidEye\\20170829_3163722_RE_5m_8bit_RGB_RD.tif, file index: 41\n", + "Processing file RapidEye\\20170831_3162817_RE_5m_8bit_RGB_RD.tif, file index: 42\n", + "Processing file RapidEye\\20170904_3263706_RE_5m_8bit_RGB_RD.tif, file index: 43\n", + "Processing file RapidEye\\20170331_3162818_RE_5m_8bit_RGB_RD.tif, file index: 44\n", + "Processing file RapidEye\\20170825_3163722_RE_5m_8bit_RGB_RD.tif, file index: 45\n", + "Processing file RapidEye\\20170511_3263706_RE_5m_8bit_RGB_RD.tif, file index: 46\n", + "Processing file RapidEye\\20170323_3163721_RE_5m_8bit_RGB_RD.tif, file index: 47\n", + "Processing file RapidEye\\20170522_3163721_RE_5m_8bit_RGB_RD.tif, file index: 48\n", + "Processing file RapidEye\\20170323_3162817_RE_5m_8bit_RGB_RD.tif, file index: 49\n", + "Processing file RapidEye\\20170902_3162816_RE_5m_8bit_RGB_RD.tif, file index: 50\n", + "Processing file RapidEye\\20170827_3162816_RE_5m_8bit_RGB_RD.tif, file index: 51\n", + "Processing file RapidEye\\20170404_3162818_RE_5m_8bit_RGB_RD.tif, file index: 52\n", + "Processing file RapidEye\\20170330_3163722_RE_5m_8bit_RGB_RD.tif, file index: 53\n", + "Processing file RapidEye\\20170707_3162816_RE_5m_8bit_RGB_RD.tif, file index: 54\n", + "Processing file RapidEye\\20170409_3162818_RE_5m_8bit_RGB_RD.tif, file index: 55\n", + "Processing file RapidEye\\20170510_3162816_RE_5m_8bit_RGB_RD.tif, file index: 56\n", + "Processing file RapidEye\\20170409_3163722_RE_5m_8bit_RGB_RD.tif, file index: 57\n", + "Processing file RapidEye\\20170325_3263706_RE_5m_8bit_RGB_RD.tif, file index: 58\n", + "Processing file RapidEye\\20170420_3162818_RE_5m_8bit_RGB_RD.tif, file index: 59\n", + "Processing file RapidEye\\20170425_3163721_RE_5m_8bit_RGB_RD.tif, file index: 60\n", + "Processing file RapidEye\\20170323_3162816_RE_5m_8bit_RGB_RD.tif, file index: 61\n", + "Processing file RapidEye\\20170827_3162817_RE_5m_8bit_RGB_RD.tif, file index: 62\n", + "Processing file RapidEye\\20170904_3163721_RE_5m_8bit_RGB_RD.tif, file index: 63\n", + "Processing file RapidEye\\20170709_3163722_RE_5m_8bit_RGB_RD.tif, file index: 64\n", + "Processing file RapidEye\\20170525_3162817_RE_5m_8bit_RGB_RD.tif, file index: 65\n", + "Processing file RapidEye\\20170525_3163721_RE_5m_8bit_RGB_RD.tif, file index: 66\n", + "Processing file RapidEye\\20170327_3163722_RE_5m_8bit_RGB_RD.tif, file index: 67\n", + "Processing file RapidEye\\20170321_3162818_RE_5m_8bit_RGB_RD.tif, file index: 68\n" ] + }, + { + "data": { + "text/html": [ + "
\n", + " | system:time_start | \n", + "
---|---|
filename | \n", + "\n", + " |
20170511_3163722_RE_5m_8bit_RGB_RD | \n", + "1.494493e+09 | \n", + "
20170409_3162817_RE_5m_8bit_RGB_RD | \n", + "1.491730e+09 | \n", + "
20170409_3163721_RE_5m_8bit_RGB_RD | \n", + "1.491729e+09 | \n", + "
20170331_3263706_RE_5m_8bit_RGB_RD | \n", + "1.490952e+09 | \n", + "
20170425_3163722_RE_5m_8bit_RGB_RD | \n", + "1.493111e+09 | \n", + "