Merge pull request #893 from pierotofy/cogeogdal

Create COGEOs using GDAL

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6-stretch
+FROM debian:stretch
 MAINTAINER Piero Toffanin <[email protected]>
 
 ENV PYTHONUNBUFFERED 1
@@ -9,17 +9,20 @@ ENV PROJ_LIB=/usr/share/proj
 RUN mkdir /webodm
 WORKDIR /webodm
 
-RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash -
-RUN apt-get -qq install -y nodejs
+# Install Node.js
+RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget
+RUN wget --no-check-certificate https://deb.nodesource.com/setup_12.x -O /tmp/node.sh && bash /tmp/node.sh
+RUN apt-get -qq update && apt-get -qq install -y nodejs
 
 # Configure use of testing branch of Debian
 RUN printf "Package: *\nPin: release a=stable\nPin-Priority: 900\n" > /etc/apt/preferences.d/stable.pref
 RUN printf "Package: *\nPin: release a=testing\nPin-Priority: 750\n" > /etc/apt/preferences.d/testing.pref
 RUN printf "deb     http://ftp.us.debian.org/debian/    stable main contrib non-free\ndeb-src http://ftp.us.debian.org/debian/    stable main contrib non-free" > /etc/apt/sources.list.d/stable.list
 RUN printf "deb     http://ftp.us.debian.org/debian/    testing main contrib non-free\ndeb-src http://ftp.us.debian.org/debian/    testing main contrib non-free" > /etc/apt/sources.list.d/testing.list
 
-# Install Node.js GDAL, nginx, letsencrypt, psql
-RUN apt-get -qq update && apt-get -qq install -t testing -y binutils libproj-dev gdal-bin nginx certbot grass-core && apt-get -qq install -y gettext-base cron postgresql-client-9.6
+# Install Python3, GDAL, nginx, letsencrypt, psql
+RUN apt-get -qq update && apt-get -qq install -t testing -y --no-install-recommends python3 python3-pip git g++ python3-dev libpq-dev binutils libproj-dev gdal-bin python3-gdal nginx certbot grass-core && apt-get -qq install -y --no-install-recommends gettext-base cron postgresql-client-9.6
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2
 
 # Install pip reqs
 ADD requirements.txt /webodm/
@@ -40,6 +43,10 @@ RUN npm install --quiet -g webpack && npm install --quiet -g webpack-cli && npm
 RUN python manage.py collectstatic --noinput
 RUN bash app/scripts/plugin_cleanup.sh && echo "from app.plugins import build_plugins;build_plugins()" | python manage.py shell
 
+# Cleanup
+RUN apt-get remove -y g++ python3-dev libpq-dev && apt-get autoremove -y
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 
+
 RUN rm /webodm/webodm/secret_key.py
 
 VOLUME /webodm/app/media

app/cogeo.py

@@ -3,6 +3,9 @@
 import tempfile
 import shutil
 import rasterio
+import re
+import subprocess
+from pipes import quote
 from rio_cogeo.cogeo import cog_validate, cog_translate
 from rio_tiler.utils import has_alpha_band
 from webodm import settings
@@ -15,7 +18,14 @@ def valid_cogeo(src_path):
     :param src_path: path to GeoTIFF
     :return: true if the GeoTIFF is a cogeo, false otherwise
     """
-    return cog_validate(src_path, strict=True)
+    try:
+        from app.vendor.validate_cloud_optimized_geotiff import validate
+        warnings, errors, details = validate(src_path, full_check=True)
+        return not errors and not warnings
+    except ModuleNotFoundError:
+        logger.warning("Using legacy cog_validate (osgeo.gdal package not found)")
+        # Legacy
+        return cog_validate(src_path, strict=True)
 
 
 def assure_cogeo(src_path):
@@ -36,6 +46,90 @@ def assure_cogeo(src_path):
 
     # Not a cogeo
     logger.info("Optimizing %s as Cloud Optimized GeoTIFF" % src_path)
+
+    # Check if we have GDAL >= 3.1
+    use_legacy = False
+    gdal_version = get_gdal_version()
+    if gdal_version:
+        major, minor, build = gdal_version
+
+        # GDAL 2 and lower
+        if major <= 2:
+            use_legacy = True
+
+        # GDAL 3.0 and lower
+        if major == 3 and minor < 1:
+            use_legacy = True
+    else:
+        # This shouldn't happen
+        use_legacy = True
+
+    if use_legacy:
+        logger.warning("Using legacy implementation (GDAL >= 3.1 not found)")
+        return make_cogeo_legacy(src_path)
+    else:
+        return make_cogeo_gdal(src_path)
+
+def get_gdal_version():
+    # Bit of a hack without installing 
+    # python bindings
+    gdal_translate = shutil.which('gdal_translate')
+    if not gdal_translate:
+        return None
+
+    # Get version
+    version_output = subprocess.check_output([gdal_translate, "--version"]).decode('utf-8')
+
+    m = re.match(r"GDAL\s+([\d+])\.([\d+])\.([\d+]),\s+released", version_output)
+    if not m:
+        return None
+
+    return tuple(map(int, m.groups()))
+
+
+def make_cogeo_gdal(src_path):
+    """
+    Make src_path a Cloud Optimized GeoTIFF.
+    Requires GDAL >= 3.1
+    """
+
+    tmpfile = tempfile.mktemp('_cogeo.tif', dir=settings.MEDIA_TMP)
+    swapfile = tempfile.mktemp('_cogeo_swap.tif', dir=settings.MEDIA_TMP)
+
+    try:
+        subprocess.run(["gdal_translate", "-of", "COG",
+                        "-co", "BLOCKSIZE=256",
+                        "-co", "COMPRESS=deflate",
+                        "-co", "NUM_THREADS=ALL_CPUS",
+                        "-co", "BIGTIFF=IF_SAFER",
+                        "--config", "GDAL_NUM_THREADS", "ALL_CPUS",
+                        quote(src_path), quote(tmpfile)])
+    except Exception as e:
+        logger.warning("Cannot create Cloud Optimized GeoTIFF: %s" % str(e))
+
+    if os.path.isfile(tmpfile):
+        shutil.move(src_path, swapfile) # Move to swap location
+
+        try:
+            shutil.move(tmpfile, src_path)
+        except IOError as e:
+            logger.warning("Cannot move %s to %s: %s" % (tmpfile, src_path, str(e)))
+            shutil.move(swapfile, src_path) # Attempt to restore
+            raise e
+
+        if os.path.isfile(swapfile):
+            os.remove(swapfile)
+
+        return True
+    else:
+        return False
+
+def make_cogeo_legacy(src_path):
+    """
+    Make src_path a Cloud Optimized GeoTIFF
+    This implementation does not require GDAL >= 3.1
+    but sometimes (rarely) hangs for unknown reasons
+    """
     tmpfile = tempfile.mktemp('_cogeo.tif', dir=settings.MEDIA_TMP)
     swapfile = tempfile.mktemp('_cogeo_swap.tif', dir=settings.MEDIA_TMP)
 
@@ -77,4 +171,8 @@ def assure_cogeo(src_path):
             raise e
 
         if os.path.isfile(swapfile):
-            os.remove(swapfile)
+            os.remove(swapfile)
+
+        return True
+    else:
+        return False

app/static/app/js/components/TaskListItem.jsx

@@ -285,9 +285,21 @@ class TaskListItem extends React.Component {
       rfMap[rfParam].onClick = this.genRestartAction(rfParam);
     }
 
-    return task.can_rerun_from
+    let items = task.can_rerun_from
             .map(rf => rfMap[rf])
             .filter(rf => rf !== undefined);
+
+    if (items.length > 0 && [statusCodes.CANCELED, statusCodes.FAILED].indexOf(task.status) !== -1){
+        // Add resume "pseudo button" to help users understand
+        // how to resume a task that failed for memory/disk issues.
+        items.unshift({
+            label: "Resume Processing",
+            icon: "fa fa-bolt",
+            onClick: this.genRestartAction(task.can_rerun_from[task.can_rerun_from.length - 1])
+        });
+    }
+
+    return items;
   }
 
   genRestartAction(rerunFrom = null){