Skip to content

Commit

Permalink
Merge pull request #418 from OpenDroneMap/volume
Browse files Browse the repository at this point in the history
Volume
  • Loading branch information
pierotofy committed Mar 30, 2018
2 parents 23a01bd + db4710c commit 7b6efd6
Show file tree
Hide file tree
Showing 58 changed files with 662 additions and 94 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ RUN printf "deb http://mirror.steadfast.net/debian/ stable main contrib n
RUN printf "deb http://mirror.steadfast.net/debian/ testing main contrib non-free\ndeb-src http://mirror.steadfast.net/debian/ testing main contrib non-free" > /etc/apt/sources.list.d/testing.list

# Install Node.js GDAL, nginx, letsencrypt, psql
RUN apt-get -qq update && apt-get -qq install -t testing -y binutils libproj-dev gdal-bin nginx && apt-get -qq install -y gettext-base cron certbot postgresql-client-9.6
RUN apt-get -qq update && apt-get -qq install -t testing -y binutils libproj-dev gdal-bin nginx grass-core && apt-get -qq install -y gettext-base cron certbot postgresql-client-9.6


# Install pip reqs
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ Developer, I'm looking to build an app that will stay behind a firewall and just
- [X] 2D Map Display
- [X] 3D Model Display
- [ ] NDVI display
- [ ] Volumetric Measurements
- [X] Volumetric Measurements
- [X] Cluster management and setup.
- [ ] Mission Planner
- [X] Plugins/Webhooks System
Expand Down
15 changes: 7 additions & 8 deletions app/api/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ class TaskIDsSerializer(serializers.BaseSerializer):
def to_representation(self, obj):
return obj.id


class TaskSerializer(serializers.ModelSerializer):
project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all())
processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all())
Expand Down Expand Up @@ -193,15 +192,15 @@ class TaskNestedView(APIView):
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', 'console_output', )
permission_classes = (IsAuthenticatedOrReadOnly, )

def get_and_check_task(self, request, pk, project_pk, annotate={}):
def get_and_check_task(self, request, pk, annotate={}):
try:
task = self.queryset.annotate(**annotate).get(pk=pk, project=project_pk)
task = self.queryset.annotate(**annotate).get(pk=pk)
except (ObjectDoesNotExist, ValidationError):
raise exceptions.NotFound()

# Check for permissions, unless the task is public
if not task.public:
get_and_check_project(request, project_pk)
get_and_check_project(request, task.project.id)

return task

Expand All @@ -211,7 +210,7 @@ def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="")
"""
Get a tile image
"""
task = self.get_and_check_task(request, pk, project_pk)
task = self.get_and_check_task(request, pk)
tile_path = task.get_tile_path(tile_type, z, x, y)
if os.path.isfile(tile_path):
tile = open(tile_path, "rb")
Expand All @@ -225,7 +224,7 @@ def get(self, request, pk=None, project_pk=None, tile_type=""):
"""
Get tile.json for this tasks's asset type
"""
task = self.get_and_check_task(request, pk, project_pk)
task = self.get_and_check_task(request, pk)

extent_map = {
'orthophoto': task.orthophoto_extent,
Expand Down Expand Up @@ -256,7 +255,7 @@ def get(self, request, pk=None, project_pk=None, asset=""):
"""
Downloads a task asset (if available)
"""
task = self.get_and_check_task(request, pk, project_pk)
task = self.get_and_check_task(request, pk)

# Check and download
try:
Expand Down Expand Up @@ -284,7 +283,7 @@ def get(self, request, pk=None, project_pk=None, unsafe_asset_path=""):
"""
Downloads a task asset (if available)
"""
task = self.get_and_check_task(request, pk, project_pk)
task = self.get_and_check_task(request, pk)

# Check for directory traversal attacks
try:
Expand Down
5 changes: 4 additions & 1 deletion app/api/urls.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from django.conf.urls import url, include

from app.api.presets import PresetViewSet
from app.plugins import get_api_url_patterns
from .projects import ProjectViewSet
from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets
from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView
Expand Down Expand Up @@ -31,4 +32,6 @@

url(r'^auth/', include('rest_framework.urls')),
url(r'^token-auth/', obtain_jwt_token),
]
]

urlpatterns += get_api_url_patterns()
46 changes: 32 additions & 14 deletions app/boot.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import kombu
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User, Group
from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.core.files import File
from django.db.utils import ProgrammingError
from guardian.shortcuts import assign_perm
Expand Down Expand Up @@ -35,6 +35,10 @@ def boot():
if settings.DEBUG:
logger.warning("Debug mode is ON (for development this is OK)")

# Make sure our app/media/tmp folder exists
if not os.path.exists(settings.MEDIA_TMP):
os.mkdir(settings.MEDIA_TMP)

# Check default group
try:
default_group, created = Group.objects.get_or_create(name='Default')
Expand All @@ -60,18 +64,7 @@ def boot():
# Add permission to view processing nodes
default_group.permissions.add(Permission.objects.get(codename="view_processingnode"))

# Add default presets
Preset.objects.get_or_create(name='DSM + DTM', system=True,
options=[{'name': 'dsm', 'value': True}, {'name': 'dtm', 'value': True}, {'name': 'mesh-octree-depth', 'value': 11}])
Preset.objects.get_or_create(name='Fast Orthophoto', system=True,
options=[{'name': 'fast-orthophoto', 'value': True}])
Preset.objects.get_or_create(name='High Quality', system=True,
options=[{'name': 'dsm', 'value': True},
{'name': 'mesh-octree-depth', 'value': "12"},
{'name': 'dem-resolution', 'value': "0.04"},
{'name': 'orthophoto-resolution', 'value': "40"},
])
Preset.objects.get_or_create(name='Default', system=True, options=[{'name': 'dsm', 'value': True}, {'name': 'mesh-octree-depth', 'value': 11}])
add_default_presets()

# Add settings
default_theme, created = Theme.objects.get_or_create(name='Default')
Expand Down Expand Up @@ -101,4 +94,29 @@ def boot():


except ProgrammingError:
logger.warning("Could not touch the database. If running a migration, this is expected.")
logger.warning("Could not touch the database. If running a migration, this is expected.")


def add_default_presets():
try:
Preset.objects.update_or_create(name='DSM + DTM', system=True,
defaults={
'options': [{'name': 'dsm', 'value': True}, {'name': 'dtm', 'value': True},
{'name': 'mesh-octree-depth', 'value': 6}]})
Preset.objects.update_or_create(name='Fast Orthophoto', system=True,
defaults={'options': [{'name': 'fast-orthophoto', 'value': True}]})
Preset.objects.update_or_create(name='High Quality', system=True,
defaults={'options': [{'name': 'dsm', 'value': True},
{'name': 'mesh-octree-depth', 'value': "12"},
{'name': 'dem-resolution', 'value': "0.04"},
{'name': 'orthophoto-resolution', 'value': "40"},
]})
Preset.objects.update_or_create(name='Default', system=True,
defaults={'options': [{'name': 'dsm', 'value': True},
{'name': 'mesh-octree-depth', 'value': 6}]})
except MultipleObjectsReturned:
# Mostly to handle a legacy code problem where
# multiple system presets with the same name were
# created if we changed the options
Preset.objects.filter(system=True).delete()
add_default_presets()
2 changes: 1 addition & 1 deletion app/models/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -578,7 +578,7 @@ def set_failure(self, error_message):
self.status = status_codes.FAILED
self.pending_action = None
self.save()

def find_all_files_matching(self, regex):
directory = full_task_directory_path(self.id, self.project.id)
return [os.path.join(directory, f) for f in os.listdir(directory) if
Expand Down
42 changes: 38 additions & 4 deletions app/plugins/functions.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
import logging
import importlib
import subprocess

import django
import json
Expand All @@ -13,30 +14,57 @@

def register_plugins():
for plugin in get_active_plugins():

# Check for package.json in public directory
# and run npm install if needed
if plugin.path_exists("public/package.json") and not plugin.path_exists("public/node_modules"):
logger.info("Running npm install for {}".format(plugin.get_name()))
subprocess.call(['npm', 'install'], cwd=plugin.get_path("public"))

# Check for webpack.config.js (if we need to build it)
if plugin.path_exists("public/webpack.config.js") and not plugin.path_exists("public/build"):
logger.info("Running webpack for {}".format(plugin.get_name()))
subprocess.call(['webpack'], cwd=plugin.get_path("public"))

plugin.register()
logger.info("Registered {}".format(plugin))


def get_url_patterns():
def get_app_url_patterns():
"""
@return the patterns to expose the /public directory of each plugin (if needed)
@return the patterns to expose the /public directory of each plugin (if needed) and
each mount point
"""
url_patterns = []
for plugin in get_active_plugins():
for mount_point in plugin.mount_points():
for mount_point in plugin.app_mount_points():
url_patterns.append(url('^plugins/{}/{}'.format(plugin.get_name(), mount_point.url),
mount_point.view,
*mount_point.args,
**mount_point.kwargs))

if plugin.has_public_path():
if plugin.path_exists("public"):
url_patterns.append(url('^plugins/{}/(.*)'.format(plugin.get_name()),
django.views.static.serve,
{'document_root': plugin.get_path("public")}))

return url_patterns

def get_api_url_patterns():
"""
@return the patterns to expose the plugin API mount points (if any)
"""
url_patterns = []
for plugin in get_active_plugins():
for mount_point in plugin.api_mount_points():
url_patterns.append(url('^plugins/{}/{}'.format(plugin.get_name(), mount_point.url),
mount_point.view,
*mount_point.args,
**mount_point.kwargs))

return url_patterns


plugins = None
def get_active_plugins():
# Cache plugins search
Expand Down Expand Up @@ -86,6 +114,12 @@ def get_active_plugins():
return plugins


def get_plugin_by_name(name):
plugins = get_active_plugins()
res = list(filter(lambda p: p.get_name() == name, plugins))
return res[0] if res else None


def get_plugins_path():
current_path = os.path.dirname(os.path.realpath(__file__))
return os.path.abspath(os.path.join(current_path, "..", "..", "plugins"))
Expand Down
117 changes: 117 additions & 0 deletions app/plugins/grass_engine.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
import logging
import shutil
import tempfile
import subprocess
import os
import geojson

from string import Template

from webodm import settings

logger = logging.getLogger('app.logger')

class GrassEngine:
def __init__(self):
self.grass_binary = shutil.which('grass7') or \
shutil.which('grass72') or \
shutil.which('grass74') or \
shutil.which('grass76')

if self.grass_binary is None:
logger.warning("Could not find a GRASS 7 executable. GRASS scripts will not work.")
else:
logger.info("Initializing GRASS engine using {}".format(self.grass_binary))

def create_context(self, serialized_context = {}):
if self.grass_binary is None: raise GrassEngineException("GRASS engine is unavailable")
return GrassContext(self.grass_binary, **serialized_context)


class GrassContext:
def __init__(self, grass_binary, tmpdir = None, template_args = {}, location = None):
self.grass_binary = grass_binary
if tmpdir is None:
tmpdir = os.path.basename(tempfile.mkdtemp('_grass_engine', dir=settings.MEDIA_TMP))
self.tmpdir = tmpdir
self.template_args = template_args
self.location = location

def get_cwd(self):
return os.path.join(settings.MEDIA_TMP, self.tmpdir)

def add_file(self, filename, source, use_as_location=False):
param = os.path.splitext(filename)[0] # filename without extension

dst_path = os.path.abspath(os.path.join(self.get_cwd(), filename))
with open(dst_path, 'w') as f:
f.write(source)
self.template_args[param] = dst_path

if use_as_location:
self.set_location(self.template_args[param])

return dst_path

def add_param(self, param, value):
self.template_args[param] = value

def set_location(self, location):
"""
:param location: either a "epsg:XXXXX" string or a path to a geospatial file defining the location
"""
if not location.startswith('epsg:'):
location = os.path.abspath(location)
self.location = location

def execute(self, script):
"""
:param script: path to .grass script
:return: script output
"""
if self.location is None: raise GrassEngineException("Location is not set")

script = os.path.abspath(script)

# Create grass script via template substitution
try:
with open(script) as f:
script_content = f.read()
except FileNotFoundError:
raise GrassEngineException("Script does not exist: {}".format(script))

tmpl = Template(script_content)

# Write script to disk
with open(os.path.join(self.get_cwd(), 'script.sh'), 'w') as f:
f.write(tmpl.substitute(self.template_args))

# Execute it
p = subprocess.Popen([self.grass_binary, '-c', self.location, 'location', '--exec', 'sh', 'script.sh'],
cwd=self.get_cwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()

out = out.decode('utf-8').strip()
err = err.decode('utf-8').strip()

if p.returncode == 0:
return out
else:
raise GrassEngineException("Could not execute GRASS script {} from {}: {}".format(script, self.get_cwd(), err))

def serialize(self):
return {
'tmpdir': self.tmpdir,
'template_args': self.template_args,
'location': self.location
}

def __del__(self):
# Cleanup
if os.path.exists(self.get_cwd()):
shutil.rmtree(self.get_cwd())

class GrassEngineException(Exception):
pass

grass = GrassEngine()
2 changes: 1 addition & 1 deletion app/plugins/mount_point.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ def __init__(self, url, view, *args, **kwargs):
"""
:param url: path to mount this view to, relative to plugins directory
:param view: Django view
:param view: Django/DjangoRestFramework view
:param args: extra args to pass to url() call
:param kwargs: extra kwargs to pass to url() call
"""
Expand Down
Loading

0 comments on commit 7b6efd6

Please sign in to comment.