Merge remote-tracking branch 'origin/develop'

# Resolved Conflicts:
#	.gitignore
#	Views.py

Views.py untested — I took the origin/develop changes over the master
changes.

.gitignore

@@ -1,20 +1,32 @@
 *.pyc
 *.[oa]
 *~
+*.zip
+*.png
+.Python
+bin/
 media/t*png
 media/profpic*png
 todo.txt
 settings.py
 temp.txt
 probmap
 
-*.zip
+arc-models/*.pdf
+arc-models/*_flymake.py
+arc-models/diffusionwriteup.synctex.gz
 
-*.png
+*.7
 
-arc-models/lognorm_summary.pdf
+junk.py
 
-arc-models/motion_model_flymake.py
+*.rel
+*.aux
+*.bbl
+*.bib
+*.blg
+*.log
+*.toc
 
 .idea/*
 .idea*

Views.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# -*- coding: UTF-8 -*-
+# -*- coding: utf-8 -*-
 # -*- mode: python -*-
 # -*- py-indent-offset: 4 -*-
 
@@ -57,9 +57,10 @@ def get_sorted_models(all_models, condition=lambda model: True):
     ''' Return list of rated models, highest-rated first.
         Uses avgrating attribute and operator.attrgetter method. '''
 
-    rated_models = list(model for model in all_models
-        if model.avgrating != 'unrated' and condition(model))
-    return sorted(rated_models, key=attrgetter('avgrating'), reverse=True)
+    rated_models = [model for model in all_models
+                if model.model_avgrating != 'unrated'
+                and condition(model)]
+    return sorted(rated_models, key=attrgetter('model_avgrating'), reverse=True)
 
 
 def confidence_interval(scores):
@@ -79,10 +80,10 @@ def confidence_interval(scores):
         lowerbound = round(avg-halfwidth, 4)
         upperbound = round(avg+halfwidth, 4)
         assert lowerbound < avg < upperbound
-        return (lowerbound, upperbound)
+        return (lowerbound, upperbound, avg)
     except:
         print >> sys.stderr, 'No 95%% CI. N=%d, avg=%6.2f, std=%6.2f' % (N, avg, stdev)
-        return (0, 0)
+        return (-1, 1, scores[0])
 
 
 def check_account_fields(fields, new_user=True):
@@ -754,7 +755,7 @@ def model_access(request):
         request_to_input(request.session, input_dic, 'info', 'error')
         return render_to_response('ModelScreen.html',input_dic)
 
-    # If comming frm account
+    # If coming frm account
     else:
         selection = request.GET['model_in']
         if selection == '0':
@@ -963,42 +964,42 @@ def completed_test(request):
 
 @login_required
 def leaderboard(request):
+    '''Generate the data for the main leaderboard: Average, 95% CI, and N.
+
+    Handles filter choice of 'case', 'category', and usual 'model'.
+
+    '''
     input_dict = dict(csrf(request))
     models = list(Model.objects.filter(completed_cases__gt=0))
     filter_choice = request.POST.get('filter')
-
     instname = lambda model: model.account_set.all()[0].institution_name
+
     if filter_choice == 'case':
-        model_data, case_name = list(), request.POST.get('case_name')
+        model_data, case_name = [], request.POST.get('case_name')
+        input_dict['msg'] = 'of case "%s"' % case_name
         for model in models:
             try:
                 test = model.tests.get(test_name=case_name, active=False)
                 model_data.append((instname(model), model.name_id,
                     round(float(test.test_rating), 3), '', 1))
             except Test.DoesNotExist:
                 pass
-        input_dict['msg'] = 'of case "%s"' % case_name
-    elif filter_choice == 'category':
-        model_data, category = list(), request.POST.get('case_category')
-        for model in models:
-            all_tests = model.tests.filter(active=False)
-            valid_tests = list(test for test in all_tests if test.test_case.subject_category == category)
-            if len(valid_tests) > 0:
-                scores = list(float(test.test_rating) for test in valid_tests)
-                lowerbound, upperbound = (-1, 1) if len(scores) == 1 else confidence_interval(scores)
-                model_data.append((instname(model), model.name_id,
-                    round(sum(scores) / len(scores), 3), '[%5.3f, %5.3f]' % (lowerbound, upperbound), len(scores)))
-        input_dict['msg'] = 'of category "%s"' % category
     else:
-        model_data = list()
+        if filter_choice == 'category':
+            category = request.POST.get('case_category')
+            input_dict['msg'] = 'of "%s"' % category
+        model_data = []
         for model in models:
-            all_tests = model.tests.filter(active=False)
-            scores = list(float(test.test_rating) for test in all_tests)
-            lowerbound, upperbound = (-1, 1)
-            if len(scores) > 1:
-                lowerbound, upperbound = confidence_interval(scores)
-            model_data.append((instname(model), model.name_id,
-                round(float(model.avgrating), 3), '[%5.3f, %5.3f]' % (lowerbound, upperbound), len(model.tests.all())))
+            tests = model.model_tests.filter(Active=False)
+            if filter_choice == 'category':
+                tests = [t for t in tests if t.test_case.subject_category == category]
+            if len(tests) > 0:
+                scores = [float(test.test_rating) for test in tests]
+                lowerbound, upperbound, avg = confidence_interval(scores)
+                model_data.append((instname(model), model.model_nameID,
+                    round(avg, 2), '[%5.2f, %5.2f]' % (lowerbound, upperbound), len(scores)))
+                model.model_avgrating = '%5.3f' % avg
+                model.save()
 
     input_dict['case_names'] = Case.objects.values_list('case_name', flat=True)
     input_dict['case_categories'] = set(case.subject_category for case in Case.objects.all())
@@ -1009,7 +1010,11 @@ def leaderboard(request):
 
 @login_required
 def Leader_model(request):
-    '''Create the leaderboard.'''
+    '''Duplicate(?) function to create leaderboard.
+
+    TODO: figure out what calls this and why it can't use previous.
+
+    '''
     sorted_models = get_sorted_models(Model.objects.all())
     # Build Leaderboard
     inputlist = []
@@ -1018,24 +1023,20 @@ def Leader_model(request):
         account = model.account_set.all()[0]
         institution = account.institution_name
         username = account.username
-        name = model.name_id
-        rating = float(model.avgrating)
-        tests = model.tests.all()
-        finished_tests = [test for test in tests if not test.active]
-        N = len(finished_tests)
+        name = model.model_nameID
+
+        tests = model.model_tests.all()
+        finished_tests = [test for test in tests if not test.Active]
         scores = [float(x.test_rating) for x in finished_tests]
+        N = len(scores)
+        lowerbound, upperbound, avg = confidence_interval(scores)
 
         # Build case, depending on sample size
-        case = [institution, name, '%5.3f'%rating, N, username]
-        if N <= 1:
-            case.extend([-1, 1, True])  # std=False, sm.sample=True
-        else:
-            lowerbound, upperbound = confidence_interval(scores)
-            if N < 10:  # small sample=True
-                case.extend([lowerbound, upperbound, True])
-            else: # small sample = False
-                case.extend([lowerbound, upperbound, False])
-        #print >> sys.stderr, case
+        case = [institution, name, '%5.3f'%avg, N, username]
+        if N < 10:  # small sample=True
+            case.extend([lowerbound, upperbound, True])
+        else: # small sample = False
+            case.extend([lowerbound, upperbound, False])
         inputlist.append(case)
 
     # Prepare variables to send to HTML template
@@ -1815,7 +1816,7 @@ def switchboard_toscenario(request):
             if N < 2:
                 entry.extend([-1, 1, True])  # std=False, sm.sample=True
             else:
-                lowerbound, upperbound = confidence_interval(scores)
+                lowerbound, upperbound, avg = confidence_interval(scores)
                 if N < 10:  # small sample=True
                     entry.extend([lowerbound, upperbound, True])
                 else: # small sample = False

arc-models/diffusionwriteup.tex

@@ -0,0 +1,56 @@
+\documentclass[]{article}
+
+%opening
+\title{Modeling Lost Person Movement as Diffusion Using Elevation and Land Cover}
+\author{Eric Cawi}
+
+\begin{document}
+
+\maketitle
+
+\begin{abstract}
+This work creates a model for lost person movement based on gas diffusion. to be filled in
+\end{abstract}
+\tableofcontents
+
+We use Burgess and Darken's \ycite{burgess_realistic_2004} gas diffusion model of path planning to create probability maps for lost hikers. Where Burgess and Darken sample the diffusion gradient to find discrete likely paths from source to target, we use the diffusion gradient itself as a probability map, and find it is competitive with the Euclidean ring model in \cite{koester_lost_2008}.
+
+\section{Create Working Gas Model}
+
+\textbf{Insert diagram here}. 
+
+First, the area is divided into cells. Each cell has a concentrationThen the elevation data is transformed onto the interval $[0,.25]$, which represents the maximum transfer of gas from that cell to the another cell. For example, a cell with a value of .25 and a concentration of 1 (units) can transfer up to .25 of its gas to each of the four neighboring cells. This ensures the conservation of concentration in the simulation. This simulation also allows for sources and sinks to be placed at arbitrary grid cells. Burgess and Darken run this simulation until a relative equilibrium is reached (mathematically taking the limit as time goes to infinity), and then use the gradient of the final concentration to generate possible paths of approach for military applications. Note-using the gradient to generate the avenues of approach is solving the transport equation-hard to solve numerically.
+
+For search and rescue, I am more interested in the final gas concentration. Modeling the last known point as a source, the gas model simulates the overall movement of many hikers from the last known point (assertion, want to prove this). I use the source because it models a spike of probability at the LKP, and as B/D used originally it represents a continuous stream of people moving around the material
+
+In this model, I use elevation/land cover data as the speed at which the gas is allowed to propagate. Because the arrays involved are very large, I downsample by a factor of 4 on the NED dataset, then resample up to 5001x5001 pixels using Matlab's default interpolation- i think it's bicubic. Currently I'm running out to a huge number of iterations because I haven't calibrated the transformations to match different categories/mean distances yet.
+\section{Transformations from elevation/land cover to impedance}
+Elevation- using Tobler's hiking function
+Land Cover - using 1-Don's values
+\section{Diffusion Equation with source}
+The diffusion equation models the overall random walks of particles diffusing through materials. At each point there is some diffusion constant determining the maximum velocity of the concentration. Mathematically, freely diffusing gas like the Burgess Darken model can be modeled with this equation. In 1-D with a constant diffusion coefficient, the solution is a normal distribution, and generalizations give similarly smooth distributions. In our case the diffusion is given by:
+\[\frac{dc(x,y,t)}{dt}=\nabla \cdot(D(x,y)\nabla c(x,y,t)=\]
+\[(\frac{dD(x,y)}{dx},\frac{dD(x,y)}{dy})\cdot\nabla c(x,y,t) + D(x,y)\Delta c(x,y,t)\]
+Current work: finite difference scheme for this equation.
+\section{Boundary Considerations}
+I'm considering using the boundary of the map as a sink and seeing what happens, or having some other way of modeling the interactions with the outside layer because in the current state we might be getting innaccurate reflection. Boundary conditions will be important for the PDE formulation as well.
+
+\section{Static Model Preliminary Testing}
+Done on 10 arizona cases I had elevation data for. Observations- On flat terrain the concentration behaves like regular diffusion, i.e. a smooth version of the distance rings. Otherwise, it avoids big changes in elevation, as expected. E.g. if a subject doesn't start on a mountain then the mountain will have lower probability. However, a 2 or 3 of the hikers tested were going for the mountain, which makes sense because they were probably trying to hike on the mountain, but this model isn't going to catch that. Fixing this would require more information about the subject, for example "They were heading towards Mt. Awesome when they got lost." Since the odds are they moved in the direction of Mt. Awesome, a sink dropped at the intended location would skew the concentration in that direction. But that isn't available all the time.
+Not done for land cover yet
+\section{Scaling with Bob's Book}
+not done
+\section{Testing/fitting on Yosemite Cases}
+Prediction- lots of mountains in yosemite, so we should see higher probabilities around ridges/trails that stay at the same elevation. I would like to somehow set up this as a batch
+
+\section{Comparison with diffusion Models}
+Compare results and speed.
+\section{Discussion}
+Goal: An easy GIS ready application/A diffusion model that responds to terrain and is about as good as the distance model, which means that the search team now has more information.
+
+\section{References}
+\bibliographystyle{plain}
+\bibliography{diffusionwriteup}
+
+%Need Citations
+\end{document}

arc-models/gas_move.m

@@ -0,0 +1,38 @@
+function gas  = gas_move(con,speed_mat,src_vec, sink_vec,num_it)
+%con- initial concentrations of the gas
+%speed_mat- matrix of the max percentage transference,takes values between
+%0 and .25-each cell can get rid of at most 25% of it's gas
+%src_vec = locations of sources in the thing
+%sink_vec= locations of sinks in the domain
+%num_it- number of iterations to transfer
+[m,n] = size(con);%also the size of speed matrix
+if ~isempty(src_vec)
+    src_inds = sub2ind([m,n],src_vec(:,1),src_vec(:,2));
+end
+if ~isempty(sink_vec)
+    sink_inds = sub2ind([m,n],sink_vec(:,1),sink_vec(:,2));
+end
+for i = 1:num_it
+    %create left, right, up, and down transition matrices, keeping in mind
+    %the borders only have things from 2-3 directions
+    left = speed_mat(1:m,1:n-1).*(con(1:m,1:n-1)-con(1:m,2:n));%represents the contribution of the cells coming from the left
+    right = speed_mat(1:m,2:n).*(con(1:m,2:n)-con(1:m,1:n-1));%contribution of cells coming from the right
+    up = speed_mat(1:m-1,1:n).*(con(1:m-1,1:n)-con(2:m,1:n));%contribution from the cells above
+    down = speed_mat(2:m,1:n).*(con(2:m,1:n)-con(1:m-1,1:n));%contribution from the below cells
+    %update concentration
+    con(1:m,2:n) = con(1:m,2:n)+left;
+    con(1:m,1:n-1) = con(1:m,1:n-1)+right;
+    con(2:m,1:n) = con(2:m,1:n)+up;
+    con(1:m-1,1:n)=con(1:m-1,1:n)+down;
+    if ~isempty(src_vec)
+        con(src_inds)=1;%fills up the source(s)
+    end
+    if ~isempty(sink_vec)
+        con(sink_inds)=0;%drains sink
+    end
+end
+%at the end number of desired iterations, gas is the final concentration
+gas = con;
+
+
+

arc-models/land2speed.m

@@ -0,0 +1,13 @@
+function speed_mat = land2speed(data,land_cover_class)
+%converts the land cover data to impedance values
+%land cover class has two columns, 1 is the values taken by the raster. The second is impedance values. In my case i'm scaling down to [0,.25] for the gas model
+temp = data;
+for i = 1:length(land_cover_class(:,1))
+    inds = find(data == land_cover_class(i,1));% find all of the points that have a particular classification
+    temp(inds) = land_cover_class(i,2); %reset all of the data
+end
+%get rid of any border values by setting unknown things to 0 go
+inds = find(temp>.25);
+temp(inds) = 0;
+speed_mat = temp;
+end