Merge pull request #166 from eEcoLiDAR/development

Development

CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 0.4.1 - 2020-05-20
+## Added:
+- select_equal filter accepts list of values to compare to the points' attributes
+- also the attribute-based filter functions optionally return a mask to allow filter combinations
+
+## Fixed:
+- bug in writing/reading 'None' as parameter in the PLY comments
+
 ## 0.4.0 - 2020-05-13
 ## Added:
 - build_volume module

CITATION.cff

@@ -52,7 +52,7 @@ authors:
     family-names: Koma
     given-names: Zsófia
 cff-version: "1.0.3"
-date-released: 2020-05-13
+date-released: 2020-05-20
 doi: "10.5281/zenodo.1219422"
 keywords:
   - "airborne laser scanning"
@@ -62,5 +62,5 @@ keywords:
 license: "Apache-2.0"
 message: "If you use this software, please cite it using these metadata."
 title: "Laserchicken: toolkit for ALS point clouds"
-version: "0.4.0"
+version: "0.4.1"
 ...

README.md

@@ -24,7 +24,7 @@ pip install laserchicken
 * Create .zenodo.json file from CITATION.cff (using cffconvert)  
 ```cffconvert --validate```  
 ```cffconvert --ignore-suspect-keys --outputformat zenodo --outfile .zenodo.json```
-* Set new version number in laserchicken/_version.py
+* Set new version number in laserchicken/_version.txt
 * Check that documentation uses the correct version
 * Edit Changelog (based on commits in https://github.com/eecolidar/laserchicken/compare/v0.3.2...master)
 * Test if package can be installed with pip (`pip install .`)

laserchicken/_version.txt

@@ -1 +1 @@
-0.4.0
+0.4.1

laserchicken/filter.py

@@ -16,51 +16,64 @@
 from laserchicken.utils import copy_point_cloud, add_metadata
 
 
-def select_equal(point_cloud, attribute, value):
+def select_equal(point_cloud, attribute, value, return_mask=False):
     """
     Return the selection of the input point cloud that contains only points with a given attribute equal to some value.
+    If a list of values is given, select the points corresponding to any of the provided values.
 
     :param point_cloud: Input point cloud.
     :param attribute: The attribute name used for selection
-    :param value: The value to compare the attribute to
-    :return: A new point cloud containing only the selected points
+    :param value: The value(s) to compare the attribute to
+    :param return_mask: If true, return the mask corresponding to the selection
+    :return:
     """
     _check_valid_arguments(attribute, point_cloud)
-    mask = point_cloud[point][attribute]['data'] == value
+    # broadcast using shape of the values
+    mask = point_cloud[point][attribute]['data'] == np.array(value)[..., None]
+    if mask.ndim > 1:
+        mask = np.any(mask, axis=0)  # reduce
+    if return_mask:
+        return mask
     point_cloud_filtered = copy_point_cloud(point_cloud, mask)
     add_metadata(point_cloud_filtered, sys.modules[__name__],
                  {'attribute': attribute, 'value': value})
     return point_cloud_filtered
 
 
-def select_above(point_cloud, attribute, threshold):
+def select_above(point_cloud, attribute, threshold, return_mask=False):
     """
     Return the selection of the input point cloud that contains only points with a given attribute above some value.
 
     :param point_cloud: Input point cloud
     :param attribute: The attribute name used for selection
     :param threshold: The threshold value used for selection
-    :return: A new point cloud containing only the selected points
+    :param return_mask: If true, return the mask corresponding to the selection
+    :return:
     """
     _check_valid_arguments(attribute, point_cloud)
     mask = point_cloud[point][attribute]['data'] > threshold
+    if return_mask:
+        return mask
     point_cloud_filtered = copy_point_cloud(point_cloud, mask)
     add_metadata(point_cloud_filtered, sys.modules[__name__],
                  {'attribute': attribute, 'threshold': threshold})
     return point_cloud_filtered
 
 
-def select_below(point_cloud, attribute, threshold):
+def select_below(point_cloud, attribute, threshold, return_mask=False):
     """
     Return the selection of the input point cloud that contains only points with a given attribute below some value.
 
     :param point_cloud: Input point cloud
     :param attribute: The attribute name used for selection
     :param threshold: The threshold value used for selection
-    :return: A new point cloud containing only the selected points
+    :param return_mask: If true, return the mask corresponding to the selection
+    :return:
     """
     _check_valid_arguments(attribute, point_cloud)
     mask = point_cloud[point][attribute]['data'] < threshold
+    if return_mask:
+        return mask
     point_cloud_filtered = copy_point_cloud(point_cloud, mask)
     add_metadata(point_cloud_filtered, sys.modules[__name__],
                  {'attribute': attribute, 'threshold': threshold})

laserchicken/io/ply_read.py

@@ -1,6 +1,8 @@
 import ast
+import json
 import numpy as np
-from dateutil import parser
+
+from json.decoder import JSONDecodeError
 from struct import unpack, calcsize
 
 from laserchicken.io.utils import convert_to_short_type, convert_to_single_character_type
@@ -78,12 +80,13 @@ def _read_header_line(ply, is_binary=False):
 
 def _read_log(comments):
     try:
-        log = ast.literal_eval(' '.join(comments)) if comments else []
-    except SyntaxError:  # Log can't be read. Maybe a ply file with 'regular' comments and no log.
-        log = []
-    for i, entry in enumerate(log):
-        if 'time' in entry:
-            entry['time'] = parser.parse(entry['time'])
+        log = json.loads(' '.join(comments)) if comments else []
+    except JSONDecodeError:
+        try:
+            # legacy: comments for laserchicken < 0.4.0
+            log = ast.literal_eval(' '.join(comments)) if comments else []
+        except SyntaxError:  # Log can't be read. Maybe a ply file with 'regular' comments and no log.
+            log = []
     return log
 
 

laserchicken/io/ply_write.py

@@ -99,31 +99,10 @@ def _write_comment(pc, ply):
 
     head = 'comment [\n'
     tail = 'comment ]\n'
-    formatted_entries = ',\n'.join(['comment ' + json.dumps(_stringify(entry), sort_keys=True) for entry in log]) + '\n'
+    formatted_entries = ',\n'.join(['comment ' + json.dumps(entry, sort_keys=True) for entry in log]) + '\n'
     ply.write(head + formatted_entries + tail)
 
 
-def _stringify(entry):
-    copy = {}
-    for key, value in _sort_by_key(entry):
-        if isinstance(value, dict):
-            copy[key] = _stringify(value)
-        elif isinstance(value, list):
-            copy[key] = [_stringify(entry) if isinstance(entry, dict) else entry for entry in value]
-        else:
-            if key == 'time':
-                copy[key] = str(value)
-            else:
-                copy[key] = value
-    return copy
-
-
-def _sort_by_key(entry):
-    key_value_pairs = list(entry.items())
-    key_value_pairs.sort(key=lambda key_value_pair: key_value_pair[0])
-    return key_value_pairs
-
-
 def _write_header_elements(pc, attributes, ply, element_name, get_num_elements=None):
     if element_name in pc:
         num_elements = get_num_elements(pc[element_name]) if get_num_elements else 1

laserchicken/io/test_read_ply.py

@@ -1,3 +1,4 @@
+import dateutil
 import os
 import shutil
 import unittest
@@ -89,8 +90,10 @@ def test_correctModulesLogged(self):
     def test_correctTimesLogged(self):
         log = load(self.test_file_path)['log']
 
-        self.assertListEqual([2018, 1, 18, 16, 1, 0, 3, 18, -1], list(log[0]['time'].timetuple()))
-        self.assertListEqual([2018, 1, 18, 16, 3, 0, 3, 18, -1], list(log[1]['time'].timetuple()))
+        self.assertListEqual([2018, 1, 18, 16, 1, 0, 3, 18, -1],
+                             list(dateutil.parser.parse(log[0]['time']).timetuple()))
+        self.assertListEqual([2018, 1, 18, 16, 3, 0, 3, 18, -1],
+                             list(dateutil.parser.parse(log[1]['time']).timetuple()))
 
     def setUp(self):
         os.mkdir(self._test_dir)

laserchicken/test_filter.py

@@ -48,6 +48,27 @@ def test_selectEqual_outputCorrect():
         pc_out = select_equal(pc_in, 'return', 1)
         assert_equal(len(pc_out[point]['x']['data']), 2)
 
+    @staticmethod
+    def test_selectEqual_multipleValues():
+        """ Correct number of results. """
+        pc_in = get_test_data()
+        pc_out = select_equal(pc_in, 'return', [1, 2])
+        assert_equal(len(pc_out[point]['x']['data']), 3)
+
+    @staticmethod
+    def test_selectEqual_maskCorrect():
+        """ Correct number of results. """
+        pc_in = get_test_data()
+        mask_out = select_equal(pc_in, 'return', 1, return_mask=True)
+        assert_equal(mask_out, np.array([1,1,0], dtype=bool))
+
+    @staticmethod
+    def test_selectEqual_maskEmpty():
+        """ Correct number of results. """
+        pc_in = get_test_data()
+        mask_out = select_equal(pc_in, 'return', 3, return_mask=True)
+        assert_equal(sum(mask_out), 0)
+
 
 class TestSelectBelow(unittest.TestCase):
     @staticmethod
@@ -88,6 +109,20 @@ def test_selectBelow_onlyOnePoint():
         assert_almost_equal(pc_out[point]['y']['data'][0], 2.1)
         assert_almost_equal(pc_out[point]['z']['data'][0], 3.1)
         assert_almost_equal(pc_out[point]['return']['data'][0], 1)
+
+    @staticmethod
+    def test_selectBelow_maskCorrect():
+        """ Correct number of results. """
+        pc_in = get_test_data()
+        mask_out = select_below(pc_in, 'return', 2, return_mask=True)
+        assert_equal(mask_out, np.array([1,1,0], dtype=bool))
+
+    @staticmethod
+    def test_selectBelow_maskAll():
+        """ Correct number of results. """
+        pc_in = get_test_data()
+        mask_out = select_below(pc_in, 'return', 3, return_mask=True)
+        assert_equal(sum(mask_out), 3)
 
 
 class TestSelectAbove(unittest.TestCase):
@@ -128,6 +163,20 @@ def test_selectBelow_onlyOnePoint():
         assert_almost_equal(pc_out[point]['y']['data'][0], 2.3)
         assert_almost_equal(pc_out[point]['z']['data'][0], 3.3)
         assert_almost_equal(pc_out[point]['return']['data'][0], 2)
+
+    @staticmethod
+    def test_selectAbove_maskCorrect():
+        """ Correct number of results. """
+        pc_in = get_test_data()
+        mask_out = select_above(pc_in, 'return', 1, return_mask=True)
+        assert_equal(mask_out, np.array([0,0,1], dtype=bool))
+
+    @staticmethod
+    def test_selectAbove_maskAll():
+        """ Correct number of results. """
+        pc_in = get_test_data()
+        mask_out = select_above(pc_in, 'return', 0, return_mask=True)
+        assert_equal(sum(mask_out), 3)
 
 
 class TestSelectPolygonWKT(unittest.TestCase):

laserchicken/utils.py

@@ -140,7 +140,7 @@ def add_metadata(point_cloud, module, params):
     :param params:
     :return:
     """
-    msg = {"time": datetime.datetime.utcnow(),
+    msg = {"time": str(datetime.datetime.utcnow()),
            "module": module.__name__ if hasattr(module, "__name__") else str(module)}
     if any(params):
         msg["parameters"] = params

tutorial.ipynb

@@ -44,15 +44,15 @@
        "          -0.24100002, -0.24000002])},\n",
        "  'raw_classification': {'type': 'uint8',\n",
        "   'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n",
+       "  'intensity': {'type': 'uint16',\n",
+       "   'data': array([ 41, 152,  12, ...,  10,  15,  10], dtype=uint16)},\n",
        "  'gps_time': {'type': 'float64',\n",
        "   'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n",
-       "          78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n",
-       "  'intensity': {'type': 'uint16',\n",
-       "   'data': array([ 41, 152,  12, ...,  10,  15,  10], dtype=uint16)}},\n",
-       " 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n",
+       "          78563778.28828931, 78563778.3107884 , 78563778.32578015])}},\n",
+       " 'log': [{'time': '2020-05-19 14:56:55.083339',\n",
        "   'module': 'laserchicken.io.load',\n",
        "   'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n",
-       "   'version': '0.3.2'}]}"
+       "   'version': '0.4.0'}]}"
       ]
      },
      "execution_count": 2,
@@ -91,21 +91,21 @@
        "          -0.24100002, -0.24000002])},\n",
        "  'raw_classification': {'type': 'uint8',\n",
        "   'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n",
+       "  'intensity': {'type': 'uint16',\n",
+       "   'data': array([ 41, 152,  12, ...,  10,  15,  10], dtype=uint16)},\n",
        "  'gps_time': {'type': 'float64',\n",
        "   'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n",
        "          78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n",
-       "  'intensity': {'type': 'uint16',\n",
-       "   'data': array([ 41, 152,  12, ...,  10,  15,  10], dtype=uint16)},\n",
        "  'normalized_height': {'type': 'float64',\n",
        "   'data': array([1.236, 1.311, 1.317, ..., 1.336, 1.336, 1.337])}},\n",
-       " 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n",
+       " 'log': [{'time': '2020-05-19 14:56:55.083339',\n",
        "   'module': 'laserchicken.io.load',\n",
        "   'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n",
-       "   'version': '0.3.2'},\n",
-       "  {'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 473466),\n",
+       "   'version': '0.4.0'},\n",
+       "  {'time': '2020-05-19 14:56:55.149038',\n",
        "   'module': 'laserchicken.normalize',\n",
        "   'parameters': {'cell_size': None},\n",
-       "   'version': '0.3.2'}]}"
+       "   'version': '0.4.0'}]}"
       ]
      },
      "execution_count": 3,
@@ -137,21 +137,21 @@
        "          -0.24100002, -0.24000002])},\n",
        "  'raw_classification': {'type': 'uint8',\n",
        "   'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n",
+       "  'intensity': {'type': 'uint16',\n",
+       "   'data': array([ 41, 152,  12, ...,  10,  15,  10], dtype=uint16)},\n",
        "  'gps_time': {'type': 'float64',\n",
        "   'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n",
        "          78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n",
-       "  'intensity': {'type': 'uint16',\n",
-       "   'data': array([ 41, 152,  12, ...,  10,  15,  10], dtype=uint16)},\n",
        "  'normalized_height': {'type': 'float64',\n",
        "   'data': array([1.236, 1.311, 1.317, ..., 1.336, 1.336, 1.337])}},\n",
-       " 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n",
+       " 'log': [{'time': '2020-05-19 14:56:55.083339',\n",
        "   'module': 'laserchicken.io.load',\n",
        "   'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n",
-       "   'version': '0.3.2'},\n",
-       "  {'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 473466),\n",
+       "   'version': '0.4.0'},\n",
+       "  {'time': '2020-05-19 14:56:55.149038',\n",
        "   'module': 'laserchicken.normalize',\n",
        "   'parameters': {'cell_size': None},\n",
-       "   'version': '0.3.2'}]}"
+       "   'version': '0.4.0'}]}"
       ]
      },
      "execution_count": 4,
@@ -245,13 +245,13 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Cylinder size in Bytes: 1214225560.6124551\n",
+      "Cylinder size in Bytes: 1262794583.0369534\n",
       "Memory size in Bytes: 17179869184\n",
       "Start tree creation\n",
       "Done with env tree creation\n",
       "Done with target tree creation\n",
-      "Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\"Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\" took 0.24 seconds\n",
-      "Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\"Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\" took 0.18 seconds\n",
+      "Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\"Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\" took 0.22 seconds\n",
+      "Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\"Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\" took 0.21 seconds\n",
       "The following unrequested features were calculated as a side effect, but will not be returned: ['normal_vector_3', 'normal_vector_2', 'normal_vector_1', 'eigenv_3', 'eigenv_2', 'eigenv_1', 'coeff_var_z']\n"
      ]
     }
@@ -574,12 +574,12 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Cylinder size in Bytes: 1214225560.6124551\n",
+      "Cylinder size in Bytes: 1262794583.0369534\n",
       "Memory size in Bytes: 17179869184\n",
       "Start tree creation\n",
       "Done with env tree creation\n",
       "Done with target tree creation\n",
-      "Extracting feature(s) \"['band_ratio_1<normalized_height<2']\"Extracting feature(s) \"['band_ratio_1<normalized_height<2']\" took 0.07 seconds\n"
+      "Extracting feature(s) \"['band_ratio_1<normalized_height<2']\"Extracting feature(s) \"['band_ratio_1<normalized_height<2']\" took 0.08 seconds\n"
      ]
     }
    ],
@@ -623,7 +623,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.8.2"
   }
  },
  "nbformat": 4,