Merge branch 'main' into 270-add-option-for-silcam-simulator-to-simul…

…ate-different-particles-rather-than-just-black-spheres

pyopia/__init__.py

@@ -1 +1 @@
-__version__ = '2.5.3'
+__version__ = '2.5.6'

pyopia/cli.py

@@ -132,8 +132,9 @@ def process(config_filename: str, num_chunks: int = 1):
 
         progress.console.print("[blue]OBTAIN FILE LIST")
         raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files'])
-        average_window = pipeline_config['steps']['correctbackground'].get('average_window', 0)
-        bgshift_function = pipeline_config['steps']['correctbackground'].get('bgshift_function', 'pass')
+        conf_corrbg = pipeline_config['steps'].get('correctbackground', dict())
+        average_window = conf_corrbg.get('average_window', 0)
+        bgshift_function = conf_corrbg.get('bgshift_function', 'pass')
         raw_files.prepare_chunking(num_chunks, average_window, bgshift_function)
 
         progress.console.print('[blue]PREPARE FOLDERS')

pyopia/io.py

@@ -213,16 +213,13 @@ def combine_stats_netcdf_files(path_to_data, prefix='*'):
                                coords='minimal', compat='override') as ds:
         xstats = ds.load()
 
-    # Check if we have image statistics in the last file, if so, load it.
-    # The last file should contain the entire time series of processed images.
+    # Check if we have image statistics, if so, load it.
     try:
-        ds = xarray.open_dataset(sorted_filelist[-1], group='image_stats')
+        with xarray.open_mfdataset(sorted_filelist, group='image_stats') as ds:
+            image_stats = ds.load()
     except OSError:
+        logger.info('Could get image_stats from netcdf files for merging, returning None for this.')
         image_stats = None
-    else:
-        image_stats = ds.load()
-    finally:
-        ds.close()
 
     return xstats, image_stats
 
@@ -393,7 +390,7 @@ def load_toml(toml_file):
     return settings
 
 
-def StatsH5():
+def StatsH5(**kwargs):
     '''.. deprecated:: 2.4.8
         :class:`pyopia.io.StatsH5` will be removed in version 3.0.0, it is replaced by
         :class:`pyopia.io.StatsToDisc`.
@@ -433,4 +430,4 @@ def StatsH5():
         append = true
     '''
     logger.warning('StatsH5 will be removed in version 3.0.0, it is replaced by pyopia.io.StatsToDisc')
-    return StatsToDisc()
+    return StatsToDisc(**kwargs)

pyopia/statistics.py

@@ -763,12 +763,14 @@ def make_timeseries_vd(stats, pixel_size, path_length, time_reference):
     similar to Sequoia LISST-100 output,
     and exportable to things like Excel or csv.
 
-    Note: If zero particles are detected within the stats daraframe,
+    Note
+    ----
+    If zero particles are detected within the stats daraframe,
     then the volume concentration should be reported as zero for that
     time. For this function to have awareness of these times, it requires
     time_reference variable. If you use `stats['timestamp'].unique()` for this,
     then you are assuming you have at least one particle per image.
-    It is better to use image_stats['datetime'].values instead, which can be obtained from
+    It is better to use `image_stats['timestamp'].values` instead, which can be obtained from
     :func:`pyopia.io.load_image_stats`
 
     Parameters
@@ -790,9 +792,12 @@ def make_timeseries_vd(stats, pixel_size, path_length, time_reference):
     Example
     -------
     .. code-block:: python
+        path_length = 40  # for a 40mm long path length
+
         time_series_vd = pyopia.statistics.make_timeseries_vd(stats,
                                 settings['general']['pixel_size'],
-                                path_length=40)
+                                path_length,
+                                image_stats['timestamp'].values)
 
         # particle diameters
         dias = np.array(time_series_vd.columns[0:52], dtype=float)
@@ -807,7 +812,7 @@ def make_timeseries_vd(stats, pixel_size, path_length, time_reference):
         time = pd.to_datetime(time_series_vd['Time'].values)
 
         # time-series of total volume concentration
-        vc = np.sum(vdarray, axis=1)
+        volume_concentration = np.sum(vdarray, axis=1)
     '''
     sample_volume = get_sample_volume(pixel_size, path_length=path_length)