Merge branch 'main' into weji/reject_sonata_extension

.gitlab-ci.yml

@@ -6,16 +6,15 @@ include:
       - /ci/jobs/publish-package.yml
       - /ci/jobs/docs.yml
       - /ci/jobs/publish-docs.yml
+  - project: hpc/gitlab-upload-logs
+    file: enable-upload.yml
 
 workflow:
   rules:
-    # Rules taken from official docs to avoid duplicated pipelines
-    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
-      when: never
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
-      when: never
-    - when: always
+    - if: '$CI_EXTERNAL_PULL_REQUEST_IID'
+    # re-define because the imported rules will be ignored
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: $CI_PIPELINE_SOURCE == "web"  # run for manually started pipelines from -/pipelines/new
 
 variables:
   BLUECONFIGS_BRANCH:
@@ -61,7 +60,7 @@ blueconfig_tests:
   needs:
     - set_alt_branches
   rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '$CI_EXTERNAL_PULL_REQUEST_IID'
       changes:
         - neurodamus/**/*
     - when: manual

.readthedocs.yml

@@ -0,0 +1,21 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3"
+
+python:
+  install:
+    - method: pip
+      path: .
+    - requirements: requirements_docs.txt

CHANGES.rst

@@ -2,6 +2,18 @@
 Change Log
 ==========
 
+Version 2.16.0
+==============
+**26 July 2023**
+
+*New Features*
+  * [BBPBGLIB-1036] Pure SONATA reader for gap junctions
+  * [BBPBGLIB-984] Option to keep Cell axon during init
+
+*Improvements*
+  * [BBPBGLIB-1035] Drop Synapsetool. Migrate all synapses loading to libsonata (no syn2 support)
+
+
 Version 2.15.3
 ===============
 **14 July 2023**

README.rst

@@ -44,7 +44,7 @@ Build special with mod files
   mkdir mods
   cp -r mod-files-from-released-circuit mods/
   cp -r neurodamus-install-prefix/share/mod/* mods/
-  nrnivmodl -incflags '-I <include-paths-of-our-dependencies>' -loadflags '-L <libs-paths-for-linking>'' mod
+  nrnivmodl -incflags '-I <include-paths-of-our-dependencies>' -loadflags '-L <libs-paths-for-linking>' mod
 
 Examples
 ========

core/mod/CoreConfig.mod

@@ -282,9 +282,9 @@ VERBATIM
     char* simConf = (char*)alloca(strlen(outputdir) + CONFIG_FILENAME_TOTAL_LEN_MAX);
     sprintf(simConf, "%s/%s", outputdir, SIM_CONFIG_FILE);
 # if CORENRN_CLI11
-    char *argv[] = {"", "--read-config", simConf, "--skip-mpi-finalize", "--mpi", NULL, NULL, NULL, NULL};
+    char *argv[] = {"coreneuron", "--read-config", simConf, "--skip-mpi-finalize", "--mpi", NULL, NULL, NULL, NULL};
 # else
-    char *argv[] = {"", "--read-config", simConf, "--skip-mpi-finalize", "-mpi", NULL, NULL, NULL, NULL};
+    char *argv[] = {"coreneuron", "--read-config", simConf, "--skip-mpi-finalize", "-mpi", NULL, NULL, NULL, NULL};
 # endif
     int argc = 5;
     int argIndex=1;

docs/api/modules.rst

@@ -8,3 +8,7 @@ neurodamus
   neurodamus.core
   neurodamus.io
   neurodamus.utils
+  subpackages/core
+  subpackages/io
+  subpackages/top
+  subpackages/utils

docs/api/subpackages/io.rst

@@ -16,7 +16,7 @@ neurodamus.io.cell\_readers
 
       load_mvd3
       load_ncs
-      load_nodes
+      load_nodes_mvd3
       load_combo_metypes
       split_round_robin
 

docs/api/subpackages/top.rst

@@ -54,6 +54,7 @@ neurodamus.connection\_manager
 
    .. autoclass:: SynapseRuleManager
       :members:
+      :noindex:
       :inherited-members:
 
 

docs/conf.py

@@ -9,7 +9,7 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-# sys.path.insert(0, os.path.abspath())
+sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration ------------------------------------------------
 

docs/development.rst

@@ -180,7 +180,7 @@ You can clone ``neurodamus-py`` and use ``spack develop`` as follows::
 
         spack env deactivate
         spack env activate -p neurodamus
-        This tends to manifest itself in import errors: ModuleNotFoundError: No module named 'neurodamus'.
+        # This tends to manifest itself in import errors: ModuleNotFoundError: No module named 'neurodamus'.
 
 .. note::
 
@@ -190,7 +190,7 @@ You can clone ``neurodamus-py`` and use ``spack develop`` as follows::
 
         spack uninstall neurodamus-neocortex
         spack install neurodamus-neocortex
-        This is required since changing the sources in neurodamus-py doesn't trigger Spack to reinstall neurodamus-neocortex.
+        # This is required since changing the sources in neurodamus-py doesn't trigger Spack to reinstall neurodamus-neocortex.
 
 .. note::
 

docs/index.rst

@@ -17,6 +17,7 @@ Contents
    Changelog <changes>
    Module Reference <api/modules>
    Developer Documentation <architecture>
+   Developing a Custom Neurodamus <development>
 
 
 Indices and tables

neurodamus/cell_distributor.py

@@ -103,12 +103,14 @@ class CellManagerBase(_CellManager):
 
     CellType = NotImplemented  # please override
     """The underlying Cell type class
+
     signature:
         __init__(self, gid, cell_info, circuit_conf)
     """
 
     _node_loader = None
     """Default function implementing the loading of nodes data, a.k.a. MVD
+
     signature:
         load(circuit_conf, gidvec, stride=1, stride_offset=0)
     """
@@ -660,6 +662,7 @@ class LoadBalance:
 
     NOTE: Given the heavy costs of computing load balance, some state files are created
     which allow the balance info to be reused. These are
+
      - cx_{TARGET}.dat: File with complexity information for the cells of a given target
      - cx_{TARGET}.{CPU_COUNT}.dat: The file assigning cells/pieces to individual CPUs ranks.
 

neurodamus/gap_junction.py

@@ -8,7 +8,8 @@
 
 from .connection_manager import ConnectionManagerBase
 from .core.configuration import ConfigurationError
-from .io.synapse_reader import SynapseReader, SynReaderNRN, SynapseParameters, FormatNotSupported
+from .io.synapse_reader import SynapseReader, SynReaderNRN, SonataReader, SynapseParameters, \
+    FormatNotSupported
 from .utils import compat
 from .utils.logging import log_verbose
 
@@ -17,40 +18,9 @@ class GapJunctionConnParameters(SynapseParameters):
     # Attribute names of synapse parameters, consistent with the normal synapses
     _synapse_fields = ("sgid", "isec", "offset", "weight", "D", "F", "ipt", "location")
 
-    # Actual fields to read from conectivity files
-    _gj_v1_fields = [
-        "connected_neurons_pre",
-        "morpho_section_id_post",
-        "morpho_offset_segment_post",
-        "conductance",
-        "junction_id_pre",
-        "junction_id_post",
-        "morpho_segment_id_post",
-
-    ]
-    _gj_v2_fields = [
-        "connected_neurons_pre",
-        "morpho_section_id_post",
-        "morpho_section_fraction_post",  # v2 field
-        "conductance",
-        "junction_id_pre",
-        "junction_id_post"
-    ]
-    # SONATA fields, see conversion map in spykfunc/schema.py and
-    # synapse-tool Naming::get_property_mapping
-    _gj_sonata_fields = [
-        "connected_neurons_pre",
-        "morpho_section_id_post",
-        "morpho_section_fraction_post",
-        "conductance",
-        "efferent_junction_id",
-        "afferent_junction_id"
-    ]
-
     @classmethod
     def create_array(cls, length):
         npa = np.recarray(length, cls.dtype)
-        npa.ipt = -1
         npa.location = 0.5
         return npa
 
@@ -65,13 +35,27 @@ class GapJunctionSynapseReader(SynapseReader):
     def create(cls, syn_src, conn_type, population=None, *args, **kw):
         """Instantiates a synapse reader, giving preference to GapJunctionSynToolReader
         """
+        if fn := cls._get_sonata_circuit(syn_src):
+            log_verbose("[SynReader] Using SonataReader.")
+            return GapJunctionSonataReader(fn, conn_type, population, **kw)
+
         # Syn2 support dropped (July 2023)
         if not ospath.isdir(syn_src) and not syn_src.endswith(".h5"):
             raise FormatNotSupported("File: {syn_src}")
         logging.info("[GapJunctionSynReader] Attempting legacy hdf5 reader.")
         return SynReaderNRN(syn_src, conn_type, None, *args, **kw)
 
 
+class GapJunctionSonataReader(SonataReader):
+    Parameters = GapJunctionConnParameters
+    parameter_mapping = {
+        "weight": "conductance",
+        "D": "efferent_junction_id",
+        "F": "afferent_junction_id",
+    }
+    # "isec", "ipt", "offset" are custom parameters as in base class
+
+
 class GapJunctionManager(ConnectionManagerBase):
     """
     The GapJunctionManager is similar to the SynapseRuleManager. It will

neurodamus/io/synapse_reader.py

@@ -12,17 +12,6 @@
 from ..utils.logging import log_verbose
 
 
-def _get_sonata_circuit(path):
-    """Returns a SONATA edge file in path if present
-    """
-    if path.endswith(".h5"):
-        import h5py
-        f = h5py.File(path, 'r')
-        if "edges" in f:
-            return path
-    return None
-
-
 def _constrained_hill(K_half, y):
     K_half_fourth = K_half**4
     y_fourth = y**4
@@ -179,6 +168,17 @@ def has_property(self, field_name):
         """Checks whether source data has the given additional field.
         """
 
+    @staticmethod
+    def _get_sonata_circuit(path):
+        """Returns a SONATA edge file in path if present
+        """
+        if path.endswith(".h5"):
+            import h5py
+            f = h5py.File(path, 'r')
+            if "edges" in f:
+                return path
+        return None
+
     @classmethod
     def create(cls, syn_src, conn_type=SYNAPSES, population=None, *args, **kw):
         """Instantiates a synapse reader, giving preference to SonataReader
@@ -188,7 +188,7 @@ def create(cls, syn_src, conn_type=SYNAPSES, population=None, *args, **kw):
             return cls(syn_src, conn_type, population, *args, **kw)
 
         kw["verbose"] = (MPI.rank == 0)
-        if fn := _get_sonata_circuit(syn_src):
+        if fn := cls._get_sonata_circuit(syn_src):
             log_verbose("[SynReader] Using SonataReader.")
             return SonataReader(fn, conn_type, population, **kw)
 

neurodamus/node.py

@@ -25,7 +25,7 @@
 from .cell_distributor import LoadBalance, LoadBalanceMode
 from .connection_manager import SynapseRuleManager, edge_node_pop_names
 from .gap_junction import GapJunctionManager
-from .replay import SpikeManager
+from .replay import MissingSpikesPopulationError, SpikeManager
 from .stimulus_manager import StimulusManager
 from .modification_manager import ModificationManager
 from .neuromodulation_manager import NeuroModulationManager
@@ -734,7 +734,6 @@ def enable_replay(self):
     def _enable_replay(self, source, target, stim_conf, tshift=.0, delay=.0,
                        connectivity_type=None):
         spike_filepath = find_input_file(stim_conf["SpikeFile"])
-        spike_manager = SpikeManager(spike_filepath, tshift)  # Disposable
         ptype_cls = EngineBase.connection_types.get(connectivity_type)
         src_target = self.target_manager.get_target(source)
         dst_target = self.target_manager.get_target(target)
@@ -743,6 +742,13 @@ def _enable_replay(self, source, target, stim_conf, tshift=.0, delay=.0,
             pop_offsets, alias_pop = CircuitManager.read_population_offsets(read_virtual_pop=True)
 
         for src_pop in src_target.population_names:
+            try:
+                log_verbose("Loading replay spikes for population '%s'", src_pop)
+                spike_manager = SpikeManager(spike_filepath, tshift, src_pop)  # Disposable
+            except MissingSpikesPopulationError:
+                logging.info("  > No replay for src population: '%s'", src_pop)
+                continue
+
             for dst_pop in dst_target.population_names:
                 src_pop_str, dst_pop_str = src_pop or "(base)", dst_pop or "(base)"
 
@@ -751,10 +757,9 @@ def _enable_replay(self, source, target, stim_conf, tshift=.0, delay=.0,
                         else pop_offsets[alias_pop[src_pop]]
                 else:
                     conn_manager = self._circuits.get_edge_manager(src_pop, dst_pop, ptype_cls)
-                    if not conn_manager:
-                        logging.error("No edge manager found among populations %s -> %s",
-                                      src_pop_str, dst_pop_str)
-                        raise ConfigurationError("Unknown replay pathway. Check Source / Target")
+                    if not conn_manager and SimConfig.cli_options.restrict_connectivity >= 1:
+                        continue
+                    assert conn_manager, f"Missing edge manager for {src_pop_str} -> {dst_pop_str}"
                     src_pop_offset = conn_manager.src_pop_offset
 
                 logging.info("=> Population pathway %s -> %s. Source offset: %d",