From 8263f1d70995d5815c30691d2b14ef4a840ff356 Mon Sep 17 00:00:00 2001 From: Weina Ji Date: Mon, 17 Jul 2023 11:28:33 +0000 Subject: [PATCH 1/7] [BBPBGLIB-1036] Implement SONATA synapse reader for gap junction ## Context Recently we have dropped the synapse-tool dependency for reading synapse parameters. SynReaderNRN is used to read the old format nrn edges files, and SonataReader is to read sonata edges. This PR is to implement a sonata reader to read gap junction synapse parameters. ## Scope In `gap_junection.py`, a GapJunctionSonata reader derived from the SonataReader. The Sonata parameters for gap junction are `efferent_junction_id` and `afferent_junction_id`. ## Testing Add a unit test to test GapJunctionSynapseReader for both NRN and SONATA edges files. ## Review * [x] PR description is complete * [x] Coding style (imports, function length, New functions, classes or files) are good * [x] Unit/Scientific test added * [ ] Updated Readme, in-code, developer documentation --- neurodamus/gap_junction.py | 48 ++++++------------ neurodamus/io/synapse_reader.py | 36 ++++++------- .../mini_thalamus_sonata/gapjunction/README | 2 + .../mini_thalamus_sonata/gapjunction/edges.h5 | Bin 0 -> 169312 bytes tests/test_synapse_reader.py | 48 ++++++++++++++++++ 5 files changed, 84 insertions(+), 50 deletions(-) create mode 100644 tests/simulations/mini_thalamus_sonata/gapjunction/README create mode 100644 tests/simulations/mini_thalamus_sonata/gapjunction/edges.h5 create mode 100644 tests/test_synapse_reader.py diff --git a/neurodamus/gap_junction.py b/neurodamus/gap_junction.py index 9d5dd67e..314568b1 100644 --- a/neurodamus/gap_junction.py +++ b/neurodamus/gap_junction.py @@ -8,7 +8,8 @@ from .connection_manager import ConnectionManagerBase from .core.configuration import ConfigurationError -from .io.synapse_reader import SynapseReader, SynReaderNRN, SynapseParameters, FormatNotSupported +from .io.synapse_reader import SynapseReader, SynReaderNRN, SonataReader, SynapseParameters, \ + FormatNotSupported from .utils import compat from .utils.logging import log_verbose @@ -17,40 +18,9 @@ class GapJunctionConnParameters(SynapseParameters): # Attribute names of synapse parameters, consistent with the normal synapses _synapse_fields = ("sgid", "isec", "offset", "weight", "D", "F", "ipt", "location") - # Actual fields to read from conectivity files - _gj_v1_fields = [ - "connected_neurons_pre", - "morpho_section_id_post", - "morpho_offset_segment_post", - "conductance", - "junction_id_pre", - "junction_id_post", - "morpho_segment_id_post", - - ] - _gj_v2_fields = [ - "connected_neurons_pre", - "morpho_section_id_post", - "morpho_section_fraction_post", # v2 field - "conductance", - "junction_id_pre", - "junction_id_post" - ] - # SONATA fields, see conversion map in spykfunc/schema.py and - # synapse-tool Naming::get_property_mapping - _gj_sonata_fields = [ - "connected_neurons_pre", - "morpho_section_id_post", - "morpho_section_fraction_post", - "conductance", - "efferent_junction_id", - "afferent_junction_id" - ] - @classmethod def create_array(cls, length): npa = np.recarray(length, cls.dtype) - npa.ipt = -1 npa.location = 0.5 return npa @@ -65,6 +35,10 @@ class GapJunctionSynapseReader(SynapseReader): def create(cls, syn_src, conn_type, population=None, *args, **kw): """Instantiates a synapse reader, giving preference to GapJunctionSynToolReader """ + if fn := cls._get_sonata_circuit(syn_src): + log_verbose("[SynReader] Using SonataReader.") + return GapJunctionSonataReader(fn, conn_type, population, **kw) + # Syn2 support dropped (July 2023) if not ospath.isdir(syn_src) and not syn_src.endswith(".h5"): raise FormatNotSupported("File: {syn_src}") @@ -72,6 +46,16 @@ def create(cls, syn_src, conn_type, population=None, *args, **kw): return SynReaderNRN(syn_src, conn_type, None, *args, **kw) +class GapJunctionSonataReader(SonataReader): + Parameters = GapJunctionConnParameters + parameter_mapping = { + "weight": "conductance", + "D": "efferent_junction_id", + "F": "afferent_junction_id", + } + # "isec", "ipt", "offset" are custom parameters as in base class + + class GapJunctionManager(ConnectionManagerBase): """ The GapJunctionManager is similar to the SynapseRuleManager. It will diff --git a/neurodamus/io/synapse_reader.py b/neurodamus/io/synapse_reader.py index 17a928c9..968b1eb3 100644 --- a/neurodamus/io/synapse_reader.py +++ b/neurodamus/io/synapse_reader.py @@ -12,23 +12,6 @@ from ..utils.logging import log_verbose -def _get_sonata_circuit(path): - """Returns a SONATA edge file in path if present - """ - if os.path.isdir(path): - filename = os.path.join(path, "edges.sonata") - if os.path.exists(filename): - return filename - elif path.endswith(".sonata"): - return path - elif path.endswith(".h5"): - import h5py - f = h5py.File(path, 'r') - if "edges" in f: - return path - return None - - def _constrained_hill(K_half, y): K_half_fourth = K_half**4 y_fourth = y**4 @@ -185,6 +168,23 @@ def has_property(self, field_name): """Checks whether source data has the given additional field. """ + @staticmethod + def _get_sonata_circuit(path): + """Returns a SONATA edge file in path if present + """ + if os.path.isdir(path): + filename = os.path.join(path, "edges.sonata") + if os.path.exists(filename): + return filename + elif path.endswith(".sonata"): + return path + elif path.endswith(".h5"): + import h5py + f = h5py.File(path, 'r') + if "edges" in f: + return path + return None + @classmethod def create(cls, syn_src, conn_type=SYNAPSES, population=None, *args, **kw): """Instantiates a synapse reader, giving preference to SonataReader @@ -194,7 +194,7 @@ def create(cls, syn_src, conn_type=SYNAPSES, population=None, *args, **kw): return cls(syn_src, conn_type, population, *args, **kw) kw["verbose"] = (MPI.rank == 0) - if fn := _get_sonata_circuit(syn_src): + if fn := cls._get_sonata_circuit(syn_src): log_verbose("[SynReader] Using SonataReader.") return SonataReader(fn, conn_type, population, **kw) diff --git a/tests/simulations/mini_thalamus_sonata/gapjunction/README b/tests/simulations/mini_thalamus_sonata/gapjunction/README new file mode 100644 index 00000000..6853bf31 --- /dev/null +++ b/tests/simulations/mini_thalamus_sonata/gapjunction/README @@ -0,0 +1,2 @@ +Extract connections with target gid = 0 from the gap junction edges file in +/gpfs/bbp.cscs.ch/project/proj12/SIT/thalamus_sonata/networks/edges/thalamus_neurons__thalamus_neurons__electrical_synapse/edges.h5 diff --git a/tests/simulations/mini_thalamus_sonata/gapjunction/edges.h5 b/tests/simulations/mini_thalamus_sonata/gapjunction/edges.h5 new file mode 100644 index 0000000000000000000000000000000000000000..b15e58e5e62c8097fbd4ad287bb351141ddb356d GIT binary patch literal 169312 zcmeI*4|G)3oxt%s`5!_g5vjrUlptVYku~i?n?*D8()%cBvA{vjHt2GOG&4zX>150w zH&Ilyh_MSrx7C97R5-i!(4)tLt+@W_QaA@~a9b9RyXvB=o;|h(*Y(dHw~9yCeQ)l) z2{X)0oDe26lkY_4H}}5#-n;kn?!3A0`X;%f$^J(9lnbT^_fb+J3PeTZrFr^PKi1uy z7s)bdvwWR?;n8tG$33dJ@I-!usM7hBdi$~ER{{!xFOiu)3v^}WrM%FKhzTL4vzG9 zbDlNZyK?yo+wH@q-&N~=JN*8L>BnY0VzfY_^T5htRW7XG-=m!e-Yr+Zf&x6>xs?-&1I@hefs^a&I1Jr$Fm(5s`8OR zjq}p_GnF>|V(`yZy2Koy%T#*Mq*apVEEO>4fdhK{`l3FrbzeKX4N~WU;^FZ?o$j}$ zTgCQcOIN3(CuX1>Q%%#4`+Ad?tJCW>cPB5`q05!rsLGl8)#>!gn^k^=E>}M-rb02v zYMpK}E5s~MHR%yW>Ka2=%%iVxJsy>DuHK-zj?H(jd)+1O?$ub#ij*jHQf_`;zS>-g z-cEjAUY{>gLJT=WG&6h?Q$`0e|I15KUEX8 zs(P*Rn=93ZRi331XZV~3kD)eI$Ft_4der!MER`#+i1_OMcB;SIjM!1Vc{k|dr!7}a z7h>-y{oIrzB7gt_2q1s}0tg_000IagfWX-o@Q6Y+|DrGaN1q?6ij=Rcp5@$w;XbOx z`6c4~n{`!_%o8>LP?d4#MbxU1^nhM7GlH6TEEt~1YM!0EUe$W>-Z{z3)$7Fv<|Z$< zQZL?gQSx$`zBG;fs`deekY2x((XWp@;t3B8= zl6LQx@%QwHgMN3Z5Rr6yS7&cnWt^H01%iEM`|Cr2^{vjGHfCPaG7D0SsUJT*kM2xs ztC?-J=OpLH!GBYAikdeT;?5Lnnx5ik;&JN#5lP}v|H+7lnRa$^a>ZlJpQ9b+sx#-< z`EKO>@>%=Gyb*^S>>nw6bS%ngk1@Yrnm!(J$=Gq{c%QXXKR=JzcgK<}@HON54h?;0 zO>*%~dhxE>c0&2b%aWgIl(@fkn&n;shR$idJ3F`tL~em&xnv7dA1 zJUpo1#dQmkznx#xiw7=EUhZAJct>6Ga^^n8Jzq{hZTaPr)v~nHY)9@7F8D=3gJ1e@$cHF_JA#%MW+DUw1k^-<{Km&Ts#oTVFNJb82~&-`UpI9qbXUouNQ)Yfnq4)m__} z4>IkHso!vZO#M#F*Y!IsU)S%H{HXe!mapqKlHc3a*5b@;Y53FfHT-G$8ooQcJ{0tC z2!`5wI^1@uE-1v1o5gTsHe{NQtr{2vOJ)fdJH%mmd?f{Q}CHcXx*X1e}B`>#9 zm#hA2@^T%z+|l{T%Wcx--u;W@<#yWM*Kgj(qk@tOte8X3vZf=o|`s>}p;{*Nc333aC)*yfY0tg_000IagfB*sr zAb{@g8P_D8-i+CSZQ zk^RA&1@=GY&9LjwFSTp#DYJiJ)!6l0X4`KaonrqiSY==G$GP@_8AWzu={ff8-z>1( zo+-C;(S|l5fB*srAb!jBH!`fEswSBmb;ETEpIQ|E_Z%Ng#j#0tg_0KrRU!5{JdB z;uYuTS#exEFMcmhh}*;#ald$4+#y~N$DHLn5I_I{1Q0+Vw*(T+7wnoAznwc~#Gfxv zchkpThg<{@KmY**5Xf$UlDUt0KdLGAc9mPc*?+vp`=yf?`wmk_dVrX+<)Br-pfAgy>A_~ zrh9i<`?lZWTXg6zN3M87(w`Rp}&5mIF z`TUaj#mPkg0R#|00D&A3xOb=fIPA-mm;zgE_uFORVW!FmAbgw6_NakW0jFO|8*6c7Eh0)&xt$>u~5gQI$ojUavfLexJJjTbX=<=0RaRM zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~fr$`Ebiduv;`p`cdo=$2c6srO zlZyZX2q1s}0y!XXLfj^{i2KFU;tuhGI3^B@L(acf#VgLwv*NgTUi>}>T1gWSKmY** z5J2Emfkg8KiT=LrogQA0R#|00D;^Rcy%D?y}x;<_rABM`<68d-=z=snWz^)(zm;SIdL*jMV`=l!Ur*xOZZ`8qH9iFezZ7yAxe8umW?@f6>x{nGcu z=A+)+QlT{nAb$UP*#~S1>R&15K=Uyit zS$w^`yR1?EJU=X#e$X%f^B*pg|9&zo10OcXzpLLWi)UXaD?fL=eBi5Jmo1CKvSxq3 zd}a0JGLzawT@XM30R#|0009ILKmY**5Xd%xROj=n7BZioZO54UB7gt_2q18#0`nHt z*q6LF+kWh_GP`^4OnYtla*_B{lXhPt3Oe=Ylf3wtJ?1 z=%#sg`%jDQKQv6U-`QVmZ+WTQKGS-zJ^~0JfB*srAbhPI{ZCtm>y5ru4$TE{hL;wK<5I_I{1Q0*~0R#|00D-Xyc$~j{?{Qa+ zJh`DNvLTO0Wt^)wYhu2rh!m|T5yJV;d03I>LMP>}k(ZYjS+PWKC*RY2gZe)JcSg#M zJjJF*mQ3WUv+pr9IXs+$_}ThL-^Og}9~0R(**5|RATXW+pJv!&Jli&&+tR}$VeV$g z*BdX;u~5e%9gB5TpMgS@>hu&H%XFNoW4Vs!=r~Qs3LPtT^f;p|w=QawCPbw(9&;ac zH>#Sgyjex}J`Hy?7khXkg`KB?TUEJglR=zR0!>RASGo;u)~&ABF|7NKPKxf%-mcc5 zzo*mR)6&%*>@oQ&<>sprCf~eIs!c{rOw@nh)%&f`{c^Pa``TvuZw&3EN+#;Rb-Mo; zj8JDF=wBajJ8D-;s6E)N7eo?nZZOd97BJ;U>%UhnUtxQkPXTjYAJqNUtQXsltM!KK zx2yhY%B|DsO?Rn$v;WBwJ)$V`Ii{Z6PpaBT(>-eXK$y=-qZ6ARo`-PO`fS&kwOiD- zc101NtJijq%fsJ%#OVj&sO9NO#P+MB-#0CfUxR9$UHil2<>IfM`?=S>~>r-`PXCYpMhXzFRAsi%pi zo+g@lnrP~kY4tJnGJd9hCf`J}+(a|JHqnf;O*G?e6V15WL^J+2(Tu}QG~;m-V~?k) zClLVz5I`V30_h)D4rcVYGST&R?DfZJ*V`pOQX`S-^c+U^l3@aguD5r^T)&R@^>+H- t&mFg`p1v#U2xRJY;)gN*QI&QQ)jDJ4kIj`tY>CO4PJh2D8OY>*_5ZvtCb9qk literal 0 HcmV?d00001 diff --git a/tests/test_synapse_reader.py b/tests/test_synapse_reader.py new file mode 100644 index 00000000..b44d3feb --- /dev/null +++ b/tests/test_synapse_reader.py @@ -0,0 +1,48 @@ +import numpy as np +import numpy.testing as npt +import os +import pytest +from neurodamus.gap_junction import GapJunctionSynapseReader +from pathlib import Path + + +pytestmark = [ + pytest.mark.forked, + pytest.mark.skipif( + not os.environ.get("NEURODAMUS_NEOCORTEX_ROOT"), + reason="Test requires loading a neocortex model to run" + ) +] + + +def test_gapjunction_synreaderNRN(): + nrn_file = "/gpfs/bbp.cscs.ch/project/proj12/jenkins/cellular/circuit-scx-v5-gapjunctions/gap_junctions/nrn_gj.h5" # noqa + nrn_reader = GapJunctionSynapseReader.create(nrn_file, 1) + syn_params_nrn = nrn_reader._load_synapse_parameters(100124) + # check reading of sgid, junction_id_pre and junction_id_post + ref_sgids = np.array([94669., 94723., 95634., 95823., 96581., + 97338., 97455., 98139., 98432., 100725., + 101360., 101506., 101696., 101696., 191567.]) + ref_junction_id_pre = np.array([735., 736., 29., 36., 51., + 77., 744., 134., 148., 286., + 322., 337., 355., 356., 681.]) + ref_junction_id_post = np.array([1251., 1259., 617., 1354., 1002., + 1756., 1027., 924., 709., 624., + 1050., 521., 592., 593., 590.]) + npt.assert_allclose(syn_params_nrn.sgid, ref_sgids) + npt.assert_allclose(syn_params_nrn.D, ref_junction_id_pre) + npt.assert_allclose(syn_params_nrn.F, ref_junction_id_post) + + +def test_gapjunction_sonata_reader(): + SIM_DIR = Path(__file__).parent.absolute() / "simulations" + sonata_file = str(SIM_DIR / "mini_thalamus_sonata/gapjunction/edges.h5") + sonata_reader = GapJunctionSynapseReader.create(sonata_file, 1) + syn_params_sonata = sonata_reader._load_synapse_parameters(1) + ref_junction_id_pre = np.array([10257., 43930., 226003., 298841., 324744., + 1094745., 1167632., 1172523., 1260104.]) + ref_junction_id_post = np.array([14., 52., 71., 76., 78., 84., 89., 90., 93.]) + ref_weight = np.array([0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2]) + npt.assert_allclose(syn_params_sonata.D, ref_junction_id_pre) + npt.assert_allclose(syn_params_sonata.F, ref_junction_id_post) + npt.assert_allclose(syn_params_sonata.weight, ref_weight) From f3a139355bdfb612de4d94672035d7b492884aff Mon Sep 17 00:00:00 2001 From: Erik Heeren Date: Tue, 18 Jul 2023 17:44:17 +0200 Subject: [PATCH 2/7] Update pipeline for GitHub pull request triggering --- .gitlab-ci.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef6f04e7..8ad47a53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,13 +9,10 @@ include: workflow: rules: - # Rules taken from official docs to avoid duplicated pipelines - - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS' - when: never - - when: always + - if: '$CI_EXTERNAL_PULL_REQUEST_IID' + # re-define because the imported rules will be ignored + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: $CI_PIPELINE_SOURCE == "web" # run for manually started pipelines from -/pipelines/new variables: BLUECONFIGS_BRANCH: @@ -61,7 +58,7 @@ blueconfig_tests: needs: - set_alt_branches rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_EXTERNAL_PULL_REQUEST_IID' changes: - neurodamus/**/* - when: manual From 3ec31e06127f6bc07247f65155b7852ae4df1ea4 Mon Sep 17 00:00:00 2001 From: Erik Heeren Date: Wed, 19 Jul 2023 12:15:04 +0200 Subject: [PATCH 3/7] Fixes for documentation builds --- .readthedocs.yml | 21 +++++++++++++++++++++ README.rst | 2 +- docs/api/modules.rst | 4 ++++ docs/api/subpackages/io.rst | 2 +- docs/api/subpackages/top.rst | 1 + docs/conf.py | 2 +- docs/development.rst | 4 ++-- docs/index.rst | 1 + neurodamus/cell_distributor.py | 3 +++ requirements_docs.txt | 3 +++ 10 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 requirements_docs.txt diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..d44da486 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,21 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +sphinx: + configuration: docs/conf.py + fail_on_warning: true + +build: + os: ubuntu-22.04 + tools: + python: "3" + +python: + install: + - method: pip + path: . + - requirements: requirements_docs.txt diff --git a/README.rst b/README.rst index a32c3368..219dcc4d 100644 --- a/README.rst +++ b/README.rst @@ -44,7 +44,7 @@ Build special with mod files mkdir mods cp -r mod-files-from-released-circuit mods/ cp -r neurodamus-install-prefix/share/mod/* mods/ - nrnivmodl -incflags '-I ' -loadflags '-L '' mod + nrnivmodl -incflags '-I ' -loadflags '-L ' mod Examples ======== diff --git a/docs/api/modules.rst b/docs/api/modules.rst index a57ae55c..063b3e83 100644 --- a/docs/api/modules.rst +++ b/docs/api/modules.rst @@ -8,3 +8,7 @@ neurodamus neurodamus.core neurodamus.io neurodamus.utils + subpackages/core + subpackages/io + subpackages/top + subpackages/utils diff --git a/docs/api/subpackages/io.rst b/docs/api/subpackages/io.rst index bc1cbad4..381da353 100644 --- a/docs/api/subpackages/io.rst +++ b/docs/api/subpackages/io.rst @@ -16,7 +16,7 @@ neurodamus.io.cell\_readers load_mvd3 load_ncs - load_nodes + load_nodes_mvd3 load_combo_metypes split_round_robin diff --git a/docs/api/subpackages/top.rst b/docs/api/subpackages/top.rst index 319f44d7..8c9aa76f 100644 --- a/docs/api/subpackages/top.rst +++ b/docs/api/subpackages/top.rst @@ -54,6 +54,7 @@ neurodamus.connection\_manager .. autoclass:: SynapseRuleManager :members: + :noindex: :inherited-members: diff --git a/docs/conf.py b/docs/conf.py index ebd47ef9..7fc66571 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,7 +9,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath()) +sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ diff --git a/docs/development.rst b/docs/development.rst index da4f36cc..4a4e9767 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -180,7 +180,7 @@ You can clone ``neurodamus-py`` and use ``spack develop`` as follows:: spack env deactivate spack env activate -p neurodamus - This tends to manifest itself in import errors: ModuleNotFoundError: No module named 'neurodamus'. + # This tends to manifest itself in import errors: ModuleNotFoundError: No module named 'neurodamus'. .. note:: @@ -190,7 +190,7 @@ You can clone ``neurodamus-py`` and use ``spack develop`` as follows:: spack uninstall neurodamus-neocortex spack install neurodamus-neocortex - This is required since changing the sources in neurodamus-py doesn't trigger Spack to reinstall neurodamus-neocortex. + # This is required since changing the sources in neurodamus-py doesn't trigger Spack to reinstall neurodamus-neocortex. .. note:: diff --git a/docs/index.rst b/docs/index.rst index 4ad9024c..7a52e41b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ Contents Changelog Module Reference Developer Documentation + Developing a Custom Neurodamus Indices and tables diff --git a/neurodamus/cell_distributor.py b/neurodamus/cell_distributor.py index 682b96d9..38b2fe57 100644 --- a/neurodamus/cell_distributor.py +++ b/neurodamus/cell_distributor.py @@ -103,12 +103,14 @@ class CellManagerBase(_CellManager): CellType = NotImplemented # please override """The underlying Cell type class + signature: __init__(self, gid, cell_info, circuit_conf) """ _node_loader = None """Default function implementing the loading of nodes data, a.k.a. MVD + signature: load(circuit_conf, gidvec, stride=1, stride_offset=0) """ @@ -660,6 +662,7 @@ class LoadBalance: NOTE: Given the heavy costs of computing load balance, some state files are created which allow the balance info to be reused. These are + - cx_{TARGET}.dat: File with complexity information for the cells of a given target - cx_{TARGET}.{CPU_COUNT}.dat: The file assigning cells/pieces to individual CPUs ranks. diff --git a/requirements_docs.txt b/requirements_docs.txt new file mode 100644 index 00000000..20f6100a --- /dev/null +++ b/requirements_docs.txt @@ -0,0 +1,3 @@ +sphinx>=2.0.0 +sphinx-bluebrain-theme +sphinx-autorun From f6665abf4e40a0171de90d7ab1e7a62564e893ae Mon Sep 17 00:00:00 2001 From: Erik Heeren Date: Mon, 24 Jul 2023 09:14:06 +0200 Subject: [PATCH 4/7] Include gitlab-upload-logs --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ad47a53..c921eadb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,8 @@ include: - /ci/jobs/publish-package.yml - /ci/jobs/docs.yml - /ci/jobs/publish-docs.yml + - project: hpc/gitlab-upload-logs + file: enable-upload.yml workflow: rules: From f388c355fa0bd5128f5a1f2754461e109e150273 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Tue, 25 Jul 2023 11:27:36 +0200 Subject: [PATCH 5/7] Update change log for 2.16.0 --- CHANGES.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 67d7b165..1f707811 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,18 @@ Change Log ========== +Version 2.16.0 +============== +**26 July 2023** + +*New Features* + * [BBPBGLIB-1036] Pure SONATA reader for gap junctions + * [BBPBGLIB-984] Option to keep Cell axon during init + +*Improvements* + * [BBPBGLIB-1035] Drop Synapsetool. Migrate all synapses loading to libsonata (no syn2 support) + + Version 2.15.3 =============== **14 July 2023** From 9606979655becd2790f7c949dc4e4a0205545f79 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Wed, 2 Aug 2023 11:57:47 +0200 Subject: [PATCH 6/7] [BBPBGLIB-1044] Sonata Replay (#8) * implement a simple reader for Sonata spikes * Integrate with replay manager --- neurodamus/node.py | 17 +++++--- neurodamus/replay.py | 28 ++++++++++--- neurodamus/target_manager.py | 2 +- tests/sample_data/out.h5 | Bin 0 -> 9744 bytes tests/scientific/test_replay.py | 70 ++++++++++++++++++++++++++++---- tests/test_replay_manager.py | 19 +++++++++ 6 files changed, 114 insertions(+), 22 deletions(-) create mode 100644 tests/sample_data/out.h5 create mode 100644 tests/test_replay_manager.py diff --git a/neurodamus/node.py b/neurodamus/node.py index 5a3aee7d..b207ad41 100644 --- a/neurodamus/node.py +++ b/neurodamus/node.py @@ -25,7 +25,7 @@ from .cell_distributor import LoadBalance, LoadBalanceMode from .connection_manager import SynapseRuleManager, edge_node_pop_names from .gap_junction import GapJunctionManager -from .replay import SpikeManager +from .replay import MissingSpikesPopulationError, SpikeManager from .stimulus_manager import StimulusManager from .modification_manager import ModificationManager from .neuromodulation_manager import NeuroModulationManager @@ -734,7 +734,6 @@ def enable_replay(self): def _enable_replay(self, source, target, stim_conf, tshift=.0, delay=.0, connectivity_type=None): spike_filepath = find_input_file(stim_conf["SpikeFile"]) - spike_manager = SpikeManager(spike_filepath, tshift) # Disposable ptype_cls = EngineBase.connection_types.get(connectivity_type) src_target = self.target_manager.get_target(source) dst_target = self.target_manager.get_target(target) @@ -743,6 +742,13 @@ def _enable_replay(self, source, target, stim_conf, tshift=.0, delay=.0, pop_offsets, alias_pop = CircuitManager.read_population_offsets(read_virtual_pop=True) for src_pop in src_target.population_names: + try: + log_verbose("Loading replay spikes for population '%s'", src_pop) + spike_manager = SpikeManager(spike_filepath, tshift, src_pop) # Disposable + except MissingSpikesPopulationError: + logging.info(" > No replay for src population: '%s'", src_pop) + continue + for dst_pop in dst_target.population_names: src_pop_str, dst_pop_str = src_pop or "(base)", dst_pop or "(base)" @@ -751,10 +757,9 @@ def _enable_replay(self, source, target, stim_conf, tshift=.0, delay=.0, else pop_offsets[alias_pop[src_pop]] else: conn_manager = self._circuits.get_edge_manager(src_pop, dst_pop, ptype_cls) - if not conn_manager: - logging.error("No edge manager found among populations %s -> %s", - src_pop_str, dst_pop_str) - raise ConfigurationError("Unknown replay pathway. Check Source / Target") + if not conn_manager and SimConfig.cli_options.restrict_connectivity >= 1: + continue + assert conn_manager, f"Missing edge manager for {src_pop_str} -> {dst_pop_str}" src_pop_offset = conn_manager.src_pop_offset logging.info("=> Population pathway %s -> %s. Source offset: %d", diff --git a/neurodamus/replay.py b/neurodamus/replay.py index 14d0b02d..eaae6e2b 100644 --- a/neurodamus/replay.py +++ b/neurodamus/replay.py @@ -3,6 +3,7 @@ """ from __future__ import absolute_import import os +import h5py import logging import numpy from .utils.logging import log_verbose @@ -10,7 +11,7 @@ from .utils.timeit import timeit -class SpikeManager(object): +class SpikeManager: """ Holds and manages gid spike time information, specially for Replay. A SynapseReplay stim can be used for a single gid that has all the synapses instantiated. @@ -22,7 +23,7 @@ class SpikeManager(object): _ascii_spike_dtype = [('time', 'double'), ('gid', 'uint32')] @timeit(name="Replay init") - def __init__(self, spike_filename, delay=0): + def __init__(self, spike_filename, delay=0, population=None): """Constructor for SynapseReplay. Args: @@ -32,10 +33,10 @@ def __init__(self, spike_filename, delay=0): """ self._gid_fire_events = None # Nd.distributedSpikes = 0 # Wonder the effects of this - self.open_spike_file(spike_filename, delay) + self.open_spike_file(spike_filename, delay, population) # - def open_spike_file(self, filename, delay): + def open_spike_file(self, filename, delay, population=None): """Opens a given spike file. Args: @@ -46,7 +47,9 @@ def open_spike_file(self, filename, delay): # TODO: filename should be able to handle relative paths, # using the Run.CurrentDir as an initial path # _read_spikes_xxx shall return numpy arrays - if filename.endswith(".bin"): + if filename.endswith(".h5"): + tvec, gidvec = self._read_spikes_sonata(filename, population) + elif filename.endswith(".bin"): tvec, gidvec = self._read_spikes_binary(filename) else: tvec, gidvec = self._read_spikes_ascii(filename) @@ -56,6 +59,15 @@ def open_spike_file(self, filename, delay): self._store_events(tvec, gidvec) + @classmethod + def _read_spikes_sonata(cls, filename, population): + spikes_file = h5py.File(filename, "r") + # File should have been validated earlier + spikes = spikes_file.get("spikes/" + population) + if spikes is None: + raise MissingSpikesPopulationError("Spikes population not found: " + population) + return spikes["timestamps"][...], spikes["node_ids"][...] + @classmethod def _read_spikes_ascii(cls, filename): log_verbose("Reading ascii spike file %s", filename) @@ -99,7 +111,6 @@ def _read_spikes_binary(filename): return tvec, gidvec # - @timeit(name="BinEvents") def _store_events(self, tvec, gidvec): """Stores the events in the _gid_fire_events GroupedMultiMap. @@ -150,3 +161,8 @@ def dump_ascii(self, f, gid_offset=None): numpy.savetxt(f, expanded_ds, fmt='%.3lf\t%d') log_verbose("Replay: Written %d entries", len(expanded_ds)) + + +class MissingSpikesPopulationError(Exception): + """An exception triggered when a given node population is not found, we may want to handle""" + pass diff --git a/neurodamus/target_manager.py b/neurodamus/target_manager.py index 82a24208..133ea14c 100644 --- a/neurodamus/target_manager.py +++ b/neurodamus/target_manager.py @@ -131,7 +131,7 @@ def _is_sonata_file(file_name): def _try_open_start_target(self, circuit): start_target_file = os.path.join(circuit.CircuitPath, "start.target") if not os.path.isfile(start_target_file): - logging.warning("Circuit %s start.target not available! Skipping", circuit._name) + log_verbose("Circuit %s start.target not available! Skipping", circuit._name) else: self.parser.open(start_target_file, False) self._has_hoc_targets = True diff --git a/tests/sample_data/out.h5 b/tests/sample_data/out.h5 new file mode 100644 index 0000000000000000000000000000000000000000..01c5bb8c15f4f7b771cdd40052bba65ee0829283 GIT binary patch literal 9744 zcmeHNPe_zO6rXQ5b=5UBq}C$Y%*;q|VYg5z_xWda529On@DgSCZ#CCN*RV@?=wc-Z zU0OkiPJ0SZg3uu-!e9%-OVGtbgvY>BCwuRkd8VH(o1zEVnZ@tT`!nyG`OTX*A2Ut| zTUrb9&gR)14#CBvDn*sDGN|~W(Un$1&`*deB|9VBq6F)P-4vVBBBc|Xs*ql_b17Z1 zkXEybU$Dta*X=-nthp&uWhU0MWbI#XS)*wtAP@{RhG?AKYOow@D=K~%gYjs1xNn>y zNtPfh4oFkQZ^$kkMpayKN_qYea@<6H<<2 zV1s?qI~|=Z6vLc1t5n>wUuzui(|yL68$xb$Ja0v6AFIum5JVLc>GD9ZKpL{2i~&#& z)I`9V2l#vHmD(Q|rUk!6m3QaLb$%&3MT zJzrFErgoeG($GA}y%0=`EaXUTGi+u<2l}Ck-AJg+?lR@3F+NVI)+Ee(zSwJBSMQx{ zxi{s!pD5g0pQ%#UOwL?9zU1Gi9`bsGx`Ev+RQw>M zYhd9ux?lCc+f{T|>4UXww6F8Q9P$0v;A0)OtFD7ZqRx6)e~8W8^)N}a53NX#6UnTg4 ziAXp`TT&^mJ6c=Y^BfuS;-PgvMM^9rYisUoH+9=&q~rWBvbFP(REY0-y+|PRQzM`e z&=n}RQD;M4(y_8Ea~o3YAd86pQ{}s2y|Bu0fAKa zDeL7sogemn3f}`*??aC29w6O$_s-Gc`8A%z(fJx3ucbThrnYv&`X4tWO@H3Kzz6Ko leAkoL<A8tmnE`aO~<{Vn64) Date: Thu, 3 Aug 2023 14:58:31 +0200 Subject: [PATCH 7/7] Update CoreConfig.mod for latest CoreNEURON changes (#18) - Parameter passed to solve_core should contain name of executable/process name as first argument in argv[]. - This is required for changes in neuronsimulator/nrn/pull/2429. Co-authored-by: Pramod S Kumbhar --- core/mod/CoreConfig.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/mod/CoreConfig.mod b/core/mod/CoreConfig.mod index 979670a0..a3d84714 100644 --- a/core/mod/CoreConfig.mod +++ b/core/mod/CoreConfig.mod @@ -282,9 +282,9 @@ VERBATIM char* simConf = (char*)alloca(strlen(outputdir) + CONFIG_FILENAME_TOTAL_LEN_MAX); sprintf(simConf, "%s/%s", outputdir, SIM_CONFIG_FILE); # if CORENRN_CLI11 - char *argv[] = {"", "--read-config", simConf, "--skip-mpi-finalize", "--mpi", NULL, NULL, NULL, NULL}; + char *argv[] = {"coreneuron", "--read-config", simConf, "--skip-mpi-finalize", "--mpi", NULL, NULL, NULL, NULL}; # else - char *argv[] = {"", "--read-config", simConf, "--skip-mpi-finalize", "-mpi", NULL, NULL, NULL, NULL}; + char *argv[] = {"coreneuron", "--read-config", simConf, "--skip-mpi-finalize", "-mpi", NULL, NULL, NULL, NULL}; # endif int argc = 5; int argIndex=1;