From c67de2ceb27f1836f3d8819b7950f24abb4bb0f9 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Mon, 29 Jan 2024 12:44:55 -0500 Subject: [PATCH 01/72] extend with some minimal inheritance --- src/nwb_benchmarks/benchmarks/streaming.py | 129 +++++------------- src/nwb_benchmarks/core/__init__.py | 0 .../core/base_benchmarks/__init__.py | 1 + .../_time_series_benchmarks.py | 10 ++ src/nwb_benchmarks/core/common/__init__.py | 1 + .../core/common/_nwbfile_read.py | 38 ++++++ 6 files changed, 81 insertions(+), 98 deletions(-) create mode 100644 src/nwb_benchmarks/core/__init__.py create mode 100644 src/nwb_benchmarks/core/base_benchmarks/__init__.py create mode 100644 src/nwb_benchmarks/core/base_benchmarks/_time_series_benchmarks.py create mode 100644 src/nwb_benchmarks/core/common/__init__.py create mode 100644 src/nwb_benchmarks/core/common/_nwbfile_read.py diff --git a/src/nwb_benchmarks/benchmarks/streaming.py b/src/nwb_benchmarks/benchmarks/streaming.py index f3308dd..f6297ba 100644 --- a/src/nwb_benchmarks/benchmarks/streaming.py +++ b/src/nwb_benchmarks/benchmarks/streaming.py @@ -1,124 +1,57 @@ -"""Basic benchmarks for NWB.""" -import warnings +"""Basic benchmarks for stream NWB files and their contents.""" +from ..core.base_benchmarks import AcquisitionTimeSeriesSliceBenchmark +from ..core.common import read_nwbfile_fsspec, read_nwbfile_remfile, read_nwbfile_ros3 -import fsspec -import h5py -import pynwb -import remfile -from fsspec.asyn import reset_lock - -# Useful if running in verbose mode -warnings.filterwarnings(action="ignore", message="No cached namespaces found in .*") -warnings.filterwarnings(action="ignore", message="Ignoring cached namespace .*") - - -class FileReadStreaming: - """A basic benchmark for streaming an NWB file from the DANDI archive.""" +class FsspecNoCacheFileReadBenchmark: repeat = 1 + s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" def setup(self): - # Random IBL raw data file; not that many groups - self.s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - - def time_fsspec_no_cache(self): - reset_lock() - fsspec.get_filesystem_class("https").clear_instance_cache() - filesystem = fsspec.filesystem("https") - - with filesystem.open(path=self.s3_url, mode="rb") as byte_stream: - with h5py.File(name=byte_stream) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() + self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_fsspec(s3_url=self.s3_url) - # Must be done at this level since teardown occurs outside of repetitions - # reset_lock() - # fsspec.get_filesystem_class("https").clear_instance_cache() - - def time_ros3(self): - ros3_form = self.s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") - with pynwb.NWBHDF5IO(path=ros3_form, mode="r", load_namespaces=True, driver="ros3") as io: - nwbfile = io.read() - - def time_remfile(self): - byte_stream = remfile.File(url=self.s3_url) - with h5py.File(name=byte_stream) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() - - # def teardown(self): - # reset_lock() - # fsspec.get_filesystem_class("https").clear_instance_cache() +class RemfileFileReadBenchmark: + repeat = 1 + s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" -class ElectricalSeriesStreamingROS3: - """ - A basic benchmark for streaming raw ecephys data. + def time_file_read(self): + self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_remfile(s3_url=self.s3_url) - Needs separate setup per class to only time slicing operation. - """ +class Ros3FileReadBenchmark: repeat = 1 + s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" def setup(self): - self.s3_url = "s3://dandiarchive/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - self.acquisition_path = "ElectricalSeriesAp" - self.slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - self.io = pynwb.NWBHDF5IO(path=self.s3_url, mode="r", load_namespaces=True, driver="ros3") - self.nwbfile = self.io.read() + self.nwbfile, self.io = read_nwbfile_ros3(s3_url=self.s3_url) - def time_ros3(self): - self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] - - -class ElectricalSeriesStreamingFsspec: - """ - A basic benchmark for streaming raw ecephys data. - - Needs separate setup per class to only time slicing operation. - """ +class FsspecNoCacheSliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): repeat = 1 + s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" + acquisition_path = "ElectricalSeriesAp" + slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB def setup(self): - self.s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - self.acquisition_path = "ElectricalSeriesAp" - self.slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - - reset_lock() - fsspec.get_filesystem_class("https").clear_instance_cache() + self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_fsspec(s3_url=self.s3_url) - self.filesystem = fsspec.filesystem("https") - self.byte_stream = self.filesystem.open(path=self.s3_url, mode="rb") - self.file = h5py.File(name=self.byte_stream) - self.io = pynwb.NWBHDF5IO(file=self.file, mode="r", load_namespaces=True) - self.nwbfile = self.io.read() - - def time_fsspec_no_cache(self): - self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] - - # def teardown(self): - # reset_lock() - # fsspec.get_filesystem_class("https").clear_instance_cache() +class Ros3SliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): + repeat = 1 + s3_url = "s3://dandiarchive/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" + acquisition_path = "ElectricalSeriesAp" + slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB -class ElectricalSeriesStreamingRemfile: - """ - A basic benchmark for streaming raw ecephys data. + def setup(self): + self.nwbfile, self.io = read_nwbfile_ros3(s3_url=self.s3_url) - Needs separate setup per class to only time slicing operation. - """ +class RemfileSliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): repeat = 1 + s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" + acquisition_path = "ElectricalSeriesAp" + slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB def setup(self): - self.s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - self.acquisition_path = "ElectricalSeriesAp" - self.slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - self.byte_stream = remfile.File(url=self.s3_url) - self.file = h5py.File(name=self.byte_stream) - self.io = pynwb.NWBHDF5IO(file=self.file, mode="r", load_namespaces=True) - self.nwbfile = self.io.read() - - def time_remfile(self): - self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] + self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_remfile(s3_url=self.s3_url) diff --git a/src/nwb_benchmarks/core/__init__.py b/src/nwb_benchmarks/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nwb_benchmarks/core/base_benchmarks/__init__.py b/src/nwb_benchmarks/core/base_benchmarks/__init__.py new file mode 100644 index 0000000..8d721a4 --- /dev/null +++ b/src/nwb_benchmarks/core/base_benchmarks/__init__.py @@ -0,0 +1 @@ +from ._time_series_benchmarks import AcquisitionTimeSeriesSliceBenchmark diff --git a/src/nwb_benchmarks/core/base_benchmarks/_time_series_benchmarks.py b/src/nwb_benchmarks/core/base_benchmarks/_time_series_benchmarks.py new file mode 100644 index 0000000..37edf03 --- /dev/null +++ b/src/nwb_benchmarks/core/base_benchmarks/_time_series_benchmarks.py @@ -0,0 +1,10 @@ +class AcquisitionTimeSeriesSliceBenchmark: + s3_url: str + acquisition_path: str + slice_range: tuple # TODO: enhance annotation + + def time_slice(self): + # Store as self._temp to avoid tracking garbage collection as well + self._temp = self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] + + # TODO: add additional trackers here diff --git a/src/nwb_benchmarks/core/common/__init__.py b/src/nwb_benchmarks/core/common/__init__.py new file mode 100644 index 0000000..ec5a1e2 --- /dev/null +++ b/src/nwb_benchmarks/core/common/__init__.py @@ -0,0 +1 @@ +from ._nwbfile_read import read_nwbfile_fsspec, read_nwbfile_remfile, read_nwbfile_ros3 diff --git a/src/nwb_benchmarks/core/common/_nwbfile_read.py b/src/nwb_benchmarks/core/common/_nwbfile_read.py new file mode 100644 index 0000000..1d222c9 --- /dev/null +++ b/src/nwb_benchmarks/core/common/_nwbfile_read.py @@ -0,0 +1,38 @@ +import warnings + +import fsspec +import h5py +import pynwb +import remfile +from fsspec.asyn import reset_lock + +# Useful if running in verbose mode +warnings.filterwarnings(action="ignore", message="No cached namespaces found in .*") +warnings.filterwarnings(action="ignore", message="Ignoring cached namespace .*") + + +def read_nwbfile_fsspec(s3_url: str) -> tuple: # TODO: enhance annotation, add cache option + reset_lock() + fsspec.get_filesystem_class("https").clear_instance_cache() + filesystem = fsspec.filesystem("https") + + byte_stream = filesystem.open(path=s3_url, mode="rb") + file = h5py.File(name=byte_stream) + io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) + nwbfile = io.read() + return (nwbfile, io, file, byte_stream) + + +def read_nwbfile_ros3(s3_url: str) -> tuple: # TODO: enhance annotation + ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") + io = pynwb.NWBHDF5IO(path=ros3_form, mode="r", load_namespaces=True, driver="ros3") + nwbfile = io.read() + return (nwbfile, io) + + +def read_nwbfile_remfile(s3_url: str) -> tuple: # TODO: enhance annotation + byte_stream = remfile.File(url=s3_url) + file = h5py.File(name=byte_stream) + io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) + nwbfile = io.read() + return (nwbfile, io, file, byte_stream) From 1875cbb489687890bc8dcdfa21e9c03fc81a35f0 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Mon, 29 Jan 2024 12:48:52 -0500 Subject: [PATCH 02/72] remove the inheritance --- src/nwb_benchmarks/benchmarks/streaming.py | 23 ++++++++++++++----- .../core/base_benchmarks/__init__.py | 1 - .../_time_series_benchmarks.py | 10 -------- 3 files changed, 17 insertions(+), 17 deletions(-) delete mode 100644 src/nwb_benchmarks/core/base_benchmarks/__init__.py delete mode 100644 src/nwb_benchmarks/core/base_benchmarks/_time_series_benchmarks.py diff --git a/src/nwb_benchmarks/benchmarks/streaming.py b/src/nwb_benchmarks/benchmarks/streaming.py index f6297ba..9074294 100644 --- a/src/nwb_benchmarks/benchmarks/streaming.py +++ b/src/nwb_benchmarks/benchmarks/streaming.py @@ -1,5 +1,4 @@ """Basic benchmarks for stream NWB files and their contents.""" -from ..core.base_benchmarks import AcquisitionTimeSeriesSliceBenchmark from ..core.common import read_nwbfile_fsspec, read_nwbfile_remfile, read_nwbfile_ros3 @@ -7,7 +6,7 @@ class FsspecNoCacheFileReadBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - def setup(self): + def time_file_read(self): self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_fsspec(s3_url=self.s3_url) @@ -23,11 +22,11 @@ class Ros3FileReadBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - def setup(self): + def time_file_read(self): self.nwbfile, self.io = read_nwbfile_ros3(s3_url=self.s3_url) -class FsspecNoCacheSliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): +class FsspecNoCacheSliceBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" acquisition_path = "ElectricalSeriesAp" @@ -36,8 +35,12 @@ class FsspecNoCacheSliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): def setup(self): self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_fsspec(s3_url=self.s3_url) + def time_slice(self): + # Store as self._temp to avoid tracking garbage collection as well + self._temp = self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] -class Ros3SliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): + +class Ros3SliceBenchmark: repeat = 1 s3_url = "s3://dandiarchive/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" acquisition_path = "ElectricalSeriesAp" @@ -46,8 +49,12 @@ class Ros3SliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): def setup(self): self.nwbfile, self.io = read_nwbfile_ros3(s3_url=self.s3_url) + def time_slice(self): + # Store as self._temp to avoid tracking garbage collection as well + self._temp = self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] + -class RemfileSliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): +class RemfileSliceBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" acquisition_path = "ElectricalSeriesAp" @@ -55,3 +62,7 @@ class RemfileSliceBenchmark(AcquisitionTimeSeriesSliceBenchmark): def setup(self): self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_remfile(s3_url=self.s3_url) + + def time_slice(self): + # Store as self._temp to avoid tracking garbage collection as well + self._temp = self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] diff --git a/src/nwb_benchmarks/core/base_benchmarks/__init__.py b/src/nwb_benchmarks/core/base_benchmarks/__init__.py deleted file mode 100644 index 8d721a4..0000000 --- a/src/nwb_benchmarks/core/base_benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from ._time_series_benchmarks import AcquisitionTimeSeriesSliceBenchmark diff --git a/src/nwb_benchmarks/core/base_benchmarks/_time_series_benchmarks.py b/src/nwb_benchmarks/core/base_benchmarks/_time_series_benchmarks.py deleted file mode 100644 index 37edf03..0000000 --- a/src/nwb_benchmarks/core/base_benchmarks/_time_series_benchmarks.py +++ /dev/null @@ -1,10 +0,0 @@ -class AcquisitionTimeSeriesSliceBenchmark: - s3_url: str - acquisition_path: str - slice_range: tuple # TODO: enhance annotation - - def time_slice(self): - # Store as self._temp to avoid tracking garbage collection as well - self._temp = self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] - - # TODO: add additional trackers here From 4bed307ca42987f2c4d69e6b26a17775b3cc9ec1 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Mon, 29 Jan 2024 12:49:40 -0500 Subject: [PATCH 03/72] fix name of test --- src/nwb_benchmarks/benchmarks/streaming.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/streaming.py b/src/nwb_benchmarks/benchmarks/streaming.py index f6297ba..f55b508 100644 --- a/src/nwb_benchmarks/benchmarks/streaming.py +++ b/src/nwb_benchmarks/benchmarks/streaming.py @@ -7,7 +7,7 @@ class FsspecNoCacheFileReadBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - def setup(self): + def time_file_read(self): self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_fsspec(s3_url=self.s3_url) @@ -23,7 +23,7 @@ class Ros3FileReadBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - def setup(self): + def time_file_read(self): self.nwbfile, self.io = read_nwbfile_ros3(s3_url=self.s3_url) From b10ff185a06b8353e6ddee634ea7f5b274312a56 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:36:50 -0500 Subject: [PATCH 04/72] from discussion (fix later) --- src/nwb_benchmarks/benchmarks/streaming.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/streaming.py b/src/nwb_benchmarks/benchmarks/streaming.py index 9074294..cd2bb96 100644 --- a/src/nwb_benchmarks/benchmarks/streaming.py +++ b/src/nwb_benchmarks/benchmarks/streaming.py @@ -2,6 +2,9 @@ from ..core.common import read_nwbfile_fsspec, read_nwbfile_remfile, read_nwbfile_ros3 +def slice_dataset(nwbfile, acquisition_path, slice_range): + return nwbfile.acquisition[self.acquisition_path].data[self.slice_range] + class FsspecNoCacheFileReadBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" @@ -37,7 +40,7 @@ def setup(self): def time_slice(self): # Store as self._temp to avoid tracking garbage collection as well - self._temp = self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] + self._temp = slice_dataset(nwbfile=self.nwbfile, ...) class Ros3SliceBenchmark: @@ -51,7 +54,7 @@ def setup(self): def time_slice(self): # Store as self._temp to avoid tracking garbage collection as well - self._temp = self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] + self._temp = slice_dataset(nwbfile=self.nwbfile, ...) class RemfileSliceBenchmark: @@ -65,4 +68,4 @@ def setup(self): def time_slice(self): # Store as self._temp to avoid tracking garbage collection as well - self._temp = self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] + self._temp = slice_dataset(nwbfile=self.nwbfile, ...) From ea018736a0e6e6f9e17edc998aec59beb5a1a81a Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Thu, 15 Feb 2024 18:12:51 -0500 Subject: [PATCH 05/72] mostly refactored --- .gitignore | 2 +- asv.conf.json | 4 +- environments/nwb_benchmarks.yaml | 2 + .../{streaming.py => ibl_chunking.py} | 57 +++++++++--- .../benchmarks/remote_file_reading.py | 71 +++++++++++++++ .../benchmarks/remote_slicing.py | 65 ++++++++++++++ src/nwb_benchmarks/command_line_interface.py | 4 +- src/nwb_benchmarks/core/__init__.py | 23 +++++ src/nwb_benchmarks/core/_dandi.py | 24 +++++ src/nwb_benchmarks/core/_nwb_helpers.py | 14 +++ src/nwb_benchmarks/core/_streaming.py | 90 +++++++++++++++++++ src/nwb_benchmarks/core/common/__init__.py | 1 - .../core/common/_nwbfile_read.py | 38 -------- src/nwb_benchmarks/scripts/__init__.py | 0 src/nwb_benchmarks/scripts/mock_ecephys.py | 53 +++++++++++ .../scripts/reduce_ibl_session.py | 66 ++++++++++++++ src/nwb_benchmarks/setup/__init__.py | 4 + ...igure_machine.py => _configure_machine.py} | 45 ++++++---- 18 files changed, 486 insertions(+), 77 deletions(-) rename src/nwb_benchmarks/benchmarks/{streaming.py => ibl_chunking.py} (56%) create mode 100644 src/nwb_benchmarks/benchmarks/remote_file_reading.py create mode 100644 src/nwb_benchmarks/benchmarks/remote_slicing.py create mode 100644 src/nwb_benchmarks/core/_dandi.py create mode 100644 src/nwb_benchmarks/core/_nwb_helpers.py create mode 100644 src/nwb_benchmarks/core/_streaming.py delete mode 100644 src/nwb_benchmarks/core/common/__init__.py delete mode 100644 src/nwb_benchmarks/core/common/_nwbfile_read.py create mode 100644 src/nwb_benchmarks/scripts/__init__.py create mode 100644 src/nwb_benchmarks/scripts/mock_ecephys.py create mode 100644 src/nwb_benchmarks/scripts/reduce_ibl_session.py rename src/nwb_benchmarks/setup/{configure_machine.py => _configure_machine.py} (88%) diff --git a/.gitignore b/.gitignore index d89bc7f..fc1fda3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # ASV -.asv/* +.asv/intermediate_results # Dataset file / log file types: /**/*.log diff --git a/asv.conf.json b/asv.conf.json index 37539e3..9d26d0d 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -10,11 +10,11 @@ "branches": ["main"], "environment_type": "conda", "conda_environment_file": "environments/nwb_benchmarks.yaml", - "results_dir": ".asv/results", + "results_dir": ".asv/intermediate_results", "html_dir": ".asv/html", // These are surprisingly slow operations; the timeout must be extended - "default_benchmark_timeout": 600, + "default_benchmark_timeout": 1800, // `asv` will cache results of the recent builds in each // environment, making them faster to install next time. This is diff --git a/environments/nwb_benchmarks.yaml b/environments/nwb_benchmarks.yaml index e06e4bb..53efbe5 100644 --- a/environments/nwb_benchmarks.yaml +++ b/environments/nwb_benchmarks.yaml @@ -10,9 +10,11 @@ dependencies: - pip: - setuptools - asv + - parameterized - numba>=0.58.1 # Pin to specific version for cuda import - psutil - pynwb + - dandi - fsspec - requests - aiohttp diff --git a/src/nwb_benchmarks/benchmarks/streaming.py b/src/nwb_benchmarks/benchmarks/ibl_chunking.py similarity index 56% rename from src/nwb_benchmarks/benchmarks/streaming.py rename to src/nwb_benchmarks/benchmarks/ibl_chunking.py index cd2bb96..eadc32a 100644 --- a/src/nwb_benchmarks/benchmarks/streaming.py +++ b/src/nwb_benchmarks/benchmarks/ibl_chunking.py @@ -1,16 +1,49 @@ """Basic benchmarks for stream NWB files and their contents.""" -from ..core.common import read_nwbfile_fsspec, read_nwbfile_remfile, read_nwbfile_ros3 +import parameterized + +from ..core import ( + get_s3_url, + read_hdf5_fsspec_no_cache, + read_hdf5_nwbfile_fsspec_no_cache, + read_hdf5_nwbfile_remfile, + read_hdf5_nwbfile_ros3, + read_hdf5_remfile, + read_hdf5_ros3, +) + +ibl_chunk_experiments = [ + dict( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ) +] + + +@parameterized.parameterized_class( + [ + dict( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ) + ] +) +class FsspecNoCacheHDF5FileReadBenchmark: + repeat = 1 + + def setup(self): + self.s3_url = get_s3_url(dandiset_id=self.dand, dandi_path=self.dandi_path) + + def time_file_read(self): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=self.s3_url) -def slice_dataset(nwbfile, acquisition_path, slice_range): - return nwbfile.acquisition[self.acquisition_path].data[self.slice_range] -class FsspecNoCacheFileReadBenchmark: +class FsspecNoCacheNWBFileReadBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" def time_file_read(self): - self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_fsspec(s3_url=self.s3_url) + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=self.s3_url) class RemfileFileReadBenchmark: @@ -32,40 +65,36 @@ def time_file_read(self): class FsspecNoCacheSliceBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - acquisition_path = "ElectricalSeriesAp" + params = [()] slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB def setup(self): self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_fsspec(s3_url=self.s3_url) def time_slice(self): - # Store as self._temp to avoid tracking garbage collection as well - self._temp = slice_dataset(nwbfile=self.nwbfile, ...) + """Note: store as self._temp to avoid tracking garbage collection as well.""" + self._temp = self.nwbfile.acquisition["ElectricalSeriesAp"].data[self.slice_range] class Ros3SliceBenchmark: repeat = 1 s3_url = "s3://dandiarchive/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - acquisition_path = "ElectricalSeriesAp" slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB def setup(self): self.nwbfile, self.io = read_nwbfile_ros3(s3_url=self.s3_url) def time_slice(self): - # Store as self._temp to avoid tracking garbage collection as well - self._temp = slice_dataset(nwbfile=self.nwbfile, ...) + self._temp = self.nwbfile.acquisition["ElectricalSeriesAp"].data[self.slice_range] class RemfileSliceBenchmark: repeat = 1 s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - acquisition_path = "ElectricalSeriesAp" slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB def setup(self): self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_remfile(s3_url=self.s3_url) def time_slice(self): - # Store as self._temp to avoid tracking garbage collection as well - self._temp = slice_dataset(nwbfile=self.nwbfile, ...) + self._temp = self.nwbfile.acquisition["ElectricalSeriesAp"].data[self.slice_range] diff --git a/src/nwb_benchmarks/benchmarks/remote_file_reading.py b/src/nwb_benchmarks/benchmarks/remote_file_reading.py new file mode 100644 index 0000000..efaed9a --- /dev/null +++ b/src/nwb_benchmarks/benchmarks/remote_file_reading.py @@ -0,0 +1,71 @@ +"""Basic benchmarks for stream NWB files and their contents.""" + +import parameterized + +from ..core import ( + get_s3_url, + read_hdf5_fsspec_no_cache, + read_hdf5_nwbfile_fsspec_no_cache, + read_hdf5_nwbfile_remfile, + read_hdf5_nwbfile_ros3, + read_hdf5_remfile, + read_hdf5_ros3, +) + +remote_read_test_files = [ + dict(s3_url=get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb")), + dict( + s3_url=get_s3_url( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ) # Not the best example for testing a theory about file read; should probably replace with something simpler + ), +] + + +@parameterized.parameterized_class(remote_read_test_files) +class FsspecNoCacheDirectFileReadBenchmark: + repeat = 1 + + def time_file_read(self): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=self.s3_url) + + +@parameterized.parameterized_class(remote_read_test_files) +class FsspecNoCacheNWBFileReadBenchmark: + repeat = 1 + + def time_file_read(self): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=self.s3_url) + + +@parameterized.parameterized_class(remote_read_test_files) +class RemfileDirectFileReadBenchmark: + repeat = 1 + + def time_file_read(self): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_remfile(s3_url=self.s3_url) + + +@parameterized.parameterized_class(remote_read_test_files) +class RemfileNWBFileReadBenchmark: + repeat = 1 + + def time_file_read(self): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=self.s3_url) + + +@parameterized.parameterized_class(remote_read_test_files) +class Ros3DirectFileReadBenchmark: + repeat = 1 + + def time_file_read(self): + self.nwbfile, self.io = read_hdf5_ros3(s3_url=self.s3_url) + + +@parameterized.parameterized_class(remote_read_test_files) +class Ros3NWBFileReadBenchmark: + repeat = 1 + + def time_file_read(self): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=self.s3_url) diff --git a/src/nwb_benchmarks/benchmarks/remote_slicing.py b/src/nwb_benchmarks/benchmarks/remote_slicing.py new file mode 100644 index 0000000..35eef07 --- /dev/null +++ b/src/nwb_benchmarks/benchmarks/remote_slicing.py @@ -0,0 +1,65 @@ +"""Basic benchmarks for stream NWB files and their contents.""" + +import parameterized + +from ..core import ( + get_object_by_name, + get_s3_url, + read_hdf5_nwbfile_fsspec_no_cache, + read_hdf5_nwbfile_remfile, + read_hdf5_nwbfile_ros3, +) + +# TODO: add the others +# TODO: could also expand a range of slice styles relative to chunking pattern +remote_slicing_test_files = [ + dict( + s3_url=get_s3_url( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ), + slice_range=(slice(0, 30_000), slice(0, 384)), # ~23 MB + ), +] + + +@parameterized.parameterized_class(remote_slicing_test_files) +class FsspecNoCacheContinuousSliceBenchmark: + repeat = 1 + + def setup(self): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=self.s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") + self.data_to_slice = self.neurodata_object.data + + def time_slice(self): + """Note: store as self._temp to avoid tracking garbage collection as well.""" + self._temp = self.data[self.slice_range] + + +@parameterized.parameterized_class(remote_slicing_test_files) +class RemfileContinuousSliceBenchmark: + repeat = 1 + + def setup(self): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=self.s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") + self.data_to_slice = self.neurodata_object.data + + def time_slice(self): + """Note: store as self._temp to avoid tracking garbage collection as well.""" + self._temp = self.data[self.slice_range] + + +@parameterized.parameterized_class(remote_slicing_test_files) +class Ros3ContinuousSliceBenchmark: + repeat = 1 + + def setup(self): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=self.s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") + self.data_to_slice = self.neurodata_object.data + + def time_slice(self): + """Note: store as self._temp to avoid tracking garbage collection as well.""" + self._temp = self.data[self.slice_range] diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index c0191b3..c9a7c8f 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -1,11 +1,11 @@ -"""Simple wrapper around `asv run` for conveneience.""" +"""Simple wrapper around `asv run` for convenience.""" import locale import subprocess import sys def main(): - """Simple wrapper around `asv run` for conveneience.""" + """Simple wrapper around `asv run` for convenience.""" # TODO: swap to click if len(sys.argv) <= 1: print("No command provided. Please specify a command (e.g. 'run').") diff --git a/src/nwb_benchmarks/core/__init__.py b/src/nwb_benchmarks/core/__init__.py index e69de29..40d6ee4 100644 --- a/src/nwb_benchmarks/core/__init__.py +++ b/src/nwb_benchmarks/core/__init__.py @@ -0,0 +1,23 @@ +"""Exposed imports to the `core` submodule.""" + +from ._dandi import get_s3_url +from ._nwb_helpers import get_object_by_name +from ._streaming import ( + read_hdf5_fsspec_no_cache, + read_hdf5_nwbfile_fsspec_no_cache, + read_hdf5_nwbfile_ros3, + read_hdf5_remfile, + read_hdf5_ros3, + read_nwbfile_remfile, +) + +__all__ = [ + "read_hdf5_fsspec_no_cache", + "read_hdf5_nwbfile_fsspec_no_cache", + "read_hdf5_ros3", + "read_hdf5_nwbfile_ros3", + "read_hdf5_remfile", + "read_nwbfile_remfile", + "get_s3_url", + "get_object_by_name", +] diff --git a/src/nwb_benchmarks/core/_dandi.py b/src/nwb_benchmarks/core/_dandi.py new file mode 100644 index 0000000..555ee94 --- /dev/null +++ b/src/nwb_benchmarks/core/_dandi.py @@ -0,0 +1,24 @@ +from dandi.dandiapi import DandiAPIClient + + +def get_s3_url(dandiset_id: str, dandi_path: str) -> str: + """ + Helper function to get S3 url form that fsspec/remfile expect from basic info about a file on DANDI. + + Parameters + ---------- + dandiset_id : string + Six-digit identifier of the dandiset. + For the NWB Benchmark project, the primary ones are 000717 (HDF5) and 000719 (Zarr). + dandi_path : string + The relative path of the file to the dandiset. + For example, "sub-". + """ + assert len(dandiset_id) == 6, f"The specified 'dandiset_id' ({dandiset_id}) should be the six-digit identifier." + + client = DandiAPIClient() + dandiset = client.get_dandiset(dandiset_id=dandiset_id) + asset = dandiset.get_asset_by_path(path=dandi_path) + + s3_url = asset.get_content_url(follow_redirects=1, strip_query=True) + return s3_url diff --git a/src/nwb_benchmarks/core/_nwb_helpers.py b/src/nwb_benchmarks/core/_nwb_helpers.py new file mode 100644 index 0000000..7aa809f --- /dev/null +++ b/src/nwb_benchmarks/core/_nwb_helpers.py @@ -0,0 +1,14 @@ +from typing import Any + +import pynwb + + +def get_object_by_name(nwbfile: pynwb.NWBFile, object_name: str) -> Any: + """Simple helper function to retrieve a neurodata object by its name, if it is unique.""" + object_names = [neurodata_object.name for neurodata_object in nwbfile.objects] + assert len(object_names) == len( + set(object_names) + ), "Some object names in the NWBFile are not unique! Unable to do a lookup by name." + assert object_name in object_names, f"The specified object name ({object_name}) is not in the NWBFile." + + return next(neurodata_object for neurodata_object in nwbfile.objects if neurodata_object.name == object_name) diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py new file mode 100644 index 0000000..94a91ab --- /dev/null +++ b/src/nwb_benchmarks/core/_streaming.py @@ -0,0 +1,90 @@ +import time +import warnings +from typing import Callable, Tuple, Union + +import fsspec +import h5py +import pynwb +import remfile +from fsspec.asyn import reset_lock + +# Useful if running in verbose mode +warnings.filterwarnings(action="ignore", message="No cached namespaces found in .*") +warnings.filterwarnings(action="ignore", message="Ignoring cached namespace .*") + + +def read_hdf5_fsspec_no_cache( + s3_url: str, +) -> Tuple[h5py.File, fsspec.implementations.http.HTTPFile]: + """Load the raw HDF5 file from an S3 URL using fsspec without a cache; does not formally read the NWB file.""" + reset_lock() + fsspec.get_filesystem_class("https").clear_instance_cache() + filesystem = fsspec.filesystem("https") + + byte_stream = filesystem.open(path=s3_url, mode="rb") + file = h5py.File(name=byte_stream) + return (file, byte_stream) + + +def read_hdf5_nwbfile_fsspec_no_cache( + s3_url: str, +) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO, h5py.File, fsspec.implementations.http.HTTPFile]: + """Read an HDF5 NWB file from an S3 URL using fsspec without a cache.""" + (file, byte_stream) = read_hdf5_fsspec_no_cache(s3_url=s3_url) + io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) + nwbfile = io.read() + return (nwbfile, io, file, byte_stream) + + +def robust_s3_read( + command: Callable, + max_retries: int = 10, + command_args: Union[list, None] = None, + command_kwargs: Union[dict, None] = None, +): + """ + Attempt the command (usually acting on an S3 IO) up to the number of max_retries using exponential backoff. + + Usually a good idea to use this to wrap any operations performed using the ROS3 driver. + """ + command_args = command_args or [] + command_kwargs = command_kwargs or dict() + for retry in range(max_retries): + try: + return command(*command_args, **command_kwargs) + except Exception as exc: + if "curl" in str(exc): # 'cannot curl request' can show up in potentially many different error types + time.sleep(0.1 * 2**retry) + else: + raise exc + raise TimeoutError(f"Unable to complete the command ({command.__name__}) after {max_retries} attempts!") + + +def read_hdf5_ros3(s3_url: str) -> h5py.File: + """Load the raw HDF5 file from an S3 URL using ROS3 driver; does not formally read the NWB file.""" + ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") + file = robust_s3_read(command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3")) + return file + + +def read_hdf5_nwbfile_ros3(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO]: + """Read an HDF5 NWB file from an S3 URL using the ROS3 driver from h5py.""" + ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") + io = pynwb.NWBHDF5IO(path=ros3_form, mode="r", load_namespaces=True, driver="ros3") + nwbfile = robust_s3_read(command=io.read) + return (nwbfile, io) + + +def read_hdf5_remfile(s3_url: str) -> Tuple[h5py.File, remfile.File]: + """Load the raw HDF5 file from an S3 URL using remfile; does not formally read the NWB file.""" + byte_stream = remfile.File(url=s3_url) + file = h5py.File(name=byte_stream) + return (file, byte_stream) + + +def read_nwbfile_remfile(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO, h5py.File, remfile.File]: + """Read an HDF5 NWB file from an S3 URL using the ROS3 driver from h5py.""" + (file, byte_stream) = read_hdf5_remfile(s3_url=s3_url) + io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) + nwbfile = io.read() + return (nwbfile, io, file, byte_stream) diff --git a/src/nwb_benchmarks/core/common/__init__.py b/src/nwb_benchmarks/core/common/__init__.py deleted file mode 100644 index ec5a1e2..0000000 --- a/src/nwb_benchmarks/core/common/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from ._nwbfile_read import read_nwbfile_fsspec, read_nwbfile_remfile, read_nwbfile_ros3 diff --git a/src/nwb_benchmarks/core/common/_nwbfile_read.py b/src/nwb_benchmarks/core/common/_nwbfile_read.py deleted file mode 100644 index 1d222c9..0000000 --- a/src/nwb_benchmarks/core/common/_nwbfile_read.py +++ /dev/null @@ -1,38 +0,0 @@ -import warnings - -import fsspec -import h5py -import pynwb -import remfile -from fsspec.asyn import reset_lock - -# Useful if running in verbose mode -warnings.filterwarnings(action="ignore", message="No cached namespaces found in .*") -warnings.filterwarnings(action="ignore", message="Ignoring cached namespace .*") - - -def read_nwbfile_fsspec(s3_url: str) -> tuple: # TODO: enhance annotation, add cache option - reset_lock() - fsspec.get_filesystem_class("https").clear_instance_cache() - filesystem = fsspec.filesystem("https") - - byte_stream = filesystem.open(path=s3_url, mode="rb") - file = h5py.File(name=byte_stream) - io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) - nwbfile = io.read() - return (nwbfile, io, file, byte_stream) - - -def read_nwbfile_ros3(s3_url: str) -> tuple: # TODO: enhance annotation - ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") - io = pynwb.NWBHDF5IO(path=ros3_form, mode="r", load_namespaces=True, driver="ros3") - nwbfile = io.read() - return (nwbfile, io) - - -def read_nwbfile_remfile(s3_url: str) -> tuple: # TODO: enhance annotation - byte_stream = remfile.File(url=s3_url) - file = h5py.File(name=byte_stream) - io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) - nwbfile = io.read() - return (nwbfile, io, file, byte_stream) diff --git a/src/nwb_benchmarks/scripts/__init__.py b/src/nwb_benchmarks/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nwb_benchmarks/scripts/mock_ecephys.py b/src/nwb_benchmarks/scripts/mock_ecephys.py new file mode 100644 index 0000000..b975e84 --- /dev/null +++ b/src/nwb_benchmarks/scripts/mock_ecephys.py @@ -0,0 +1,53 @@ +"""Scipt for creating a simpler version of an IBL file.""" + +import pathlib + +import pynwb +from pynwb.testing.mock.ecephys import ( + mock_Device, + mock_ElectricalSeries, + mock_ElectrodeGroup, + mock_ElectrodeTable, +) +from pynwb.testing.mock.file import mock_NWBFile + +hdf5_benchmark_dandiset_folder = pathlib.Path("E:/nwb_benchmark_data/000717") # Path to your local copy of 000717 + + +# Create simple mock +session_id = "ecephys1" +nwbfile = mock_NWBFile( + session_id=session_id, + subject=pynwb.file.Subject(subject_id="mock", species="Mus musculus", age="P20D", sex="O"), +) + +device = mock_Device() +nwbfile.add_device(device) +electrode_group = mock_ElectrodeGroup(device=device) +nwbfile.electrode_groups = {electrode_group.name: electrode_group} +electrodes = mock_ElectrodeTable(group=electrode_group) +nwbfile.electrodes = electrodes + + +electrical_series = mock_ElectricalSeries( + electrodes=pynwb.ecephys.DynamicTableRegion( + name="electrodes", description="", data=list(range(5)), table=electrodes + ) +) +# nwbfile.add_device(electrical_series.electrodes[0]["group"][0].device) +# nwbfile.electrode_groups = {electrical_series.electrodes[0]["group"].name: electrical_series.electrodes[0]["group"]} +# nwbfile.electrodes = electrical_series.electrodes +nwbfile.add_acquisition(electrical_series) + +# Write the new NWB file +# Could modify above to produce variations; should document that in dandi description as well as session description +dandi_filename_description = f"" + +simplified_output_subject_folder = hdf5_benchmark_dandiset_folder / "sub-mock" +simplified_output_subject_folder.mkdir(exist_ok=True) +simplified_output_file_path = ( + simplified_output_subject_folder / f"sub-mock_ses-{session_id}{dandi_filename_description}.nwb" +) + +with pynwb.NWBHDF5IO(path=simplified_output_file_path, mode="w") as io: + io.write(nwbfile) diff --git a/src/nwb_benchmarks/scripts/reduce_ibl_session.py b/src/nwb_benchmarks/scripts/reduce_ibl_session.py new file mode 100644 index 0000000..e5a9965 --- /dev/null +++ b/src/nwb_benchmarks/scripts/reduce_ibl_session.py @@ -0,0 +1,66 @@ +"""Scipt for creating a simpler version of an IBL file.""" + +import pathlib +import uuid + +import lazy_ops # Not yet a part of core nwb_benchmarks environment; will have to install additionally for now +import neuroconv # Not yet a part of core nwb_benchmarks environment; will have to install additionally for now +import pynwb + +source_folder = pathlib.Path("E:/nwb_benchmark_data") +original_file_path = ( + source_folder / "sub-CSH-ZAD-001_ses-3e7ae7c0-fe8b-487c-9354-036236fa1010-chunking-13653-384_behavior+ecephys.nwb" +) +hdf5_benchmark_dandiset_folder = pathlib.Path("E:/nwb_benchmark_data/000717") # Path to your local copy of 000717 + +# Read source NWBFile +io = pynwb.NWBHDF5IO(path=original_file_path, mode="r", load_namespaces=True) +source_nwbfile = io.read() + +session_id = source_nwbfile.session_id.split("-")[0] # Shorten for readability; should still be enough to be unique +source_electrical_series = source_nwbfile.acquisition["ElectricalSeriesAp"] + +# rate (Hz) * sec * min * chan * itemsize ; e.g., 30_000 * 60 * 10 * 384 * 2 / 1e9 ~ 14 GB +reduce_frame_bound = 30_000 * 60 * 10 +lazy_source_dataset = lazy_ops.DatasetView(dataset=source_electrical_series.data) +reduced_source_dataset = lazy_source_dataset.lazy_slice[(slice(0, reduce_frame_bound), slice(0, 384))] +new_chunks = source_electrical_series.data.chunks # Just setting equal for now, but in theory could change + +# Make output NWBFile - copy over minimal values +simplified_output_nwbfile = pynwb.NWBFile( + session_description="Minimal copy of a dataset from an IBL electrical series.", + session_id=session_id, + session_start_time=source_nwbfile.session_start_time, + identifier=str(uuid.uuid4()), + subject=pynwb.file.Subject( + subject_id=source_nwbfile.subject.subject_id, + date_of_birth=source_nwbfile.subject.date_of_birth, + sex=source_nwbfile.subject.sex, + species=source_nwbfile.subject.species, + ), +) + + +simplified_output_time_series = pynwb.ecephys.TimeSeries( + name="ElectricalSeriesAp", + description="Reduced copy of the data from the ElectricalSeriesAp object from the IBL session specified in the ID.", + data=neuroconv.tools.hdmf.SliceableDataChunkIterator( + data=reduced_source_dataset, buffer_gb=0.5, chunk_shape=new_chunks, display_progress=True + ), + rate=source_electrical_series.rate, + conversion=source_electrical_series.conversion, + unit=source_electrical_series.unit, +) +simplified_output_nwbfile.add_acquisition(simplified_output_time_series) + +# Write the new NWB file +dandi_filename_description = f"_desc-{reduce_frame_bound}-frames-{new_chunks[0]}-by-{new_chunks[1]}-chunking" + +simplified_output_subject_folder = hdf5_benchmark_dandiset_folder / "sub-IBL-ecephys" +simplified_output_subject_folder.mkdir(exist_ok=True) +simplified_output_file_path = ( + simplified_output_subject_folder / f"sub-IBL-ecephys_ses-{session_id}{dandi_filename_description}.nwb" +) + +with pynwb.NWBHDF5IO(path=simplified_output_file_path, mode="w") as io: + io.write(simplified_output_nwbfile) diff --git a/src/nwb_benchmarks/setup/__init__.py b/src/nwb_benchmarks/setup/__init__.py index e69de29..aa49cd6 100644 --- a/src/nwb_benchmarks/setup/__init__.py +++ b/src/nwb_benchmarks/setup/__init__.py @@ -0,0 +1,4 @@ +"""Exposed imports to the `setup` submodule.""" +from ._configure_machine import collect_machine_info, customize_asv_machine_file + +__all__ = ["collect_machine_info", "customize_asv_machine_file"] diff --git a/src/nwb_benchmarks/setup/configure_machine.py b/src/nwb_benchmarks/setup/_configure_machine.py similarity index 88% rename from src/nwb_benchmarks/setup/configure_machine.py rename to src/nwb_benchmarks/setup/_configure_machine.py index 0e33f85..549d6a8 100644 --- a/src/nwb_benchmarks/setup/configure_machine.py +++ b/src/nwb_benchmarks/setup/_configure_machine.py @@ -1,4 +1,5 @@ """Add information to the ASV machine parameters.""" + import hashlib import json import os @@ -6,29 +7,16 @@ import platform import sys import warnings +from typing import Any, Dict import psutil from numba import cuda -def customize_asv_machine_file(file_path: pathlib.Path) -> None: - """Modify ASV machine file in-place with additional values.""" - with open(file=default_asv_machine_file_path, mode="r") as io: - machine_file_info = json.load(fp=io) - - # Assume there's only one machine configured per installation - default_machine_name = next(key for key in machine_file_info.keys() if key != "version") - - # Add flag for detecting if this modification script has been run on the file already - if machine_file_info[default_machine_name].get("custom", False): - return - - default_machine_info = machine_file_info[default_machine_name] - custom_machine_info = dict(custom=True) +def collect_machine_info() -> Dict[str, Dict[str, Any]]: + """Collect attributes for uniquely identifying a system and providing metadata associated with performance.""" + custom_machine_info = dict() - # Add additional tracking parameters from each library - # Availability and values can different from system to system so best to simply store all info possible - custom_machine_info["defaults"] = default_machine_info # Defaults tends to have blanks custom_machine_info["os"] = dict(cpu_count=os.cpu_count()) custom_machine_info["sys"] = dict(platform=sys.platform) custom_machine_info["platform"] = dict( @@ -62,9 +50,28 @@ def customize_asv_machine_file(file_path: pathlib.Path) -> None: except Exception as exception: raise exception - # Required key at the outer level - # This is really the 'ID' of the machines logs + +def customize_asv_machine_file(file_path: pathlib.Path) -> None: + """Modify ASV machine file in-place with additional values.""" + with open(file=default_asv_machine_file_path, mode="r") as io: + machine_file_info = json.load(fp=io) + + # Assume there's only one machine configured per installation + default_machine_name = next(key for key in machine_file_info.keys() if key != "version") + + # Add flag for detecting if this modification script has been run on the file already + if machine_file_info[default_machine_name].get("custom", False): + return + + default_machine_info = machine_file_info[default_machine_name] + custom_machine_info = collect_machine_info() + + custom_machine_info["defaults"] = default_machine_info # Defaults tends to have blanks + + # Required keys at the outer level + # The 'machine' key is really the 'ID' of the machines logs # Needs to be unique for a combined results database + custom_machine_info.update(custom=True) custom_machine_hash = hashlib.sha1( string=bytes(json.dumps(obj=custom_machine_info, sort_keys=True), "utf-8") ).hexdigest() From dd8eb575dc127de7f52d7796cf11155a5394bc39 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Thu, 15 Feb 2024 21:31:32 -0500 Subject: [PATCH 06/72] debugs and ensure machine is up to date --- src/nwb_benchmarks/benchmarks/ibl_chunking.py | 2 +- .../benchmarks/remote_file_reading.py | 2 +- .../benchmarks/remote_slicing.py | 2 +- src/nwb_benchmarks/command_line_interface.py | 10 ++++ src/nwb_benchmarks/core/__init__.py | 4 +- src/nwb_benchmarks/core/_streaming.py | 35 ++++++------ src/nwb_benchmarks/setup/__init__.py | 8 ++- .../setup/_configure_machine.py | 56 ++++++++++++++++--- 8 files changed, 88 insertions(+), 31 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/ibl_chunking.py b/src/nwb_benchmarks/benchmarks/ibl_chunking.py index eadc32a..6a6327d 100644 --- a/src/nwb_benchmarks/benchmarks/ibl_chunking.py +++ b/src/nwb_benchmarks/benchmarks/ibl_chunking.py @@ -2,7 +2,7 @@ import parameterized -from ..core import ( +from nwb_benchmarks.core import ( get_s3_url, read_hdf5_fsspec_no_cache, read_hdf5_nwbfile_fsspec_no_cache, diff --git a/src/nwb_benchmarks/benchmarks/remote_file_reading.py b/src/nwb_benchmarks/benchmarks/remote_file_reading.py index efaed9a..7e3c7ef 100644 --- a/src/nwb_benchmarks/benchmarks/remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/remote_file_reading.py @@ -2,7 +2,7 @@ import parameterized -from ..core import ( +from nwb_benchmarks.core import ( get_s3_url, read_hdf5_fsspec_no_cache, read_hdf5_nwbfile_fsspec_no_cache, diff --git a/src/nwb_benchmarks/benchmarks/remote_slicing.py b/src/nwb_benchmarks/benchmarks/remote_slicing.py index 35eef07..272c611 100644 --- a/src/nwb_benchmarks/benchmarks/remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/remote_slicing.py @@ -2,7 +2,7 @@ import parameterized -from ..core import ( +from nwb_benchmarks.core import ( get_object_by_name, get_s3_url, read_hdf5_nwbfile_fsspec_no_cache, diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index c9a7c8f..19f5f12 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -1,8 +1,12 @@ """Simple wrapper around `asv run` for convenience.""" + import locale +import pathlib import subprocess import sys +from .setup import customize_asv_machine_file, ensure_machine_info_current + def main(): """Simple wrapper around `asv run` for convenience.""" @@ -22,6 +26,12 @@ def main(): if command == "run": commit_hash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("ascii").strip() + default_asv_machine_file_path = pathlib.Path.home() / ".asv-machine.json" + if default_asv_machine_file_path.exists(): + ensure_machine_info_current(file_path=default_asv_machine_file_path) + else: + customize_asv_machine_file(file_path=default_asv_machine_file_path) + cmd = [ "asv", "run", diff --git a/src/nwb_benchmarks/core/__init__.py b/src/nwb_benchmarks/core/__init__.py index 40d6ee4..afd1504 100644 --- a/src/nwb_benchmarks/core/__init__.py +++ b/src/nwb_benchmarks/core/__init__.py @@ -5,10 +5,10 @@ from ._streaming import ( read_hdf5_fsspec_no_cache, read_hdf5_nwbfile_fsspec_no_cache, + read_hdf5_nwbfile_remfile, read_hdf5_nwbfile_ros3, read_hdf5_remfile, read_hdf5_ros3, - read_nwbfile_remfile, ) __all__ = [ @@ -17,7 +17,7 @@ "read_hdf5_ros3", "read_hdf5_nwbfile_ros3", "read_hdf5_remfile", - "read_nwbfile_remfile", + "read_hdf5_nwbfile_remfile", "get_s3_url", "get_object_by_name", ] diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py index 94a91ab..d70e0a8 100644 --- a/src/nwb_benchmarks/core/_streaming.py +++ b/src/nwb_benchmarks/core/_streaming.py @@ -7,6 +7,7 @@ import pynwb import remfile from fsspec.asyn import reset_lock +from fsspec.implementations.http import HTTPFile # Useful if running in verbose mode warnings.filterwarnings(action="ignore", message="No cached namespaces found in .*") @@ -15,7 +16,7 @@ def read_hdf5_fsspec_no_cache( s3_url: str, -) -> Tuple[h5py.File, fsspec.implementations.http.HTTPFile]: +) -> Tuple[h5py.File, HTTPFile]: """Load the raw HDF5 file from an S3 URL using fsspec without a cache; does not formally read the NWB file.""" reset_lock() fsspec.get_filesystem_class("https").clear_instance_cache() @@ -28,7 +29,7 @@ def read_hdf5_fsspec_no_cache( def read_hdf5_nwbfile_fsspec_no_cache( s3_url: str, -) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO, h5py.File, fsspec.implementations.http.HTTPFile]: +) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO, h5py.File, HTTPFile]: """Read an HDF5 NWB file from an S3 URL using fsspec without a cache.""" (file, byte_stream) = read_hdf5_fsspec_no_cache(s3_url=s3_url) io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) @@ -36,6 +37,21 @@ def read_hdf5_nwbfile_fsspec_no_cache( return (nwbfile, io, file, byte_stream) +def read_hdf5_remfile(s3_url: str) -> Tuple[h5py.File, remfile.File]: + """Load the raw HDF5 file from an S3 URL using remfile; does not formally read the NWB file.""" + byte_stream = remfile.File(url=s3_url) + file = h5py.File(name=byte_stream) + return (file, byte_stream) + + +def read_hdf5_nwbfile_remfile(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO, h5py.File, remfile.File]: + """Read an HDF5 NWB file from an S3 URL using the ROS3 driver from h5py.""" + (file, byte_stream) = read_hdf5_remfile(s3_url=s3_url) + io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) + nwbfile = io.read() + return (nwbfile, io, file, byte_stream) + + def robust_s3_read( command: Callable, max_retries: int = 10, @@ -73,18 +89,3 @@ def read_hdf5_nwbfile_ros3(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO] io = pynwb.NWBHDF5IO(path=ros3_form, mode="r", load_namespaces=True, driver="ros3") nwbfile = robust_s3_read(command=io.read) return (nwbfile, io) - - -def read_hdf5_remfile(s3_url: str) -> Tuple[h5py.File, remfile.File]: - """Load the raw HDF5 file from an S3 URL using remfile; does not formally read the NWB file.""" - byte_stream = remfile.File(url=s3_url) - file = h5py.File(name=byte_stream) - return (file, byte_stream) - - -def read_nwbfile_remfile(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO, h5py.File, remfile.File]: - """Read an HDF5 NWB file from an S3 URL using the ROS3 driver from h5py.""" - (file, byte_stream) = read_hdf5_remfile(s3_url=s3_url) - io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) - nwbfile = io.read() - return (nwbfile, io, file, byte_stream) diff --git a/src/nwb_benchmarks/setup/__init__.py b/src/nwb_benchmarks/setup/__init__.py index aa49cd6..6f6121b 100644 --- a/src/nwb_benchmarks/setup/__init__.py +++ b/src/nwb_benchmarks/setup/__init__.py @@ -1,4 +1,8 @@ """Exposed imports to the `setup` submodule.""" -from ._configure_machine import collect_machine_info, customize_asv_machine_file +from ._configure_machine import ( + collect_machine_info, + customize_asv_machine_file, + ensure_machine_info_current, +) -__all__ = ["collect_machine_info", "customize_asv_machine_file"] +__all__ = ["collect_machine_info", "customize_asv_machine_file", "ensure_machine_info_current"] diff --git a/src/nwb_benchmarks/setup/_configure_machine.py b/src/nwb_benchmarks/setup/_configure_machine.py index 549d6a8..e655e93 100644 --- a/src/nwb_benchmarks/setup/_configure_machine.py +++ b/src/nwb_benchmarks/setup/_configure_machine.py @@ -6,6 +6,7 @@ import pathlib import platform import sys +import unittest import warnings from typing import Any, Dict @@ -20,7 +21,7 @@ def collect_machine_info() -> Dict[str, Dict[str, Any]]: custom_machine_info["os"] = dict(cpu_count=os.cpu_count()) custom_machine_info["sys"] = dict(platform=sys.platform) custom_machine_info["platform"] = dict( - architecture=platform.architecture(), + architecture=list(platform.architecture()), # Must be cast as a list for later assertions against JSON machine=platform.machine(), platform=platform.platform(), processor=platform.processor(), @@ -31,7 +32,7 @@ def collect_machine_info() -> Dict[str, Dict[str, Any]]: number_of_threads=psutil.cpu_count(logical=True), total_virtual_memory=psutil.virtual_memory().total, total_swap_memory=psutil.swap_memory().total, - disk_partitions=psutil.disk_partitions(), + disk_partitions=[disk_partition._asdict() for disk_partition in psutil.disk_partitions()], ) # TODO: psutil does have some socket stuff in .net_connections, is that useful at all? @@ -50,22 +51,27 @@ def collect_machine_info() -> Dict[str, Dict[str, Any]]: except Exception as exception: raise exception + return custom_machine_info -def customize_asv_machine_file(file_path: pathlib.Path) -> None: + +def customize_asv_machine_file(file_path: pathlib.Path, overwrite: bool = False) -> None: """Modify ASV machine file in-place with additional values.""" - with open(file=default_asv_machine_file_path, mode="r") as io: + with open(file=file_path, mode="r") as io: machine_file_info = json.load(fp=io) # Assume there's only one machine configured per installation default_machine_name = next(key for key in machine_file_info.keys() if key != "version") # Add flag for detecting if this modification script has been run on the file already - if machine_file_info[default_machine_name].get("custom", False): + if not overwrite and machine_file_info[default_machine_name].get("custom", False): return - default_machine_info = machine_file_info[default_machine_name] custom_machine_info = collect_machine_info() + if overwrite and "defaults" in machine_file_info[default_machine_name]: + default_machine_info = machine_file_info[default_machine_name]["defaults"] + else: + default_machine_info = machine_file_info[default_machine_name] custom_machine_info["defaults"] = default_machine_info # Defaults tends to have blanks # Required keys at the outer level @@ -82,10 +88,46 @@ def customize_asv_machine_file(file_path: pathlib.Path) -> None: "version": machine_file_info["version"], } - with open(file=default_asv_machine_file_path, mode="w") as io: + with open(file=file_path, mode="w") as io: json.dump(fp=io, obj=custom_file_info, indent=4) +def ensure_machine_info_current(file_path: pathlib.Path): + """ + Even something as simple as adjusting the disk partitions could affect performance. + + If out of date, automatically trigger regeneration of machine info and hash. + Will also likely be affected by ipcfg. + """ + current_machine_info = collect_machine_info() + + # Assume there's only one machine configured per installation + with open(file=file_path, mode="r") as io: + machine_file = json.load(fp=io) + + default_machine_name = next(key for key in machine_file.keys() if key != "version") + machine_info_from_file = machine_file[default_machine_name] + + # Only asserting agains the custom stuff we grab + machine_info_from_file.pop("defaults") + machine_info_from_file.pop("machine") + machine_info_from_file.pop("custom") + + with open(file=file_path.parent / "test.json", mode="w") as io: + json.dump(fp=io, obj=current_machine_info, indent=4) + + if machine_info_from_file == current_machine_info: + return + + # If debugging is ever necessary in the future, best way I found to summarize differences was + # test = unittest.TestCase() + # test.maxDiff = None + # test.assertDictEqual(d1=machine_info_from_file, d2=current_machine_info) + + warnings.warn("The current machine info is out of date! Automatically updating the file.", stacklevel=2) + customize_asv_machine_file(file_path=file_path, overwrite=True) + + if __name__ == "__main__": home_directory = pathlib.Path.home() From 8d0091a14840b13695ea6fc7aa53b334555bccdb Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Thu, 15 Feb 2024 21:43:00 -0500 Subject: [PATCH 07/72] debugs --- src/nwb_benchmarks/benchmarks/ibl_chunking.py | 100 ------------------ .../benchmarks/remote_slicing.py | 3 +- src/nwb_benchmarks/command_line_interface.py | 2 +- src/nwb_benchmarks/core/__init__.py | 2 + src/nwb_benchmarks/core/_streaming.py | 2 +- 5 files changed, 6 insertions(+), 103 deletions(-) delete mode 100644 src/nwb_benchmarks/benchmarks/ibl_chunking.py diff --git a/src/nwb_benchmarks/benchmarks/ibl_chunking.py b/src/nwb_benchmarks/benchmarks/ibl_chunking.py deleted file mode 100644 index 6a6327d..0000000 --- a/src/nwb_benchmarks/benchmarks/ibl_chunking.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Basic benchmarks for stream NWB files and their contents.""" - -import parameterized - -from nwb_benchmarks.core import ( - get_s3_url, - read_hdf5_fsspec_no_cache, - read_hdf5_nwbfile_fsspec_no_cache, - read_hdf5_nwbfile_remfile, - read_hdf5_nwbfile_ros3, - read_hdf5_remfile, - read_hdf5_ros3, -) - -ibl_chunk_experiments = [ - dict( - dandiset_id="000717", - dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", - ) -] - - -@parameterized.parameterized_class( - [ - dict( - dandiset_id="000717", - dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", - ) - ] -) -class FsspecNoCacheHDF5FileReadBenchmark: - repeat = 1 - - def setup(self): - self.s3_url = get_s3_url(dandiset_id=self.dand, dandi_path=self.dandi_path) - - def time_file_read(self): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=self.s3_url) - - -class FsspecNoCacheNWBFileReadBenchmark: - repeat = 1 - s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - - def time_file_read(self): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=self.s3_url) - - -class RemfileFileReadBenchmark: - repeat = 1 - s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - - def time_file_read(self): - self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_remfile(s3_url=self.s3_url) - - -class Ros3FileReadBenchmark: - repeat = 1 - s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - - def time_file_read(self): - self.nwbfile, self.io = read_nwbfile_ros3(s3_url=self.s3_url) - - -class FsspecNoCacheSliceBenchmark: - repeat = 1 - s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - params = [()] - slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - - def setup(self): - self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_fsspec(s3_url=self.s3_url) - - def time_slice(self): - """Note: store as self._temp to avoid tracking garbage collection as well.""" - self._temp = self.nwbfile.acquisition["ElectricalSeriesAp"].data[self.slice_range] - - -class Ros3SliceBenchmark: - repeat = 1 - s3_url = "s3://dandiarchive/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - - def setup(self): - self.nwbfile, self.io = read_nwbfile_ros3(s3_url=self.s3_url) - - def time_slice(self): - self._temp = self.nwbfile.acquisition["ElectricalSeriesAp"].data[self.slice_range] - - -class RemfileSliceBenchmark: - repeat = 1 - s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - - def setup(self): - self.nwbfile, self.io, self.file, self.bytestream = read_nwbfile_remfile(s3_url=self.s3_url) - - def time_slice(self): - self._temp = self.nwbfile.acquisition["ElectricalSeriesAp"].data[self.slice_range] diff --git a/src/nwb_benchmarks/benchmarks/remote_slicing.py b/src/nwb_benchmarks/benchmarks/remote_slicing.py index 272c611..d997001 100644 --- a/src/nwb_benchmarks/benchmarks/remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/remote_slicing.py @@ -8,6 +8,7 @@ read_hdf5_nwbfile_fsspec_no_cache, read_hdf5_nwbfile_remfile, read_hdf5_nwbfile_ros3, + robust_ros3_read, ) # TODO: add the others @@ -62,4 +63,4 @@ def setup(self): def time_slice(self): """Note: store as self._temp to avoid tracking garbage collection as well.""" - self._temp = self.data[self.slice_range] + self._temp = robust_ros3_read(command=self.data.__getitem__, command_args=(self.slice_range,)) diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index 19f5f12..7b7a6de 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -41,7 +41,7 @@ def main(): commit_hash, ] if debug_mode: - cmd.extend(["--verbose", "--show-std-err"]) + cmd.extend(["--verbose", "--show-stderr"]) if bench_mode: cmd.extend(["--bench", specific_benchmark_pattern]) diff --git a/src/nwb_benchmarks/core/__init__.py b/src/nwb_benchmarks/core/__init__.py index afd1504..497546c 100644 --- a/src/nwb_benchmarks/core/__init__.py +++ b/src/nwb_benchmarks/core/__init__.py @@ -9,6 +9,7 @@ read_hdf5_nwbfile_ros3, read_hdf5_remfile, read_hdf5_ros3, + robust_ros3_read, ) __all__ = [ @@ -20,4 +21,5 @@ "read_hdf5_nwbfile_remfile", "get_s3_url", "get_object_by_name", + "robust_ros3_read", ] diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py index d70e0a8..e82f150 100644 --- a/src/nwb_benchmarks/core/_streaming.py +++ b/src/nwb_benchmarks/core/_streaming.py @@ -52,7 +52,7 @@ def read_hdf5_nwbfile_remfile(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5 return (nwbfile, io, file, byte_stream) -def robust_s3_read( +def robust_ros3_read( command: Callable, max_retries: int = 10, command_args: Union[list, None] = None, From 245bb02f79702dfa54b6c8044e7d3e21f6318f85 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 16 Feb 2024 11:09:02 -0500 Subject: [PATCH 08/72] more debugs --- README.md | 2 +- environments/nwb_benchmarks.yaml | 4 -- .../benchmarks/remote_file_reading.py | 67 ++++++++++--------- .../benchmarks/remote_slicing.py | 53 ++++++++------- src/nwb_benchmarks/core/_nwb_helpers.py | 6 +- src/nwb_benchmarks/core/_streaming.py | 4 +- 6 files changed, 71 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 33cb9be..d638450 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ cd nwb_benchmarks Setup the environment... ``` -conda env create -f environments/nwb_benchmarks.yaml +conda env create --file environments/nwb_benchmarks.yaml --no-default-packages conda activate nwb_benchmarks ``` diff --git a/environments/nwb_benchmarks.yaml b/environments/nwb_benchmarks.yaml index 53efbe5..ab33e01 100644 --- a/environments/nwb_benchmarks.yaml +++ b/environments/nwb_benchmarks.yaml @@ -1,7 +1,4 @@ name: nwb_benchmarks -#channels: -# - defaults -# - conda-forge dependencies: - python = 3.11.7 - git = 2.40.1 @@ -10,7 +7,6 @@ dependencies: - pip: - setuptools - asv - - parameterized - numba>=0.58.1 # Pin to specific version for cuda import - psutil - pynwb diff --git a/src/nwb_benchmarks/benchmarks/remote_file_reading.py b/src/nwb_benchmarks/benchmarks/remote_file_reading.py index 7e3c7ef..e0ea9fb 100644 --- a/src/nwb_benchmarks/benchmarks/remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/remote_file_reading.py @@ -1,7 +1,5 @@ """Basic benchmarks for stream NWB files and their contents.""" -import parameterized - from nwb_benchmarks.core import ( get_s3_url, read_hdf5_fsspec_no_cache, @@ -12,60 +10,65 @@ read_hdf5_ros3, ) -remote_read_test_files = [ - dict(s3_url=get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb")), - dict( - s3_url=get_s3_url( - dandiset_id="000717", - dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", - ) # Not the best example for testing a theory about file read; should probably replace with something simpler - ), +param_names = ["s3_url"] +params = [ + get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb"), + get_s3_url( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ), # Not the best example for testing a theory about file read; should probably replace with something simpler ] -@parameterized.parameterized_class(remote_read_test_files) class FsspecNoCacheDirectFileReadBenchmark: repeat = 1 + param_names = param_names + params = params - def time_file_read(self): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=self.s3_url) + def time_file_read(self, s3_url: str): + self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) -@parameterized.parameterized_class(remote_read_test_files) -class FsspecNoCacheNWBFileReadBenchmark: +class RemfileDirectFileReadBenchmark: repeat = 1 + param_names = param_names + params = params - def time_file_read(self): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=self.s3_url) + def time_file_read(self, s3_url: str): + self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) -@parameterized.parameterized_class(remote_read_test_files) -class RemfileDirectFileReadBenchmark: +class Ros3DirectFileReadBenchmark: repeat = 1 + param_names = param_names + params = params - def time_file_read(self): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_remfile(s3_url=self.s3_url) + def time_file_read(self, s3_url: str): + self.file = read_hdf5_ros3(s3_url=s3_url) -@parameterized.parameterized_class(remote_read_test_files) -class RemfileNWBFileReadBenchmark: +class FsspecNoCacheNWBFileReadBenchmark: repeat = 1 + param_names = param_names + params = params - def time_file_read(self): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=self.s3_url) + def time_file_read(self, s3_url: str): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) -@parameterized.parameterized_class(remote_read_test_files) -class Ros3DirectFileReadBenchmark: +class RemfileNWBFileReadBenchmark: repeat = 1 + param_names = param_names + params = params - def time_file_read(self): - self.nwbfile, self.io = read_hdf5_ros3(s3_url=self.s3_url) + def time_file_read(self, s3_url: str): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) -@parameterized.parameterized_class(remote_read_test_files) class Ros3NWBFileReadBenchmark: repeat = 1 + param_names = param_names + params = params - def time_file_read(self): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=self.s3_url) + def time_file_read(self, s3_url: str): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/remote_slicing.py b/src/nwb_benchmarks/benchmarks/remote_slicing.py index d997001..c7a1743 100644 --- a/src/nwb_benchmarks/benchmarks/remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/remote_slicing.py @@ -1,6 +1,6 @@ """Basic benchmarks for stream NWB files and their contents.""" -import parameterized +from typing import Tuple from nwb_benchmarks.core import ( get_object_by_name, @@ -13,54 +13,59 @@ # TODO: add the others # TODO: could also expand a range of slice styles relative to chunking pattern -remote_slicing_test_files = [ - dict( - s3_url=get_s3_url( +param_names = ["s3_url", "object_name", "slice_range"] +params = ( + [ + get_s3_url( dandiset_id="000717", dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", - ), - slice_range=(slice(0, 30_000), slice(0, 384)), # ~23 MB - ), -] + ) + ], + ["ElectricalSeriesAp"], + [(slice(0, 30_000), slice(0, 384))], # ~23 MB +) -@parameterized.parameterized_class(remote_slicing_test_files) class FsspecNoCacheContinuousSliceBenchmark: repeat = 1 + param_names = param_names + params = params - def setup(self): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=self.s3_url) - self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def time_slice(self): + def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" - self._temp = self.data[self.slice_range] + self._temp = self.data_to_slice[slice_range] -@parameterized.parameterized_class(remote_slicing_test_files) class RemfileContinuousSliceBenchmark: repeat = 1 + param_names = param_names + params = params - def setup(self): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=self.s3_url) + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") self.data_to_slice = self.neurodata_object.data - def time_slice(self): + def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" - self._temp = self.data[self.slice_range] + self._temp = self.data_to_slice[slice_range] -@parameterized.parameterized_class(remote_slicing_test_files) class Ros3ContinuousSliceBenchmark: repeat = 1 + param_names = param_names + params = params - def setup(self): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=self.s3_url) + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") self.data_to_slice = self.neurodata_object.data - def time_slice(self): + def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" - self._temp = robust_ros3_read(command=self.data.__getitem__, command_args=(self.slice_range,)) + self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) diff --git a/src/nwb_benchmarks/core/_nwb_helpers.py b/src/nwb_benchmarks/core/_nwb_helpers.py index 7aa809f..dd8d30f 100644 --- a/src/nwb_benchmarks/core/_nwb_helpers.py +++ b/src/nwb_benchmarks/core/_nwb_helpers.py @@ -5,10 +5,12 @@ def get_object_by_name(nwbfile: pynwb.NWBFile, object_name: str) -> Any: """Simple helper function to retrieve a neurodata object by its name, if it is unique.""" - object_names = [neurodata_object.name for neurodata_object in nwbfile.objects] + object_names = [neurodata_object.name for neurodata_object in nwbfile.objects.values()] assert len(object_names) == len( set(object_names) ), "Some object names in the NWBFile are not unique! Unable to do a lookup by name." assert object_name in object_names, f"The specified object name ({object_name}) is not in the NWBFile." - return next(neurodata_object for neurodata_object in nwbfile.objects if neurodata_object.name == object_name) + return next( + neurodata_object for neurodata_object in nwbfile.objects.values() if neurodata_object.name == object_name + ) diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py index e82f150..d171d6c 100644 --- a/src/nwb_benchmarks/core/_streaming.py +++ b/src/nwb_benchmarks/core/_streaming.py @@ -79,7 +79,7 @@ def robust_ros3_read( def read_hdf5_ros3(s3_url: str) -> h5py.File: """Load the raw HDF5 file from an S3 URL using ROS3 driver; does not formally read the NWB file.""" ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") - file = robust_s3_read(command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3")) + file = robust_ros3_read(command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3")) return file @@ -87,5 +87,5 @@ def read_hdf5_nwbfile_ros3(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO] """Read an HDF5 NWB file from an S3 URL using the ROS3 driver from h5py.""" ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") io = pynwb.NWBHDF5IO(path=ros3_form, mode="r", load_namespaces=True, driver="ros3") - nwbfile = robust_s3_read(command=io.read) + nwbfile = robust_ros3_read(command=io.read) return (nwbfile, io) From e59195908b88531a080a97f0fd0511051e266408 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Feb 2024 16:12:00 +0000 Subject: [PATCH 09/72] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/nwb_benchmarks/setup/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nwb_benchmarks/setup/__init__.py b/src/nwb_benchmarks/setup/__init__.py index 6f6121b..aeedaa9 100644 --- a/src/nwb_benchmarks/setup/__init__.py +++ b/src/nwb_benchmarks/setup/__init__.py @@ -1,4 +1,5 @@ """Exposed imports to the `setup` submodule.""" + from ._configure_machine import ( collect_machine_info, customize_asv_machine_file, From 7c83ccc57668420946d9eb174ca3b1e7ba2c69ac Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 16 Feb 2024 14:18:03 -0500 Subject: [PATCH 10/72] refactor to new style; some debugs for Windows --- .../benchmarks/netbench_streaming.py | 39 ++- .../benchmarks/netperf/__init__.py | 0 .../benchmarks/netperf/profile.py | 312 ------------------ src/nwb_benchmarks/core/__init__.py | 8 + .../_base_network_benchmark.py} | 15 +- .../core/_capture_connections.py | 69 ++++ src/nwb_benchmarks/core/_network_profiler.py | 84 +++++ .../core/_network_statistics.py | 130 ++++++++ .../scripts/demo_network_profiling.py | 53 +++ .../setup/_configure_machine.py | 12 +- 10 files changed, 380 insertions(+), 342 deletions(-) delete mode 100644 src/nwb_benchmarks/benchmarks/netperf/__init__.py delete mode 100644 src/nwb_benchmarks/benchmarks/netperf/profile.py rename src/nwb_benchmarks/{benchmarks/netperf/benchmarks.py => core/_base_network_benchmark.py} (87%) create mode 100644 src/nwb_benchmarks/core/_capture_connections.py create mode 100644 src/nwb_benchmarks/core/_network_profiler.py create mode 100644 src/nwb_benchmarks/core/_network_statistics.py create mode 100644 src/nwb_benchmarks/scripts/demo_network_profiling.py diff --git a/src/nwb_benchmarks/benchmarks/netbench_streaming.py b/src/nwb_benchmarks/benchmarks/netbench_streaming.py index ebd1076..7dd02d9 100644 --- a/src/nwb_benchmarks/benchmarks/netbench_streaming.py +++ b/src/nwb_benchmarks/benchmarks/netbench_streaming.py @@ -10,7 +10,7 @@ from fsspec.asyn import reset_lock from fsspec.implementations.cached import CachingFileSystem -from .netperf.benchmarks import NetworkBenchmarkBase +from nwb_benchmarks.core import BaseNetworkBenchmark # Useful if running in verbose mode warnings.filterwarnings(action="ignore", message="No cached namespaces found in .*") @@ -22,8 +22,8 @@ # S3_NWB = "https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a" # ICEphys -class NetworkBenchmarkRos3Read(NetworkBenchmarkBase): - """Benchmark NWB file read with Ros3""" +class NetworkBenchmarkRos3Read(BaseNetworkBenchmark): + """Benchmark NWB file read with ROS3.""" s3_url = S3_NWB repeat = 1 @@ -63,12 +63,11 @@ def track_total_time(self, cache): return cache["total_time"] -class NetworkBenchmarkRemFileRead(NetworkBenchmarkBase): - """Benchmark NWB file read with RemFile""" +class NetworkBenchmarkRemFileRead(BaseNetworkBenchmark): + """Benchmark NWB file read with RemFile.""" s3_url = S3_NWB repeat = 1 - unit = "Bytes" timeout = 1200 def setup_cache(self): @@ -84,30 +83,46 @@ def test_case(self): def track_bytes_downloaded(self, cache): return cache["bytes_downloaded"] + track_bytes_downloaded.unit = "bytes" + def track_bytes_uploaded(self, cache): return cache["bytes_uploaded"] + track_bytes_uploaded.unit = "bytes" + def track_bytes_total(self, cache): return cache["bytes_total"] + track_bytes_total.unit = "bytes" + def track_num_packets(self, cache): return cache["num_packets"] + track_num_packets.unit = "count" + def track_num_packets_downloaded(self, cache): return cache["num_packets_downloaded"] + track_num_packets_downloaded.unit = "count" + def track_num_packets_uploaded(self, cache): return cache["num_packets_uploaded"] + track_num_packets_uploaded.unit = "count" + def track_total_transfer_time(self, cache): return cache["total_transfer_time"] + track_total_transfer_time.unit = "seconds" + def track_total_time(self, cache): return cache["total_time"] + track_total_time.unit = "seconds" + -class NetworkBenchmarkRemFileWithCacheRead(NetworkBenchmarkBase): - """Benchmark NWB file read with RemFile using a remfile.DiskCache as a temporary cache""" +class NetworkBenchmarkRemFileWithCacheRead(BaseNetworkBenchmark): + """Benchmark NWB file read with RemFile using a remfile.DiskCache as a temporary cache.""" s3_url = S3_NWB repeat = 1 @@ -151,8 +166,8 @@ def track_total_time(self, cache): return cache["total_time"] -class NetworkBenchmarkFsspecWithCacheFileRead(NetworkBenchmarkBase): - """Benchmark NWB file read with fsspec using CachingFileSystem""" +class NetworkBenchmarkFsspecWithCacheFileRead(BaseNetworkBenchmark): + """Benchmark NWB file read with fsspec using CachingFileSystem.""" s3_url = S3_NWB repeat = 1 @@ -200,8 +215,8 @@ def track_total_time(self, cache): return cache["total_time"] -class NetworkBenchmarkFsspecFileRead(NetworkBenchmarkBase): - """Benchmark NWB file read with fsspec (no extra cache)""" +class NetworkBenchmarkFsspecFileRead(BaseNetworkBenchmark): + """Benchmark NWB file read with fsspec (no extra cache).""" s3_url = S3_NWB repeat = 1 diff --git a/src/nwb_benchmarks/benchmarks/netperf/__init__.py b/src/nwb_benchmarks/benchmarks/netperf/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/nwb_benchmarks/benchmarks/netperf/profile.py b/src/nwb_benchmarks/benchmarks/netperf/profile.py deleted file mode 100644 index 94f5df9..0000000 --- a/src/nwb_benchmarks/benchmarks/netperf/profile.py +++ /dev/null @@ -1,312 +0,0 @@ -""" -Example script to test capturing network traffic for remote data reads for NWB - -NOTE: This requires sudo/root access on macOS and AIX -""" - -import os -import subprocess -import tempfile -import time -from threading import Thread - -import numpy as np -import psutil -import pyshark -from pynwb import NWBHDF5IO - - -class CaptureConnections(Thread): - """ - Thread class used to listen for connections on this machine - and collecting the mapping of connections to process pid's - """ - - def __init__(self): - super(CaptureConnections, self).__init__() - self.__connection_to_pid = {} # map each pair of connection ports to the corresponding process ID (PID) - self.__run_capture_connections = False # Used to control the capture_connections thread - - @property - def connection_to_pid(self): - """ - Dict mapping of connections to process pid's. - The keys are tuples of 2 ports, and the values are the process id - """ - return self.__connection_to_pid - - def get_connections_for_pid(self, pid): - """ - Get list of all the connection for a given pid from `self.connection_to_pid` - """ - return [k for k, v in self.connection_to_pid.items() if v == pid] - - def start(self): - """Start the capture thread""" - self.__run_capture_connections = True - super(CaptureConnections, self).start() - - def stop(self): - """Stop the capture thread""" - self.__run_capture_connections = False - - def run(self): - """Run the capture thread""" - while self.__run_capture_connections: - # using psutil, we can grab each connection's source and destination ports - # and their process ID - for connection in psutil.net_connections(): # NOTE: This requires sudo/root access on macOS and AIX - if connection.laddr and connection.raddr and connection.pid: - # if local address, remote address and PID are in the connection - # add them to our global dictionary - self.connection_to_pid[(connection.laddr.port, connection.raddr.port)] = connection.pid - self.connection_to_pid[(connection.raddr.port, connection.laddr.port)] = connection.pid - # check how much sleep-time we should use - time.sleep(0.2) - - @staticmethod - def get_local_addresses(): - """Get list of all local addresses""" - addresses = [ - psutil.net_if_addrs()[interface][0].address # get the address of the interface - for interface in psutil.net_if_addrs() # for all network interfaces - if psutil.net_if_addrs()[interface][0].address # if the interface has a valid address - ] - return addresses - - -class NetProfiler: - """ - Use TShark command line interface in a subprocess to capture network traffic packets - in the background. - """ - - def __init__(self, capture_filename=None): - self.__capture_filename = capture_filename - self.__tshark_process = None - self.__packets = None - if self.__capture_filename is None: - self.__capture_filename = tempfile.NamedTemporaryFile(mode="w", delete=False) - - def __del__(self): - self.stop_capture() - if os.path.exists(self.capture_filename): - os.remove(self.capture_filename) - - @property - def packets(self): - """List of all packets captured""" - if self.__packets is None: - try: - cap = pyshark.FileCapture(self.capture_filename) - self.__packets = [packet for packet in cap] - del cap - except: - pass - return self.__packets - - @property - def capture_filename(self): - """Filename of the capture file""" - if isinstance(self.__capture_filename, str): - return self.__capture_filename - else: - return self.__capture_filename.name - - def start_capture(self): - """Start the capture with tshark in a subprocess""" - tsharkCall = ["tshark", "-w", self.capture_filename] - self.__tshark_process = subprocess.Popen(tsharkCall, stderr=subprocess.DEVNULL) # , - time.sleep(0.2) # not sure if this is needed but just to be safe - - def stop_capture(self): - """Stop the capture with tshark in a subprocess""" - if self.__tshark_process is not None: - self.__tshark_process.terminate() - self.__tshark_process.kill() - del self.__tshark_process - self.__tshark_process = None - - def get_packets_for_connections(self, pid_connections: list): - """ - Get packets for all connections in the given pid_connections list. - To get the local connection we can use CaptureConntections to - simultaneously capture all connections with psutils and then use - `connections_thread.get_connections_for_pid(os.getpid())` to get - the local connections. - """ - pid_packets = [] - try: - for packet in self.packets: - if hasattr(packet, "tcp"): - ports = int(str(packet.tcp.srcport)), int(str(packet.tcp.dstport)) - if ports in pid_connections: - pid_packets.append(packet) - except Exception: # pyshark.capture.capture.TSharkCrashException: - pass - return pid_packets - - -class NetStats: - """Compute basic statistics about network packets captures with tshark/pyshark""" - - @staticmethod - def num_packets(packets: list): - """Total number of packets""" - return len(packets) - - @staticmethod - def get_web_traffic_packets(packets): - """ - Get all HTTP and HTTPS packets from the packets captured. - - :param packet: raw packet from a pcap file. - :return: specific packet details - """ - web_packets = [ - packet - for packet in packets - if hasattr(packet, "tcp") and packet[packet.transport_layer].dstport in ["80", "443"] - ] - return web_packets - - @staticmethod - def transfer_time(packets): - """Sum of all time_delta's between packets""" - return float(np.sum([float(p.tcp.time_delta) for p in packets])) - - @staticmethod - def num_packets_downloaded(packets: list, local_addresses: list = None): - """Total number of packets downloaded""" - if local_addresses is None: - local_addresses = CaptureConnections.get_local_addresses() - downloaded = [ - packet - for packet in packets # check all packets - if packet.ip.src not in local_addresses # the source address is not ours so it's download - ] - return int(len(downloaded)) - - @staticmethod - def num_packets_uploaded(packets: list, local_addresses: list = None): - """Total number of packets uploaded (e.g., HTTP requests)""" - if local_addresses is None: - local_addresses = CaptureConnections.get_local_addresses() - uploaded = [ - packet - for packet in packets # check all packets - if packet.ip.src in local_addresses # the source address is ours so it's upload - ] - return int(len(uploaded)) - - @staticmethod - def total_bytes(packets: list): - """Total number of bytes in the packets""" - total_bytes = np.sum([len(packet) for packet in packets]) - return int(total_bytes) - - @staticmethod - def bytes_downloaded(packets: list, local_addresses: list = None): - """Total number of bytes from downloaded packets""" - if local_addresses is None: - local_addresses = CaptureConnections.get_local_addresses() - bytes_downloaded = np.sum( - [ - len(packet) # size of packet in bytes - for packet in packets # check all packets - if packet.ip.src not in local_addresses # the source address is not ours so it's download - ] - ) - return int(bytes_downloaded) - - @staticmethod - def bytes_uploaded(packets: list, local_addresses: list = None): - """Total number of bytes from uploaded packets""" - if local_addresses is None: - local_addresses = CaptureConnections.get_local_addresses() - bytes_uploaded = np.sum( - [ - len(packet) # size of packet in bytes - for packet in packets # check all packets - if packet.ip.src in local_addresses # the source address is ours so it's upload - ] - ) - return int(bytes_uploaded) - - @staticmethod - def bytes_to_str(bytes: int) -> str: - """Format the size in bytes as a human-readable string""" - for unit in ["", "K", "M", "G", "T", "P"]: - if bytes < 1024: - return f"{bytes:.2f}{unit}B" - bytes /= 1024 - - @classmethod - def get_stats(cls, packets: list, local_addresses: list = None): - """ - Calculate all the statistics and return them as a dictionary - """ - if local_addresses is None: - local_addresses = CaptureConnections.get_local_addresses() - stats = { - "bytes_downloaded": cls.bytes_downloaded(packets=packets, local_addresses=local_addresses), - "bytes_uploaded": cls.bytes_uploaded(packets=packets, local_addresses=local_addresses), - "bytes_total": cls.total_bytes(packets=packets), - "num_packets": cls.num_packets(packets=packets), - "num_packets_downloaded": cls.num_packets_downloaded(packets=packets, local_addresses=local_addresses), - "num_packets_uploaded": cls.num_packets_uploaded(packets=packets, local_addresses=local_addresses), - "num_web_packets": len(cls.get_web_traffic_packets(packets)), - "total_transfer_time": cls.transfer_time(packets=packets), - } - return stats - - @classmethod - def print_stats(cls, packets: list, local_addresses: list = None): - """Print all the statistics""" - stats = cls.get_stats(packets=packets, local_addresses=local_addresses) - for k, v in stats.items(): - print(f"{k}: {v}") - - -if __name__ == "__main__": - ### - # This would be part of the setUp() of a test - ## - # start the capture_connections() function to update the current connections of this machine - connections_thread = CaptureConnections() - connections_thread.start() - time.sleep(0.2) # not sure if this is needed but just to be safe - - # start capturing the raw packets by running the tshark commandline tool in a subprocess - netprofiler = NetProfiler() - netprofiler.start_capture() - - ### - # This would be the unit test - ### - # Read the NWB data file from DANDI - t0 = time.time() - s3_path = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" - with NWBHDF5IO(s3_path, mode="r", driver="ros3") as io: - nwbfile = io.read() - test_data = nwbfile.acquisition["ts_name"].data[:] - t1 = time.time() - total_time = t1 - t0 - - ### - # This part would go into tearDown to stop the capture and compute statistics - ### - # Stop capturing packets and connections - netprofiler.stop_capture() - connections_thread.stop() - - # get the connections for the PID of this process - pid_connections = connections_thread.get_connections_for_pid(os.getpid()) - - # Parse packets and filter out all the packets for this process pid by matching with the pid_connections - pid_packets = netprofiler.get_packets_for_connections(pid_connections) - - # Print basic connection statistics - print("num_connections:", int(len(pid_connections) / 2.0)) - NetStats.print_stats(packets=pid_packets) - print("total_time:", total_time) diff --git a/src/nwb_benchmarks/core/__init__.py b/src/nwb_benchmarks/core/__init__.py index 497546c..e093193 100644 --- a/src/nwb_benchmarks/core/__init__.py +++ b/src/nwb_benchmarks/core/__init__.py @@ -1,6 +1,10 @@ """Exposed imports to the `core` submodule.""" +from ._base_network_benchmark import BaseNetworkBenchmark +from ._capture_connections import CaptureConnections from ._dandi import get_s3_url +from ._network_profiler import NetworkProfiler +from ._network_statistics import NetworkStatistics from ._nwb_helpers import get_object_by_name from ._streaming import ( read_hdf5_fsspec_no_cache, @@ -13,6 +17,10 @@ ) __all__ = [ + "BaseNetworkBenchmark", + "CaptureConnections", + "NetworkProfiler", + "NetworkStatistics", "read_hdf5_fsspec_no_cache", "read_hdf5_nwbfile_fsspec_no_cache", "read_hdf5_ros3", diff --git a/src/nwb_benchmarks/benchmarks/netperf/benchmarks.py b/src/nwb_benchmarks/core/_base_network_benchmark.py similarity index 87% rename from src/nwb_benchmarks/benchmarks/netperf/benchmarks.py rename to src/nwb_benchmarks/core/_base_network_benchmark.py index 5a8b40a..1defe44 100644 --- a/src/nwb_benchmarks/benchmarks/netperf/benchmarks.py +++ b/src/nwb_benchmarks/core/_base_network_benchmark.py @@ -1,17 +1,16 @@ -""" -Base classes for implementing benchmarks for evaluating -network performance metrics for streaming read of NWB files. -""" +"""Base class for implementing benchmarks for evaluating network performance metrics for streaming read of NWB files.""" import os import time from asv_runner.benchmarks.mark import SkipNotImplemented -from .profile import CaptureConnections, NetProfiler, NetStats +from ._capture_connections import CaptureConnections +from ._network_profiler import NetworkProfiler +from ._network_statistics import NetworkStatistics -class NetworkBenchmarkBase: +class BaseNetworkBenchmark: """ Base class for network performance metrics for streaming data access. @@ -68,7 +67,7 @@ def start_net_capture(self): time.sleep(0.2) # not sure if this is needed but just to be safe # start capturing the raw packets by running the tshark commandline tool in a subprocess - self.netprofiler = NetProfiler() + self.netprofiler = NetworkProfiler() self.netprofiler.start_capture() def stop_netcapture(self): @@ -81,4 +80,4 @@ def stop_netcapture(self): # Parse packets and filter out all the packets for this process pid by matching with the pid_connections self.pid_packets = self.netprofiler.get_packets_for_connections(self.pid_connections) # Compute all the network statistics - self.net_stats = NetStats.get_stats(packets=self.pid_packets) + self.net_stats = NetworkStatistics.get_stats(packets=self.pid_packets) diff --git a/src/nwb_benchmarks/core/_capture_connections.py b/src/nwb_benchmarks/core/_capture_connections.py new file mode 100644 index 0000000..43e0697 --- /dev/null +++ b/src/nwb_benchmarks/core/_capture_connections.py @@ -0,0 +1,69 @@ +""" +Class definition for capturing network traffic when remotely reading data from an NWB file. + +NOTE: This requires sudo/root access on macOS and AIX. +""" + +import time +from threading import Thread + +import psutil + + +class CaptureConnections(Thread): + """ + Thread class used to listen to connections on this machine. + + Collects a mapping of connections to process PIDs. + """ + + def __init__(self): + super(CaptureConnections, self).__init__() + self.__connection_to_pid = {} # map each pair of connection ports to the corresponding process ID (PID) + self.__run_capture_connections = False # Used to control the capture_connections thread + + @property + def connection_to_pid(self): + """ + Dict mapping of connections to process pid's. + + The keys are tuples of 2 ports, and the values are the process id + """ + return self.__connection_to_pid + + def get_connections_for_pid(self, pid: int): + """Get list of all the connection for a given pid from `self.connection_to_pid`.""" + return [k for k, v in self.connection_to_pid.items() if v == pid] + + def start(self): + """Start the capture thread.""" + self.__run_capture_connections = True + super(CaptureConnections, self).start() + + def stop(self): + """Stop the capture thread.""" + self.__run_capture_connections = False + + def run(self): + """Run the capture thread.""" + while self.__run_capture_connections: + # using psutil, we can grab each connection's source and destination ports + # and their process ID + for connection in psutil.net_connections(): # NOTE: This requires sudo/root access on macOS and AIX + if connection.laddr and connection.raddr and connection.pid: + # if local address, remote address and PID are in the connection + # add them to our global dictionary + self.connection_to_pid[(connection.laddr.port, connection.raddr.port)] = connection.pid + self.connection_to_pid[(connection.raddr.port, connection.laddr.port)] = connection.pid + # check how much sleep-time we should use + time.sleep(0.2) + + @staticmethod + def get_local_addresses() -> list: + """Get list of all local addresses.""" + addresses = [ + psutil.net_if_addrs()[interface][0].address # get the address of the interface + for interface in psutil.net_if_addrs() # for all network interfaces + if psutil.net_if_addrs()[interface][0].address # if the interface has a valid address + ] + return addresses diff --git a/src/nwb_benchmarks/core/_network_profiler.py b/src/nwb_benchmarks/core/_network_profiler.py new file mode 100644 index 0000000..180fd8e --- /dev/null +++ b/src/nwb_benchmarks/core/_network_profiler.py @@ -0,0 +1,84 @@ +""" +Class definition for capturing network traffic when remotely reading data from an NWB file. + +NOTE: This requires sudo/root access on macOS and AIX. +""" + +import pathlib +import subprocess +import tempfile +import time +import warnings +from typing import Union + +import pyshark + + +class NetworkProfiler: + """Use TShark command line interface in a subprocess to capture network traffic packets in the background.""" + + def __init__(self, capture_file_path: Union[pathlib.Path, None] = None): + self.__tshark_process = None + self.__packets = None + + self.capture_file_path = capture_file_path + if self.capture_file_path is None: + self.capture_file = tempfile.NamedTemporaryFile(mode="w", delete=False) + self.capture_file_path = pathlib.Path(self.capture_file.name) + + def __del__(self): + """Stop capture and cleanup temporary file.""" + self.stop_capture() + try: + self.capture_file_path.unlink(missing_ok=True) + except PermissionError: + warnings.warn("Unable to clean temporary network profiling files! Please clean temp directory manually.") + + @property + def packets(self): + """List of all packets captured.""" + if self.__packets is None: + try: + cap = pyshark.FileCapture(self.capture_file_path) + self.__packets = [packet for packet in cap] + del cap + except Exception: + pass + return self.__packets + + def start_capture(self, tshark_exe_path: Union[pathlib.Path, None] = None): + """Start the capture with tshark in a subprocess.""" + tshark_exe_path = tshark_exe_path or "tshark" + tsharkCall = [str(tshark_exe_path), "-w", str(self.capture_file_path)] + self.__tshark_process = subprocess.Popen(tsharkCall, stderr=subprocess.DEVNULL) + time.sleep(0.2) # not sure if this is needed but just to be safe + + def stop_capture(self): + """Stop the capture with tshark in a subprocess.""" + if self.__tshark_process is not None: + self.__tshark_process.terminate() + self.__tshark_process.kill() + del self.__tshark_process + self.__tshark_process = None + if hasattr(self, "capture_file"): + del self.capture_file + + def get_packets_for_connections(self, pid_connections: list): + """ + Get packets for all connections in the given pid_connections list. + + To get the local connection we can use CaptureConntections to + simultaneously capture all connections with psutils and then use + `connections_thread.get_connections_for_pid(os.getpid())` to get + the local connections. + """ + pid_packets = [] + try: + for packet in self.packets: + if hasattr(packet, "tcp"): + ports = int(str(packet.tcp.srcport)), int(str(packet.tcp.dstport)) + if ports in pid_connections: + pid_packets.append(packet) + except Exception: # pyshark.capture.capture.TSharkCrashException: + pass + return pid_packets diff --git a/src/nwb_benchmarks/core/_network_statistics.py b/src/nwb_benchmarks/core/_network_statistics.py new file mode 100644 index 0000000..64d7722 --- /dev/null +++ b/src/nwb_benchmarks/core/_network_statistics.py @@ -0,0 +1,130 @@ +"""Class for summary and display of basic network statistics.""" + +import numpy as np + +from ._capture_connections import CaptureConnections + + +class NetworkStatistics: + """Compute basic statistics about network packets captures with tshark/pyshark.""" + + @staticmethod + def num_packets(packets: list): + """Total number of packets.""" + return len(packets) + + @staticmethod + def get_web_traffic_packets(packets: list) -> list: + """ + Get all HTTP and HTTPS packets from the packets captured. + + Parameters + ---------- + packet : + Raw packet from a pcap file. + + Returns + ------- + Specific packet details. + """ + web_packets = [ + packet + for packet in packets + if hasattr(packet, "tcp") and packet[packet.transport_layer].dstport in ["80", "443"] + ] + return web_packets + + @staticmethod + def transfer_time(packets: list) -> float: + """Sum of all time_delta's between packets.""" + return float(np.sum([float(p.tcp.time_delta) for p in packets])) + + @staticmethod + def num_packets_downloaded(packets: list, local_addresses: list = None): + """Total number of packets downloaded.""" + if local_addresses is None: + local_addresses = CaptureConnections.get_local_addresses() + downloaded = [ + packet + for packet in packets # check all packets + if packet.ip.src not in local_addresses # the source address is not ours so it's download + ] + return int(len(downloaded)) + + @staticmethod + def num_packets_uploaded(packets: list, local_addresses: list = None): + """Total number of packets uploaded (e.g., HTTP requests).""" + if local_addresses is None: + local_addresses = CaptureConnections.get_local_addresses() + uploaded = [ + packet + for packet in packets # check all packets + if packet.ip.src in local_addresses # the source address is ours so it's upload + ] + return int(len(uploaded)) + + @staticmethod + def total_bytes(packets: list): + """Total number of bytes in the packets.""" + total_bytes = np.sum([len(packet) for packet in packets]) + return int(total_bytes) + + @staticmethod + def bytes_downloaded(packets: list, local_addresses: list = None): + """Total number of bytes from downloaded packets.""" + if local_addresses is None: + local_addresses = CaptureConnections.get_local_addresses() + bytes_downloaded = np.sum( + [ + len(packet) # size of packet in bytes + for packet in packets # check all packets + if packet.ip.src not in local_addresses # the source address is not ours so it's download + ] + ) + return int(bytes_downloaded) + + @staticmethod + def bytes_uploaded(packets: list, local_addresses: list = None): + """Total number of bytes from uploaded packets.""" + if local_addresses is None: + local_addresses = CaptureConnections.get_local_addresses() + bytes_uploaded = np.sum( + [ + len(packet) # size of packet in bytes + for packet in packets # check all packets + if packet.ip.src in local_addresses # the source address is ours so it's upload + ] + ) + return int(bytes_uploaded) + + @staticmethod + def bytes_to_str(size_in_bytes: int) -> str: + """Format the size in bytes as a human-readable string.""" + for unit in ["", "K", "M", "G", "T", "P"]: + if size_in_bytes < 1024: + return f"{bytes:.2f}{unit}B" + size_in_bytes /= 1024 + + @classmethod + def get_stats(cls, packets: list, local_addresses: list = None): + """Calculate all the statistics and return them as a dictionary.""" + if local_addresses is None: + local_addresses = CaptureConnections.get_local_addresses() + stats = { + "bytes_downloaded": cls.bytes_downloaded(packets=packets, local_addresses=local_addresses), + "bytes_uploaded": cls.bytes_uploaded(packets=packets, local_addresses=local_addresses), + "bytes_total": cls.total_bytes(packets=packets), + "num_packets": cls.num_packets(packets=packets), + "num_packets_downloaded": cls.num_packets_downloaded(packets=packets, local_addresses=local_addresses), + "num_packets_uploaded": cls.num_packets_uploaded(packets=packets, local_addresses=local_addresses), + "num_web_packets": len(cls.get_web_traffic_packets(packets)), + "total_transfer_time": cls.transfer_time(packets=packets), + } + return stats + + @classmethod + def print_stats(cls, packets: list, local_addresses: list = None): + """Print all the statistics.""" + stats = cls.get_stats(packets=packets, local_addresses=local_addresses) + for k, v in stats.items(): + print(f"{k}: {v}") diff --git a/src/nwb_benchmarks/scripts/demo_network_profiling.py b/src/nwb_benchmarks/scripts/demo_network_profiling.py new file mode 100644 index 0000000..0794565 --- /dev/null +++ b/src/nwb_benchmarks/scripts/demo_network_profiling.py @@ -0,0 +1,53 @@ +"""Demo of network profiling tools.""" + +import os +import pathlib +import time + +import pynwb + +from nwb_benchmarks.core import CaptureConnections, NetworkProfiler, NetworkStatistics + +### +# This would be part of the setUp() of a test +## +# start the capture_connections() function to update the current connections of this machine +connections_thread = CaptureConnections() +connections_thread.start() +time.sleep(0.2) # not sure if this is needed but just to be safe + +# start capturing the raw packets by running the tshark commandline tool in a subprocess +netprofiler = NetworkProfiler() + +# If `tshark` is not available as a global command, specify the pathlib.Path pointing to the .exe +netprofiler.start_capture(tshark_exe_path=pathlib.Path("D:/Wireshark/tshark.exe")) + +### +# This would be the unit test +### +# Read the NWB data file from DANDI +t0 = time.time() +s3_path = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" +with pynwb.NWBHDF5IO(s3_path, mode="r", driver="ros3") as io: + nwbfile = io.read() + test_data = nwbfile.acquisition["ts_name"].data[:] +t1 = time.time() +total_time = t1 - t0 + +### +# This part would go into tearDown to stop the capture and compute statistics +### +# Stop capturing packets and connections +netprofiler.stop_capture() +connections_thread.stop() + +# get the connections for the PID of this process +pid_connections = connections_thread.get_connections_for_pid(os.getpid()) + +# Parse packets and filter out all the packets for this process pid by matching with the pid_connections +pid_packets = netprofiler.get_packets_for_connections(pid_connections) + +# Print basic connection statistics +print("num_connections:", int(len(pid_connections) / 2.0)) +NetworkStatistics.print_stats(packets=pid_packets) +print("total_time:", total_time) diff --git a/src/nwb_benchmarks/setup/_configure_machine.py b/src/nwb_benchmarks/setup/_configure_machine.py index e655e93..181c553 100644 --- a/src/nwb_benchmarks/setup/_configure_machine.py +++ b/src/nwb_benchmarks/setup/_configure_machine.py @@ -6,7 +6,6 @@ import pathlib import platform import sys -import unittest import warnings from typing import Any, Dict @@ -120,18 +119,11 @@ def ensure_machine_info_current(file_path: pathlib.Path): return # If debugging is ever necessary in the future, best way I found to summarize differences was + # import unittest + # # test = unittest.TestCase() # test.maxDiff = None # test.assertDictEqual(d1=machine_info_from_file, d2=current_machine_info) warnings.warn("The current machine info is out of date! Automatically updating the file.", stacklevel=2) customize_asv_machine_file(file_path=file_path, overwrite=True) - - -if __name__ == "__main__": - home_directory = pathlib.Path.home() - - # This should theoretically always be the location across systems - default_asv_machine_file_path = home_directory / ".asv-machine.json" - - customize_asv_machine_file(file_path=default_asv_machine_file_path) From f7891ced34d64b0cb877d53ad0ff9d372add4853 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 16 Feb 2024 14:57:45 -0500 Subject: [PATCH 11/72] integrate directly into other benchmarks --- .../benchmarks/netbench_streaming.py | 261 ------------------ .../benchmarks/remote_file_reading.py | 32 ++- .../benchmarks/remote_slicing.py | 16 +- src/nwb_benchmarks/benchmarks/streaming.py | 125 --------- .../core/_base_network_benchmark.py | 83 +++--- 5 files changed, 88 insertions(+), 429 deletions(-) delete mode 100644 src/nwb_benchmarks/benchmarks/netbench_streaming.py delete mode 100644 src/nwb_benchmarks/benchmarks/streaming.py diff --git a/src/nwb_benchmarks/benchmarks/netbench_streaming.py b/src/nwb_benchmarks/benchmarks/netbench_streaming.py deleted file mode 100644 index 7dd02d9..0000000 --- a/src/nwb_benchmarks/benchmarks/netbench_streaming.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Basic benchmarks for network performance metrics for streaming read of NWB files.""" - -import tempfile -import warnings - -import fsspec -import h5py -import pynwb -import remfile -from fsspec.asyn import reset_lock -from fsspec.implementations.cached import CachingFileSystem - -from nwb_benchmarks.core import BaseNetworkBenchmark - -# Useful if running in verbose mode -warnings.filterwarnings(action="ignore", message="No cached namespaces found in .*") -warnings.filterwarnings(action="ignore", message="Ignoring cached namespace .*") - -S3_NWB = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" # Small test NWB file -# S3_NWB = "https://dandiarchive.s3.amazonaws.com/blobs/c49/311/c493119b-4b99-4b14-bc03-65bb28cfbd29" # ECephy+Behavior -# S3_NWB = "https://dandiarchive.s3.amazonaws.com/blobs/38c/c24/38cc240b-77c5-418a-9040-a7f582ff6541" # Ophys -# S3_NWB = "https://dandiarchive.s3.amazonaws.com/blobs/c98/3a4/c983a4e1-097a-402c-bda8-e6a41cb7e24a" # ICEphys - - -class NetworkBenchmarkRos3Read(BaseNetworkBenchmark): - """Benchmark NWB file read with ROS3.""" - - s3_url = S3_NWB - repeat = 1 - unit = "Bytes" - timeout = 1200 - - def setup_cache(self): - return self.compute_test_case_metrics() - - def test_case(self): - with pynwb.NWBHDF5IO(self.s3_url, mode="r", driver="ros3") as io: - nwbfile = io.read() - # test_data = nwbfile.acquisition["ts_name"].data[:] - - def track_bytes_downloaded(self, cache): - return cache["bytes_downloaded"] - - def track_bytes_uploaded(self, cache): - return cache["bytes_uploaded"] - - def track_bytes_total(self, cache): - return cache["bytes_total"] - - def track_num_packets(self, cache): - return cache["num_packets"] - - def track_num_packets_downloaded(self, cache): - return cache["num_packets_downloaded"] - - def track_num_packets_uploaded(self, cache): - return cache["num_packets_uploaded"] - - def track_total_transfer_time(self, cache): - return cache["total_transfer_time"] - - def track_total_time(self, cache): - return cache["total_time"] - - -class NetworkBenchmarkRemFileRead(BaseNetworkBenchmark): - """Benchmark NWB file read with RemFile.""" - - s3_url = S3_NWB - repeat = 1 - timeout = 1200 - - def setup_cache(self): - return self.compute_test_case_metrics() - - def test_case(self): - byte_stream = remfile.File(url=self.s3_url) - with h5py.File(name=byte_stream) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() - # test_data = nwbfile.acquisition["ts_name"].data[:] - - def track_bytes_downloaded(self, cache): - return cache["bytes_downloaded"] - - track_bytes_downloaded.unit = "bytes" - - def track_bytes_uploaded(self, cache): - return cache["bytes_uploaded"] - - track_bytes_uploaded.unit = "bytes" - - def track_bytes_total(self, cache): - return cache["bytes_total"] - - track_bytes_total.unit = "bytes" - - def track_num_packets(self, cache): - return cache["num_packets"] - - track_num_packets.unit = "count" - - def track_num_packets_downloaded(self, cache): - return cache["num_packets_downloaded"] - - track_num_packets_downloaded.unit = "count" - - def track_num_packets_uploaded(self, cache): - return cache["num_packets_uploaded"] - - track_num_packets_uploaded.unit = "count" - - def track_total_transfer_time(self, cache): - return cache["total_transfer_time"] - - track_total_transfer_time.unit = "seconds" - - def track_total_time(self, cache): - return cache["total_time"] - - track_total_time.unit = "seconds" - - -class NetworkBenchmarkRemFileWithCacheRead(BaseNetworkBenchmark): - """Benchmark NWB file read with RemFile using a remfile.DiskCache as a temporary cache.""" - - s3_url = S3_NWB - repeat = 1 - unit = "Bytes" - timeout = 1200 - - def setup_cache(self): - return self.compute_test_case_metrics() - - def test_case(self): - byte_stream = remfile.File(url=self.s3_url) - with tempfile.TemporaryDirectory() as tmpdirname: - disk_cache = remfile.DiskCache(tmpdirname) - with h5py.File(name=byte_stream, disk_cache=disk_cache) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() - # test_data = nwbfile.acquisition["ts_name"].data[:] - - def track_bytes_downloaded(self, cache): - return cache["bytes_downloaded"] - - def track_bytes_uploaded(self, cache): - return cache["bytes_uploaded"] - - def track_bytes_total(self, cache): - return cache["bytes_total"] - - def track_num_packets(self, cache): - return cache["num_packets"] - - def track_num_packets_downloaded(self, cache): - return cache["num_packets_downloaded"] - - def track_num_packets_uploaded(self, cache): - return cache["num_packets_uploaded"] - - def track_total_transfer_time(self, cache): - return cache["total_transfer_time"] - - def track_total_time(self, cache): - return cache["total_time"] - - -class NetworkBenchmarkFsspecWithCacheFileRead(BaseNetworkBenchmark): - """Benchmark NWB file read with fsspec using CachingFileSystem.""" - - s3_url = S3_NWB - repeat = 1 - unit = "Bytes" - timeout = 1200 - - def setup_cache(self): - return self.compute_test_case_metrics() - - def test_case(self): - reset_lock() - fsspec.get_filesystem_class("https").clear_instance_cache() - filesystem = fsspec.filesystem("https") - # create a cache to save downloaded data to disk (optional) - cached_filesystem = CachingFileSystem(fs=filesystem) # Use temporary storage that will be cleaned up - - with cached_filesystem.open(path=self.s3_url, mode="rb") as byte_stream: - with h5py.File(name=byte_stream) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() - # test_data = nwbfile.acquisition["ts_name"].data[:] - - def track_bytes_downloaded(self, cache): - return cache["bytes_downloaded"] - - def track_bytes_uploaded(self, cache): - return cache["bytes_uploaded"] - - def track_bytes_total(self, cache): - return cache["bytes_total"] - - def track_num_packets(self, cache): - return cache["num_packets"] - - def track_num_packets_downloaded(self, cache): - return cache["num_packets_downloaded"] - - def track_num_packets_uploaded(self, cache): - return cache["num_packets_uploaded"] - - def track_total_transfer_time(self, cache): - return cache["total_transfer_time"] - - def track_total_time(self, cache): - return cache["total_time"] - - -class NetworkBenchmarkFsspecFileRead(BaseNetworkBenchmark): - """Benchmark NWB file read with fsspec (no extra cache).""" - - s3_url = S3_NWB - repeat = 1 - unit = "Bytes" - timeout = 1200 - - def setup_cache(self): - return self.compute_test_case_metrics() - - def test_case(self): - reset_lock() - fsspec.get_filesystem_class("https").clear_instance_cache() - filesystem = fsspec.filesystem("https") - with filesystem.open(path=self.s3_url, mode="rb") as byte_stream: - with h5py.File(name=byte_stream) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() - # test_data = nwbfile.acquisition["ts_name"].data[:] - - def track_bytes_downloaded(self, cache): - return cache["bytes_downloaded"] - - def track_bytes_uploaded(self, cache): - return cache["bytes_uploaded"] - - def track_bytes_total(self, cache): - return cache["bytes_total"] - - def track_num_packets(self, cache): - return cache["num_packets"] - - def track_num_packets_downloaded(self, cache): - return cache["num_packets_downloaded"] - - def track_num_packets_uploaded(self, cache): - return cache["num_packets_uploaded"] - - def track_total_transfer_time(self, cache): - return cache["total_transfer_time"] - - def track_total_time(self, cache): - return cache["total_time"] diff --git a/src/nwb_benchmarks/benchmarks/remote_file_reading.py b/src/nwb_benchmarks/benchmarks/remote_file_reading.py index e0ea9fb..f048e7f 100644 --- a/src/nwb_benchmarks/benchmarks/remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/remote_file_reading.py @@ -1,6 +1,7 @@ """Basic benchmarks for stream NWB files and their contents.""" from nwb_benchmarks.core import ( + BaseNetworkBenchmark, get_s3_url, read_hdf5_fsspec_no_cache, read_hdf5_nwbfile_fsspec_no_cache, @@ -17,10 +18,11 @@ dandiset_id="000717", dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", ), # Not the best example for testing a theory about file read; should probably replace with something simpler + "https://dandiarchive.s3.amazonaws.com/ros3test.nwb", # The original small test NWB file ] -class FsspecNoCacheDirectFileReadBenchmark: +class FsspecNoCacheDirectFileReadBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params @@ -28,8 +30,11 @@ class FsspecNoCacheDirectFileReadBenchmark: def time_file_read(self, s3_url: str): self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) + def operation_to_track_network_activity_of(self, s3_url: str): + self.time_file_read(s3_url=s3_url) -class RemfileDirectFileReadBenchmark: + +class RemfileDirectFileReadBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params @@ -37,8 +42,11 @@ class RemfileDirectFileReadBenchmark: def time_file_read(self, s3_url: str): self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) + def operation_to_track_network_activity_of(self, s3_url: str): + self.time_file_read(s3_url=s3_url) + -class Ros3DirectFileReadBenchmark: +class Ros3DirectFileReadBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params @@ -46,8 +54,11 @@ class Ros3DirectFileReadBenchmark: def time_file_read(self, s3_url: str): self.file = read_hdf5_ros3(s3_url=s3_url) + def operation_to_track_network_activity_of(self, s3_url: str): + self.time_file_read(s3_url=s3_url) + -class FsspecNoCacheNWBFileReadBenchmark: +class FsspecNoCacheNWBFileReadBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params @@ -55,8 +66,11 @@ class FsspecNoCacheNWBFileReadBenchmark: def time_file_read(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) + def operation_to_track_network_activity_of(self, s3_url: str): + self.time_file_read(s3_url=s3_url) -class RemfileNWBFileReadBenchmark: + +class RemfileNWBFileReadBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params @@ -64,11 +78,17 @@ class RemfileNWBFileReadBenchmark: def time_file_read(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) + def operation_to_track_network_activity_of(self, s3_url: str): + self.time_file_read(s3_url=s3_url) + -class Ros3NWBFileReadBenchmark: +class Ros3NWBFileReadBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params def time_file_read(self, s3_url: str): self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) + + def operation_to_track_network_activity_of(self, s3_url: str): + self.time_file_read(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/remote_slicing.py b/src/nwb_benchmarks/benchmarks/remote_slicing.py index c7a1743..d63fabf 100644 --- a/src/nwb_benchmarks/benchmarks/remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/remote_slicing.py @@ -3,6 +3,7 @@ from typing import Tuple from nwb_benchmarks.core import ( + BaseNetworkBenchmark, get_object_by_name, get_s3_url, read_hdf5_nwbfile_fsspec_no_cache, @@ -26,7 +27,7 @@ ) -class FsspecNoCacheContinuousSliceBenchmark: +class FsspecNoCacheContinuousSliceBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params @@ -40,8 +41,11 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = self.data_to_slice[slice_range] + def operation_to_track_network_activity_of(self, s3_url: str, slice_range: Tuple[slice]): + self.time_slice(s3_url=s3_url, slice_range=slice_range) -class RemfileContinuousSliceBenchmark: + +class RemfileContinuousSliceBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params @@ -55,8 +59,11 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = self.data_to_slice[slice_range] + def operation_to_track_network_activity_of(self, s3_url: str, slice_range: Tuple[slice]): + self.time_slice(s3_url=s3_url, slice_range=slice_range) + -class Ros3ContinuousSliceBenchmark: +class Ros3ContinuousSliceBenchmark(BaseNetworkBenchmark): repeat = 1 param_names = param_names params = params @@ -69,3 +76,6 @@ def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) + + def operation_to_track_network_activity_of(self, s3_url: str, slice_range: Tuple[slice]): + self.time_slice(s3_url=s3_url, slice_range=slice_range) diff --git a/src/nwb_benchmarks/benchmarks/streaming.py b/src/nwb_benchmarks/benchmarks/streaming.py deleted file mode 100644 index a938c32..0000000 --- a/src/nwb_benchmarks/benchmarks/streaming.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Basic benchmarks for NWB.""" - -import warnings - -import fsspec -import h5py -import pynwb -import remfile -from fsspec.asyn import reset_lock - -# Useful if running in verbose mode -warnings.filterwarnings(action="ignore", message="No cached namespaces found in .*") -warnings.filterwarnings(action="ignore", message="Ignoring cached namespace .*") - - -class FileReadStreaming: - """A basic benchmark for streaming an NWB file from the DANDI archive.""" - - repeat = 1 - - def setup(self): - # Random IBL raw data file; not that many groups - self.s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - - def time_fsspec_no_cache(self): - reset_lock() - fsspec.get_filesystem_class("https").clear_instance_cache() - filesystem = fsspec.filesystem("https") - - with filesystem.open(path=self.s3_url, mode="rb") as byte_stream: - with h5py.File(name=byte_stream) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() - - # Must be done at this level since teardown occurs outside of repetitions - # reset_lock() - # fsspec.get_filesystem_class("https").clear_instance_cache() - - def time_ros3(self): - ros3_form = self.s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") - with pynwb.NWBHDF5IO(path=ros3_form, mode="r", load_namespaces=True, driver="ros3") as io: - nwbfile = io.read() - - def time_remfile(self): - byte_stream = remfile.File(url=self.s3_url) - with h5py.File(name=byte_stream) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() - - # def teardown(self): - # reset_lock() - # fsspec.get_filesystem_class("https").clear_instance_cache() - - -class ElectricalSeriesStreamingROS3: - """ - A basic benchmark for streaming raw ecephys data. - - Needs separate setup per class to only time slicing operation. - """ - - repeat = 1 - - def setup(self): - self.s3_url = "s3://dandiarchive/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - self.acquisition_path = "ElectricalSeriesAp" - self.slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - self.io = pynwb.NWBHDF5IO(path=self.s3_url, mode="r", load_namespaces=True, driver="ros3") - self.nwbfile = self.io.read() - - def time_ros3(self): - self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] - - -class ElectricalSeriesStreamingFsspec: - """ - A basic benchmark for streaming raw ecephys data. - - Needs separate setup per class to only time slicing operation. - """ - - repeat = 1 - - def setup(self): - self.s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - self.acquisition_path = "ElectricalSeriesAp" - self.slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - - reset_lock() - fsspec.get_filesystem_class("https").clear_instance_cache() - - self.filesystem = fsspec.filesystem("https") - self.byte_stream = self.filesystem.open(path=self.s3_url, mode="rb") - self.file = h5py.File(name=self.byte_stream) - self.io = pynwb.NWBHDF5IO(file=self.file, mode="r", load_namespaces=True) - self.nwbfile = self.io.read() - - def time_fsspec_no_cache(self): - self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] - - # def teardown(self): - # reset_lock() - # fsspec.get_filesystem_class("https").clear_instance_cache() - - -class ElectricalSeriesStreamingRemfile: - """ - A basic benchmark for streaming raw ecephys data. - - Needs separate setup per class to only time slicing operation. - """ - - repeat = 1 - - def setup(self): - self.s3_url = "https://dandiarchive.s3.amazonaws.com/blobs/8c5/65f/8c565f28-e5fc-43fe-8fb7-318ad2081319" - self.acquisition_path = "ElectricalSeriesAp" - self.slice_range = (slice(0, 30_000), slice(0, 384)) # ~23 MB - self.byte_stream = remfile.File(url=self.s3_url) - self.file = h5py.File(name=self.byte_stream) - self.io = pynwb.NWBHDF5IO(file=self.file, mode="r", load_namespaces=True) - self.nwbfile = self.io.read() - - def time_remfile(self): - self.nwbfile.acquisition[self.acquisition_path].data[self.slice_range] diff --git a/src/nwb_benchmarks/core/_base_network_benchmark.py b/src/nwb_benchmarks/core/_base_network_benchmark.py index 1defe44..926d8e6 100644 --- a/src/nwb_benchmarks/core/_base_network_benchmark.py +++ b/src/nwb_benchmarks/core/_base_network_benchmark.py @@ -12,44 +12,18 @@ class BaseNetworkBenchmark: """ - Base class for network performance metrics for streaming data access. - - Child classes must implement: - - test_case : implementing the test case to be run - - setup_cache: when implemented here in the base class, asv will run the setup_cache function - only once for all child classes. As such, each child class must implement its - own setup_cache, which should typically just consists of a call to - ``return self.compute_test_case_metrics()``. - - track_...: methods defining the metrics to be tracked. These functions usually just retrieve - the corresponding metric from the cache and return the result. For example: - - .. code-block:: python - - def track_bytes_downloaded(self, cache): # unit is Bytes - return cache["bytes_downloaded"] - - def track_bytes_uploaded(self, cache): # unit is Bytes - return cache["bytes_uploaded"] - - def track_bytes_total(self, cache): # unit is Bytes - return cache["bytes_total"] - - def track_num_packets(self, cache): # unit is Count - return cache["num_packets"] - - def track_num_packets_downloaded(self, cache): # unit is Count - return cache["num_packets_downloaded"] - - def track_num_packets_uploaded(self, cache): # unit is Count - return cache["num_packets_uploaded"] + Base class for ASV Benchmark network performance metrics for streaming data access. + Child classes must implement the `operation_to_track_network_activity_of`, + which replaces the usual time_ or mem_ operation being tracked. """ - s3_url: str = None - - def test_case(self): + def operation_to_track_network_activity_of(self): raise SkipNotImplemented() + def setup_cache(self): + return self.compute_test_case_metrics() + def compute_test_case_metrics(self): self.start_net_capture() t0 = time.time() @@ -57,7 +31,7 @@ def compute_test_case_metrics(self): t1 = time.time() total_time = t1 - t0 self.stop_netcapture() - self.net_stats["total_time"] = total_time + self.net_stats["network_total_time"] = total_time return self.net_stats def start_net_capture(self): @@ -81,3 +55,44 @@ def stop_netcapture(self): self.pid_packets = self.netprofiler.get_packets_for_connections(self.pid_connections) # Compute all the network statistics self.net_stats = NetworkStatistics.get_stats(packets=self.pid_packets) + + def track_bytes_downloaded(self, cache): + return cache["bytes_downloaded"] + + track_bytes_downloaded.unit = "bytes" + + def track_bytes_uploaded(self, cache): + return cache["bytes_uploaded"] + + track_bytes_uploaded.unit = "bytes" + + def track_bytes_total(self, cache): + return cache["bytes_total"] + + track_bytes_total.unit = "bytes" + + def track_num_packets(self, cache): + return cache["num_packets"] + + track_num_packets.unit = "count" + + def track_num_packets_downloaded(self, cache): + return cache["num_packets_downloaded"] + + track_num_packets_downloaded.unit = "count" + + def track_num_packets_uploaded(self, cache): + return cache["num_packets_uploaded"] + + track_num_packets_uploaded.unit = "count" + + def track_total_transfer_time(self, cache): + return cache["total_transfer_time"] + + track_total_transfer_time.unit = "seconds" + + def track_network_total_time(self, cache): + """Technically different from the official time_ approach.""" + return cache["network_total_time"] + + track_network_total_time.unit = "seconds" From 964b8356b221dd02fd810a68982a685a6dceb534 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 16 Feb 2024 15:03:36 -0500 Subject: [PATCH 12/72] pass keyword arguments --- src/nwb_benchmarks/core/_base_network_benchmark.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/nwb_benchmarks/core/_base_network_benchmark.py b/src/nwb_benchmarks/core/_base_network_benchmark.py index 926d8e6..5371676 100644 --- a/src/nwb_benchmarks/core/_base_network_benchmark.py +++ b/src/nwb_benchmarks/core/_base_network_benchmark.py @@ -21,13 +21,15 @@ class BaseNetworkBenchmark: def operation_to_track_network_activity_of(self): raise SkipNotImplemented() - def setup_cache(self): - return self.compute_test_case_metrics() + def setup_cache(self, **keyword_arguments): + return self.compute_test_case_metrics(**keyword_arguments) - def compute_test_case_metrics(self): + def compute_test_case_metrics(self, **keyword_arguments): + """Strategy essentially mimics a single benchmark execution (setup + run), but only once (ignores `repeat`).""" self.start_net_capture() t0 = time.time() - self.test_case() + self.setup(**keyword_arguments) + self.operation_to_track_network_activity_of(**keyword_arguments) t1 = time.time() total_time = t1 - t0 self.stop_netcapture() From b57c2982211059c8d3a530d8e0f9b7305be8f04f Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 16 Feb 2024 15:23:12 -0500 Subject: [PATCH 13/72] less redirection --- src/nwb_benchmarks/benchmarks/remote_file_reading.py | 12 ++++++------ src/nwb_benchmarks/benchmarks/remote_slicing.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/remote_file_reading.py b/src/nwb_benchmarks/benchmarks/remote_file_reading.py index f048e7f..6a5c0e2 100644 --- a/src/nwb_benchmarks/benchmarks/remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/remote_file_reading.py @@ -31,7 +31,7 @@ def time_file_read(self, s3_url: str): self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) def operation_to_track_network_activity_of(self, s3_url: str): - self.time_file_read(s3_url=s3_url) + self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) class RemfileDirectFileReadBenchmark(BaseNetworkBenchmark): @@ -43,7 +43,7 @@ def time_file_read(self, s3_url: str): self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) def operation_to_track_network_activity_of(self, s3_url: str): - self.time_file_read(s3_url=s3_url) + self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) class Ros3DirectFileReadBenchmark(BaseNetworkBenchmark): @@ -55,7 +55,7 @@ def time_file_read(self, s3_url: str): self.file = read_hdf5_ros3(s3_url=s3_url) def operation_to_track_network_activity_of(self, s3_url: str): - self.time_file_read(s3_url=s3_url) + self.file = read_hdf5_ros3(s3_url=s3_url) class FsspecNoCacheNWBFileReadBenchmark(BaseNetworkBenchmark): @@ -67,7 +67,7 @@ def time_file_read(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) def operation_to_track_network_activity_of(self, s3_url: str): - self.time_file_read(s3_url=s3_url) + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) class RemfileNWBFileReadBenchmark(BaseNetworkBenchmark): @@ -79,7 +79,7 @@ def time_file_read(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) def operation_to_track_network_activity_of(self, s3_url: str): - self.time_file_read(s3_url=s3_url) + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) class Ros3NWBFileReadBenchmark(BaseNetworkBenchmark): @@ -91,4 +91,4 @@ def time_file_read(self, s3_url: str): self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) def operation_to_track_network_activity_of(self, s3_url: str): - self.time_file_read(s3_url=s3_url) + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/remote_slicing.py b/src/nwb_benchmarks/benchmarks/remote_slicing.py index d63fabf..4b9b09d 100644 --- a/src/nwb_benchmarks/benchmarks/remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/remote_slicing.py @@ -41,8 +41,8 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = self.data_to_slice[slice_range] - def operation_to_track_network_activity_of(self, s3_url: str, slice_range: Tuple[slice]): - self.time_slice(s3_url=s3_url, slice_range=slice_range) + def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self._temp = self.data_to_slice[slice_range] class RemfileContinuousSliceBenchmark(BaseNetworkBenchmark): @@ -59,8 +59,8 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = self.data_to_slice[slice_range] - def operation_to_track_network_activity_of(self, s3_url: str, slice_range: Tuple[slice]): - self.time_slice(s3_url=s3_url, slice_range=slice_range) + def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self._temp = self.data_to_slice[slice_range] class Ros3ContinuousSliceBenchmark(BaseNetworkBenchmark): @@ -77,5 +77,5 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) - def operation_to_track_network_activity_of(self, s3_url: str, slice_range: Tuple[slice]): - self.time_slice(s3_url=s3_url, slice_range=slice_range) + def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self._temp = self.data_to_slice[slice_range] From be0ce473daa38311f999bdf3b8a4660e2c938c31 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sat, 17 Feb 2024 00:56:08 -0500 Subject: [PATCH 14/72] finishing refactoring; split time and network for bench flag; always append samples; limit rounds to 1 and increase repeats --- .../network_tracking_remote_file_reading.py | 101 ++++++++++++++++++ .../network_tracking_remote_slicing.py | 84 +++++++++++++++ .../benchmarks/remote_file_reading.py | 94 ---------------- .../benchmarks/time_remote_file_reading.py | 65 +++++++++++ ...mote_slicing.py => time_remote_slicing.py} | 22 ++-- src/nwb_benchmarks/command_line_interface.py | 3 +- .../core/_base_network_benchmark.py | 48 +++++---- src/nwb_benchmarks/core/_streaming.py | 4 +- .../scripts/demo_network_profiling.py | 5 + 9 files changed, 291 insertions(+), 135 deletions(-) create mode 100644 src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py create mode 100644 src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py delete mode 100644 src/nwb_benchmarks/benchmarks/remote_file_reading.py create mode 100644 src/nwb_benchmarks/benchmarks/time_remote_file_reading.py rename src/nwb_benchmarks/benchmarks/{remote_slicing.py => time_remote_slicing.py} (77%) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py new file mode 100644 index 0000000..4ee6e7d --- /dev/null +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -0,0 +1,101 @@ +"""Basic benchmarks for stream NWB files and their contents.""" + +from nwb_benchmarks.core import ( + BaseNetworkBenchmark, + get_s3_url, + read_hdf5_fsspec_no_cache, + read_hdf5_nwbfile_fsspec_no_cache, + read_hdf5_nwbfile_remfile, + read_hdf5_nwbfile_ros3, + read_hdf5_remfile, + read_hdf5_ros3, +) + +# Network base does not yet support params +# param_names = ["s3_url"] +# params = [ +# get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb"), +# get_s3_url( +# dandiset_id="000717", +# dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", +# ), # Not the best example for testing a theory about file read; should probably replace with something simpler +# "https://dandiarchive.s3.amazonaws.com/ros3test.nwb", # The original small test NWB file +# ] + + +class FsspecNoCacheDirectFileReadBenchmark(BaseNetworkBenchmark): + s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" + + def setup_cache(self): + super().setup_cache(s3_url=self.s3_url) + + def setup(self, s3_url: str): + pass + + def operation_to_track_network_activity_of(self, s3_url: str): + self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) + + +class RemfileDirectFileReadBenchmark(BaseNetworkBenchmark): + s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" + + def setup_cache(self): + super().setup_cache(s3_url=self.s3_url) + + def setup(self, s3_url: str): + pass + + def operation_to_track_network_activity_of(self, s3_url: str): + self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) + + +class Ros3DirectFileReadBenchmark(BaseNetworkBenchmark): + s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" + + def setup_cache(self): + super().setup_cache(s3_url=self.s3_url) + + def setup(self, s3_url: str): + pass + + def operation_to_track_network_activity_of(self, s3_url: str): + self.file = read_hdf5_ros3(s3_url=s3_url) + + +class FsspecNoCacheNWBFileReadBenchmark(BaseNetworkBenchmark): + s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" + + def setup_cache(self): + super().setup_cache(s3_url=self.s3_url) + + def setup(self, s3_url: str): + pass + + def operation_to_track_network_activity_of(self, s3_url: str): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) + + +class RemfileNWBFileReadBenchmark(BaseNetworkBenchmark): + s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" + + def setup_cache(self): + super().setup_cache(s3_url=self.s3_url) + + def setup(self, s3_url: str): + pass + + def operation_to_track_network_activity_of(self, s3_url: str): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) + + +class Ros3NWBFileReadBenchmark(BaseNetworkBenchmark): + s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" + + def setup_cache(self): + super().setup_cache(s3_url=self.s3_url) + + def setup(self, s3_url: str): + pass + + def operation_to_track_network_activity_of(self, s3_url: str): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py new file mode 100644 index 0000000..9ea2daf --- /dev/null +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -0,0 +1,84 @@ +"""Basic benchmarks for stream NWB files and their contents.""" + +from typing import Tuple + +from nwb_benchmarks.core import ( + BaseNetworkBenchmark, + get_object_by_name, + get_s3_url, + read_hdf5_nwbfile_fsspec_no_cache, + read_hdf5_nwbfile_remfile, + read_hdf5_nwbfile_ros3, + robust_ros3_read, +) + +# Network base does not yet support params +# Might have more luck using parameterized.paramterize_class +# param_names = ["s3_url", "object_name", "slice_range"] +# params = ( +# [ +# get_s3_url( +# dandiset_id="000717", +# dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", +# ) +# ], +# ["ElectricalSeriesAp"], +# [(slice(0, 30_000), slice(0, 384))], # ~23 MB +# ) +s3_url = get_s3_url( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", +) +object_name = "ElectricalSeriesAp" +slice_range = (slice(0, 30_000), slice(0, 384)) + + +class FsspecNoCacheContinuousSliceBenchmark(BaseNetworkBenchmark): + s3_url = s3_url + object_name = object_name + slice_range = slice_range + + def setup_cache(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + super().setup_cache(s3_url=self.s3_url, object_name=self.object_name, slice_range=self.slice_range) + + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) + self.data_to_slice = self.neurodata_object.data + + def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self._temp = self.data_to_slice[slice_range] + + +class RemfileContinuousSliceBenchmark(BaseNetworkBenchmark): + s3_url = s3_url + object_name = object_name + slice_range = slice_range + + def setup_cache(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + super().setup_cache(s3_url=self.s3_url, object_name=self.object_name, slice_range=self.slice_range) + + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) + self.data_to_slice = self.neurodata_object.data + + def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self._temp = self.data_to_slice[slice_range] + + +class Ros3ContinuousSliceBenchmark(BaseNetworkBenchmark): + s3_url = s3_url + object_name = object_name + slice_range = slice_range + + def setup_cache(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + super().setup_cache(s3_url=self.s3_url, object_name=self.object_name, slice_range=self.slice_range) + + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) + self.data_to_slice = self.neurodata_object.data + + def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) diff --git a/src/nwb_benchmarks/benchmarks/remote_file_reading.py b/src/nwb_benchmarks/benchmarks/remote_file_reading.py deleted file mode 100644 index 6a5c0e2..0000000 --- a/src/nwb_benchmarks/benchmarks/remote_file_reading.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Basic benchmarks for stream NWB files and their contents.""" - -from nwb_benchmarks.core import ( - BaseNetworkBenchmark, - get_s3_url, - read_hdf5_fsspec_no_cache, - read_hdf5_nwbfile_fsspec_no_cache, - read_hdf5_nwbfile_remfile, - read_hdf5_nwbfile_ros3, - read_hdf5_remfile, - read_hdf5_ros3, -) - -param_names = ["s3_url"] -params = [ - get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb"), - get_s3_url( - dandiset_id="000717", - dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", - ), # Not the best example for testing a theory about file read; should probably replace with something simpler - "https://dandiarchive.s3.amazonaws.com/ros3test.nwb", # The original small test NWB file -] - - -class FsspecNoCacheDirectFileReadBenchmark(BaseNetworkBenchmark): - repeat = 1 - param_names = param_names - params = params - - def time_file_read(self, s3_url: str): - self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) - - def operation_to_track_network_activity_of(self, s3_url: str): - self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) - - -class RemfileDirectFileReadBenchmark(BaseNetworkBenchmark): - repeat = 1 - param_names = param_names - params = params - - def time_file_read(self, s3_url: str): - self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) - - def operation_to_track_network_activity_of(self, s3_url: str): - self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) - - -class Ros3DirectFileReadBenchmark(BaseNetworkBenchmark): - repeat = 1 - param_names = param_names - params = params - - def time_file_read(self, s3_url: str): - self.file = read_hdf5_ros3(s3_url=s3_url) - - def operation_to_track_network_activity_of(self, s3_url: str): - self.file = read_hdf5_ros3(s3_url=s3_url) - - -class FsspecNoCacheNWBFileReadBenchmark(BaseNetworkBenchmark): - repeat = 1 - param_names = param_names - params = params - - def time_file_read(self, s3_url: str): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) - - def operation_to_track_network_activity_of(self, s3_url: str): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) - - -class RemfileNWBFileReadBenchmark(BaseNetworkBenchmark): - repeat = 1 - param_names = param_names - params = params - - def time_file_read(self, s3_url: str): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) - - def operation_to_track_network_activity_of(self, s3_url: str): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) - - -class Ros3NWBFileReadBenchmark(BaseNetworkBenchmark): - repeat = 1 - param_names = param_names - params = params - - def time_file_read(self, s3_url: str): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) - - def operation_to_track_network_activity_of(self, s3_url: str): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py new file mode 100644 index 0000000..9123bb3 --- /dev/null +++ b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py @@ -0,0 +1,65 @@ +"""Basic benchmarks for stream NWB files and their contents.""" + +from nwb_benchmarks.core import ( + get_s3_url, + read_hdf5_fsspec_no_cache, + read_hdf5_nwbfile_fsspec_no_cache, + read_hdf5_nwbfile_remfile, + read_hdf5_nwbfile_ros3, + read_hdf5_remfile, + read_hdf5_ros3, +) + +param_names = ["s3_url"] +params = [ + get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb"), + get_s3_url( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ), # Not the best example for testing a theory about file read; should probably replace with something simpler + "https://dandiarchive.s3.amazonaws.com/ros3test.nwb", # The original small test NWB file +] + + +class DirectFileReadBenchmark: + """ + Time the direct read of the HDF5-backend files with `h5py` using each streaming method. + + There is no formal parsing of the `pynwb.NWBFile` object. + + Note: in all cases, store the in-memory objects to avoid timing garbage collection steps. + """ + + repeat = 3 + param_names = param_names + params = params + + def time_read_hdf5_fsspec_no_cache(self, s3_url: str): + self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) + + def time_read_hdf5_remfile(self, s3_url: str): + self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) + + def time_read_hdf5_ros3(self, s3_url: str): + self.file = read_hdf5_ros3(s3_url=s3_url) + + +class NWBFileReadBenchmark: + """ + Time the read of the HDF5-backend files with `pynwb` using each streaming method. + + Note: in all cases, store the in-memory objects to avoid timing garbage collection steps. + """ + + repeat = 3 + param_names = param_names + params = params + + def time_read_hdf5_nwbfile_fsspec_no_cache(self, s3_url: str): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) + + def time_read_hdf5_nwbfile_remfile(self, s3_url: str): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) + + def time_read_hdf5_nwbfile_ros3(self, s3_url: str): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py similarity index 77% rename from src/nwb_benchmarks/benchmarks/remote_slicing.py rename to src/nwb_benchmarks/benchmarks/time_remote_slicing.py index 4b9b09d..d4d12ab 100644 --- a/src/nwb_benchmarks/benchmarks/remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -3,7 +3,6 @@ from typing import Tuple from nwb_benchmarks.core import ( - BaseNetworkBenchmark, get_object_by_name, get_s3_url, read_hdf5_nwbfile_fsspec_no_cache, @@ -27,8 +26,8 @@ ) -class FsspecNoCacheContinuousSliceBenchmark(BaseNetworkBenchmark): - repeat = 1 +class FsspecNoCacheContinuousSliceBenchmark: + repeat = 3 param_names = param_names params = params @@ -41,12 +40,9 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = self.data_to_slice[slice_range] - def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self._temp = self.data_to_slice[slice_range] - -class RemfileContinuousSliceBenchmark(BaseNetworkBenchmark): - repeat = 1 +class RemfileContinuousSliceBenchmark: + repeat = 3 param_names = param_names params = params @@ -59,12 +55,9 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = self.data_to_slice[slice_range] - def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self._temp = self.data_to_slice[slice_range] - -class Ros3ContinuousSliceBenchmark(BaseNetworkBenchmark): - repeat = 1 +class Ros3ContinuousSliceBenchmark: + repeat = 3 param_names = param_names params = params @@ -76,6 +69,3 @@ def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) - - def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self._temp = self.data_to_slice[slice_range] diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index 7b7a6de..429fc62 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -35,8 +35,9 @@ def main(): cmd = [ "asv", "run", + "-a rounds=1", "--python=same", - "--record-samples", + "--append-samples", "--set-commit-hash", commit_hash, ] diff --git a/src/nwb_benchmarks/core/_base_network_benchmark.py b/src/nwb_benchmarks/core/_base_network_benchmark.py index 5371676..9e7ece0 100644 --- a/src/nwb_benchmarks/core/_base_network_benchmark.py +++ b/src/nwb_benchmarks/core/_base_network_benchmark.py @@ -14,18 +14,20 @@ class BaseNetworkBenchmark: """ Base class for ASV Benchmark network performance metrics for streaming data access. - Child classes must implement the `operation_to_track_network_activity_of`, - which replaces the usual time_ or mem_ operation being tracked. + Child classes must implement: + 1) The `operation_to_track_network_activity_of`, which replaces the usual time_ or mem_ operation being tracked. + 2) The `setup_cache` method to explicitly recieve the necessary keyword arguments to be passed to the operation. """ - def operation_to_track_network_activity_of(self): + def operation_to_track_network_activity_of(self, **keyword_arguments): raise SkipNotImplemented() - def setup_cache(self, **keyword_arguments): - return self.compute_test_case_metrics(**keyword_arguments) + def setup_cache(self, **keyword_arguments) -> dict: + """ + Strategy essentially mimics a single benchmark execution (setup + run). - def compute_test_case_metrics(self, **keyword_arguments): - """Strategy essentially mimics a single benchmark execution (setup + run), but only once (ignores `repeat`).""" + Only supports single operations (ignores `repeat` and prohibits `params`). + """ self.start_net_capture() t0 = time.time() self.setup(**keyword_arguments) @@ -58,43 +60,43 @@ def stop_netcapture(self): # Compute all the network statistics self.net_stats = NetworkStatistics.get_stats(packets=self.pid_packets) - def track_bytes_downloaded(self, cache): - return cache["bytes_downloaded"] + def track_bytes_downloaded(self, net_stats: dict): + return net_stats["bytes_downloaded"] track_bytes_downloaded.unit = "bytes" - def track_bytes_uploaded(self, cache): - return cache["bytes_uploaded"] + def track_bytes_uploaded(self, net_stats: dict): + return net_stats["bytes_uploaded"] track_bytes_uploaded.unit = "bytes" - def track_bytes_total(self, cache): - return cache["bytes_total"] + def track_bytes_total(self, net_stats: dict): + return net_stats["bytes_total"] track_bytes_total.unit = "bytes" - def track_num_packets(self, cache): - return cache["num_packets"] + def track_num_packets(self, net_stats: dict): + return net_stats["num_packets"] track_num_packets.unit = "count" - def track_num_packets_downloaded(self, cache): - return cache["num_packets_downloaded"] + def track_num_packets_downloaded(self, net_stats: dict): + return net_stats["num_packets_downloaded"] track_num_packets_downloaded.unit = "count" - def track_num_packets_uploaded(self, cache): - return cache["num_packets_uploaded"] + def track_num_packets_uploaded(self, net_stats: dict): + return net_stats["num_packets_uploaded"] track_num_packets_uploaded.unit = "count" - def track_total_transfer_time(self, cache): - return cache["total_transfer_time"] + def track_total_transfer_time(self, net_stats: dict): + return net_stats["total_transfer_time"] track_total_transfer_time.unit = "seconds" - def track_network_total_time(self, cache): + def track_network_total_time(self, net_stats: dict): """Technically different from the official time_ approach.""" - return cache["network_total_time"] + return net_stats["network_total_time"] track_network_total_time.unit = "seconds" diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py index d171d6c..7ba6a73 100644 --- a/src/nwb_benchmarks/core/_streaming.py +++ b/src/nwb_benchmarks/core/_streaming.py @@ -79,7 +79,9 @@ def robust_ros3_read( def read_hdf5_ros3(s3_url: str) -> h5py.File: """Load the raw HDF5 file from an S3 URL using ROS3 driver; does not formally read the NWB file.""" ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") - file = robust_ros3_read(command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3")) + file = robust_ros3_read( + command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3", aws_region=bytes("us-east-2", "ascii")) + ) return file diff --git a/src/nwb_benchmarks/scripts/demo_network_profiling.py b/src/nwb_benchmarks/scripts/demo_network_profiling.py index 0794565..9261bbf 100644 --- a/src/nwb_benchmarks/scripts/demo_network_profiling.py +++ b/src/nwb_benchmarks/scripts/demo_network_profiling.py @@ -20,6 +20,11 @@ netprofiler = NetworkProfiler() # If `tshark` is not available as a global command, specify the pathlib.Path pointing to the .exe +# Otherwise, set to None or do not pass into `.start_capture` + +# tshark_exe_path=pathlib.Path("D:/Wireshark/tshark.exe") +tshark_exe_path = None + netprofiler.start_capture(tshark_exe_path=pathlib.Path("D:/Wireshark/tshark.exe")) ### From fcbc1ee93503008ba8f59846a44877d416592628 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sat, 17 Feb 2024 00:59:37 -0500 Subject: [PATCH 15/72] does not like --attributes flag --- src/nwb_benchmarks/benchmarks/time_remote_file_reading.py | 2 ++ src/nwb_benchmarks/benchmarks/time_remote_slicing.py | 3 +++ src/nwb_benchmarks/command_line_interface.py | 1 - 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py index 9123bb3..4a183c1 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py @@ -30,6 +30,7 @@ class DirectFileReadBenchmark: Note: in all cases, store the in-memory objects to avoid timing garbage collection steps. """ + rounds = 1 repeat = 3 param_names = param_names params = params @@ -51,6 +52,7 @@ class NWBFileReadBenchmark: Note: in all cases, store the in-memory objects to avoid timing garbage collection steps. """ + rounds = 1 repeat = 3 param_names = param_names params = params diff --git a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py index d4d12ab..115a9c9 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -27,6 +27,7 @@ class FsspecNoCacheContinuousSliceBenchmark: + rounds = 1 repeat = 3 param_names = param_names params = params @@ -42,6 +43,7 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): class RemfileContinuousSliceBenchmark: + rounds = 1 repeat = 3 param_names = param_names params = params @@ -57,6 +59,7 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): class Ros3ContinuousSliceBenchmark: + rounds = 1 repeat = 3 param_names = param_names params = params diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index 429fc62..17261ae 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -35,7 +35,6 @@ def main(): cmd = [ "asv", "run", - "-a rounds=1", "--python=same", "--append-samples", "--set-commit-hash", From b2e7a58afb821b5963e96f911d6c68a7ebb29e09 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sat, 17 Feb 2024 16:46:11 -0500 Subject: [PATCH 16/72] add results --- .gitignore | 3 +- ...c211421a12aeede580b602fde9cda8143b0aa.json | 946 ++++++++++++++++++ ...05e6bc6fe1582a6a348d5cfddde3e86882815.json | 175 ++++ ...c211421a12aeede580b602fde9cda8143b0aa.json | 75 ++ src/nwb_benchmarks/command_line_interface.py | 31 +- src/nwb_benchmarks/core/_network_profiler.py | 4 +- .../scripts/demo_network_profiling.py | 6 +- src/nwb_benchmarks/setup/__init__.py | 3 +- src/nwb_benchmarks/setup/_reduce_results.py | 102 ++ 9 files changed, 1336 insertions(+), 9 deletions(-) create mode 100644 results/environment-509c211421a12aeede580b602fde9cda8143b0aa.json create mode 100644 results/machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json create mode 100644 results/timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json create mode 100644 src/nwb_benchmarks/setup/_reduce_results.py diff --git a/.gitignore b/.gitignore index fc1fda3..af2e23b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ -# ASV +# ASV and customization .asv/intermediate_results +.asv/.raw_environment_info.txt # Dataset file / log file types: /**/*.log diff --git a/results/environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/environment-509c211421a12aeede580b602fde9cda8143b0aa.json new file mode 100644 index 0000000..c46ab88 --- /dev/null +++ b/results/environment-509c211421a12aeede580b602fde9cda8143b0aa.json @@ -0,0 +1,946 @@ +{ + "3.11.7 | packaged by Anaconda, Inc. | (main, Dec 15 2023, 18:05:47) [MSC v.1916 64 bit (AMD64)]": [ + { + "name": "aiobotocore", + "version": "2.11.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "aiohttp", + "version": "3.9.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "aioitertools", + "version": "0.11.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "aiosignal", + "version": "1.3.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "appdirs", + "version": "1.4.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "arrow", + "version": "1.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "asciitree", + "version": "0.3.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "asttokens", + "version": "2.4.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "asv", + "version": "0.6.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "asv-runner", + "version": "0.2.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "attrs", + "version": "23.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "bidsschematools", + "version": "0.7.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "blas", + "version": "1.0", + "build": "mkl" + }, + { + "name": "botocore", + "version": "1.34.34", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "bzip2", + "version": "1.0.8", + "build": "he774522_0" + }, + { + "name": "ca-certificates", + "version": "2023.12.12", + "build": "haa95532_0" + }, + { + "name": "cached-property", + "version": "1.5.2", + "build": "py_0" + }, + { + "name": "certifi", + "version": "2024.2.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "cfgv", + "version": "3.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "charset-normalizer", + "version": "3.3.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ci-info", + "version": "0.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "click", + "version": "8.1.7", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "click-didyoumean", + "version": "0.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "cloudpickle", + "version": "3.0.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "colorama", + "version": "0.4.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "comm", + "version": "0.2.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "dandi", + "version": "0.59.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "dandischema", + "version": "0.8.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "debugpy", + "version": "1.8.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "decorator", + "version": "5.1.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "dill", + "version": "0.3.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "distlib", + "version": "0.3.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "dnspython", + "version": "2.5.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "email-validator", + "version": "2.1.0.post1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "etelemetry", + "version": "0.3.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "executing", + "version": "2.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "fasteners", + "version": "0.19", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "filelock", + "version": "3.13.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "fqdn", + "version": "1.5.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "frozenlist", + "version": "1.4.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "fscacher", + "version": "0.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "fsspec", + "version": "2024.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "git", + "version": "2.40.1", + "build": "haa95532_1" + }, + { + "name": "h5py", + "version": "3.10.0", + "build": "nompi_py311h7195302_101", + "channel": "conda-forge" + }, + { + "name": "hdf5", + "version": "1.14.3", + "build": "nompi_h73e8ff5_100", + "channel": "conda-forge" + }, + { + "name": "hdmf", + "version": "3.12.1.dev7+g9b3282a", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "humanize", + "version": "4.9.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "identify", + "version": "2.5.34", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "idna", + "version": "3.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "importlib-metadata", + "version": "7.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "intel-openmp", + "version": "2023.1.0", + "build": "h59b6b97_46320" + }, + { + "name": "interleave", + "version": "0.2.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ipykernel", + "version": "6.29.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ipython", + "version": "8.21.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "isodate", + "version": "0.6.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "isoduration", + "version": "20.11.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jaraco-classes", + "version": "3.3.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jedi", + "version": "0.19.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jmespath", + "version": "1.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "joblib", + "version": "1.3.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "json5", + "version": "0.9.14", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jsonpointer", + "version": "2.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jsonschema", + "version": "4.21.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jsonschema-specifications", + "version": "2023.12.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jupyter-client", + "version": "8.6.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jupyter-core", + "version": "5.7.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "keyring", + "version": "24.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "keyrings-alt", + "version": "5.0.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "libaec", + "version": "1.1.2", + "build": "h63175ca_1", + "channel": "conda-forge" + }, + { + "name": "libcurl", + "version": "8.5.0", + "build": "h86230a5_0" + }, + { + "name": "libffi", + "version": "3.4.4", + "build": "hd77b12b_0" + }, + { + "name": "libssh2", + "version": "1.10.0", + "build": "he2ea4bf_2" + }, + { + "name": "libzlib", + "version": "1.2.13", + "build": "hcfcfb64_5", + "channel": "conda-forge" + }, + { + "name": "llvmlite", + "version": "0.42.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "lxml", + "version": "5.1.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "matplotlib-inline", + "version": "0.1.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "mkl", + "version": "2023.1.0", + "build": "h6b88ed4_46358" + }, + { + "name": "mkl-service", + "version": "2.4.0", + "build": "py311h2bbff1b_1" + }, + { + "name": "mkl_fft", + "version": "1.3.8", + "build": "py311h2bbff1b_0" + }, + { + "name": "mkl_random", + "version": "1.2.4", + "build": "py311h59b6b97_0" + }, + { + "name": "more-itertools", + "version": "10.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "multidict", + "version": "6.0.5", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "natsort", + "version": "8.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "nest-asyncio", + "version": "1.6.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "nodeenv", + "version": "1.8.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "numba", + "version": "0.59.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "numcodecs", + "version": "0.12.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "numpy", + "version": "1.26.3", + "build": "py311hdab7c0b_0" + }, + { + "name": "numpy-base", + "version": "1.26.3", + "build": "py311hd01c5d8_0" + }, + { + "name": "nwb-benchmarks", + "version": "0.1.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "nwbinspector", + "version": "0.4.33", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "openssl", + "version": "3.2.1", + "build": "hcfcfb64_0", + "channel": "conda-forge" + }, + { + "name": "packaging", + "version": "23.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pandas", + "version": "2.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "parso", + "version": "0.8.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pip", + "version": "23.3.1", + "build": "py311haa95532_0" + }, + { + "name": "platformdirs", + "version": "4.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pre-commit", + "version": "3.6.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "prompt-toolkit", + "version": "3.0.43", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "psutil", + "version": "5.9.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pure-eval", + "version": "0.2.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pycryptodomex", + "version": "3.20.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pydantic", + "version": "1.10.14", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pygments", + "version": "2.17.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pympler", + "version": "1.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pynwb", + "version": "2.5.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pyout", + "version": "0.7.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pyshark", + "version": "0.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "python", + "version": "3.11.7", + "build": "he1021f5_0" + }, + { + "name": "python-dateutil", + "version": "2.8.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "python_abi", + "version": "3.11", + "build": "2_cp311", + "channel": "conda-forge" + }, + { + "name": "pytz", + "version": "2024.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pywin32", + "version": "306", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pywin32-ctypes", + "version": "0.2.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pyyaml", + "version": "6.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pyzmq", + "version": "25.1.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "referencing", + "version": "0.33.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "remfile", + "version": "0.1.10", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "requests", + "version": "2.31.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "rfc3339-validator", + "version": "0.1.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "rfc3987", + "version": "1.3.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "rpds-py", + "version": "0.18.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ruamel-yaml", + "version": "0.18.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ruamel-yaml-clib", + "version": "0.2.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "s3fs", + "version": "2024.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "scipy", + "version": "1.12.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "semantic-version", + "version": "2.10.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "setuptools", + "version": "69.1.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "six", + "version": "1.16.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "spyder-kernels", + "version": "2.5.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "sqlite", + "version": "3.41.2", + "build": "h2bbff1b_0" + }, + { + "name": "stack-data", + "version": "0.6.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tabulate", + "version": "0.9.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tbb", + "version": "2021.8.0", + "build": "h59b6b97_0" + }, + { + "name": "tenacity", + "version": "8.2.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "termcolor", + "version": "2.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tk", + "version": "8.6.12", + "build": "h2bbff1b_0" + }, + { + "name": "tomli", + "version": "2.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tornado", + "version": "6.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tqdm", + "version": "4.66.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "traitlets", + "version": "5.14.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "types-python-dateutil", + "version": "2.8.19.20240106", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "typing-extensions", + "version": "4.9.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tzdata", + "version": "2024.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ucrt", + "version": "10.0.20348.0", + "build": "haa95532_0" + }, + { + "name": "uri-template", + "version": "1.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "urllib3", + "version": "2.0.7", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "vc", + "version": "14.2", + "build": "h21ff451_1" + }, + { + "name": "vc14_runtime", + "version": "14.38.33130", + "build": "h82b7239_18", + "channel": "conda-forge" + }, + { + "name": "virtualenv", + "version": "20.25.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "vs2015_runtime", + "version": "14.38.33130", + "build": "hcb4865c_18", + "channel": "conda-forge" + }, + { + "name": "wcwidth", + "version": "0.2.13", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "webcolors", + "version": "1.13", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "wheel", + "version": "0.41.2", + "build": "py311haa95532_0" + }, + { + "name": "wrapt", + "version": "1.16.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "xz", + "version": "5.4.5", + "build": "h8cc25b3_0" + }, + { + "name": "yarl", + "version": "1.9.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "zarr", + "version": "2.17.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "zarr-checksum", + "version": "0.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "zipp", + "version": "3.17.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "zlib", + "version": "1.2.13", + "build": "hcfcfb64_5", + "channel": "conda-forge" + } + ] +} diff --git a/results/machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json b/results/machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json new file mode 100644 index 0000000..5645669 --- /dev/null +++ b/results/machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json @@ -0,0 +1,175 @@ +{ + "cuda": { + "gpu_name": "NVIDIA GeForce RTX 2080", + "gpu_specifications": { + "ASYNC_ENGINE_COUNT": 6, + "CAN_MAP_HOST_MEMORY": 1, + "CAN_USE_HOST_POINTER_FOR_REGISTERED_MEM": 0, + "CLOCK_RATE": 1710000, + "COMPUTE_CAPABILITY_MAJOR": 7, + "COMPUTE_CAPABILITY_MINOR": 5, + "COMPUTE_MODE": 0, + "COMPUTE_PREEMPTION_SUPPORTED": 1, + "CONCURRENT_KERNELS": 1, + "CONCURRENT_MANAGED_ACCESS": 0, + "COOPERATIVE_LAUNCH": 1, + "COOPERATIVE_MULTI_DEVICE_LAUNCH": 0, + "ECC_ENABLED": 0, + "GLOBAL_L1_CACHE_SUPPORTED": 1, + "GLOBAL_MEMORY_BUS_WIDTH": 256, + "GPU_OVERLAP": 1, + "HOST_NATIVE_ATOMIC_SUPPORTED": 0, + "INTEGRATED": 0, + "IS_MULTI_GPU_BOARD": 0, + "KERNEL_EXEC_TIMEOUT": 1, + "L2_CACHE_SIZE": 4194304, + "LOCAL_L1_CACHE_SUPPORTED": 1, + "MANAGED_MEMORY": 1, + "MAX_BLOCK_DIM_X": 1024, + "MAX_BLOCK_DIM_Y": 1024, + "MAX_BLOCK_DIM_Z": 64, + "MAX_GRID_DIM_X": 2147483647, + "MAX_GRID_DIM_Y": 65535, + "MAX_GRID_DIM_Z": 65535, + "MAX_MAX_TEXTURE_2D_MIPMAPPED_HEIGHT": 32768, + "MAX_PITCH": 2147483647, + "MAX_REGISTERS_PER_BLOCK": 65536, + "MAX_REGISTERS_PER_MULTIPROCESSOR": 65536, + "MAX_SHARED_MEMORY_PER_BLOCK": 49152, + "MAX_SHARED_MEMORY_PER_BLOCK_OPTIN": 65536, + "MAX_SHARED_MEMORY_PER_MULTIPROCESSOR": 65536, + "MAX_SURFACE_1D_LAYERED_LAYERS": 2048, + "MAX_SURFACE_1D_LAYERED_WIDTH": 32768, + "MAX_SURFACE_1D_WIDTH": 32768, + "MAX_SURFACE_2D_HEIGHT": 65536, + "MAX_SURFACE_2D_LAYERED_HEIGHT": 32768, + "MAX_SURFACE_2D_LAYERED_LAYERS": 2048, + "MAX_SURFACE_2D_LAYERED_WIDTH": 32768, + "MAX_SURFACE_2D_WIDTH": 131072, + "MAX_SURFACE_3D_DEPTH": 16384, + "MAX_SURFACE_3D_HEIGHT": 16384, + "MAX_SURFACE_3D_WIDTH": 16384, + "MAX_SURFACE_CUBEMAP_LAYERED_LAYERS": 2046, + "MAX_SURFACE_CUBEMAP_LAYERED_WIDTH": 32768, + "MAX_SURFACE_CUBEMAP_WIDTH": 32768, + "MAX_TEXTURE_1D_LAYERED_LAYERS": 2048, + "MAX_TEXTURE_1D_LAYERED_WIDTH": 32768, + "MAX_TEXTURE_1D_LINEAR_WIDTH": 268435456, + "MAX_TEXTURE_1D_MIPMAPPED_WIDTH": 32768, + "MAX_TEXTURE_1D_WIDTH": 131072, + "MAX_TEXTURE_2D_GATHER_HEIGHT": 32768, + "MAX_TEXTURE_2D_GATHER_WIDTH": 32768, + "MAX_TEXTURE_2D_HEIGHT": 65536, + "MAX_TEXTURE_2D_LAYERED_HEIGHT": 32768, + "MAX_TEXTURE_2D_LAYERED_LAYERS": 2048, + "MAX_TEXTURE_2D_LAYERED_WIDTH": 32768, + "MAX_TEXTURE_2D_LINEAR_HEIGHT": 65000, + "MAX_TEXTURE_2D_LINEAR_PITCH": 2097120, + "MAX_TEXTURE_2D_LINEAR_WIDTH": 131072, + "MAX_TEXTURE_2D_MIPMAPPED_WIDTH": 32768, + "MAX_TEXTURE_2D_WIDTH": 131072, + "MAX_TEXTURE_3D_DEPTH": 16384, + "MAX_TEXTURE_3D_DEPTH_ALT": 32768, + "MAX_TEXTURE_3D_HEIGHT": 16384, + "MAX_TEXTURE_3D_HEIGHT_ALT": 8192, + "MAX_TEXTURE_3D_WIDTH": 16384, + "MAX_TEXTURE_3D_WIDTH_ALT": 8192, + "MAX_TEXTURE_CUBEMAP_LAYERED_LAYERS": 2046, + "MAX_TEXTURE_CUBEMAP_LAYERED_WIDTH": 32768, + "MAX_TEXTURE_CUBEMAP_WIDTH": 32768, + "MAX_THREADS_PER_BLOCK": 1024, + "MAX_THREADS_PER_MULTI_PROCESSOR": 1024, + "MEMORY_CLOCK_RATE": 7000000, + "MULTIPROCESSOR_COUNT": 46, + "MULTI_GPU_BOARD_GROUP_ID": 0, + "PAGEABLE_MEMORY_ACCESS": 0, + "PCI_BUS_ID": 1, + "PCI_DEVICE_ID": 0, + "PCI_DOMAIN_ID": 0, + "SINGLE_TO_DOUBLE_PRECISION_PERF_RATIO": 32, + "STREAM_PRIORITIES_SUPPORTED": 1, + "SURFACE_ALIGNMENT": 512, + "TCC_DRIVER": 0, + "TEXTURE_ALIGNMENT": 512, + "TEXTURE_PITCH_ALIGNMENT": 32, + "TOTAL_CONSTANT_MEMORY": 65536, + "UNIFIED_ADDRESSING": 1, + "WARP_SIZE": 32 + } + }, + "custom": true, + "defaults": { + "arch": "AMD64", + "cpu": "Intel(R) Core(TM) i7-9700K CPU @ 3.60GHz", + "machine": "DESKTOP-QJKETS8", + "num_cpu": "8", + "os": "Windows 10", + "ram": "" + }, + "machine": "9d805e6bc6fe1582a6a348d5cfddde3e86882815", + "os": { + "cpu_count": 8 + }, + "platform": { + "architecture": [ + "64bit", + "WindowsPE" + ], + "machine": "AMD64", + "platform": "Windows-10-10.0.19045-SP0", + "processor": "Intel64 Family 6 Model 158 Stepping 12, GenuineIntel", + "system": "Windows" + }, + "psutil": { + "disk_partitions": [ + { + "device": "C:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "C:\\", + "opts": "rw,fixed" + }, + { + "device": "D:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "D:\\", + "opts": "rw,fixed" + }, + { + "device": "E:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "E:\\", + "opts": "rw,fixed" + }, + { + "device": "F:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "F:\\", + "opts": "rw,fixed" + }, + { + "device": "G:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "G:\\", + "opts": "rw,fixed" + } + ], + "number_of_processes": 8, + "number_of_threads": 8, + "total_swap_memory": 17118638080, + "total_virtual_memory": 17118638080 + }, + "sys": { + "platform": "win32" + }, + "version": 1 +} diff --git a/results/timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json new file mode 100644 index 0000000..8b652fa --- /dev/null +++ b/results/timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json @@ -0,0 +1,75 @@ +{ + "version": 2, + "timestamp": "2024_02_17_00_59_37", + "commit_hash": "fcbc1ee93503008ba8f59846a44877d416592628", + "environment_hash": "509c211421a12aeede580b602fde9cda8143b0aa", + "machine_hash": "9d805e6bc6fe1582a6a348d5cfddde3e86882815", + "results": { + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.395349400001578, + 16.374563500052318, + 0.6053932000068016, + 0.6525553000392392, + 0.5618528000195511, + 0.5382418999797665, + 12.189247599977534, + 16.17910980002489, + 14.66157190001104, + 0.5886657999944873, + 0.746196600026451, + 12.95918340003118, + 0.6654667999828234, + 13.60833990003448, + 0.6666267000255175, + 0.5994374000001699, + 1.0131437000236474, + 0.9876768999965861 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 4.920633600035217, + 0.5284106999752112, + 0.9341197999892756, + 5.714047100045718, + 0.5253001000382937, + 0.5296251999679953, + 8.583959300012793, + 0.5409118999959901, + 0.540925600042101, + 8.369992599997204, + 9.031809899955988, + 0.5621228999807499, + 0.5253351999563165, + 0.514035000000149, + 0.5971826000022702, + 0.536596899968572, + 0.5279040000168607, + 6.630319699994288 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.006150400033220649, + 0.0063019999652169645, + 0.008725399966351688, + 0.0057545999879948795, + 0.0057687999797053635, + 0.005848100001458079, + 0.006170000007841736, + 0.005869700049515814, + 0.009840199956670403, + 0.0063875000341795385, + 0.0062386999488808215, + 0.007613099995069206, + 0.007296500029042363, + 0.006599900021683425, + 0.006746999977622181, + 0.007564299972727895, + 0.006594400038011372, + 0.007813100004568696 + ] + } + } +} diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index 17261ae..741ed52 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -1,11 +1,16 @@ """Simple wrapper around `asv run` for convenience.""" import locale +import os import pathlib import subprocess import sys -from .setup import customize_asv_machine_file, ensure_machine_info_current +from .setup import ( + customize_asv_machine_file, + ensure_machine_info_current, + reduce_results, +) def main(): @@ -32,6 +37,13 @@ def main(): else: customize_asv_machine_file(file_path=default_asv_machine_file_path) + # Save latest environment list from conda (most thorough) + # subprocess tends to have issues inheriting `conda` entrypoint + asv_root = pathlib.Path(__file__).parent.parent.parent / ".asv" + raw_environment_info_file_path = asv_root / ".raw_environment_info.txt" + os.system(f"conda list > {raw_environment_info_file_path}") + + # Deploy ASV cmd = [ "asv", "run", @@ -45,12 +57,27 @@ def main(): if bench_mode: cmd.extend(["--bench", specific_benchmark_pattern]) - process = subprocess.Popen(cmd, stdout=subprocess.PIPE) # , bufsize=1) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) encoding = locale.getpreferredencoding() # This is how ASV chooses to encode the output for line in iter(process.stdout.readline, b""): print(line.decode(encoding).strip("\n")) process.stdout.close() process.wait() + + globbed_json_file_paths = [ + path + for path in pathlib.Path(asv_root / "intermediate_results").rglob("*.json") + if not any(path.stem == skip_stems for skip_stems in ["benchmarks", "machine"]) + ] + assert len(globbed_json_file_paths) == 1, ( + "More than one intermediate result found, likely as a result of a previous failure to reduce the results! " + "Please manually remove these." + ) + raw_results_file_path = globbed_json_file_paths[0] + reduce_results( + raw_results_file_path=raw_results_file_path, raw_environment_info_file_path=raw_environment_info_file_path + ) + else: print(f"{command} is an invalid command.") diff --git a/src/nwb_benchmarks/core/_network_profiler.py b/src/nwb_benchmarks/core/_network_profiler.py index 180fd8e..88c52e9 100644 --- a/src/nwb_benchmarks/core/_network_profiler.py +++ b/src/nwb_benchmarks/core/_network_profiler.py @@ -60,8 +60,8 @@ def stop_capture(self): self.__tshark_process.kill() del self.__tshark_process self.__tshark_process = None - if hasattr(self, "capture_file"): - del self.capture_file + # if hasattr(self, "capture_file"): + # del self.capture_file def get_packets_for_connections(self, pid_connections: list): """ diff --git a/src/nwb_benchmarks/scripts/demo_network_profiling.py b/src/nwb_benchmarks/scripts/demo_network_profiling.py index 9261bbf..3ab3e43 100644 --- a/src/nwb_benchmarks/scripts/demo_network_profiling.py +++ b/src/nwb_benchmarks/scripts/demo_network_profiling.py @@ -22,10 +22,10 @@ # If `tshark` is not available as a global command, specify the pathlib.Path pointing to the .exe # Otherwise, set to None or do not pass into `.start_capture` -# tshark_exe_path=pathlib.Path("D:/Wireshark/tshark.exe") -tshark_exe_path = None +tshark_exe_path = pathlib.Path("D:/Wireshark/tshark.exe") +# tshark_exe_path = None -netprofiler.start_capture(tshark_exe_path=pathlib.Path("D:/Wireshark/tshark.exe")) +netprofiler.start_capture(tshark_exe_path=tshark_exe_path) ### # This would be the unit test diff --git a/src/nwb_benchmarks/setup/__init__.py b/src/nwb_benchmarks/setup/__init__.py index aeedaa9..97f058f 100644 --- a/src/nwb_benchmarks/setup/__init__.py +++ b/src/nwb_benchmarks/setup/__init__.py @@ -5,5 +5,6 @@ customize_asv_machine_file, ensure_machine_info_current, ) +from ._reduce_results import reduce_results -__all__ = ["collect_machine_info", "customize_asv_machine_file", "ensure_machine_info_current"] +__all__ = ["collect_machine_info", "customize_asv_machine_file", "ensure_machine_info_current", "reduce_results"] diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py new file mode 100644 index 0000000..10e992e --- /dev/null +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -0,0 +1,102 @@ +"""Default ASV result file is very inefficient - this routine simplifies it for sharing.""" + +import collections +import datetime +import hashlib +import json +import pathlib +import shutil +import sys +from typing import Dict, List + + +def _parse_environment_info(raw_environment_info: List[str]) -> Dict[str, List[[Dict[str, str]]]]: + """Turn the results of `conda list` printout to a JSON dictionary.""" + header_stripped = raw_environment_info[3:] + newline_stripped = [line.rstrip("\n") for line in header_stripped] + # Tried several regex but none quite did the trick in all cases + spacing_stripped = [[x for x in line.split(" ") if x] for line in newline_stripped] + + keys = ["name", "version", "build", "channel"] + parsed_environment = { + sys.version: [{key: value for key, value in zip(keys, values)} for values in spacing_stripped] + } + + return parsed_environment + + +def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_file_path: pathlib.Path): + """Default ASV result file is very inefficient - this routine simplifies it for sharing.""" + with open(file=raw_results_file_path, mode="r") as io: + raw_results_info = json.load(fp=io) + with open(file=raw_environment_info_file_path, mode="r") as io: + raw_environment_info = io.readlines() + parsed_environment_info = _parse_environment_info(raw_environment_info=raw_environment_info) + + # In timestamp, replace separators with underscores for file path + unix_time_to_datetime = str(datetime.datetime.fromtimestamp(raw_results_info["date"] / 1e3)) + timestamp = unix_time_to_datetime.replace(" ", "_").replace("-", "_").replace(":", "_") + + environment_hash = hashlib.sha1(string=bytes(json.dumps(obj=parsed_environment_info), "utf-8")).hexdigest() + machine_hash = raw_results_info["params"]["machine"] + + reduced_results = dict() + for test_case, raw_results_list in raw_results_info["results"].items(): + if len(raw_results_list) != 12: # Only successful runs results in this length + continue + + flattened_joint_params = collections.defaultdict(list) + for param_types in raw_results_list[1]: # Will naturally iterate in correct order + for param_value_index, param_value in enumerate(param_types): + flattened_joint_params[param_value_index].append(param_value) + serialized_flattened_joint_params = [ + str(tuple(joint_params)) for joint_params in flattened_joint_params.values() + ] + + assert len(serialized_flattened_joint_params) == len(raw_results_list[11]), ( + f"Length mismatch between flattened joint parameters ({serialized_flattened_joint_params} and result " + f"samples ({len(raw_results_list[11])}) in test case '{test_case}'! " + "Please raise an issue and share your intermediate results file." + ) + reduced_results.update( + { + test_case: { + joint_params: raw_result + for joint_params, raw_result in zip(serialized_flattened_joint_params, raw_results_list[11]) + } + } + ) + reduced_results_info = dict( + version=raw_results_info["version"], + timestamp=timestamp, + commit_hash=raw_results_info["commit_hash"], + environment_hash=environment_hash, + machine_hash=machine_hash, + results=reduced_results, + ) + + # Save reduced results to main .asv folder + # 'raw' results go to nwb_benchmarks/.asv/intermediate_results//.json + # 'processed' results go to nwb_benchmarks/results + main_results_folder = raw_results_file_path.parent.parent.parent.parent / "results" + parsed_results_file = ( + main_results_folder / f"timestamp-{timestamp}_machine-{machine_hash}_environment-{environment_hash}.json" + ) + main_results_folder.mkdir(parents=True, exist_ok=True) + + with open(file=parsed_results_file, mode="w") as io: + json.dump(obj=reduced_results_info, fp=io, indent=4) + + # Copy machine file to main results + machine_info_file_path = raw_results_file_path.parent / "machine.json" + machine_info_copy_file_path = main_results_folder / f"machine-{machine_hash}.json" + if not machine_info_copy_file_path.exists(): + shutil.copyfile(src=machine_info_file_path, dst=machine_info_copy_file_path) + + # Save parsed environment info within machine subdirectory of .asv + parsed_environment_file_path = main_results_folder / f"environment-{environment_hash}.json" + if not parsed_environment_file_path.exists(): + with open(file=parsed_environment_file_path, mode="w") as io: + json.dump(obj=parsed_environment_info, fp=io, indent=4) + + raw_results_file_path.unlink() From f07ce45a1a05bd60a4cafa8a9aa14632a4f83fad Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sat, 17 Feb 2024 16:55:10 -0500 Subject: [PATCH 17/72] update README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d638450..6364f53 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ Many of the current tests can take several minutes to complete; the entire suite To run only a single benchmark, use the `--bench ` flag. +To contribute your results back to the project, just be sure to `git add` and `commit` the results in the main `results` folder. + +Note: Each result file should be single to double-digit KB in size; if we ever reach the point where this is prohibitive to store on GitHub itself, then we will investigate other upload strategies and purge the folder from the repository history. + ## Building the documentation To install the additional packages required to build the docs execute the command ... From 72cdfdea4be13a0299337df49f707636b0c03ebb Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sat, 17 Feb 2024 17:27:55 -0500 Subject: [PATCH 18/72] accidentally committed prior to split point --- src/nwb_benchmarks/command_line_interface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index 741ed52..31004ff 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -48,7 +48,7 @@ def main(): "asv", "run", "--python=same", - "--append-samples", + "--record-samples", "--set-commit-hash", commit_hash, ] @@ -70,8 +70,8 @@ def main(): if not any(path.stem == skip_stems for skip_stems in ["benchmarks", "machine"]) ] assert len(globbed_json_file_paths) == 1, ( - "More than one intermediate result found, likely as a result of a previous failure to reduce the results! " - "Please manually remove these." + "A single intermediate result was not found, likely as a result of a previous failure to reduce " + "the results! Please manually remove these." ) raw_results_file_path = globbed_json_file_paths[0] reduce_results( From 11a75ee6ee57ad7a858a8f7397b51375e19643f3 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sat, 17 Feb 2024 23:24:36 -0500 Subject: [PATCH 19/72] start docs --- README.md | 41 +-------------- docs/development.rst | 27 ++++++---- docs/index.rst | 4 +- docs/running_benchmarks.rst | 52 +++++++++++++++++++ docs/setup.rst | 27 +++++++--- docs/using_asv.rst | 11 ---- docs/writing_benchmarks.rst | 46 +++++++++++----- src/nwb_benchmarks/command_line_interface.py | 15 ++++-- .../setup/_configure_machine.py | 3 -- 9 files changed, 136 insertions(+), 90 deletions(-) create mode 100644 docs/running_benchmarks.rst delete mode 100644 docs/using_asv.rst diff --git a/README.md b/README.md index 6364f53..33a3e98 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,8 @@ # nwb_benchmarks -Benchmark suite for NWB performances using [airspeed velocity](https://asv.readthedocs.io/en/stable/). +Benchmark suite for NWB performances using a customization of [airspeed velocity](https://asv.readthedocs.io/en/stable/). -## Getting Started -To get started, clone this repo... - -``` -git clone https://github.com/neurodatawithoutborders/nwb_benchmarks.git -cd nwb_benchmarks -``` - -Setup the environment... - -``` -conda env create --file environments/nwb_benchmarks.yaml --no-default-packages -conda activate nwb_benchmarks -``` - -Configure tracking of our custom machine-dependent parameters by calling... - -``` -asv machine --yes -python src/nwb_benchmarks/setup/configure_machine.py -``` - -Please note that we do not currently distinguish any configurations based on your internet; as such there may be difference observed from the same machine in the results database if that machine is a laptop that runs the testing suite on a wide variety of internet qualities. - -## Running Benchmarks - -To run the full benchmark suite, please ensure you are not running any additional heavy processes in the background to avoid interference or bottlenecks, then execute the command... - -``` -nwb_benchmarks run -``` - -Many of the current tests can take several minutes to complete; the entire suite can take 10 or more minutes. Grab some coffee, read a book, or better yet (when the suite becomes larger) just leave it to run overnight. - -To run only a single benchmark, use the `--bench ` flag. - -To contribute your results back to the project, just be sure to `git add` and `commit` the results in the main `results` folder. - -Note: Each result file should be single to double-digit KB in size; if we ever reach the point where this is prohibitive to store on GitHub itself, then we will investigate other upload strategies and purge the folder from the repository history. ## Building the documentation diff --git a/docs/development.rst b/docs/development.rst index 1ddb5c2..bc75dd2 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -3,13 +3,20 @@ Development This section covers advanced details of managing the operation of the AirSpeed Velocity testing suite. -- TODO: add section on environment matrices and current `python=same` -- TODO: add section on custom network packet tracking -- TODO: add section outlining the approach of the machine customization - -.. Indices and tables -.. ================== -.. -.. * :ref:`genindex` -.. * :ref:`modindex` -.. * :ref:`search` + +Customized Machine Header +------------------------- + + +Customized Call to Run +---------------------- + + +Customized Parsing of Results +----------------------------- + + +Network Tracking +---------------- + +Please contact Oliver Ruebel for details. diff --git a/docs/index.rst b/docs/index.rst index e65d4bb..8b8887c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ nwb_benchmarks ============== -This project is an effort to establish and understand, in a robust and reproducible manner, the principles underlying optimized file storage patterns for reading and writing NWB files from both local filesystems and the cloud (in particular, AWS S3). +This project is an effort to understand, in a robust and reproducible manner, the principles underlying optimized file storage patterns for reading and writing NWB files from both local filesystems and remotely from the cloud (in particular, AWS S3 buckets). Funding is provided by NOSI ... @@ -10,7 +10,7 @@ Funding is provided by NOSI ... :caption: Contents setup - using_asv + running_benchmarks writing_benchmarks development diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst new file mode 100644 index 0000000..380f3f6 --- /dev/null +++ b/docs/running_benchmarks.rst @@ -0,0 +1,52 @@ +Running the Benchmarks +====================== + +Before running the benchmark suite, please ensure you are not running any additional heavy processes in the background to avoid interference or bottlenecks. + +To run the full benchmark suite, simply call... + +.. code-block:: + + nwb_benchmarks run + +Many of the current tests can take several minutes to complete; the entire suite will take many times that. Grab some coffee, read a book, or better yet (when the suite becomes larger) just leave it to run overnight. + + +Additional Flags +---------------- + +Subset of the Suite +~~~~~~~~~~~~~~~~~~~ + +To run only a single benchmark suite (a single file in the ``benchmarks`` directory), use the command... + +.. code-block:: + + nwb_benchmarks run --bench + +For example, + +.. code-block:: + + nwb_benchmarks run --bench time_remote_slicing + +Debug mode +~~~~~~~~~~ + +If you want to get a full traceback to examine why a new test might be failing, simply add the flag... + +.. code-block:: + + nwb_benchmarks run --debug + + +Contributing Results +-------------------- + +To contribute your results back to the project, all you have to do is `git add` and `commit` the results in the `results` folder. + +Then, open a PR to merge the results to the `main` branch. + +.. note:: + + Each result file should be single to double-digit KB in size; if we ever reach the point where this is prohibitive to store on GitHub itself, then we will investigate other upload strategies and purge the folder from the repository history. diff --git a/docs/setup.rst b/docs/setup.rst index ccfd6a6..47e990d 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -1,11 +1,22 @@ Setup ===== -TODO: move from README - -.. Indices and tables -.. ================== -.. -.. * :ref:`genindex` -.. * :ref:`modindex` -.. * :ref:`search` +To get started, clone this repo... + +.. code-block:: + + git clone https://github.com/neurodatawithoutborders/nwb_benchmarks.git + cd nwb_benchmarks + +Setup a completely fresh environment... + +.. code-block:: + + conda env create --file environments/nwb_benchmarks.yaml --no-default-packages + conda activate nwb_benchmarks + +Setup initial machine configuration values with + +.. code-block:: + + nwb_benchmarks setup diff --git a/docs/using_asv.rst b/docs/using_asv.rst deleted file mode 100644 index 11b0c94..0000000 --- a/docs/using_asv.rst +++ /dev/null @@ -1,11 +0,0 @@ -Using ASV -========= - -- TODO: move from README - -.. Indices and tables -.. ================== -.. -.. * :ref:`genindex` -.. * :ref:`modindex` -.. * :ref:`search` diff --git a/docs/writing_benchmarks.rst b/docs/writing_benchmarks.rst index 5acbc69..962221f 100644 --- a/docs/writing_benchmarks.rst +++ b/docs/writing_benchmarks.rst @@ -1,16 +1,36 @@ Writing Benchmarks ================== -Have an idea for how to speed up read or write from a local or remote NWB file? This section explains how to write your own benchmark to prove it robustly across platforms, architectures, and environments. - -- TODO: cover standard prefixes -- TODO: cover standard setup values -- TODO: cover custom setup/teardown trackers -- TODO: cover params - -.. Indices and tables -.. ================== -.. -.. * :ref:`genindex` -.. * :ref:`modindex` -.. * :ref:`search` +Have an idea for how to speed up read or write from a local or remote NWB file? + +This section explains how to write your own benchmark to prove it robustly across platforms, architectures, and environments. + + +Standard Prefixes +----------------- + +Just like how ``pytest`` automatically detects and runs any function or method leading with the keyphrase ``test_``, AirSpeed Velocity runs timing tests for anything prefixed with ``time_``, tracks peak memory via prefix ``peakmem_``, custom values, such as our functions for network traffic, with ``track_`` (this must return the value being tracked), and many others. Check out the full listing in the `primary AirSpeed Velocity documentation `_. + +A single tracking function should perform only the minimal operations you wish to time. It is also capable of tracking only a single value. The philosophy for this is to avoid interference from cross-measurements; that is, the act of tracking memory of the operation may impact how much overall time it takes that process to complete, so you would not want to simultaneously measure both time and memory. + + +Class Structure +--------------- + +A single benchmark suite is a file within the ``benchmarks`` folder. It contains one or more benchmark classes. It is not itself important that the word 'Benchmark' be in the name of the class; only the prefix on the function matters. + +The class has several attributes, the most important of which are ``round``, ``repeat``, and ``timeout``. All functions in a class can be repeated in round-robin fashion using ``round > 1``; the philsophy here is to 'average out' variation on the system over time and may not always be relevant to increase. Each function in a suite is repeated ``repeat`` number of times to get an estimate of the standard deviation of the operation. Every function in the suite has at most ``timout`` number of seconds to complete, otherwise it will count as a failure. + +Similar to ``unittest.TestCase`` classes, these have a ``setup`` and ``teardown`` method which call before and after execution of every ``round`` and every ``repeat`` for every tracking function (such as timing) in the class. ``setup`` should therefore be as light as possible since it will be repeated so often, though sometimes even a minimal setup can still take time (such as reading a large remote NWB file using a suboptimal method). In some cases, ``setup_cache`` is a method that can be defined, and runs only once per class to precompute some operation, such as the creation of a fake dataset for testing on local disk. + +.. note:: + + Be careful to assign objects fetched by operations within the tracking functions; otherwise, you may unintentionally track the garbage collection step triggered when the reference count of the return value reaches zero in the namespace. For relatively heavy I/O operations this can be non-negligible. + +Finally, you can leverage ``params`` and ``param_names`` to perform a structured iteration over many inputs to the operations. ``param_names`` is a list of length equal to the number of inputs you wish to pass to an operation. ``params`` is a list of lists; the outer list being of equal length to the number of inputs, and each inner list being equal in length to the number of different cases to iterate over. + +.. note:: + + This structure for ``params`` can be very inconvenient to specify; if you desire a helper function that would instead take a flat list of dictionaries to serve as keyword arguments for all the iteration cases, please request it on our issues board. + +For more advanced details, refer to the `primary AirSpeed Velocity documentation `_. diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index 31004ff..ff36309 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -13,7 +13,7 @@ ) -def main(): +def main() -> None: """Simple wrapper around `asv run` for convenience.""" # TODO: swap to click if len(sys.argv) <= 1: @@ -28,10 +28,19 @@ def main(): if bench_mode: specific_benchmark_pattern = flags_list[flags_list.index("--bench") + 1] - if command == "run": + default_asv_machine_file_path = pathlib.Path.home() / ".asv-machine.json" + if command == "setup": + if default_asv_machine_file_path.exists(): + ensure_machine_info_current(file_path=default_asv_machine_file_path) + return + + process = subprocess.Popen(["asv", "machine", "--yes"], stdout=subprocess.PIPE) + process.wait() + + customize_asv_machine_file(file_path=default_asv_machine_file_path) + elif command == "run": commit_hash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("ascii").strip() - default_asv_machine_file_path = pathlib.Path.home() / ".asv-machine.json" if default_asv_machine_file_path.exists(): ensure_machine_info_current(file_path=default_asv_machine_file_path) else: diff --git a/src/nwb_benchmarks/setup/_configure_machine.py b/src/nwb_benchmarks/setup/_configure_machine.py index 181c553..22236cd 100644 --- a/src/nwb_benchmarks/setup/_configure_machine.py +++ b/src/nwb_benchmarks/setup/_configure_machine.py @@ -112,9 +112,6 @@ def ensure_machine_info_current(file_path: pathlib.Path): machine_info_from_file.pop("machine") machine_info_from_file.pop("custom") - with open(file=file_path.parent / "test.json", mode="w") as io: - json.dump(fp=io, obj=current_machine_info, indent=4) - if machine_info_from_file == current_machine_info: return From a38e41ef02d00594290c17c5dc2243fb90facbeb Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sun, 18 Feb 2024 13:08:15 -0500 Subject: [PATCH 20/72] finished first pass --- README.md | 10 +++--- docs/development.rst | 60 ++++++++++++++++++++++++++++++- docs/index.rst | 2 +- docs/writing_benchmarks.rst | 72 +++++++++++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 33a3e98..d155a57 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,20 @@ -# nwb_benchmarks +# NWB Benchmarks Benchmark suite for NWB performances using a customization of [airspeed velocity](https://asv.readthedocs.io/en/stable/). -## Building the documentation +## Building the Documentation -To install the additional packages required to build the docs execute the command ... +Public documentation can be found via `readthedocs`: https://nwb-benchmarks.readthedocs.io/en/latest/ + +To generate them locally, first install the additional packages required by executing the command... ``` pip install -r docs/requirements-rtd.txt ``` -To build the docs execute the command ... +then build the docs by executing the command... ``` mkdir -p docs/build/html diff --git a/docs/development.rst b/docs/development.rst index bc75dd2..35946b1 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -1,20 +1,78 @@ Development =========== -This section covers advanced details of managing the operation of the AirSpeed Velocity testing suite. +This section covers advanced implementation details for managing the operation of the AirSpeed Velocity testing suite. + + +Coding Style +------------ + +We use pre-commit and the pre-commit PR bot to automatically ensure usage of ``black`` and ``isort``. To setup pre-commit in your local environment, simply call... + + +.. code-block:: + + pip install pre-commit + pre-commit install + +Otherwise, please ensure all signatures and returns are annotated using the ``typing`` module. + +Writing thorough docstrings is encouraged; please follow the Numpy style. + +Import and submodule structure follows ``scikit-learn`` standard. Customized Machine Header ------------------------- +The spirit of AirSpeed Velocity to track machine specific information that could be related to benchmark performance is admirable. By calling ``asv machine`` you can generate a file on your system located at ``~/.asv-machine.json`` which is then used to uniquely tag results from your device. However, the defaults from ``asv machine --yes`` is woefully minimal. One example of the output of this call was... + +.. code-block:: json + + { + "DESKTOP-QJKETS8": { + "arch": "AMD64", + "cpu": "", + "machine": "DESKTOP-QJKETS8", + "num_cpu": "8", + "os": "Windows 10", + "ram": "" + }, + "version": 1 + } + +Not only are many of the values outright missing, but the ones that are found are not sufficient to uniquely tie to performance. For example, ``num_cpu=8`` does not distinguish between 8 Intel i5 or i9 cores, and there's a big difference between those two generations. + +Thankfully, system information like this is generally easy to grab from other Python built-ins or from the wonderful externally maintained platform-specific utilities package ``psutil``. + +As such, the functions in ``nwb_benchmarks.setup`` extend this framework by automatically tracking as many persistent system configurations as can be tracked without being overly invasive. A call to these functions is exposed to the ``nwb_benchmarks setup`` entrypoint for easy command line usage, which simply runs ``asv machine --yes`` to generate defaults and then calls the custom configuration to modify the file. + +``nwb_benchmarks run`` also checks on each run of the benchmark if the system configuration has changed. Reasons this might change are mainly the disk partitions (which can include external USB drives, which can have a big difference in performance compared to SSD and HDD plugged directly into the motherboard via PCI). + Customized Call to Run ---------------------- +By default, AirSpeed Velocity was designed primarily for scientific computing projects to track their optimization over time as measured by commits on the repo. As such, the default call to ``asv run`` has a DevOps continuous integration flavor in that it tries to spin up a unique virtual environment over a defined version matrix (both Python and constituent dependencies) each time, do a fresh checkout and pull of the Git repo, and only records the statistics aggregated over the runs of that instance. + +For this project, since our tests are a bit heavier despite using somewhat minimal code, we would wish to keep the valuable raw samples from each run. The virtual environment setup from AirSpeed Velocity can also run into compatability issues with different conda distributions and so we wish to maintain broader control over this aspect. + +These are the justifications to defining our ``nwb_benchmarks run`` command which wraps ``asv run`` with the following flags: ``--python=same`` means 'run benchmarks within the current Python/Conda environment' (do not create a separate one), which requires us to be running already from within an existing clone of the Git repo, but nonetheless requires the commit hash of that to be specified explicitly with ``--commit-hash ``, and finally ``--record-samples`` to store the values of each ``round`` and ``repeat``. + +A successful run of the benchmark produces a ``.json`` file in the ``.asv/intermediate_results`` folder, as well as provenance records of the machine and benchmark state. The name of this JSON file is usually a combination of the name of the environment and the flags passed to ``asv run``, and is not necessarily guaranteed to be different over multiple runs on the same commit hash. + Customized Parsing of Results ----------------------------- +Since our first approach to simplifying the sharing of results is to just commit them to the common GitHub repo, it was noticed that the default results files stored a lot of extraneous information. + +Since all we're really after here is the raw tracking output, some custom reduction of the original results files is performed so that only the minimal amount of information needed is actually stored in the final results files. These parsed results follow the dandi-esque name pattern ``result_timestamp-%Y-%M-%D-%H-%M-%S_machine-_environment-.json`` and are stored in the outer level ``results`` folder along with some ``info_machine-`` and ``info_environment-`` header files that are not regenerated whenever the hashes are the same. + +.. note:: + + If this ``results`` folder eventually becomes too large for Git to reasonably handle, we will explore options to share via other data storage services. + Network Tracking ---------------- diff --git a/docs/index.rst b/docs/index.rst index 8b8887c..963072a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -nwb_benchmarks +NWB Benchmarks ============== This project is an effort to understand, in a robust and reproducible manner, the principles underlying optimized file storage patterns for reading and writing NWB files from both local filesystems and remotely from the cloud (in particular, AWS S3 buckets). diff --git a/docs/writing_benchmarks.rst b/docs/writing_benchmarks.rst index 962221f..fdace75 100644 --- a/docs/writing_benchmarks.rst +++ b/docs/writing_benchmarks.rst @@ -34,3 +34,75 @@ Finally, you can leverage ``params`` and ``param_names`` to perform a structured This structure for ``params`` can be very inconvenient to specify; if you desire a helper function that would instead take a flat list of dictionaries to serve as keyword arguments for all the iteration cases, please request it on our issues board. For more advanced details, refer to the `primary AirSpeed Velocity documentation `_. + + +Philosophy +---------- + +In the spirit of PEP8, it was decided from PRs 12, 19, 20, and 21 and ensuing meetings that we should adopt an explicit functionally-based approach to structuring these classes and their methods. This will help make the project much easier to understand for people outside the team and will even reduce the amount of time it takes our main developers to read benchmarks they have not seen before or have forgotten about over an extended period of time. + +This approach means relying as little on inheritance and mixins as possible to reduce the amount of implicit knowledge required to understand a benchmark just by looking at it - instead, all instance methods of the benchmark class should be explicitly defined. + +To reduce duplicated code, it is suggested to write standalone helper functions in the ``core`` submodule and then call those functions within the benchmarks. This does mean that some redirection is still required to understand exactly how a given helper function operates, but this was deemed worth it to keep the actual size of benchmarks from inflating. + +An example of this philosophy in practice would be as follows. In this example we wish to test how long it takes to both read a small remote NWB file (from the ``s3_url``) using the ``remfile`` method as well as how long it takes to slice ~20 MB of data from the contents of a remote NWB file that has a large amount of series data. + +.. code-block:: python + from nwb_benchmarks.core import read_hdf5_nwbfile_remfile + + class NWBFileReadBenchmark: + param_names = ["s3_url"] + params = [ + "https://dandiarchive.s3.amazonaws.com/ros3test.nwb", # The original small test NWB file + ] + + def time_read_hdf5_nwbfile_remfile(self, s3_url: str): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) + + +.. code-block:: python + + from nwb_benchmarks.core import get_s3_url, read_hdf5_nwbfile_remfile + + class RemfileContinuousSliceBenchmark: + param_names = ["s3_url", "object_name", "slice_range"] + params = ( + [ + get_s3_url( # Yet another helper function for making the NWB file input easier to read + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ) + ], + ["ElectricalSeriesAp"], + [(slice(0, 30_000), slice(0, 384))], # ~23 MB + ) + + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") + self.data_to_slice = self.neurodata_object.data + + def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + """Note: store as self._temp to avoid tracking garbage collection as well.""" + self._temp = self.data_to_slice[slice_range] + +Notice how the ``read_hdf5_nwbfile_remfile`` method was used as both the main operating being timed in the first case, then reused in the ``setup`` of the of the second. By following the redirection of the function to its definition, we find it is itself a compound of another helper function for ``remfile`` usage... + +.. code-block:: python + # In nwb_benchmarks/core/_streaming.py + + def read_hdf5_remfile(s3_url: str) -> Tuple[h5py.File, remfile.File]: + """Load the raw HDF5 file from an S3 URL using remfile; does not formally read the NWB file.""" + byte_stream = remfile.File(url=s3_url) + file = h5py.File(name=byte_stream) + return (file, byte_stream) + + + def read_hdf5_nwbfile_remfile(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO, h5py.File, remfile.File]: + """Read an HDF5 NWB file from an S3 URL using the ROS3 driver from h5py.""" + (file, byte_stream) = read_hdf5_remfile(s3_url=s3_url) + io = pynwb.NWBHDF5IO(file=file, load_namespaces=True) + nwbfile = io.read() + return (nwbfile, io, file, byte_stream) + +and so we managed to save ~5 lines of code for every occurence of this logic in the benchmarks. Good choices of function names are critical to effectively communicating the actions being undertaken. Thorough annotation of signatures is likewise critical to understanding input/output relationships for these functions. From ef0023a7ccccd0547a1334545f767d30589cb2c9 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sun, 18 Feb 2024 13:12:18 -0500 Subject: [PATCH 21/72] improve results file stem --- ...onment-509c211421a12aeede580b602fde9cda8143b0aa.json} | 0 ...achine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json} | 0 ...onment-509c211421a12aeede580b602fde9cda8143b0aa.json} | 0 src/nwb_benchmarks/setup/_reduce_results.py | 9 +++++---- 4 files changed, 5 insertions(+), 4 deletions(-) rename results/{environment-509c211421a12aeede580b602fde9cda8143b0aa.json => info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json} (100%) rename results/{machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json => info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json} (100%) rename results/{timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json => results_timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json} (100%) diff --git a/results/environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json similarity index 100% rename from results/environment-509c211421a12aeede580b602fde9cda8143b0aa.json rename to results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json diff --git a/results/machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json b/results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json similarity index 100% rename from results/machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json rename to results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json diff --git a/results/timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/results_timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json similarity index 100% rename from results/timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json rename to results/results_timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 10e992e..fc6a1d7 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -35,7 +35,7 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil # In timestamp, replace separators with underscores for file path unix_time_to_datetime = str(datetime.datetime.fromtimestamp(raw_results_info["date"] / 1e3)) - timestamp = unix_time_to_datetime.replace(" ", "_").replace("-", "_").replace(":", "_") + timestamp = unix_time_to_datetime.replace(" ", "-").replace(":", "-") environment_hash = hashlib.sha1(string=bytes(json.dumps(obj=parsed_environment_info), "utf-8")).hexdigest() machine_hash = raw_results_info["params"]["machine"] @@ -80,7 +80,8 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil # 'processed' results go to nwb_benchmarks/results main_results_folder = raw_results_file_path.parent.parent.parent.parent / "results" parsed_results_file = ( - main_results_folder / f"timestamp-{timestamp}_machine-{machine_hash}_environment-{environment_hash}.json" + main_results_folder + / f"results_timestamp-{timestamp}_machine-{machine_hash}_environment-{environment_hash}.json" ) main_results_folder.mkdir(parents=True, exist_ok=True) @@ -89,12 +90,12 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil # Copy machine file to main results machine_info_file_path = raw_results_file_path.parent / "machine.json" - machine_info_copy_file_path = main_results_folder / f"machine-{machine_hash}.json" + machine_info_copy_file_path = main_results_folder / f"info_machine-{machine_hash}.json" if not machine_info_copy_file_path.exists(): shutil.copyfile(src=machine_info_file_path, dst=machine_info_copy_file_path) # Save parsed environment info within machine subdirectory of .asv - parsed_environment_file_path = main_results_folder / f"environment-{environment_hash}.json" + parsed_environment_file_path = main_results_folder / f"info_environment-{environment_hash}.json" if not parsed_environment_file_path.exists(): with open(file=parsed_environment_file_path, mode="w") as io: json.dump(obj=parsed_environment_info, fp=io, indent=4) From 6391fa207e276625b61e2dd30a09c0bcdec80b7e Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sun, 18 Feb 2024 13:17:33 -0500 Subject: [PATCH 22/72] also timestamp --- ...815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename results/{results_timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json => results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json} (100%) diff --git a/results/results_timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json similarity index 100% rename from results/results_timestamp-2024_02_17_00_59_37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json rename to results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json From 2c5f418bed9b7aa07e8d4e41a96b73a3f1981bae Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sun, 18 Feb 2024 13:18:23 -0500 Subject: [PATCH 23/72] fix rendering --- docs/writing_benchmarks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/writing_benchmarks.rst b/docs/writing_benchmarks.rst index fdace75..d2453d0 100644 --- a/docs/writing_benchmarks.rst +++ b/docs/writing_benchmarks.rst @@ -48,6 +48,7 @@ To reduce duplicated code, it is suggested to write standalone helper functions An example of this philosophy in practice would be as follows. In this example we wish to test how long it takes to both read a small remote NWB file (from the ``s3_url``) using the ``remfile`` method as well as how long it takes to slice ~20 MB of data from the contents of a remote NWB file that has a large amount of series data. .. code-block:: python + from nwb_benchmarks.core import read_hdf5_nwbfile_remfile class NWBFileReadBenchmark: @@ -89,6 +90,7 @@ An example of this philosophy in practice would be as follows. In this example w Notice how the ``read_hdf5_nwbfile_remfile`` method was used as both the main operating being timed in the first case, then reused in the ``setup`` of the of the second. By following the redirection of the function to its definition, we find it is itself a compound of another helper function for ``remfile`` usage... .. code-block:: python + # In nwb_benchmarks/core/_streaming.py def read_hdf5_remfile(s3_url: str) -> Tuple[h5py.File, remfile.File]: From b6f4691224aea22347eb960408447f35123ee1fc Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sun, 18 Feb 2024 10:47:09 -0800 Subject: [PATCH 24/72] Update docs/development.rst --- docs/development.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development.rst b/docs/development.rst index 35946b1..38d2f9d 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -77,4 +77,4 @@ Since all we're really after here is the raw tracking output, some custom reduct Network Tracking ---------------- -Please contact Oliver Ruebel for details. +Stay tuned https://github.com/NeurodataWithoutBorders/nwb_benchmarks/issues/24 From 832ddb5bb4227f5d413cad175760100b1b4a05e4 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sun, 18 Feb 2024 13:51:36 -0500 Subject: [PATCH 25/72] breakup text section around code --- docs/writing_benchmarks.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/writing_benchmarks.rst b/docs/writing_benchmarks.rst index d2453d0..f47bbe8 100644 --- a/docs/writing_benchmarks.rst +++ b/docs/writing_benchmarks.rst @@ -45,7 +45,7 @@ This approach means relying as little on inheritance and mixins as possible to r To reduce duplicated code, it is suggested to write standalone helper functions in the ``core`` submodule and then call those functions within the benchmarks. This does mean that some redirection is still required to understand exactly how a given helper function operates, but this was deemed worth it to keep the actual size of benchmarks from inflating. -An example of this philosophy in practice would be as follows. In this example we wish to test how long it takes to both read a small remote NWB file (from the ``s3_url``) using the ``remfile`` method as well as how long it takes to slice ~20 MB of data from the contents of a remote NWB file that has a large amount of series data. +An example of this philosophy in practice would be as follows. In this example we wish to test how long it takes to both read a small remote NWB file (from the ``s3_url``) using the ``remfile`` method... .. code-block:: python @@ -60,6 +60,7 @@ An example of this philosophy in practice would be as follows. In this example w def time_read_hdf5_nwbfile_remfile(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) +as well as how long it takes to slice ~20 MB of data from the contents of a remote NWB file that has a large amount of series data... .. code-block:: python @@ -87,7 +88,7 @@ An example of this philosophy in practice would be as follows. In this example w """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = self.data_to_slice[slice_range] -Notice how the ``read_hdf5_nwbfile_remfile`` method was used as both the main operating being timed in the first case, then reused in the ``setup`` of the of the second. By following the redirection of the function to its definition, we find it is itself a compound of another helper function for ``remfile`` usage... +Notice how the ``read_hdf5_nwbfile_remfile`` function (which reads an HDF5-backend ``pynwb.NWBFile`` object into memory using the ``remfile`` method) was used as both the main operation being timed in the first case, then reused in the ``setup`` of the of the second. By following the redirection of the function to its definition, we find it is itself a compound of another helper function for ``remfile`` usage... .. code-block:: python From c6cb67c16cb97fbc0d0607fd01285d2763136591 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sun, 18 Feb 2024 19:55:08 -0500 Subject: [PATCH 26/72] improve some readability --- .gitignore | 3 + .../core/_base_network_benchmark.py | 41 +++++++------- .../core/_network_statistics.py | 55 +++++++++++-------- 3 files changed, 56 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index af2e23b..65de2af 100644 --- a/.gitignore +++ b/.gitignore @@ -157,3 +157,6 @@ fabric.properties # Built Visual Studio Code Extensions *.vsix + +# Spyder +.spyproject/* diff --git a/src/nwb_benchmarks/core/_base_network_benchmark.py b/src/nwb_benchmarks/core/_base_network_benchmark.py index 9e7ece0..330c2ff 100644 --- a/src/nwb_benchmarks/core/_base_network_benchmark.py +++ b/src/nwb_benchmarks/core/_base_network_benchmark.py @@ -19,6 +19,9 @@ class BaseNetworkBenchmark: 2) The `setup_cache` method to explicitly recieve the necessary keyword arguments to be passed to the operation. """ + def setup_network_test(self, **keyword_arguments): + raise SkipNotImplemented() + def operation_to_track_network_activity_of(self, **keyword_arguments): raise SkipNotImplemented() @@ -28,15 +31,15 @@ def setup_cache(self, **keyword_arguments) -> dict: Only supports single operations (ignores `repeat` and prohibits `params`). """ + self.setup_network_test(**keyword_arguments) self.start_net_capture() t0 = time.time() - self.setup(**keyword_arguments) self.operation_to_track_network_activity_of(**keyword_arguments) t1 = time.time() total_time = t1 - t0 - self.stop_netcapture() - self.net_stats["network_total_time"] = total_time - return self.net_stats + self.stop_net_capture() + self.network_statistics["network_total_time"] = total_time + return self.network_statistics def start_net_capture(self): # start the capture_connections() function to update the current connections of this machine @@ -45,20 +48,20 @@ def start_net_capture(self): time.sleep(0.2) # not sure if this is needed but just to be safe # start capturing the raw packets by running the tshark commandline tool in a subprocess - self.netprofiler = NetworkProfiler() - self.netprofiler.start_capture() + self.network_profiler = NetworkProfiler() + self.network_profiler.start_capture() - def stop_netcapture(self): + def stop_net_capture(self): # Stop capturing packets and connections - self.netprofiler.stop_capture() + self.network_profiler.stop_capture() self.connections_thread.stop() # get the connections for the PID of this process self.pid_connections = self.connections_thread.get_connections_for_pid(os.getpid()) # Parse packets and filter out all the packets for this process pid by matching with the pid_connections - self.pid_packets = self.netprofiler.get_packets_for_connections(self.pid_connections) + self.pid_packets = self.network_profiler.get_packets_for_connections(self.pid_connections) # Compute all the network statistics - self.net_stats = NetworkStatistics.get_stats(packets=self.pid_packets) + self.network_statistics = NetworkStatistics.get_statistics(packets=self.pid_packets) def track_bytes_downloaded(self, net_stats: dict): return net_stats["bytes_downloaded"] @@ -75,20 +78,20 @@ def track_bytes_total(self, net_stats: dict): track_bytes_total.unit = "bytes" - def track_num_packets(self, net_stats: dict): - return net_stats["num_packets"] + def track_number_of_packets(self, net_stats: dict): + return net_stats["number_of_packets"] - track_num_packets.unit = "count" + track_number_of_packets.unit = "count" - def track_num_packets_downloaded(self, net_stats: dict): - return net_stats["num_packets_downloaded"] + def track_number_of_packets_downloaded(self, net_stats: dict): + return net_stats["number_of_packets_downloaded"] - track_num_packets_downloaded.unit = "count" + track_number_of_packets_downloaded.unit = "count" - def track_num_packets_uploaded(self, net_stats: dict): - return net_stats["num_packets_uploaded"] + def track_number_of_packets_uploaded(self, net_stats: dict): + return net_stats["number_of_packets_uploaded"] - track_num_packets_uploaded.unit = "count" + track_number_of_packets_uploaded.unit = "count" def track_total_transfer_time(self, net_stats: dict): return net_stats["total_transfer_time"] diff --git a/src/nwb_benchmarks/core/_network_statistics.py b/src/nwb_benchmarks/core/_network_statistics.py index 64d7722..4e7c1a7 100644 --- a/src/nwb_benchmarks/core/_network_statistics.py +++ b/src/nwb_benchmarks/core/_network_statistics.py @@ -1,5 +1,7 @@ """Class for summary and display of basic network statistics.""" +from typing import Dict, Literal, Union + import numpy as np from ._capture_connections import CaptureConnections @@ -9,7 +11,7 @@ class NetworkStatistics: """Compute basic statistics about network packets captures with tshark/pyshark.""" @staticmethod - def num_packets(packets: list): + def num_packets(packets: list) -> int: """Total number of packets.""" return len(packets) @@ -37,10 +39,10 @@ def get_web_traffic_packets(packets: list) -> list: @staticmethod def transfer_time(packets: list) -> float: """Sum of all time_delta's between packets.""" - return float(np.sum([float(p.tcp.time_delta) for p in packets])) + return float(np.sum([float(packet.tcp.time_delta) for packet in packets])) @staticmethod - def num_packets_downloaded(packets: list, local_addresses: list = None): + def number_of_packets_downloaded(packets: list, local_addresses: list = None) -> int: """Total number of packets downloaded.""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() @@ -49,10 +51,10 @@ def num_packets_downloaded(packets: list, local_addresses: list = None): for packet in packets # check all packets if packet.ip.src not in local_addresses # the source address is not ours so it's download ] - return int(len(downloaded)) + return len(downloaded) @staticmethod - def num_packets_uploaded(packets: list, local_addresses: list = None): + def num_packets_uploaded(packets: list, local_addresses: list = None) -> int: """Total number of packets uploaded (e.g., HTTP requests).""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() @@ -61,16 +63,16 @@ def num_packets_uploaded(packets: list, local_addresses: list = None): for packet in packets # check all packets if packet.ip.src in local_addresses # the source address is ours so it's upload ] - return int(len(uploaded)) + return len(uploaded) @staticmethod - def total_bytes(packets: list): + def total_bytes(packets: list) -> int: """Total number of bytes in the packets.""" total_bytes = np.sum([len(packet) for packet in packets]) return int(total_bytes) @staticmethod - def bytes_downloaded(packets: list, local_addresses: list = None): + def bytes_downloaded(packets: list, local_addresses: list = None) -> int: """Total number of bytes from downloaded packets.""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() @@ -84,7 +86,7 @@ def bytes_downloaded(packets: list, local_addresses: list = None): return int(bytes_downloaded) @staticmethod - def bytes_uploaded(packets: list, local_addresses: list = None): + def bytes_uploaded(packets: list, local_addresses: list = None) -> int: """Total number of bytes from uploaded packets.""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() @@ -98,33 +100,38 @@ def bytes_uploaded(packets: list, local_addresses: list = None): return int(bytes_uploaded) @staticmethod - def bytes_to_str(size_in_bytes: int) -> str: + def bytes_to_str(size_in_bytes: int, convention: Literal["B", "iB"] = "iB") -> str: """Format the size in bytes as a human-readable string.""" + factor = 1024 if convention == "iB" else 1000 for unit in ["", "K", "M", "G", "T", "P"]: - if size_in_bytes < 1024: - return f"{bytes:.2f}{unit}B" - size_in_bytes /= 1024 + if size_in_bytes < factor: + return f"{bytes:.2f}{unit}{convention}" + size_in_bytes /= factor @classmethod - def get_stats(cls, packets: list, local_addresses: list = None): + def get_statistics(cls, packets: list, local_addresses: list = None) -> Dict[str, Union[int, float]]: """Calculate all the statistics and return them as a dictionary.""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() - stats = { + statistics = { "bytes_downloaded": cls.bytes_downloaded(packets=packets, local_addresses=local_addresses), "bytes_uploaded": cls.bytes_uploaded(packets=packets, local_addresses=local_addresses), "bytes_total": cls.total_bytes(packets=packets), - "num_packets": cls.num_packets(packets=packets), - "num_packets_downloaded": cls.num_packets_downloaded(packets=packets, local_addresses=local_addresses), - "num_packets_uploaded": cls.num_packets_uploaded(packets=packets, local_addresses=local_addresses), - "num_web_packets": len(cls.get_web_traffic_packets(packets)), + "number_of_packets": cls.num_packets(packets=packets), + "number_of_packets_downloaded": cls.number_of_packets_downloaded( + packets=packets, local_addresses=local_addresses + ), + "number_of_packets_uploaded": cls.number_of_packets_uploaded( + packets=packets, local_addresses=local_addresses + ), + "number_of_web_packets": len(cls.get_web_traffic_packets(packets)), "total_transfer_time": cls.transfer_time(packets=packets), } - return stats + return statistics @classmethod - def print_stats(cls, packets: list, local_addresses: list = None): + def print_statistics(cls, packets: list, local_addresses: list = None): """Print all the statistics.""" - stats = cls.get_stats(packets=packets, local_addresses=local_addresses) - for k, v in stats.items(): - print(f"{k}: {v}") + statistics = cls.Calculate(packets=packets, local_addresses=local_addresses) + for key, value in statistics.items(): + print(f"{key}: {value}") From b4c1ac108cf48723dd6442953508fe5a1834cd1c Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Sun, 18 Feb 2024 21:19:10 -0500 Subject: [PATCH 27/72] refactor to context; adjust demo; adjust benchmarks --- .../network_tracking_remote_file_reading.py | 119 +++++++----------- .../network_tracking_remote_slicing.py | 77 +++++------- src/nwb_benchmarks/core/__init__.py | 4 +- .../core/_base_network_benchmark.py | 105 ---------------- .../core/_network_statistics.py | 4 +- src/nwb_benchmarks/core/_network_tracker.py | 53 ++++++++ .../scripts/demo_network_profiling.py | 53 ++------ 7 files changed, 138 insertions(+), 277 deletions(-) delete mode 100644 src/nwb_benchmarks/core/_base_network_benchmark.py create mode 100644 src/nwb_benchmarks/core/_network_tracker.py diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py index 4ee6e7d..4fb6317 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -1,7 +1,8 @@ """Basic benchmarks for stream NWB files and their contents.""" +import os + from nwb_benchmarks.core import ( - BaseNetworkBenchmark, get_s3_url, read_hdf5_fsspec_no_cache, read_hdf5_nwbfile_fsspec_no_cache, @@ -9,93 +10,59 @@ read_hdf5_nwbfile_ros3, read_hdf5_remfile, read_hdf5_ros3, + track_network_activity, ) -# Network base does not yet support params -# param_names = ["s3_url"] -# params = [ -# get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb"), -# get_s3_url( -# dandiset_id="000717", -# dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", -# ), # Not the best example for testing a theory about file read; should probably replace with something simpler -# "https://dandiarchive.s3.amazonaws.com/ros3test.nwb", # The original small test NWB file -# ] - - -class FsspecNoCacheDirectFileReadBenchmark(BaseNetworkBenchmark): - s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" - - def setup_cache(self): - super().setup_cache(s3_url=self.s3_url) - - def setup(self, s3_url: str): - pass - - def operation_to_track_network_activity_of(self, s3_url: str): - self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) - - -class RemfileDirectFileReadBenchmark(BaseNetworkBenchmark): - s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" - - def setup_cache(self): - super().setup_cache(s3_url=self.s3_url) - - def setup(self, s3_url: str): - pass - - def operation_to_track_network_activity_of(self, s3_url: str): - self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) - - -class Ros3DirectFileReadBenchmark(BaseNetworkBenchmark): - s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" - - def setup_cache(self): - super().setup_cache(s3_url=self.s3_url) - - def setup(self, s3_url: str): - pass - - def operation_to_track_network_activity_of(self, s3_url: str): - self.file = read_hdf5_ros3(s3_url=s3_url) - - -class FsspecNoCacheNWBFileReadBenchmark(BaseNetworkBenchmark): - s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" +TSHARK_PATH = os.environ.get("TSHARK_PATH", None) - def setup_cache(self): - super().setup_cache(s3_url=self.s3_url) +param_names = ["s3_url"] +params = [ + get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb"), + get_s3_url( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ), # Not the best example for testing a theory about file read; should probably replace with something simpler + "https://dandiarchive.s3.amazonaws.com/ros3test.nwb", # The original small test NWB file +] - def setup(self, s3_url: str): - pass - def operation_to_track_network_activity_of(self, s3_url: str): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) +class FsspecNoCacheDirectFileReadBenchmark: + def track_network_activity_during_read(self, s3_url: str): + with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) + return network_activity.network_statistics -class RemfileNWBFileReadBenchmark(BaseNetworkBenchmark): - s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" +class RemfileDirectFileReadBenchmark: + def track_network_activity_during_read(self, s3_url: str): + with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) + return network_activity.network_statistics - def setup_cache(self): - super().setup_cache(s3_url=self.s3_url) - def setup(self, s3_url: str): - pass +class Ros3DirectFileReadBenchmark: + def track_network_activity_during_read(self, s3_url: str): + with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + self.file = read_hdf5_ros3(s3_url=s3_url) + return network_activity.network_statistics - def operation_to_track_network_activity_of(self, s3_url: str): - self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) +class FsspecNoCacheNWBFileReadBenchmark: + def track_network_activity_during_read(self, s3_url: str): + with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) + return network_activity.network_statistics -class Ros3NWBFileReadBenchmark(BaseNetworkBenchmark): - s3_url = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" - def setup_cache(self): - super().setup_cache(s3_url=self.s3_url) +class RemfileNWBFileReadBenchmark: + def track_network_activity_during_read(self, s3_url: str): + with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) + return network_activity.network_statistics - def setup(self, s3_url: str): - pass - def operation_to_track_network_activity_of(self, s3_url: str): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) +class Ros3NWBFileReadBenchmark: + def track_network_activity_during_read(self, s3_url: str): + with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) + return network_activity.network_statistics diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index 9ea2daf..f7faf25 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -1,5 +1,6 @@ """Basic benchmarks for stream NWB files and their contents.""" +import os from typing import Tuple from nwb_benchmarks.core import ( @@ -10,75 +11,55 @@ read_hdf5_nwbfile_remfile, read_hdf5_nwbfile_ros3, robust_ros3_read, + track_network_activity, ) -# Network base does not yet support params -# Might have more luck using parameterized.paramterize_class -# param_names = ["s3_url", "object_name", "slice_range"] -# params = ( -# [ -# get_s3_url( -# dandiset_id="000717", -# dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", -# ) -# ], -# ["ElectricalSeriesAp"], -# [(slice(0, 30_000), slice(0, 384))], # ~23 MB -# ) -s3_url = get_s3_url( - dandiset_id="000717", - dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", +TSHARK_PATH = os.environ.get("TSHARK_PATH", None) + +param_names = ["s3_url", "object_name", "slice_range"] +params = ( + [ + get_s3_url( + dandiset_id="000717", + dandi_path="sub-IBL-ecephys/sub-IBL-ecephys_ses-3e7ae7c0_desc-18000000-frames-13653-by-384-chunking.nwb", + ) + ], + ["ElectricalSeriesAp"], + [(slice(0, 30_000), slice(0, 384))], # ~23 MB ) -object_name = "ElectricalSeriesAp" -slice_range = (slice(0, 30_000), slice(0, 384)) -class FsspecNoCacheContinuousSliceBenchmark(BaseNetworkBenchmark): - s3_url = s3_url - object_name = object_name - slice_range = slice_range - - def setup_cache(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - super().setup_cache(s3_url=self.s3_url, object_name=self.object_name, slice_range=self.slice_range) - +class FsspecNoCacheContinuousSliceBenchmark: def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self._temp = self.data_to_slice[slice_range] - + def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + with track_network_activity(tshark_path=TSHARK_PATH) as network_tracker: + self._temp = self.data_to_slice[slice_range] + return network_tracker.network_statistics -class RemfileContinuousSliceBenchmark(BaseNetworkBenchmark): - s3_url = s3_url - object_name = object_name - slice_range = slice_range - - def setup_cache(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - super().setup_cache(s3_url=self.s3_url, object_name=self.object_name, slice_range=self.slice_range) +class RemfileContinuousSliceBenchmark: def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self._temp = self.data_to_slice[slice_range] - - -class Ros3ContinuousSliceBenchmark(BaseNetworkBenchmark): - s3_url = s3_url - object_name = object_name - slice_range = slice_range + def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + with track_network_activity(tshark_path=TSHARK_PATH) as network_tracker: + self._temp = self.data_to_slice[slice_range] + return network_tracker.network_statistics - def setup_cache(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - super().setup_cache(s3_url=self.s3_url, object_name=self.object_name, slice_range=self.slice_range) +class Ros3ContinuousSliceBenchmark: def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def operation_to_track_network_activity_of(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) + def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + with track_network_activity(tshark_path=TSHARK_PATH) as network_tracker: + self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=slice_range) + return network_tracker.network_statistics diff --git a/src/nwb_benchmarks/core/__init__.py b/src/nwb_benchmarks/core/__init__.py index e093193..2c0483a 100644 --- a/src/nwb_benchmarks/core/__init__.py +++ b/src/nwb_benchmarks/core/__init__.py @@ -1,10 +1,10 @@ """Exposed imports to the `core` submodule.""" -from ._base_network_benchmark import BaseNetworkBenchmark from ._capture_connections import CaptureConnections from ._dandi import get_s3_url from ._network_profiler import NetworkProfiler from ._network_statistics import NetworkStatistics +from ._network_tracker import track_network_activity from ._nwb_helpers import get_object_by_name from ._streaming import ( read_hdf5_fsspec_no_cache, @@ -17,10 +17,10 @@ ) __all__ = [ - "BaseNetworkBenchmark", "CaptureConnections", "NetworkProfiler", "NetworkStatistics", + "track_network_activity", "read_hdf5_fsspec_no_cache", "read_hdf5_nwbfile_fsspec_no_cache", "read_hdf5_ros3", diff --git a/src/nwb_benchmarks/core/_base_network_benchmark.py b/src/nwb_benchmarks/core/_base_network_benchmark.py deleted file mode 100644 index 330c2ff..0000000 --- a/src/nwb_benchmarks/core/_base_network_benchmark.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Base class for implementing benchmarks for evaluating network performance metrics for streaming read of NWB files.""" - -import os -import time - -from asv_runner.benchmarks.mark import SkipNotImplemented - -from ._capture_connections import CaptureConnections -from ._network_profiler import NetworkProfiler -from ._network_statistics import NetworkStatistics - - -class BaseNetworkBenchmark: - """ - Base class for ASV Benchmark network performance metrics for streaming data access. - - Child classes must implement: - 1) The `operation_to_track_network_activity_of`, which replaces the usual time_ or mem_ operation being tracked. - 2) The `setup_cache` method to explicitly recieve the necessary keyword arguments to be passed to the operation. - """ - - def setup_network_test(self, **keyword_arguments): - raise SkipNotImplemented() - - def operation_to_track_network_activity_of(self, **keyword_arguments): - raise SkipNotImplemented() - - def setup_cache(self, **keyword_arguments) -> dict: - """ - Strategy essentially mimics a single benchmark execution (setup + run). - - Only supports single operations (ignores `repeat` and prohibits `params`). - """ - self.setup_network_test(**keyword_arguments) - self.start_net_capture() - t0 = time.time() - self.operation_to_track_network_activity_of(**keyword_arguments) - t1 = time.time() - total_time = t1 - t0 - self.stop_net_capture() - self.network_statistics["network_total_time"] = total_time - return self.network_statistics - - def start_net_capture(self): - # start the capture_connections() function to update the current connections of this machine - self.connections_thread = CaptureConnections() - self.connections_thread.start() - time.sleep(0.2) # not sure if this is needed but just to be safe - - # start capturing the raw packets by running the tshark commandline tool in a subprocess - self.network_profiler = NetworkProfiler() - self.network_profiler.start_capture() - - def stop_net_capture(self): - # Stop capturing packets and connections - self.network_profiler.stop_capture() - self.connections_thread.stop() - - # get the connections for the PID of this process - self.pid_connections = self.connections_thread.get_connections_for_pid(os.getpid()) - # Parse packets and filter out all the packets for this process pid by matching with the pid_connections - self.pid_packets = self.network_profiler.get_packets_for_connections(self.pid_connections) - # Compute all the network statistics - self.network_statistics = NetworkStatistics.get_statistics(packets=self.pid_packets) - - def track_bytes_downloaded(self, net_stats: dict): - return net_stats["bytes_downloaded"] - - track_bytes_downloaded.unit = "bytes" - - def track_bytes_uploaded(self, net_stats: dict): - return net_stats["bytes_uploaded"] - - track_bytes_uploaded.unit = "bytes" - - def track_bytes_total(self, net_stats: dict): - return net_stats["bytes_total"] - - track_bytes_total.unit = "bytes" - - def track_number_of_packets(self, net_stats: dict): - return net_stats["number_of_packets"] - - track_number_of_packets.unit = "count" - - def track_number_of_packets_downloaded(self, net_stats: dict): - return net_stats["number_of_packets_downloaded"] - - track_number_of_packets_downloaded.unit = "count" - - def track_number_of_packets_uploaded(self, net_stats: dict): - return net_stats["number_of_packets_uploaded"] - - track_number_of_packets_uploaded.unit = "count" - - def track_total_transfer_time(self, net_stats: dict): - return net_stats["total_transfer_time"] - - track_total_transfer_time.unit = "seconds" - - def track_network_total_time(self, net_stats: dict): - """Technically different from the official time_ approach.""" - return net_stats["network_total_time"] - - track_network_total_time.unit = "seconds" diff --git a/src/nwb_benchmarks/core/_network_statistics.py b/src/nwb_benchmarks/core/_network_statistics.py index 4e7c1a7..3f79a69 100644 --- a/src/nwb_benchmarks/core/_network_statistics.py +++ b/src/nwb_benchmarks/core/_network_statistics.py @@ -54,7 +54,7 @@ def number_of_packets_downloaded(packets: list, local_addresses: list = None) -> return len(downloaded) @staticmethod - def num_packets_uploaded(packets: list, local_addresses: list = None) -> int: + def number_of_packets_uploaded(packets: list, local_addresses: list = None) -> int: """Total number of packets uploaded (e.g., HTTP requests).""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() @@ -132,6 +132,6 @@ def get_statistics(cls, packets: list, local_addresses: list = None) -> Dict[str @classmethod def print_statistics(cls, packets: list, local_addresses: list = None): """Print all the statistics.""" - statistics = cls.Calculate(packets=packets, local_addresses=local_addresses) + statistics = cls.get_statistics(packets=packets, local_addresses=local_addresses) for key, value in statistics.items(): print(f"{key}: {value}") diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py new file mode 100644 index 0000000..bba7728 --- /dev/null +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -0,0 +1,53 @@ +"""Base class for implementing benchmarks for evaluating network performance metrics for streaming read of NWB files.""" + +import contextlib +import os +import pathlib +import time +from typing import Union + +from ._capture_connections import CaptureConnections +from ._network_profiler import NetworkProfiler +from ._network_statistics import NetworkStatistics + + +@contextlib.contextmanager +def track_network_activity(tshark_exe_path: Union[pathlib.Path, None] = None): + network_tracker = NetworkTracker() + + try: + network_tracker.start_network_capture(tshark_exe_path=tshark_exe_path) + time.sleep(0.3) + + t0 = time.time() + yield network_tracker + finally: + network_tracker.stop_network_capture() + + t1 = time.time() + network_total_time = t1 - t0 + network_tracker.network_statistics["network_total_time"] = network_total_time + + +class NetworkTracker: + def start_network_capture(self, tshark_exe_path: Union[pathlib.Path, None] = None): + # start the capture_connections() function to update the current connections of this machine + self.connections_thread = CaptureConnections() + self.connections_thread.start() + time.sleep(0.2) # not sure if this is needed but just to be safe + + # start capturing the raw packets by running the tshark commandline tool in a subprocess + self.network_profiler = NetworkProfiler() + self.network_profiler.start_capture(tshark_exe_path=tshark_exe_path) + + def stop_network_capture(self): + # Stop capturing packets and connections + self.network_profiler.stop_capture() + self.connections_thread.stop() + + # get the connections for the PID of this process + self.pid_connections = self.connections_thread.get_connections_for_pid(os.getpid()) + # Parse packets and filter out all the packets for this process pid by matching with the pid_connections + self.pid_packets = self.network_profiler.get_packets_for_connections(self.pid_connections) + # Compute all the network statistics + self.network_statistics = NetworkStatistics.get_statistics(packets=self.pid_packets) diff --git a/src/nwb_benchmarks/scripts/demo_network_profiling.py b/src/nwb_benchmarks/scripts/demo_network_profiling.py index 3ab3e43..30c86b7 100644 --- a/src/nwb_benchmarks/scripts/demo_network_profiling.py +++ b/src/nwb_benchmarks/scripts/demo_network_profiling.py @@ -1,58 +1,23 @@ """Demo of network profiling tools.""" -import os -import pathlib -import time +import json import pynwb -from nwb_benchmarks.core import CaptureConnections, NetworkProfiler, NetworkStatistics - -### -# This would be part of the setUp() of a test -## -# start the capture_connections() function to update the current connections of this machine -connections_thread = CaptureConnections() -connections_thread.start() -time.sleep(0.2) # not sure if this is needed but just to be safe - -# start capturing the raw packets by running the tshark commandline tool in a subprocess -netprofiler = NetworkProfiler() +from nwb_benchmarks.core import track_network_activity # If `tshark` is not available as a global command, specify the pathlib.Path pointing to the .exe # Otherwise, set to None or do not pass into `.start_capture` -tshark_exe_path = pathlib.Path("D:/Wireshark/tshark.exe") -# tshark_exe_path = None - -netprofiler.start_capture(tshark_exe_path=tshark_exe_path) +# tshark_exe_path = pathlib.Path("D:/Wireshark/tshark.exe") +tshark_exe_path = None -### -# This would be the unit test -### # Read the NWB data file from DANDI -t0 = time.time() s3_path = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" -with pynwb.NWBHDF5IO(s3_path, mode="r", driver="ros3") as io: - nwbfile = io.read() - test_data = nwbfile.acquisition["ts_name"].data[:] -t1 = time.time() -total_time = t1 - t0 - -### -# This part would go into tearDown to stop the capture and compute statistics -### -# Stop capturing packets and connections -netprofiler.stop_capture() -connections_thread.stop() - -# get the connections for the PID of this process -pid_connections = connections_thread.get_connections_for_pid(os.getpid()) - -# Parse packets and filter out all the packets for this process pid by matching with the pid_connections -pid_packets = netprofiler.get_packets_for_connections(pid_connections) +with track_network_activity(tshark_exe_path=tshark_exe_path) as network_tracker: + with pynwb.NWBHDF5IO(s3_path, mode="r", driver="ros3") as io: + nwbfile = io.read() + test_data = nwbfile.acquisition["ts_name"].data[:] # Print basic connection statistics -print("num_connections:", int(len(pid_connections) / 2.0)) -NetworkStatistics.print_stats(packets=pid_packets) -print("total_time:", total_time) +print(json.dumps(obj=network_tracker.network_statistics, indent=4)) From 0802e0b46f60721f7c7cfcad31ce7361ad99bed9 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Sun, 18 Feb 2024 22:41:00 -0500 Subject: [PATCH 28/72] adjust names to not conflict --- docs/running_benchmarks.rst | 6 +- environments/nwb_benchmarks.yaml | 3 +- ...c211421a12aeede580b602fde9cda8143b0aa.json | 946 ------------------ ...05e6bc6fe1582a6a348d5cfddde3e86882815.json | 175 ---- ...c211421a12aeede580b602fde9cda8143b0aa.json | 75 -- .../network_tracking_remote_file_reading.py | 56 +- .../network_tracking_remote_slicing.py | 30 +- src/nwb_benchmarks/command_line_interface.py | 3 + src/nwb_benchmarks/core/__init__.py | 4 +- src/nwb_benchmarks/core/_network_profiler.py | 6 +- .../core/_network_statistics.py | 46 +- src/nwb_benchmarks/core/_network_tracker.py | 8 +- .../scripts/demo_network_profiling.py | 8 +- .../setup/_configure_machine.py | 3 +- src/nwb_benchmarks/setup/_reduce_results.py | 7 + 15 files changed, 120 insertions(+), 1256 deletions(-) delete mode 100644 results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json delete mode 100644 results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json delete mode 100644 results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index 380f3f6..d23fc6a 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -3,11 +3,13 @@ Running the Benchmarks Before running the benchmark suite, please ensure you are not running any additional heavy processes in the background to avoid interference or bottlenecks. -To run the full benchmark suite, simply call... +To run the full benchmark suite, including network tracking tests (which require ``sudo`` on Mac or Linux), simply call... .. code-block:: - nwb_benchmarks run + sudo nwb_benchmarks run + +Or drop the ``sudo`` if on Windows. Running on Windows may also require you to set the ``TSHARK_PATH`` environment variable beforehand. Many of the current tests can take several minutes to complete; the entire suite will take many times that. Grab some coffee, read a book, or better yet (when the suite becomes larger) just leave it to run overnight. diff --git a/environments/nwb_benchmarks.yaml b/environments/nwb_benchmarks.yaml index dcda0f1..a8f439c 100644 --- a/environments/nwb_benchmarks.yaml +++ b/environments/nwb_benchmarks.yaml @@ -16,5 +16,6 @@ dependencies: - aiohttp - remfile - pyshark - - hdmf @ git+https://github.com/hdmf-dev/hdmf.git@expose_aws_region # required until region fix is released + - hdmf @ git+https://github.com/hdmf-dev/hdmf.git@9b3282aa4999d2922f003f1b79ec7458ea3ddc5e # required until PyNWB propagates changes + #- hdmf @ git+https://github.com/hdmf-dev/hdmf.git@expose_aws_region # required until region fix is released - -e .. diff --git a/results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json deleted file mode 100644 index c46ab88..0000000 --- a/results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json +++ /dev/null @@ -1,946 +0,0 @@ -{ - "3.11.7 | packaged by Anaconda, Inc. | (main, Dec 15 2023, 18:05:47) [MSC v.1916 64 bit (AMD64)]": [ - { - "name": "aiobotocore", - "version": "2.11.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "aiohttp", - "version": "3.9.3", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "aioitertools", - "version": "0.11.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "aiosignal", - "version": "1.3.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "appdirs", - "version": "1.4.4", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "arrow", - "version": "1.3.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "asciitree", - "version": "0.3.3", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "asttokens", - "version": "2.4.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "asv", - "version": "0.6.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "asv-runner", - "version": "0.2.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "attrs", - "version": "23.2.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "bidsschematools", - "version": "0.7.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "blas", - "version": "1.0", - "build": "mkl" - }, - { - "name": "botocore", - "version": "1.34.34", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "bzip2", - "version": "1.0.8", - "build": "he774522_0" - }, - { - "name": "ca-certificates", - "version": "2023.12.12", - "build": "haa95532_0" - }, - { - "name": "cached-property", - "version": "1.5.2", - "build": "py_0" - }, - { - "name": "certifi", - "version": "2024.2.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "cfgv", - "version": "3.4.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "charset-normalizer", - "version": "3.3.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "ci-info", - "version": "0.3.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "click", - "version": "8.1.7", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "click-didyoumean", - "version": "0.3.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "cloudpickle", - "version": "3.0.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "colorama", - "version": "0.4.6", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "comm", - "version": "0.2.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "dandi", - "version": "0.59.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "dandischema", - "version": "0.8.4", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "debugpy", - "version": "1.8.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "decorator", - "version": "5.1.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "dill", - "version": "0.3.8", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "distlib", - "version": "0.3.8", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "dnspython", - "version": "2.5.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "email-validator", - "version": "2.1.0.post1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "etelemetry", - "version": "0.3.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "executing", - "version": "2.0.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "fasteners", - "version": "0.19", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "filelock", - "version": "3.13.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "fqdn", - "version": "1.5.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "frozenlist", - "version": "1.4.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "fscacher", - "version": "0.4.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "fsspec", - "version": "2024.2.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "git", - "version": "2.40.1", - "build": "haa95532_1" - }, - { - "name": "h5py", - "version": "3.10.0", - "build": "nompi_py311h7195302_101", - "channel": "conda-forge" - }, - { - "name": "hdf5", - "version": "1.14.3", - "build": "nompi_h73e8ff5_100", - "channel": "conda-forge" - }, - { - "name": "hdmf", - "version": "3.12.1.dev7+g9b3282a", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "humanize", - "version": "4.9.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "identify", - "version": "2.5.34", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "idna", - "version": "3.6", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "importlib-metadata", - "version": "7.0.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "intel-openmp", - "version": "2023.1.0", - "build": "h59b6b97_46320" - }, - { - "name": "interleave", - "version": "0.2.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "ipykernel", - "version": "6.29.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "ipython", - "version": "8.21.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "isodate", - "version": "0.6.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "isoduration", - "version": "20.11.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "jaraco-classes", - "version": "3.3.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "jedi", - "version": "0.19.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "jmespath", - "version": "1.0.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "joblib", - "version": "1.3.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "json5", - "version": "0.9.14", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "jsonpointer", - "version": "2.4", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "jsonschema", - "version": "4.21.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "jsonschema-specifications", - "version": "2023.12.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "jupyter-client", - "version": "8.6.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "jupyter-core", - "version": "5.7.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "keyring", - "version": "24.3.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "keyrings-alt", - "version": "5.0.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "libaec", - "version": "1.1.2", - "build": "h63175ca_1", - "channel": "conda-forge" - }, - { - "name": "libcurl", - "version": "8.5.0", - "build": "h86230a5_0" - }, - { - "name": "libffi", - "version": "3.4.4", - "build": "hd77b12b_0" - }, - { - "name": "libssh2", - "version": "1.10.0", - "build": "he2ea4bf_2" - }, - { - "name": "libzlib", - "version": "1.2.13", - "build": "hcfcfb64_5", - "channel": "conda-forge" - }, - { - "name": "llvmlite", - "version": "0.42.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "lxml", - "version": "5.1.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "matplotlib-inline", - "version": "0.1.6", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "mkl", - "version": "2023.1.0", - "build": "h6b88ed4_46358" - }, - { - "name": "mkl-service", - "version": "2.4.0", - "build": "py311h2bbff1b_1" - }, - { - "name": "mkl_fft", - "version": "1.3.8", - "build": "py311h2bbff1b_0" - }, - { - "name": "mkl_random", - "version": "1.2.4", - "build": "py311h59b6b97_0" - }, - { - "name": "more-itertools", - "version": "10.2.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "multidict", - "version": "6.0.5", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "natsort", - "version": "8.4.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "nest-asyncio", - "version": "1.6.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "nodeenv", - "version": "1.8.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "numba", - "version": "0.59.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "numcodecs", - "version": "0.12.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "numpy", - "version": "1.26.3", - "build": "py311hdab7c0b_0" - }, - { - "name": "numpy-base", - "version": "1.26.3", - "build": "py311hd01c5d8_0" - }, - { - "name": "nwb-benchmarks", - "version": "0.1.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "nwbinspector", - "version": "0.4.33", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "openssl", - "version": "3.2.1", - "build": "hcfcfb64_0", - "channel": "conda-forge" - }, - { - "name": "packaging", - "version": "23.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pandas", - "version": "2.2.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "parso", - "version": "0.8.3", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pip", - "version": "23.3.1", - "build": "py311haa95532_0" - }, - { - "name": "platformdirs", - "version": "4.2.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pre-commit", - "version": "3.6.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "prompt-toolkit", - "version": "3.0.43", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "psutil", - "version": "5.9.8", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pure-eval", - "version": "0.2.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pycryptodomex", - "version": "3.20.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pydantic", - "version": "1.10.14", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pygments", - "version": "2.17.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pympler", - "version": "1.0.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pynwb", - "version": "2.5.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pyout", - "version": "0.7.3", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pyshark", - "version": "0.6", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "python", - "version": "3.11.7", - "build": "he1021f5_0" - }, - { - "name": "python-dateutil", - "version": "2.8.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "python_abi", - "version": "3.11", - "build": "2_cp311", - "channel": "conda-forge" - }, - { - "name": "pytz", - "version": "2024.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pywin32", - "version": "306", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pywin32-ctypes", - "version": "0.2.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pyyaml", - "version": "6.0.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "pyzmq", - "version": "25.1.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "referencing", - "version": "0.33.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "remfile", - "version": "0.1.10", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "requests", - "version": "2.31.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "rfc3339-validator", - "version": "0.1.4", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "rfc3987", - "version": "1.3.8", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "rpds-py", - "version": "0.18.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "ruamel-yaml", - "version": "0.18.6", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "ruamel-yaml-clib", - "version": "0.2.8", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "s3fs", - "version": "2024.2.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "scipy", - "version": "1.12.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "semantic-version", - "version": "2.10.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "setuptools", - "version": "69.1.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "six", - "version": "1.16.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "spyder-kernels", - "version": "2.5.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "sqlite", - "version": "3.41.2", - "build": "h2bbff1b_0" - }, - { - "name": "stack-data", - "version": "0.6.3", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "tabulate", - "version": "0.9.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "tbb", - "version": "2021.8.0", - "build": "h59b6b97_0" - }, - { - "name": "tenacity", - "version": "8.2.3", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "termcolor", - "version": "2.4.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "tk", - "version": "8.6.12", - "build": "h2bbff1b_0" - }, - { - "name": "tomli", - "version": "2.0.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "tornado", - "version": "6.4", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "tqdm", - "version": "4.66.2", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "traitlets", - "version": "5.14.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "types-python-dateutil", - "version": "2.8.19.20240106", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "typing-extensions", - "version": "4.9.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "tzdata", - "version": "2024.1", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "ucrt", - "version": "10.0.20348.0", - "build": "haa95532_0" - }, - { - "name": "uri-template", - "version": "1.3.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "urllib3", - "version": "2.0.7", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "vc", - "version": "14.2", - "build": "h21ff451_1" - }, - { - "name": "vc14_runtime", - "version": "14.38.33130", - "build": "h82b7239_18", - "channel": "conda-forge" - }, - { - "name": "virtualenv", - "version": "20.25.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "vs2015_runtime", - "version": "14.38.33130", - "build": "hcb4865c_18", - "channel": "conda-forge" - }, - { - "name": "wcwidth", - "version": "0.2.13", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "webcolors", - "version": "1.13", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "wheel", - "version": "0.41.2", - "build": "py311haa95532_0" - }, - { - "name": "wrapt", - "version": "1.16.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "xz", - "version": "5.4.5", - "build": "h8cc25b3_0" - }, - { - "name": "yarl", - "version": "1.9.4", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "zarr", - "version": "2.17.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "zarr-checksum", - "version": "0.4.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "zipp", - "version": "3.17.0", - "build": "pypi_0", - "channel": "pypi" - }, - { - "name": "zlib", - "version": "1.2.13", - "build": "hcfcfb64_5", - "channel": "conda-forge" - } - ] -} diff --git a/results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json b/results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json deleted file mode 100644 index 5645669..0000000 --- a/results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json +++ /dev/null @@ -1,175 +0,0 @@ -{ - "cuda": { - "gpu_name": "NVIDIA GeForce RTX 2080", - "gpu_specifications": { - "ASYNC_ENGINE_COUNT": 6, - "CAN_MAP_HOST_MEMORY": 1, - "CAN_USE_HOST_POINTER_FOR_REGISTERED_MEM": 0, - "CLOCK_RATE": 1710000, - "COMPUTE_CAPABILITY_MAJOR": 7, - "COMPUTE_CAPABILITY_MINOR": 5, - "COMPUTE_MODE": 0, - "COMPUTE_PREEMPTION_SUPPORTED": 1, - "CONCURRENT_KERNELS": 1, - "CONCURRENT_MANAGED_ACCESS": 0, - "COOPERATIVE_LAUNCH": 1, - "COOPERATIVE_MULTI_DEVICE_LAUNCH": 0, - "ECC_ENABLED": 0, - "GLOBAL_L1_CACHE_SUPPORTED": 1, - "GLOBAL_MEMORY_BUS_WIDTH": 256, - "GPU_OVERLAP": 1, - "HOST_NATIVE_ATOMIC_SUPPORTED": 0, - "INTEGRATED": 0, - "IS_MULTI_GPU_BOARD": 0, - "KERNEL_EXEC_TIMEOUT": 1, - "L2_CACHE_SIZE": 4194304, - "LOCAL_L1_CACHE_SUPPORTED": 1, - "MANAGED_MEMORY": 1, - "MAX_BLOCK_DIM_X": 1024, - "MAX_BLOCK_DIM_Y": 1024, - "MAX_BLOCK_DIM_Z": 64, - "MAX_GRID_DIM_X": 2147483647, - "MAX_GRID_DIM_Y": 65535, - "MAX_GRID_DIM_Z": 65535, - "MAX_MAX_TEXTURE_2D_MIPMAPPED_HEIGHT": 32768, - "MAX_PITCH": 2147483647, - "MAX_REGISTERS_PER_BLOCK": 65536, - "MAX_REGISTERS_PER_MULTIPROCESSOR": 65536, - "MAX_SHARED_MEMORY_PER_BLOCK": 49152, - "MAX_SHARED_MEMORY_PER_BLOCK_OPTIN": 65536, - "MAX_SHARED_MEMORY_PER_MULTIPROCESSOR": 65536, - "MAX_SURFACE_1D_LAYERED_LAYERS": 2048, - "MAX_SURFACE_1D_LAYERED_WIDTH": 32768, - "MAX_SURFACE_1D_WIDTH": 32768, - "MAX_SURFACE_2D_HEIGHT": 65536, - "MAX_SURFACE_2D_LAYERED_HEIGHT": 32768, - "MAX_SURFACE_2D_LAYERED_LAYERS": 2048, - "MAX_SURFACE_2D_LAYERED_WIDTH": 32768, - "MAX_SURFACE_2D_WIDTH": 131072, - "MAX_SURFACE_3D_DEPTH": 16384, - "MAX_SURFACE_3D_HEIGHT": 16384, - "MAX_SURFACE_3D_WIDTH": 16384, - "MAX_SURFACE_CUBEMAP_LAYERED_LAYERS": 2046, - "MAX_SURFACE_CUBEMAP_LAYERED_WIDTH": 32768, - "MAX_SURFACE_CUBEMAP_WIDTH": 32768, - "MAX_TEXTURE_1D_LAYERED_LAYERS": 2048, - "MAX_TEXTURE_1D_LAYERED_WIDTH": 32768, - "MAX_TEXTURE_1D_LINEAR_WIDTH": 268435456, - "MAX_TEXTURE_1D_MIPMAPPED_WIDTH": 32768, - "MAX_TEXTURE_1D_WIDTH": 131072, - "MAX_TEXTURE_2D_GATHER_HEIGHT": 32768, - "MAX_TEXTURE_2D_GATHER_WIDTH": 32768, - "MAX_TEXTURE_2D_HEIGHT": 65536, - "MAX_TEXTURE_2D_LAYERED_HEIGHT": 32768, - "MAX_TEXTURE_2D_LAYERED_LAYERS": 2048, - "MAX_TEXTURE_2D_LAYERED_WIDTH": 32768, - "MAX_TEXTURE_2D_LINEAR_HEIGHT": 65000, - "MAX_TEXTURE_2D_LINEAR_PITCH": 2097120, - "MAX_TEXTURE_2D_LINEAR_WIDTH": 131072, - "MAX_TEXTURE_2D_MIPMAPPED_WIDTH": 32768, - "MAX_TEXTURE_2D_WIDTH": 131072, - "MAX_TEXTURE_3D_DEPTH": 16384, - "MAX_TEXTURE_3D_DEPTH_ALT": 32768, - "MAX_TEXTURE_3D_HEIGHT": 16384, - "MAX_TEXTURE_3D_HEIGHT_ALT": 8192, - "MAX_TEXTURE_3D_WIDTH": 16384, - "MAX_TEXTURE_3D_WIDTH_ALT": 8192, - "MAX_TEXTURE_CUBEMAP_LAYERED_LAYERS": 2046, - "MAX_TEXTURE_CUBEMAP_LAYERED_WIDTH": 32768, - "MAX_TEXTURE_CUBEMAP_WIDTH": 32768, - "MAX_THREADS_PER_BLOCK": 1024, - "MAX_THREADS_PER_MULTI_PROCESSOR": 1024, - "MEMORY_CLOCK_RATE": 7000000, - "MULTIPROCESSOR_COUNT": 46, - "MULTI_GPU_BOARD_GROUP_ID": 0, - "PAGEABLE_MEMORY_ACCESS": 0, - "PCI_BUS_ID": 1, - "PCI_DEVICE_ID": 0, - "PCI_DOMAIN_ID": 0, - "SINGLE_TO_DOUBLE_PRECISION_PERF_RATIO": 32, - "STREAM_PRIORITIES_SUPPORTED": 1, - "SURFACE_ALIGNMENT": 512, - "TCC_DRIVER": 0, - "TEXTURE_ALIGNMENT": 512, - "TEXTURE_PITCH_ALIGNMENT": 32, - "TOTAL_CONSTANT_MEMORY": 65536, - "UNIFIED_ADDRESSING": 1, - "WARP_SIZE": 32 - } - }, - "custom": true, - "defaults": { - "arch": "AMD64", - "cpu": "Intel(R) Core(TM) i7-9700K CPU @ 3.60GHz", - "machine": "DESKTOP-QJKETS8", - "num_cpu": "8", - "os": "Windows 10", - "ram": "" - }, - "machine": "9d805e6bc6fe1582a6a348d5cfddde3e86882815", - "os": { - "cpu_count": 8 - }, - "platform": { - "architecture": [ - "64bit", - "WindowsPE" - ], - "machine": "AMD64", - "platform": "Windows-10-10.0.19045-SP0", - "processor": "Intel64 Family 6 Model 158 Stepping 12, GenuineIntel", - "system": "Windows" - }, - "psutil": { - "disk_partitions": [ - { - "device": "C:\\", - "fstype": "NTFS", - "maxfile": 255, - "maxpath": 260, - "mountpoint": "C:\\", - "opts": "rw,fixed" - }, - { - "device": "D:\\", - "fstype": "NTFS", - "maxfile": 255, - "maxpath": 260, - "mountpoint": "D:\\", - "opts": "rw,fixed" - }, - { - "device": "E:\\", - "fstype": "NTFS", - "maxfile": 255, - "maxpath": 260, - "mountpoint": "E:\\", - "opts": "rw,fixed" - }, - { - "device": "F:\\", - "fstype": "NTFS", - "maxfile": 255, - "maxpath": 260, - "mountpoint": "F:\\", - "opts": "rw,fixed" - }, - { - "device": "G:\\", - "fstype": "NTFS", - "maxfile": 255, - "maxpath": 260, - "mountpoint": "G:\\", - "opts": "rw,fixed" - } - ], - "number_of_processes": 8, - "number_of_threads": 8, - "total_swap_memory": 17118638080, - "total_virtual_memory": 17118638080 - }, - "sys": { - "platform": "win32" - }, - "version": 1 -} diff --git a/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json deleted file mode 100644 index 8b652fa..0000000 --- a/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "version": 2, - "timestamp": "2024_02_17_00_59_37", - "commit_hash": "fcbc1ee93503008ba8f59846a44877d416592628", - "environment_hash": "509c211421a12aeede580b602fde9cda8143b0aa", - "machine_hash": "9d805e6bc6fe1582a6a348d5cfddde3e86882815", - "results": { - "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { - "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ - 1.395349400001578, - 16.374563500052318, - 0.6053932000068016, - 0.6525553000392392, - 0.5618528000195511, - 0.5382418999797665, - 12.189247599977534, - 16.17910980002489, - 14.66157190001104, - 0.5886657999944873, - 0.746196600026451, - 12.95918340003118, - 0.6654667999828234, - 13.60833990003448, - 0.6666267000255175, - 0.5994374000001699, - 1.0131437000236474, - 0.9876768999965861 - ] - }, - "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { - "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ - 4.920633600035217, - 0.5284106999752112, - 0.9341197999892756, - 5.714047100045718, - 0.5253001000382937, - 0.5296251999679953, - 8.583959300012793, - 0.5409118999959901, - 0.540925600042101, - 8.369992599997204, - 9.031809899955988, - 0.5621228999807499, - 0.5253351999563165, - 0.514035000000149, - 0.5971826000022702, - 0.536596899968572, - 0.5279040000168607, - 6.630319699994288 - ] - }, - "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { - "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ - 0.006150400033220649, - 0.0063019999652169645, - 0.008725399966351688, - 0.0057545999879948795, - 0.0057687999797053635, - 0.005848100001458079, - 0.006170000007841736, - 0.005869700049515814, - 0.009840199956670403, - 0.0063875000341795385, - 0.0062386999488808215, - 0.007613099995069206, - 0.007296500029042363, - 0.006599900021683425, - 0.006746999977622181, - 0.007564299972727895, - 0.006594400038011372, - 0.007813100004568696 - ] - } - } -} diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py index 4fb6317..436adf5 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -4,13 +4,13 @@ from nwb_benchmarks.core import ( get_s3_url, + network_activity_tracker, read_hdf5_fsspec_no_cache, read_hdf5_nwbfile_fsspec_no_cache, read_hdf5_nwbfile_remfile, read_hdf5_nwbfile_ros3, read_hdf5_remfile, read_hdf5_ros3, - track_network_activity, ) TSHARK_PATH = os.environ.get("TSHARK_PATH", None) @@ -27,42 +27,72 @@ class FsspecNoCacheDirectFileReadBenchmark: - def track_network_activity_during_read(self, s3_url: str): - with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + + def network_activity_tracker_during_read(self, s3_url: str): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) return network_activity.network_statistics class RemfileDirectFileReadBenchmark: - def track_network_activity_during_read(self, s3_url: str): - with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + + def network_activity_tracker_during_read(self, s3_url: str): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) return network_activity.network_statistics class Ros3DirectFileReadBenchmark: - def track_network_activity_during_read(self, s3_url: str): - with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + + def network_activity_tracker_during_read(self, s3_url: str): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file = read_hdf5_ros3(s3_url=s3_url) return network_activity.network_statistics class FsspecNoCacheNWBFileReadBenchmark: - def track_network_activity_during_read(self, s3_url: str): - with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + + def network_activity_tracker_during_read(self, s3_url: str): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) return network_activity.network_statistics class RemfileNWBFileReadBenchmark: - def track_network_activity_during_read(self, s3_url: str): - with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + + def network_activity_tracker_during_read(self, s3_url: str): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) return network_activity.network_statistics class Ros3NWBFileReadBenchmark: - def track_network_activity_during_read(self, s3_url: str): - with track_network_activity(tshark_exe_path=TSHARK_PATH) as network_activity: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + + def network_activity_tracker_during_read(self, s3_url: str): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) return network_activity.network_statistics diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index f7faf25..6c2fb49 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -4,14 +4,13 @@ from typing import Tuple from nwb_benchmarks.core import ( - BaseNetworkBenchmark, get_object_by_name, get_s3_url, + network_activity_tracker, read_hdf5_nwbfile_fsspec_no_cache, read_hdf5_nwbfile_remfile, read_hdf5_nwbfile_ros3, robust_ros3_read, - track_network_activity, ) TSHARK_PATH = os.environ.get("TSHARK_PATH", None) @@ -30,36 +29,51 @@ class FsspecNoCacheContinuousSliceBenchmark: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - with track_network_activity(tshark_path=TSHARK_PATH) as network_tracker: + def network_activity_tracker_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self._temp = self.data_to_slice[slice_range] return network_tracker.network_statistics class RemfileContinuousSliceBenchmark: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - with track_network_activity(tshark_path=TSHARK_PATH) as network_tracker: + def network_activity_tracker_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self._temp = self.data_to_slice[slice_range] return network_tracker.network_statistics class Ros3ContinuousSliceBenchmark: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - with track_network_activity(tshark_path=TSHARK_PATH) as network_tracker: + def network_activity_tracker_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=slice_range) return network_tracker.network_statistics diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index ff36309..f072d6b 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -78,6 +78,9 @@ def main() -> None: for path in pathlib.Path(asv_root / "intermediate_results").rglob("*.json") if not any(path.stem == skip_stems for skip_stems in ["benchmarks", "machine"]) ] + assert ( + len(globbed_json_file_paths) > 0 + ), "No intermediate result was found, likely as a result of a failure in the benchmarks." assert len(globbed_json_file_paths) == 1, ( "A single intermediate result was not found, likely as a result of a previous failure to reduce " "the results! Please manually remove these." diff --git a/src/nwb_benchmarks/core/__init__.py b/src/nwb_benchmarks/core/__init__.py index 2c0483a..fd49fb6 100644 --- a/src/nwb_benchmarks/core/__init__.py +++ b/src/nwb_benchmarks/core/__init__.py @@ -4,7 +4,7 @@ from ._dandi import get_s3_url from ._network_profiler import NetworkProfiler from ._network_statistics import NetworkStatistics -from ._network_tracker import track_network_activity +from ._network_tracker import network_activity_tracker from ._nwb_helpers import get_object_by_name from ._streaming import ( read_hdf5_fsspec_no_cache, @@ -20,7 +20,7 @@ "CaptureConnections", "NetworkProfiler", "NetworkStatistics", - "track_network_activity", + "network_activity_tracker", "read_hdf5_fsspec_no_cache", "read_hdf5_nwbfile_fsspec_no_cache", "read_hdf5_ros3", diff --git a/src/nwb_benchmarks/core/_network_profiler.py b/src/nwb_benchmarks/core/_network_profiler.py index 88c52e9..a8c16b2 100644 --- a/src/nwb_benchmarks/core/_network_profiler.py +++ b/src/nwb_benchmarks/core/_network_profiler.py @@ -46,10 +46,10 @@ def packets(self): pass return self.__packets - def start_capture(self, tshark_exe_path: Union[pathlib.Path, None] = None): + def start_capture(self, tshark_path: Union[pathlib.Path, None] = None): """Start the capture with tshark in a subprocess.""" - tshark_exe_path = tshark_exe_path or "tshark" - tsharkCall = [str(tshark_exe_path), "-w", str(self.capture_file_path)] + tshark_path = tshark_path or "tshark" + tsharkCall = [str(tshark_path), "-w", str(self.capture_file_path)] self.__tshark_process = subprocess.Popen(tsharkCall, stderr=subprocess.DEVNULL) time.sleep(0.2) # not sure if this is needed but just to be safe diff --git a/src/nwb_benchmarks/core/_network_statistics.py b/src/nwb_benchmarks/core/_network_statistics.py index 3f79a69..1d0ad22 100644 --- a/src/nwb_benchmarks/core/_network_statistics.py +++ b/src/nwb_benchmarks/core/_network_statistics.py @@ -11,12 +11,12 @@ class NetworkStatistics: """Compute basic statistics about network packets captures with tshark/pyshark.""" @staticmethod - def num_packets(packets: list) -> int: + def total_transfer_in_number_of_packets(packets: list) -> int: """Total number of packets.""" return len(packets) @staticmethod - def get_web_traffic_packets(packets: list) -> list: + def total_traffic_in_number_of_web_packets(packets: list) -> list: """ Get all HTTP and HTTPS packets from the packets captured. @@ -37,12 +37,12 @@ def get_web_traffic_packets(packets: list) -> list: return web_packets @staticmethod - def transfer_time(packets: list) -> float: + def total_transfer_time_in_seconds(packets: list) -> float: """Sum of all time_delta's between packets.""" return float(np.sum([float(packet.tcp.time_delta) for packet in packets])) @staticmethod - def number_of_packets_downloaded(packets: list, local_addresses: list = None) -> int: + def amount_downloaded_in_number_of_packets(packets: list, local_addresses: list = None) -> int: """Total number of packets downloaded.""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() @@ -54,7 +54,7 @@ def number_of_packets_downloaded(packets: list, local_addresses: list = None) -> return len(downloaded) @staticmethod - def number_of_packets_uploaded(packets: list, local_addresses: list = None) -> int: + def amount_uploaded_in_number_of_packets(packets: list, local_addresses: list = None) -> int: """Total number of packets uploaded (e.g., HTTP requests).""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() @@ -66,38 +66,38 @@ def number_of_packets_uploaded(packets: list, local_addresses: list = None) -> i return len(uploaded) @staticmethod - def total_bytes(packets: list) -> int: + def total_transfer_in_bytes(packets: list) -> int: """Total number of bytes in the packets.""" - total_bytes = np.sum([len(packet) for packet in packets]) - return int(total_bytes) + total_transfer_in_bytes = np.sum([len(packet) for packet in packets]) + return int(total_transfer_in_bytes) @staticmethod - def bytes_downloaded(packets: list, local_addresses: list = None) -> int: + def amount_downloaded_in_bytes(packets: list, local_addresses: list = None) -> int: """Total number of bytes from downloaded packets.""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() - bytes_downloaded = np.sum( + amount_downloaded_in_bytes = np.sum( [ len(packet) # size of packet in bytes for packet in packets # check all packets if packet.ip.src not in local_addresses # the source address is not ours so it's download ] ) - return int(bytes_downloaded) + return int(amount_downloaded_in_bytes) @staticmethod - def bytes_uploaded(packets: list, local_addresses: list = None) -> int: + def amount_uploaded_in_bytes(packets: list, local_addresses: list = None) -> int: """Total number of bytes from uploaded packets.""" if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() - bytes_uploaded = np.sum( + amount_uploaded_in_bytes = np.sum( [ len(packet) # size of packet in bytes for packet in packets # check all packets if packet.ip.src in local_addresses # the source address is ours so it's upload ] ) - return int(bytes_uploaded) + return int(amount_uploaded_in_bytes) @staticmethod def bytes_to_str(size_in_bytes: int, convention: Literal["B", "iB"] = "iB") -> str: @@ -114,18 +114,20 @@ def get_statistics(cls, packets: list, local_addresses: list = None) -> Dict[str if local_addresses is None: local_addresses = CaptureConnections.get_local_addresses() statistics = { - "bytes_downloaded": cls.bytes_downloaded(packets=packets, local_addresses=local_addresses), - "bytes_uploaded": cls.bytes_uploaded(packets=packets, local_addresses=local_addresses), - "bytes_total": cls.total_bytes(packets=packets), - "number_of_packets": cls.num_packets(packets=packets), - "number_of_packets_downloaded": cls.number_of_packets_downloaded( + "amount_downloaded_in_bytes": cls.amount_downloaded_in_bytes( packets=packets, local_addresses=local_addresses ), - "number_of_packets_uploaded": cls.number_of_packets_uploaded( + "amount_uploaded_in_bytes": cls.amount_uploaded_in_bytes(packets=packets, local_addresses=local_addresses), + "total_transfer_in_bytes": cls.total_transfer_in_bytes(packets=packets), + "amount_downloaded_in_number_of_packets": cls.amount_downloaded_in_number_of_packets( packets=packets, local_addresses=local_addresses ), - "number_of_web_packets": len(cls.get_web_traffic_packets(packets)), - "total_transfer_time": cls.transfer_time(packets=packets), + "amount_uploaded_in_number_of_packets": cls.amount_uploaded_in_number_of_packets( + packets=packets, local_addresses=local_addresses + ), + "total_transfer_in_number_of_packets": cls.total_transfer_in_number_of_packets(packets=packets), + "total_traffic_in_number_of_web_packets": len(cls.total_traffic_in_number_of_web_packets(packets)), + "total_transfer_time_in_seconds": cls.total_transfer_time_in_seconds(packets=packets), } return statistics diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index bba7728..0691ec5 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -12,11 +12,11 @@ @contextlib.contextmanager -def track_network_activity(tshark_exe_path: Union[pathlib.Path, None] = None): +def network_activity_tracker(tshark_path: Union[pathlib.Path, None] = None): network_tracker = NetworkTracker() try: - network_tracker.start_network_capture(tshark_exe_path=tshark_exe_path) + network_tracker.start_network_capture(tshark_path=tshark_path) time.sleep(0.3) t0 = time.time() @@ -30,7 +30,7 @@ def track_network_activity(tshark_exe_path: Union[pathlib.Path, None] = None): class NetworkTracker: - def start_network_capture(self, tshark_exe_path: Union[pathlib.Path, None] = None): + def start_network_capture(self, tshark_path: Union[pathlib.Path, None] = None): # start the capture_connections() function to update the current connections of this machine self.connections_thread = CaptureConnections() self.connections_thread.start() @@ -38,7 +38,7 @@ def start_network_capture(self, tshark_exe_path: Union[pathlib.Path, None] = Non # start capturing the raw packets by running the tshark commandline tool in a subprocess self.network_profiler = NetworkProfiler() - self.network_profiler.start_capture(tshark_exe_path=tshark_exe_path) + self.network_profiler.start_capture(tshark_path=tshark_path) def stop_network_capture(self): # Stop capturing packets and connections diff --git a/src/nwb_benchmarks/scripts/demo_network_profiling.py b/src/nwb_benchmarks/scripts/demo_network_profiling.py index 30c86b7..560b69d 100644 --- a/src/nwb_benchmarks/scripts/demo_network_profiling.py +++ b/src/nwb_benchmarks/scripts/demo_network_profiling.py @@ -4,17 +4,17 @@ import pynwb -from nwb_benchmarks.core import track_network_activity +from nwb_benchmarks.core import network_activity_tracker # If `tshark` is not available as a global command, specify the pathlib.Path pointing to the .exe # Otherwise, set to None or do not pass into `.start_capture` -# tshark_exe_path = pathlib.Path("D:/Wireshark/tshark.exe") -tshark_exe_path = None +# tshark_path = pathlib.Path("D:/Wireshark/tshark.exe") +tshark_path = None # Read the NWB data file from DANDI s3_path = "https://dandiarchive.s3.amazonaws.com/ros3test.nwb" -with track_network_activity(tshark_exe_path=tshark_exe_path) as network_tracker: +with network_activity_tracker(tshark_path=tshark_path) as network_tracker: with pynwb.NWBHDF5IO(s3_path, mode="r", driver="ros3") as io: nwbfile = io.read() test_data = nwbfile.acquisition["ts_name"].data[:] diff --git a/src/nwb_benchmarks/setup/_configure_machine.py b/src/nwb_benchmarks/setup/_configure_machine.py index 22236cd..9d6911c 100644 --- a/src/nwb_benchmarks/setup/_configure_machine.py +++ b/src/nwb_benchmarks/setup/_configure_machine.py @@ -46,7 +46,8 @@ def collect_machine_info() -> Dict[str, Dict[str, Any]]: gpu_specifications = {gpu_attribute: getattr(device, gpu_attribute) for gpu_attribute in gpu_attributes} custom_machine_info["cuda"] = dict(gpu_name=device.name.decode("utf-8"), gpu_specifications=gpu_specifications) except cuda.cudadrv.error.CudaSupportError: - warnings.warn("No GPU detected by cuda; skipping section of custom machine info.") + # No GPU detected by cuda; skipping section of custom machine info + pass except Exception as exception: raise exception diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index fc6a1d7..ea583e8 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -66,6 +66,13 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil } } ) + + if len(reduced_results) == 0: + raise ValueError( + "The results parser failed to find any succesful results! " + "Please raise an issue and share your intermediate results file." + ) + reduced_results_info = dict( version=raw_results_info["version"], timestamp=timestamp, From d7104e39e12ac08260d9f0c377dc5b65d7449bef Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Mon, 19 Feb 2024 01:45:46 -0500 Subject: [PATCH 29/72] debug dictionary tracking; update docs; commit first network results --- docs/writing_benchmarks.rst | 14 +- environments/nwb_benchmarks.yaml | 2 +- ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 2494 +++++++++++++++++ ...0f1370ec8dc3640d91c7d0f381a6d61bd9a46.json | 101 + ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 346 +++ .../network_tracking_remote_file_reading.py | 52 +- .../network_tracking_remote_slicing.py | 31 +- .../benchmarks/time_remote_file_reading.py | 4 - .../benchmarks/time_remote_slicing.py | 6 - src/nwb_benchmarks/core/_network_tracker.py | 3 + src/nwb_benchmarks/setup/_reduce_results.py | 2 +- 11 files changed, 2977 insertions(+), 78 deletions(-) create mode 100644 results/info_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json create mode 100644 results/info_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46.json create mode 100644 results/results_timestamp-2024-02-18-22-41-00_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json diff --git a/docs/writing_benchmarks.rst b/docs/writing_benchmarks.rst index f47bbe8..ebd9c1f 100644 --- a/docs/writing_benchmarks.rst +++ b/docs/writing_benchmarks.rst @@ -13,21 +13,29 @@ Just like how ``pytest`` automatically detects and runs any function or method l A single tracking function should perform only the minimal operations you wish to time. It is also capable of tracking only a single value. The philosophy for this is to avoid interference from cross-measurements; that is, the act of tracking memory of the operation may impact how much overall time it takes that process to complete, so you would not want to simultaneously measure both time and memory. +.. note:: + + One caveat with using ``track_`` is that if you want it to return the custom values as ``samples`` similar to the timing tests, it must be wrapped in a dictionary with required outer keywords ``samples`` and ``number=None``. + Class Structure --------------- A single benchmark suite is a file within the ``benchmarks`` folder. It contains one or more benchmark classes. It is not itself important that the word 'Benchmark' be in the name of the class; only the prefix on the function matters. -The class has several attributes, the most important of which are ``round``, ``repeat``, and ``timeout``. All functions in a class can be repeated in round-robin fashion using ``round > 1``; the philsophy here is to 'average out' variation on the system over time and may not always be relevant to increase. Each function in a suite is repeated ``repeat`` number of times to get an estimate of the standard deviation of the operation. Every function in the suite has at most ``timout`` number of seconds to complete, otherwise it will count as a failure. +Every benchmark function has an attribute ``timeout`` which specified the most number of seconds the process has to complete, otherwise it will count as a failure. The global value for this is set in the ``.asv.conf.json`` file. + +Similar to ``unittest.TestCase`` classes, all benchmark classes have ``setup`` and ``teardown`` methods. Set any values that need to be established before the main benchmark cases run as attributes on the instance during ``setup``; likewise, if you save anything to disk and you don't want it to persist, remove it in the ``teardown`` step. + +Timing benchmarks have several special attributes, the most important of which are ``rounds`` and ``repeat``. All timing functions in a class can be repeated in round-robin fashion using ``rounds > 1``; the philsophy here is to 'average out' variation on the system over time and may not always be relevant to increase. Each function in a suite is repeated ``repeat`` number of times to get an estimate of the standard deviation of the operation. Every function in the suite has at most ``timout`` number of seconds to complete, otherwise it will count as a failure. -Similar to ``unittest.TestCase`` classes, these have a ``setup`` and ``teardown`` method which call before and after execution of every ``round`` and every ``repeat`` for every tracking function (such as timing) in the class. ``setup`` should therefore be as light as possible since it will be repeated so often, though sometimes even a minimal setup can still take time (such as reading a large remote NWB file using a suboptimal method). In some cases, ``setup_cache`` is a method that can be defined, and runs only once per class to precompute some operation, such as the creation of a fake dataset for testing on local disk. +For timing functions, ``setup`` and ``teardown`` will be called before and after execution of every count of ``rounds`` and ``repeat`` for every tracking function (such as timing) in the class. ``setup`` should therefore be as light as possible since it will be repeated so often, though sometimes even a minimal setup can still take time (such as reading a large remote NWB file using a suboptimal method). In some cases, ``setup_cache`` is a method that can be defined, and runs only once per class to precompute some operation, such as the creation of a fake dataset for testing on local disk. .. note:: Be careful to assign objects fetched by operations within the tracking functions; otherwise, you may unintentionally track the garbage collection step triggered when the reference count of the return value reaches zero in the namespace. For relatively heavy I/O operations this can be non-negligible. -Finally, you can leverage ``params`` and ``param_names`` to perform a structured iteration over many inputs to the operations. ``param_names`` is a list of length equal to the number of inputs you wish to pass to an operation. ``params`` is a list of lists; the outer list being of equal length to the number of inputs, and each inner list being equal in length to the number of different cases to iterate over. +Finally, you can leverage ``params`` and ``param_names`` on all benchmark types to perform a structured iteration over many inputs to the operations. ``param_names`` is a list of length equal to the number of inputs you wish to pass to an operation. ``params`` is a list of lists; the outer list being of equal length to the number of inputs, and each inner list being equal in length to the number of different cases to iterate over. If defined at the global scope of a benchmark file, all classes in that file will inherit those ``param`` and ``param_names``. .. note:: diff --git a/environments/nwb_benchmarks.yaml b/environments/nwb_benchmarks.yaml index a8f439c..6d002ba 100644 --- a/environments/nwb_benchmarks.yaml +++ b/environments/nwb_benchmarks.yaml @@ -6,7 +6,7 @@ dependencies: - pip - pip: - setuptools - - asv + - asv == 0.6.2 - numba>=0.58.1 # Pin to specific version for cuda import - psutil - pynwb diff --git a/results/info_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/info_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json new file mode 100644 index 0000000..828dd9f --- /dev/null +++ b/results/info_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -0,0 +1,2494 @@ +{ + "3.11.7 | packaged by conda-forge | (main, Dec 23 2023, 14:38:07) [Clang 16.0.6 ]": [ + { + "name": "_anaconda_depends", + "version": "2023.07", + "build": "py311_1" + }, + { + "name": "abseil-cpp", + "version": "20211102.0", + "build": "hc377ac9_0" + }, + { + "name": "aiobotocore", + "version": "2.5.0", + "build": "py311hca03da5_0" + }, + { + "name": "aiofiles", + "version": "22.1.0", + "build": "py311hca03da5_0" + }, + { + "name": "aiohttp", + "version": "3.8.3", + "build": "py311h80987f9_0" + }, + { + "name": "aioitertools", + "version": "0.7.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "aiosignal", + "version": "1.2.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "aiosqlite", + "version": "0.18.0", + "build": "py311hca03da5_0" + }, + { + "name": "alabaster", + "version": "0.7.12", + "build": "pyhd3eb1b0_0" + }, + { + "name": "anaconda-anon-usage", + "version": "0.4.1", + "build": "py311hd6b623d_0" + }, + { + "name": "anaconda-catalogs", + "version": "0.2.0", + "build": "py311hca03da5_0" + }, + { + "name": "anaconda-client", + "version": "1.12.0", + "build": "py311hca03da5_0" + }, + { + "name": "anaconda-navigator", + "version": "2.4.3", + "build": "py311hca03da5_0" + }, + { + "name": "anaconda-project", + "version": "0.11.1", + "build": "py311hca03da5_0" + }, + { + "name": "anyio", + "version": "3.5.0", + "build": "py311hca03da5_0" + }, + { + "name": "appdirs", + "version": "1.4.4", + "build": "pyhd3eb1b0_0" + }, + { + "name": "applaunchservices", + "version": "0.3.0", + "build": "py311hca03da5_0" + }, + { + "name": "appnope", + "version": "0.1.2", + "build": "py311hca03da5_1001" + }, + { + "name": "appscript", + "version": "1.1.2", + "build": "py311h80987f9_0" + }, + { + "name": "argon2-cffi", + "version": "21.3.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "argon2-cffi-bindings", + "version": "21.2.0", + "build": "py311h80987f9_0" + }, + { + "name": "arrow", + "version": "1.2.3", + "build": "py311hca03da5_1" + }, + { + "name": "arrow-cpp", + "version": "11.0.0", + "build": "py311h5aa4e29_0" + }, + { + "name": "astroid", + "version": "2.14.2", + "build": "py311hca03da5_0" + }, + { + "name": "astropy", + "version": "5.1", + "build": "py311ha0d4635_0" + }, + { + "name": "asttokens", + "version": "2.0.5", + "build": "pyhd3eb1b0_0" + }, + { + "name": "async-timeout", + "version": "4.0.2", + "build": "py311hca03da5_0" + }, + { + "name": "atomicwrites", + "version": "1.4.0", + "build": "py_0" + }, + { + "name": "attrs", + "version": "22.1.0", + "build": "py311hca03da5_0" + }, + { + "name": "automat", + "version": "20.2.0", + "build": "py_0" + }, + { + "name": "autopep8", + "version": "1.6.0", + "build": "pyhd3eb1b0_1" + }, + { + "name": "aws-c-common", + "version": "0.6.8", + "build": "h80987f9_1" + }, + { + "name": "aws-c-event-stream", + "version": "0.1.6", + "build": "h313beb8_6" + }, + { + "name": "aws-checksums", + "version": "0.1.11", + "build": "h80987f9_2" + }, + { + "name": "aws-sdk-cpp", + "version": "1.8.185", + "build": "h4a942e0_0" + }, + { + "name": "babel", + "version": "2.11.0", + "build": "py311hca03da5_0" + }, + { + "name": "backcall", + "version": "0.2.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "backports", + "version": "1.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "backports.functools_lru_cache", + "version": "1.6.4", + "build": "pyhd3eb1b0_0" + }, + { + "name": "backports.tempfile", + "version": "1.0", + "build": "pyhd3eb1b0_1" + }, + { + "name": "backports.weakref", + "version": "1.0.post1", + "build": "py_1" + }, + { + "name": "bcrypt", + "version": "3.2.0", + "build": "py311h80987f9_1" + }, + { + "name": "beautifulsoup4", + "version": "4.12.2", + "build": "py311hca03da5_0" + }, + { + "name": "binaryornot", + "version": "0.4.4", + "build": "pyhd3eb1b0_1" + }, + { + "name": "black", + "version": "23.3.0", + "build": "py311hca03da5_0" + }, + { + "name": "blas", + "version": "1.0", + "build": "openblas" + }, + { + "name": "bleach", + "version": "4.1.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "blosc", + "version": "1.21.3", + "build": "h313beb8_0" + }, + { + "name": "bokeh", + "version": "3.2.1", + "build": "py311hb6e6a13_0" + }, + { + "name": "boltons", + "version": "23.0.0", + "build": "py311hca03da5_0" + }, + { + "name": "boost-cpp", + "version": "1.73.0", + "build": "h1a28f6b_12" + }, + { + "name": "botocore", + "version": "1.29.76", + "build": "py311hca03da5_0" + }, + { + "name": "bottleneck", + "version": "1.3.5", + "build": "py311ha0d4635_0" + }, + { + "name": "brotli", + "version": "1.0.9", + "build": "h1a28f6b_7" + }, + { + "name": "brotli-bin", + "version": "1.0.9", + "build": "h1a28f6b_7" + }, + { + "name": "brotlipy", + "version": "0.7.0", + "build": "py311h80987f9_1002" + }, + { + "name": "brunsli", + "version": "0.1", + "build": "hc377ac9_1" + }, + { + "name": "bzip2", + "version": "1.0.8", + "build": "h620ffc9_4" + }, + { + "name": "c-ares", + "version": "1.19.0", + "build": "h80987f9_0" + }, + { + "name": "c-blosc2", + "version": "2.8.0", + "build": "h313beb8_0" + }, + { + "name": "ca-certificates", + "version": "2023.08.22", + "build": "hca03da5_0" + }, + { + "name": "cctools", + "version": "949.0.1", + "build": "hc179dcd_25" + }, + { + "name": "cctools_osx-arm64", + "version": "949.0.1", + "build": "h332cad3_25" + }, + { + "name": "certifi", + "version": "2023.7.22", + "build": "py311hca03da5_0" + }, + { + "name": "cffi", + "version": "1.15.1", + "build": "py311h80987f9_3" + }, + { + "name": "cfitsio", + "version": "3.470", + "build": "h7f6438f_7" + }, + { + "name": "chardet", + "version": "4.0.0", + "build": "py311hca03da5_1003" + }, + { + "name": "charls", + "version": "2.2.0", + "build": "hc377ac9_0" + }, + { + "name": "charset-normalizer", + "version": "2.0.4", + "build": "pyhd3eb1b0_0" + }, + { + "name": "click", + "version": "8.0.4", + "build": "py311hca03da5_0" + }, + { + "name": "cloudpickle", + "version": "2.2.1", + "build": "py311hca03da5_0" + }, + { + "name": "clyent", + "version": "1.2.2", + "build": "py311hca03da5_1" + }, + { + "name": "colorama", + "version": "0.4.6", + "build": "py311hca03da5_0" + }, + { + "name": "colorcet", + "version": "3.0.1", + "build": "py311hca03da5_0" + }, + { + "name": "comm", + "version": "0.1.2", + "build": "py311hca03da5_0" + }, + { + "name": "conda", + "version": "23.7.4", + "build": "py311hca03da5_0" + }, + { + "name": "conda-build", + "version": "3.26.0", + "build": "py311hca03da5_0" + }, + { + "name": "conda-content-trust", + "version": "0.2.0", + "build": "py311hca03da5_0" + }, + { + "name": "conda-index", + "version": "0.2.3", + "build": "py311hca03da5_0" + }, + { + "name": "conda-libmamba-solver", + "version": "23.5.0", + "build": "py311hca03da5_0" + }, + { + "name": "conda-pack", + "version": "0.6.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "conda-package-handling", + "version": "2.2.0", + "build": "py311hca03da5_0" + }, + { + "name": "conda-package-streaming", + "version": "0.9.0", + "build": "py311hca03da5_0" + }, + { + "name": "conda-repo-cli", + "version": "1.0.41", + "build": "py311hca03da5_0" + }, + { + "name": "conda-token", + "version": "0.4.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "conda-verify", + "version": "3.4.2", + "build": "py_1" + }, + { + "name": "constantly", + "version": "15.1.0", + "build": "py311hca03da5_0" + }, + { + "name": "contourpy", + "version": "1.0.5", + "build": "py311h48ca7d4_0" + }, + { + "name": "cookiecutter", + "version": "1.7.3", + "build": "pyhd3eb1b0_0" + }, + { + "name": "cryptography", + "version": "41.0.2", + "build": "py311h6e31b35_0" + }, + { + "name": "cssselect", + "version": "1.1.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "curl", + "version": "8.1.1", + "build": "h449679c_2" + }, + { + "name": "cycler", + "version": "0.11.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "cytoolz", + "version": "0.12.0", + "build": "py311h80987f9_0" + }, + { + "name": "dask", + "version": "2023.6.0", + "build": "py311hca03da5_0" + }, + { + "name": "dask-core", + "version": "2023.6.0", + "build": "py311hca03da5_0" + }, + { + "name": "datasets", + "version": "2.12.0", + "build": "py311hca03da5_0" + }, + { + "name": "datashader", + "version": "0.15.1", + "build": "py311hca03da5_0" + }, + { + "name": "datashape", + "version": "0.5.4", + "build": "py311hca03da5_1" + }, + { + "name": "debugpy", + "version": "1.6.7", + "build": "py311h313beb8_0" + }, + { + "name": "decorator", + "version": "5.1.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "defusedxml", + "version": "0.7.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "diff-match-patch", + "version": "20200713", + "build": "pyhd3eb1b0_0" + }, + { + "name": "dill", + "version": "0.3.6", + "build": "py311hca03da5_0" + }, + { + "name": "distributed", + "version": "2023.6.0", + "build": "py311hca03da5_0" + }, + { + "name": "docstring-to-markdown", + "version": "0.11", + "build": "py311hca03da5_0" + }, + { + "name": "docutils", + "version": "0.18.1", + "build": "py311hca03da5_3" + }, + { + "name": "entrypoints", + "version": "0.4", + "build": "py311hca03da5_0" + }, + { + "name": "et_xmlfile", + "version": "1.1.0", + "build": "py311hca03da5_0" + }, + { + "name": "executing", + "version": "0.8.3", + "build": "pyhd3eb1b0_0" + }, + { + "name": "filelock", + "version": "3.9.0", + "build": "py311hca03da5_0" + }, + { + "name": "flake8", + "version": "6.0.0", + "build": "py311hca03da5_0" + }, + { + "name": "flask", + "version": "2.2.2", + "build": "py311hca03da5_0" + }, + { + "name": "fmt", + "version": "9.1.0", + "build": "h48ca7d4_0" + }, + { + "name": "fonttools", + "version": "4.25.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "freetype", + "version": "2.12.1", + "build": "h1192e45_0" + }, + { + "name": "frozenlist", + "version": "1.3.3", + "build": "py311h80987f9_0" + }, + { + "name": "fsspec", + "version": "2023.4.0", + "build": "py311hca03da5_0" + }, + { + "name": "future", + "version": "0.18.3", + "build": "py311hca03da5_0" + }, + { + "name": "gensim", + "version": "4.3.0", + "build": "py311h6956b77_0" + }, + { + "name": "gettext", + "version": "0.21.0", + "build": "h13f89a0_1" + }, + { + "name": "gflags", + "version": "2.2.2", + "build": "hc377ac9_0" + }, + { + "name": "giflib", + "version": "5.2.1", + "build": "h80987f9_3" + }, + { + "name": "glib", + "version": "2.69.1", + "build": "h514c7bf_2" + }, + { + "name": "glob2", + "version": "0.7", + "build": "pyhd3eb1b0_0" + }, + { + "name": "glog", + "version": "0.5.0", + "build": "hc377ac9_0" + }, + { + "name": "gmp", + "version": "6.2.1", + "build": "hc377ac9_3" + }, + { + "name": "gmpy2", + "version": "2.1.2", + "build": "py311h40f64dc_0" + }, + { + "name": "greenlet", + "version": "2.0.1", + "build": "py311h313beb8_0" + }, + { + "name": "grpc-cpp", + "version": "1.46.1", + "build": "h8e4afa7_1" + }, + { + "name": "gst-plugins-base", + "version": "1.14.1", + "build": "h313beb8_1" + }, + { + "name": "gstreamer", + "version": "1.14.1", + "build": "h80987f9_1" + }, + { + "name": "h5py", + "version": "3.7.0", + "build": "py311h39e7838_0" + }, + { + "name": "hdf5", + "version": "1.12.1", + "build": "h160e8cb_2" + }, + { + "name": "heapdict", + "version": "1.0.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "holoviews", + "version": "1.17.0", + "build": "py311hca03da5_0" + }, + { + "name": "huggingface_hub", + "version": "0.15.1", + "build": "py311hca03da5_0" + }, + { + "name": "hvplot", + "version": "0.8.4", + "build": "py311hca03da5_0" + }, + { + "name": "hyperlink", + "version": "21.0.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "icu", + "version": "68.1", + "build": "hc377ac9_0" + }, + { + "name": "idna", + "version": "3.4", + "build": "py311hca03da5_0" + }, + { + "name": "imagecodecs", + "version": "2021.8.26", + "build": "py311haa897ea_2" + }, + { + "name": "imageio", + "version": "2.31.1", + "build": "py311hca03da5_0" + }, + { + "name": "imagesize", + "version": "1.4.1", + "build": "py311hca03da5_0" + }, + { + "name": "imbalanced-learn", + "version": "0.10.1", + "build": "py311hca03da5_1" + }, + { + "name": "importlib-metadata", + "version": "6.0.0", + "build": "py311hca03da5_0" + }, + { + "name": "importlib_metadata", + "version": "6.0.0", + "build": "hd3eb1b0_0" + }, + { + "name": "incremental", + "version": "21.3.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "inflection", + "version": "0.5.1", + "build": "py311hca03da5_0" + }, + { + "name": "iniconfig", + "version": "1.1.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "intake", + "version": "0.6.8", + "build": "py311hca03da5_0" + }, + { + "name": "intervaltree", + "version": "3.1.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "ipykernel", + "version": "6.19.2", + "build": "py311h37496c9_0" + }, + { + "name": "ipython", + "version": "8.12.0", + "build": "py311hca03da5_0" + }, + { + "name": "ipython_genutils", + "version": "0.2.0", + "build": "pyhd3eb1b0_1" + }, + { + "name": "ipywidgets", + "version": "8.0.4", + "build": "py311hca03da5_0" + }, + { + "name": "isort", + "version": "5.9.3", + "build": "pyhd3eb1b0_0" + }, + { + "name": "itemadapter", + "version": "0.3.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "itemloaders", + "version": "1.0.4", + "build": "pyhd3eb1b0_1" + }, + { + "name": "itsdangerous", + "version": "2.0.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "jaraco.classes", + "version": "3.2.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "jedi", + "version": "0.18.1", + "build": "py311hca03da5_1" + }, + { + "name": "jellyfish", + "version": "0.9.0", + "build": "py311h80987f9_0" + }, + { + "name": "jinja2", + "version": "3.1.2", + "build": "py311hca03da5_0" + }, + { + "name": "jinja2-time", + "version": "0.2.0", + "build": "pyhd3eb1b0_3" + }, + { + "name": "jmespath", + "version": "0.10.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "joblib", + "version": "1.2.0", + "build": "py311hca03da5_0" + }, + { + "name": "jpeg", + "version": "9e", + "build": "h80987f9_1" + }, + { + "name": "jq", + "version": "1.6", + "build": "h1a28f6b_1" + }, + { + "name": "json5", + "version": "0.9.6", + "build": "pyhd3eb1b0_0" + }, + { + "name": "jsonpatch", + "version": "1.32", + "build": "pyhd3eb1b0_0" + }, + { + "name": "jsonpointer", + "version": "2.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "jsonschema", + "version": "4.17.3", + "build": "py311hca03da5_0" + }, + { + "name": "jupyter", + "version": "1.0.0", + "build": "py311hca03da5_8" + }, + { + "name": "jupyter_client", + "version": "7.4.9", + "build": "py311hca03da5_0" + }, + { + "name": "jupyter_console", + "version": "6.6.3", + "build": "py311hca03da5_0" + }, + { + "name": "jupyter_core", + "version": "5.3.0", + "build": "py311hca03da5_0" + }, + { + "name": "jupyter_events", + "version": "0.6.3", + "build": "py311hca03da5_0" + }, + { + "name": "jupyter_server", + "version": "1.23.4", + "build": "py311hca03da5_0" + }, + { + "name": "jupyter_server_fileid", + "version": "0.9.0", + "build": "py311hca03da5_0" + }, + { + "name": "jupyter_server_ydoc", + "version": "0.8.0", + "build": "py311hca03da5_1" + }, + { + "name": "jupyter_ydoc", + "version": "0.2.4", + "build": "py311hca03da5_0" + }, + { + "name": "jupyterlab", + "version": "3.6.3", + "build": "py311hca03da5_0" + }, + { + "name": "jupyterlab_pygments", + "version": "0.1.2", + "build": "py_0" + }, + { + "name": "jupyterlab_server", + "version": "2.22.0", + "build": "py311hca03da5_0" + }, + { + "name": "jupyterlab_widgets", + "version": "3.0.5", + "build": "py311hca03da5_0" + }, + { + "name": "jxrlib", + "version": "1.1", + "build": "h1a28f6b_2" + }, + { + "name": "keyring", + "version": "23.13.1", + "build": "py311hca03da5_0" + }, + { + "name": "kiwisolver", + "version": "1.4.4", + "build": "py311h313beb8_0" + }, + { + "name": "krb5", + "version": "1.20.1", + "build": "h8380606_1" + }, + { + "name": "lazy-object-proxy", + "version": "1.6.0", + "build": "py311h80987f9_0" + }, + { + "name": "lazy_loader", + "version": "0.2", + "build": "py311hca03da5_0" + }, + { + "name": "lcms2", + "version": "2.12", + "build": "hba8e193_0" + }, + { + "name": "ld64", + "version": "530", + "build": "hb29bf3f_25" + }, + { + "name": "ld64_osx-arm64", + "version": "530", + "build": "h001ce53_25" + }, + { + "name": "ldid", + "version": "2.1.5", + "build": "h20b2a84_3" + }, + { + "name": "lerc", + "version": "3.0", + "build": "hc377ac9_0" + }, + { + "name": "libaec", + "version": "1.0.4", + "build": "hc377ac9_1" + }, + { + "name": "libarchive", + "version": "3.6.2", + "build": "h09f0540_1" + }, + { + "name": "libboost", + "version": "1.73.0", + "build": "h49e8a49_12" + }, + { + "name": "libbrotlicommon", + "version": "1.0.9", + "build": "h1a28f6b_7" + }, + { + "name": "libbrotlidec", + "version": "1.0.9", + "build": "h1a28f6b_7" + }, + { + "name": "libbrotlienc", + "version": "1.0.9", + "build": "h1a28f6b_7" + }, + { + "name": "libclang", + "version": "12.0.0", + "build": "default_hc321e17_4" + }, + { + "name": "libcurl", + "version": "8.1.1", + "build": "h0f1d93c_2" + }, + { + "name": "libcxx", + "version": "14.0.6", + "build": "h848a8c0_0" + }, + { + "name": "libdeflate", + "version": "1.17", + "build": "h80987f9_0" + }, + { + "name": "libedit", + "version": "3.1.20221030", + "build": "h80987f9_0" + }, + { + "name": "libev", + "version": "4.33", + "build": "h1a28f6b_1" + }, + { + "name": "libevent", + "version": "2.1.12", + "build": "hf27765b_0" + }, + { + "name": "libffi", + "version": "3.4.4", + "build": "hca03da5_0" + }, + { + "name": "libgfortran", + "version": "5.0.0", + "build": "11_3_0_hca03da5_28" + }, + { + "name": "libgfortran5", + "version": "11.3.0", + "build": "h009349e_28" + }, + { + "name": "libiconv", + "version": "1.16", + "build": "h1a28f6b_2" + }, + { + "name": "liblief", + "version": "0.12.3", + "build": "h313beb8_0" + }, + { + "name": "libllvm12", + "version": "12.0.0", + "build": "h12f7ac0_4" + }, + { + "name": "libllvm14", + "version": "14.0.6", + "build": "h7ec7a93_3" + }, + { + "name": "libmamba", + "version": "1.4.1", + "build": "h48ca7d4_0" + }, + { + "name": "libmambapy", + "version": "1.4.1", + "build": "py311h48ca7d4_0" + }, + { + "name": "libnghttp2", + "version": "1.52.0", + "build": "h10c0552_1" + }, + { + "name": "libopenblas", + "version": "0.3.21", + "build": "h269037a_0" + }, + { + "name": "libpng", + "version": "1.6.39", + "build": "h80987f9_0" + }, + { + "name": "libpq", + "version": "12.15", + "build": "h449679c_1" + }, + { + "name": "libprotobuf", + "version": "3.20.3", + "build": "h514c7bf_0" + }, + { + "name": "libsodium", + "version": "1.0.18", + "build": "h1a28f6b_0" + }, + { + "name": "libsolv", + "version": "0.7.22", + "build": "h98b2900_0" + }, + { + "name": "libspatialindex", + "version": "1.9.3", + "build": "hc377ac9_0" + }, + { + "name": "libssh2", + "version": "1.10.0", + "build": "h449679c_2" + }, + { + "name": "libthrift", + "version": "0.13.0", + "build": "hd358383_6" + }, + { + "name": "libtiff", + "version": "4.5.0", + "build": "h313beb8_2" + }, + { + "name": "libuv", + "version": "1.44.2", + "build": "h80987f9_0" + }, + { + "name": "libwebp", + "version": "1.2.4", + "build": "ha3663a8_1" + }, + { + "name": "libwebp-base", + "version": "1.2.4", + "build": "h80987f9_1" + }, + { + "name": "libxml2", + "version": "2.10.3", + "build": "h372ba2a_0" + }, + { + "name": "libxslt", + "version": "1.1.37", + "build": "habca612_0" + }, + { + "name": "libzopfli", + "version": "1.0.3", + "build": "hc377ac9_0" + }, + { + "name": "linkify-it-py", + "version": "2.0.0", + "build": "py311hca03da5_0" + }, + { + "name": "llvm-openmp", + "version": "14.0.6", + "build": "hc6e5704_0" + }, + { + "name": "llvmlite", + "version": "0.40.0", + "build": "py311h514c7bf_0" + }, + { + "name": "locket", + "version": "1.0.0", + "build": "py311hca03da5_0" + }, + { + "name": "lxml", + "version": "4.9.2", + "build": "py311h80987f9_0" + }, + { + "name": "lz4", + "version": "4.3.2", + "build": "py311h80987f9_0" + }, + { + "name": "lz4-c", + "version": "1.9.4", + "build": "h313beb8_0" + }, + { + "name": "lzo", + "version": "2.10", + "build": "h1a28f6b_2" + }, + { + "name": "markdown", + "version": "3.4.1", + "build": "py311hca03da5_0" + }, + { + "name": "markdown-it-py", + "version": "2.2.0", + "build": "py311hca03da5_1" + }, + { + "name": "markupsafe", + "version": "2.1.1", + "build": "py311h80987f9_0" + }, + { + "name": "matplotlib", + "version": "3.7.1", + "build": "py311hca03da5_1" + }, + { + "name": "matplotlib-base", + "version": "3.7.1", + "build": "py311h7aedaa7_1" + }, + { + "name": "matplotlib-inline", + "version": "0.1.6", + "build": "py311hca03da5_0" + }, + { + "name": "mccabe", + "version": "0.7.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "mdit-py-plugins", + "version": "0.3.0", + "build": "py311hca03da5_0" + }, + { + "name": "mdurl", + "version": "0.1.0", + "build": "py311hca03da5_0" + }, + { + "name": "mistune", + "version": "0.8.4", + "build": "py311h80987f9_1000" + }, + { + "name": "more-itertools", + "version": "8.12.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "mpc", + "version": "1.1.0", + "build": "h8c48613_1" + }, + { + "name": "mpfr", + "version": "4.0.2", + "build": "h695f6f0_1" + }, + { + "name": "mpmath", + "version": "1.3.0", + "build": "py311hca03da5_0" + }, + { + "name": "msgpack-python", + "version": "1.0.3", + "build": "py311h48ca7d4_0" + }, + { + "name": "multidict", + "version": "6.0.2", + "build": "py311h80987f9_0" + }, + { + "name": "multipledispatch", + "version": "0.6.0", + "build": "py311hca03da5_0" + }, + { + "name": "multiprocess", + "version": "0.70.14", + "build": "py311hca03da5_0" + }, + { + "name": "munkres", + "version": "1.1.4", + "build": "py_0" + }, + { + "name": "mypy_extensions", + "version": "0.4.3", + "build": "py311hca03da5_1" + }, + { + "name": "navigator-updater", + "version": "0.4.0", + "build": "py311hca03da5_0" + }, + { + "name": "nbclassic", + "version": "0.5.5", + "build": "py311hca03da5_0" + }, + { + "name": "nbclient", + "version": "0.5.13", + "build": "py311hca03da5_0" + }, + { + "name": "nbconvert", + "version": "6.5.4", + "build": "py311hca03da5_0" + }, + { + "name": "nbformat", + "version": "5.7.0", + "build": "py311hca03da5_0" + }, + { + "name": "ncurses", + "version": "6.4", + "build": "h313beb8_0" + }, + { + "name": "nest-asyncio", + "version": "1.5.6", + "build": "py311hca03da5_0" + }, + { + "name": "networkx", + "version": "3.1", + "build": "py311hca03da5_0" + }, + { + "name": "ninja", + "version": "1.10.2", + "build": "hca03da5_5" + }, + { + "name": "ninja-base", + "version": "1.10.2", + "build": "h525c30c_5" + }, + { + "name": "nltk", + "version": "3.8.1", + "build": "py311hca03da5_0" + }, + { + "name": "notebook", + "version": "6.5.4", + "build": "py311hca03da5_1" + }, + { + "name": "notebook-shim", + "version": "0.2.2", + "build": "py311hca03da5_0" + }, + { + "name": "nspr", + "version": "4.35", + "build": "h313beb8_0" + }, + { + "name": "nss", + "version": "3.89.1", + "build": "h313beb8_0" + }, + { + "name": "numba", + "version": "0.57.0", + "build": "py311h7aedaa7_0" + }, + { + "name": "numexpr", + "version": "2.8.4", + "build": "py311h6dc990b_1" + }, + { + "name": "numpy", + "version": "1.24.3", + "build": "py311hb57d4eb_0" + }, + { + "name": "numpy-base", + "version": "1.24.3", + "build": "py311h1d85a46_0" + }, + { + "name": "numpydoc", + "version": "1.5.0", + "build": "py311hca03da5_0" + }, + { + "name": "oniguruma", + "version": "6.9.7.1", + "build": "h1a28f6b_0" + }, + { + "name": "openjpeg", + "version": "2.3.0", + "build": "h7a6adac_2" + }, + { + "name": "openpyxl", + "version": "3.0.10", + "build": "py311h80987f9_0" + }, + { + "name": "openssl", + "version": "1.1.1w", + "build": "h1a28f6b_0" + }, + { + "name": "orc", + "version": "1.7.4", + "build": "hdca1487_1" + }, + { + "name": "packaging", + "version": "23.0", + "build": "py311hca03da5_0" + }, + { + "name": "pandas", + "version": "1.5.3", + "build": "py311h6956b77_0" + }, + { + "name": "pandocfilters", + "version": "1.5.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "panel", + "version": "1.2.1", + "build": "py311hca03da5_0" + }, + { + "name": "param", + "version": "1.13.0", + "build": "py311hca03da5_0" + }, + { + "name": "parsel", + "version": "1.6.0", + "build": "py311hca03da5_0" + }, + { + "name": "parso", + "version": "0.8.3", + "build": "pyhd3eb1b0_0" + }, + { + "name": "partd", + "version": "1.2.0", + "build": "pyhd3eb1b0_1" + }, + { + "name": "patch", + "version": "2.7.6", + "build": "h1a28f6b_1001" + }, + { + "name": "pathlib", + "version": "1.0.1", + "build": "pyhd3eb1b0_1" + }, + { + "name": "pathspec", + "version": "0.10.3", + "build": "py311hca03da5_0" + }, + { + "name": "patsy", + "version": "0.5.3", + "build": "py311hca03da5_0" + }, + { + "name": "pcre", + "version": "8.45", + "build": "hc377ac9_0" + }, + { + "name": "pcre2", + "version": "10.37", + "build": "h37e8eca_1" + }, + { + "name": "pep8", + "version": "1.7.1", + "build": "py311hca03da5_1" + }, + { + "name": "pexpect", + "version": "4.8.0", + "build": "pyhd3eb1b0_3" + }, + { + "name": "pickleshare", + "version": "0.7.5", + "build": "pyhd3eb1b0_1003" + }, + { + "name": "pillow", + "version": "9.4.0", + "build": "py311h313beb8_0" + }, + { + "name": "pip", + "version": "23.2.1", + "build": "py311hca03da5_0" + }, + { + "name": "pkginfo", + "version": "1.9.6", + "build": "py311hca03da5_0" + }, + { + "name": "platformdirs", + "version": "2.5.2", + "build": "py311hca03da5_0" + }, + { + "name": "plotly", + "version": "5.9.0", + "build": "py311hca03da5_0" + }, + { + "name": "pluggy", + "version": "1.0.0", + "build": "py311hca03da5_1" + }, + { + "name": "ply", + "version": "3.11", + "build": "py311hca03da5_0" + }, + { + "name": "pooch", + "version": "1.4.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "poyo", + "version": "0.5.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "prometheus_client", + "version": "0.14.1", + "build": "py311hca03da5_0" + }, + { + "name": "prompt-toolkit", + "version": "3.0.36", + "build": "py311hca03da5_0" + }, + { + "name": "prompt_toolkit", + "version": "3.0.36", + "build": "hd3eb1b0_0" + }, + { + "name": "protego", + "version": "0.1.16", + "build": "py_0" + }, + { + "name": "psutil", + "version": "5.9.0", + "build": "py311h80987f9_0" + }, + { + "name": "ptyprocess", + "version": "0.7.0", + "build": "pyhd3eb1b0_2" + }, + { + "name": "pure_eval", + "version": "0.2.2", + "build": "pyhd3eb1b0_0" + }, + { + "name": "py-cpuinfo", + "version": "8.0.0", + "build": "pyhd3eb1b0_1" + }, + { + "name": "py-lief", + "version": "0.12.3", + "build": "py311h313beb8_0" + }, + { + "name": "pyarrow", + "version": "11.0.0", + "build": "py311h7575258_0" + }, + { + "name": "pyasn1", + "version": "0.4.8", + "build": "pyhd3eb1b0_0" + }, + { + "name": "pyasn1-modules", + "version": "0.2.8", + "build": "py_0" + }, + { + "name": "pybind11-abi", + "version": "4", + "build": "hd3eb1b0_1" + }, + { + "name": "pycodestyle", + "version": "2.10.0", + "build": "py311hca03da5_0" + }, + { + "name": "pycosat", + "version": "0.6.4", + "build": "py311h80987f9_0" + }, + { + "name": "pycparser", + "version": "2.21", + "build": "pyhd3eb1b0_0" + }, + { + "name": "pyct", + "version": "0.5.0", + "build": "py311hca03da5_0" + }, + { + "name": "pycurl", + "version": "7.45.2", + "build": "py311h449679c_0" + }, + { + "name": "pydispatcher", + "version": "2.0.5", + "build": "py311hca03da5_2" + }, + { + "name": "pydocstyle", + "version": "6.3.0", + "build": "py311hca03da5_0" + }, + { + "name": "pyerfa", + "version": "2.0.0", + "build": "py311h80987f9_0" + }, + { + "name": "pyflakes", + "version": "3.0.1", + "build": "py311hca03da5_0" + }, + { + "name": "pygments", + "version": "2.15.1", + "build": "py311hca03da5_1" + }, + { + "name": "pyjwt", + "version": "2.4.0", + "build": "py311hca03da5_0" + }, + { + "name": "pylint", + "version": "2.16.2", + "build": "py311hca03da5_0" + }, + { + "name": "pylint-venv", + "version": "2.3.0", + "build": "py311hca03da5_0" + }, + { + "name": "pyls-spyder", + "version": "0.4.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "pyobjc-core", + "version": "9.0", + "build": "py311h3eb5a62_1" + }, + { + "name": "pyobjc-framework-cocoa", + "version": "9.0", + "build": "py311hb094c41_0" + }, + { + "name": "pyobjc-framework-coreservices", + "version": "9.0", + "build": "py311hdd8dd1f_0" + }, + { + "name": "pyobjc-framework-fsevents", + "version": "9.0", + "build": "py311hca03da5_0" + }, + { + "name": "pyodbc", + "version": "4.0.34", + "build": "py311h313beb8_0" + }, + { + "name": "pyopenssl", + "version": "23.2.0", + "build": "py311hca03da5_0" + }, + { + "name": "pyparsing", + "version": "3.0.9", + "build": "py311hca03da5_0" + }, + { + "name": "pyqt", + "version": "5.15.7", + "build": "py311h313beb8_0" + }, + { + "name": "pyqt5-sip", + "version": "12.11.0", + "build": "py311h313beb8_0" + }, + { + "name": "pyqtwebengine", + "version": "5.15.7", + "build": "py311h313beb8_0" + }, + { + "name": "pyrsistent", + "version": "0.18.0", + "build": "py311h80987f9_0" + }, + { + "name": "pysocks", + "version": "1.7.1", + "build": "py311hca03da5_0" + }, + { + "name": "pytables", + "version": "3.8.0", + "build": "py311he239255_2" + }, + { + "name": "pytest", + "version": "7.4.0", + "build": "py311hca03da5_0" + }, + { + "name": "python", + "version": "3.11.4", + "build": "hc0d8a6c_0" + }, + { + "name": "python-dateutil", + "version": "2.8.2", + "build": "pyhd3eb1b0_0" + }, + { + "name": "python-fastjsonschema", + "version": "2.16.2", + "build": "py311hca03da5_0" + }, + { + "name": "python-json-logger", + "version": "2.0.7", + "build": "py311hca03da5_0" + }, + { + "name": "python-libarchive-c", + "version": "2.9", + "build": "pyhd3eb1b0_1" + }, + { + "name": "python-lmdb", + "version": "1.4.1", + "build": "py311h313beb8_0" + }, + { + "name": "python-lsp-black", + "version": "1.2.1", + "build": "py311hca03da5_0" + }, + { + "name": "python-lsp-jsonrpc", + "version": "1.0.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "python-lsp-server", + "version": "1.7.2", + "build": "py311hca03da5_0" + }, + { + "name": "python-slugify", + "version": "5.0.2", + "build": "pyhd3eb1b0_0" + }, + { + "name": "python-snappy", + "version": "0.6.1", + "build": "py311h313beb8_0" + }, + { + "name": "python-xxhash", + "version": "2.0.2", + "build": "py311h80987f9_1" + }, + { + "name": "python.app", + "version": "3", + "build": "py311h80987f9_0" + }, + { + "name": "pytoolconfig", + "version": "1.2.5", + "build": "py311hca03da5_1" + }, + { + "name": "pytorch", + "version": "2.0.1", + "build": "gpu_mps_py311h0436fea_0" + }, + { + "name": "pytz", + "version": "2022.7", + "build": "py311hca03da5_0" + }, + { + "name": "pyviz_comms", + "version": "2.3.0", + "build": "py311hca03da5_0" + }, + { + "name": "pywavelets", + "version": "1.4.1", + "build": "py311h80987f9_0" + }, + { + "name": "pyyaml", + "version": "6.0", + "build": "py311h80987f9_1" + }, + { + "name": "pyzmq", + "version": "23.2.0", + "build": "py311h313beb8_0" + }, + { + "name": "qdarkstyle", + "version": "3.0.2", + "build": "pyhd3eb1b0_0" + }, + { + "name": "qstylizer", + "version": "0.2.2", + "build": "py311hca03da5_0" + }, + { + "name": "qt-main", + "version": "5.15.2", + "build": "ha2d02b5_7" + }, + { + "name": "qt-webengine", + "version": "5.15.9", + "build": "h2903aaf_4" + }, + { + "name": "qtawesome", + "version": "1.2.2", + "build": "py311hca03da5_0" + }, + { + "name": "qtconsole", + "version": "5.4.2", + "build": "py311hca03da5_0" + }, + { + "name": "qtpy", + "version": "2.2.0", + "build": "py311hca03da5_0" + }, + { + "name": "qtwebkit", + "version": "5.212", + "build": "h19f419d_5" + }, + { + "name": "queuelib", + "version": "1.5.0", + "build": "py311hca03da5_0" + }, + { + "name": "re2", + "version": "2022.04.01", + "build": "hc377ac9_0" + }, + { + "name": "readline", + "version": "8.2", + "build": "h1a28f6b_0" + }, + { + "name": "regex", + "version": "2022.7.9", + "build": "py311h80987f9_0" + }, + { + "name": "reproc", + "version": "14.2.4", + "build": "hc377ac9_1" + }, + { + "name": "reproc-cpp", + "version": "14.2.4", + "build": "hc377ac9_1" + }, + { + "name": "requests", + "version": "2.31.0", + "build": "py311hca03da5_0" + }, + { + "name": "requests-file", + "version": "1.5.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "requests-toolbelt", + "version": "1.0.0", + "build": "py311hca03da5_0" + }, + { + "name": "responses", + "version": "0.13.3", + "build": "pyhd3eb1b0_0" + }, + { + "name": "rfc3339-validator", + "version": "0.1.4", + "build": "py311hca03da5_0" + }, + { + "name": "rfc3986-validator", + "version": "0.1.1", + "build": "py311hca03da5_0" + }, + { + "name": "rope", + "version": "1.7.0", + "build": "py311hca03da5_0" + }, + { + "name": "rtree", + "version": "1.0.1", + "build": "py311hca03da5_0" + }, + { + "name": "ruamel.yaml", + "version": "0.17.21", + "build": "py311h80987f9_0" + }, + { + "name": "ruamel_yaml", + "version": "0.17.21", + "build": "py311h80987f9_0" + }, + { + "name": "s3fs", + "version": "2023.4.0", + "build": "py311hca03da5_0" + }, + { + "name": "sacremoses", + "version": "0.0.43", + "build": "pyhd3eb1b0_0" + }, + { + "name": "scikit-image", + "version": "0.20.0", + "build": "py311h313beb8_0" + }, + { + "name": "scikit-learn", + "version": "1.3.0", + "build": "py311h7aedaa7_0" + }, + { + "name": "scipy", + "version": "1.10.1", + "build": "py311hc76d9b0_1" + }, + { + "name": "scrapy", + "version": "2.8.0", + "build": "py311hca03da5_0" + }, + { + "name": "seaborn", + "version": "0.12.2", + "build": "py311hca03da5_0" + }, + { + "name": "send2trash", + "version": "1.8.0", + "build": "pyhd3eb1b0_1" + }, + { + "name": "service_identity", + "version": "18.1.0", + "build": "pyhd3eb1b0_1" + }, + { + "name": "setuptools", + "version": "68.0.0", + "build": "py311hca03da5_0" + }, + { + "name": "sip", + "version": "6.6.2", + "build": "py311h313beb8_0" + }, + { + "name": "six", + "version": "1.16.0", + "build": "pyhd3eb1b0_1" + }, + { + "name": "sleef", + "version": "3.5.1", + "build": "h80987f9_2" + }, + { + "name": "smart_open", + "version": "5.2.1", + "build": "py311hca03da5_0" + }, + { + "name": "snappy", + "version": "1.1.9", + "build": "hc377ac9_0" + }, + { + "name": "sniffio", + "version": "1.2.0", + "build": "py311hca03da5_1" + }, + { + "name": "snowballstemmer", + "version": "2.2.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "sortedcontainers", + "version": "2.4.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "soupsieve", + "version": "2.4", + "build": "py311hca03da5_0" + }, + { + "name": "sphinx", + "version": "5.0.2", + "build": "py311hca03da5_0" + }, + { + "name": "sphinxcontrib-applehelp", + "version": "1.0.2", + "build": "pyhd3eb1b0_0" + }, + { + "name": "sphinxcontrib-devhelp", + "version": "1.0.2", + "build": "pyhd3eb1b0_0" + }, + { + "name": "sphinxcontrib-htmlhelp", + "version": "2.0.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "sphinxcontrib-jsmath", + "version": "1.0.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "sphinxcontrib-qthelp", + "version": "1.0.3", + "build": "pyhd3eb1b0_0" + }, + { + "name": "sphinxcontrib-serializinghtml", + "version": "1.1.5", + "build": "pyhd3eb1b0_0" + }, + { + "name": "spyder", + "version": "5.4.3", + "build": "py311hca03da5_1" + }, + { + "name": "spyder-kernels", + "version": "2.4.3", + "build": "py311hca03da5_0" + }, + { + "name": "sqlalchemy", + "version": "1.4.39", + "build": "py311h80987f9_0" + }, + { + "name": "sqlite", + "version": "3.41.2", + "build": "h80987f9_0" + }, + { + "name": "stack_data", + "version": "0.2.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "statsmodels", + "version": "0.14.0", + "build": "py311hb9f6ed7_0" + }, + { + "name": "sympy", + "version": "1.11.1", + "build": "py311hca03da5_0" + }, + { + "name": "tabulate", + "version": "0.8.10", + "build": "py311hca03da5_0" + }, + { + "name": "tapi", + "version": "1100.0.11", + "build": "h8754e6a_1" + }, + { + "name": "tbb", + "version": "2021.8.0", + "build": "h48ca7d4_0" + }, + { + "name": "tbb4py", + "version": "2021.8.0", + "build": "py311h48ca7d4_0" + }, + { + "name": "tblib", + "version": "1.7.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "tenacity", + "version": "8.2.2", + "build": "py311hca03da5_0" + }, + { + "name": "terminado", + "version": "0.17.1", + "build": "py311hca03da5_0" + }, + { + "name": "text-unidecode", + "version": "1.3", + "build": "pyhd3eb1b0_0" + }, + { + "name": "textdistance", + "version": "4.2.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "threadpoolctl", + "version": "2.2.0", + "build": "pyh0d69192_0" + }, + { + "name": "three-merge", + "version": "0.1.1", + "build": "pyhd3eb1b0_0" + }, + { + "name": "tifffile", + "version": "2021.7.2", + "build": "pyhd3eb1b0_2" + }, + { + "name": "tinycss2", + "version": "1.2.1", + "build": "py311hca03da5_0" + }, + { + "name": "tk", + "version": "8.6.12", + "build": "hb8d0fd4_0" + }, + { + "name": "tldextract", + "version": "3.2.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "tokenizers", + "version": "0.13.2", + "build": "py311h3dd52b7_1" + }, + { + "name": "toml", + "version": "0.10.2", + "build": "pyhd3eb1b0_0" + }, + { + "name": "tomlkit", + "version": "0.11.1", + "build": "py311hca03da5_0" + }, + { + "name": "toolz", + "version": "0.12.0", + "build": "py311hca03da5_0" + }, + { + "name": "tornado", + "version": "6.3.2", + "build": "py311h80987f9_0" + }, + { + "name": "tqdm", + "version": "4.65.0", + "build": "py311hb6e6a13_0" + }, + { + "name": "traitlets", + "version": "5.7.1", + "build": "py311hca03da5_0" + }, + { + "name": "transformers", + "version": "4.29.2", + "build": "py311hca03da5_0" + }, + { + "name": "twisted", + "version": "22.10.0", + "build": "py311h80987f9_0" + }, + { + "name": "typing-extensions", + "version": "4.7.1", + "build": "py311hca03da5_0" + }, + { + "name": "typing_extensions", + "version": "4.7.1", + "build": "py311hca03da5_0" + }, + { + "name": "tzdata", + "version": "2023c", + "build": "h04d1e81_0" + }, + { + "name": "uc-micro-py", + "version": "1.0.1", + "build": "py311hca03da5_0" + }, + { + "name": "ujson", + "version": "5.4.0", + "build": "py311h313beb8_0" + }, + { + "name": "unidecode", + "version": "1.2.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "unixodbc", + "version": "2.3.11", + "build": "h1a28f6b_0" + }, + { + "name": "urllib3", + "version": "1.26.16", + "build": "py311hca03da5_0" + }, + { + "name": "utf8proc", + "version": "2.6.1", + "build": "h1a28f6b_0" + }, + { + "name": "w3lib", + "version": "1.21.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "watchdog", + "version": "2.1.6", + "build": "py311h80987f9_0" + }, + { + "name": "wcwidth", + "version": "0.2.5", + "build": "pyhd3eb1b0_0" + }, + { + "name": "webencodings", + "version": "0.5.1", + "build": "py311hca03da5_1" + }, + { + "name": "websocket-client", + "version": "0.58.0", + "build": "py311hca03da5_4" + }, + { + "name": "werkzeug", + "version": "2.2.3", + "build": "py311hca03da5_0" + }, + { + "name": "whatthepatch", + "version": "1.0.2", + "build": "py311hca03da5_0" + }, + { + "name": "wheel", + "version": "0.38.4", + "build": "py311hca03da5_0" + }, + { + "name": "widgetsnbextension", + "version": "4.0.5", + "build": "py311hca03da5_0" + }, + { + "name": "wrapt", + "version": "1.14.1", + "build": "py311h80987f9_0" + }, + { + "name": "wurlitzer", + "version": "3.0.2", + "build": "py311hca03da5_0" + }, + { + "name": "xarray", + "version": "2023.6.0", + "build": "py311hca03da5_0" + }, + { + "name": "xlwings", + "version": "0.29.1", + "build": "py311hca03da5_0" + }, + { + "name": "xxhash", + "version": "0.8.0", + "build": "h1a28f6b_3" + }, + { + "name": "xyzservices", + "version": "2022.9.0", + "build": "py311hca03da5_1" + }, + { + "name": "xz", + "version": "5.4.2", + "build": "h80987f9_0" + }, + { + "name": "y-py", + "version": "0.5.9", + "build": "py311ha6e5c4f_0" + }, + { + "name": "yaml", + "version": "0.2.5", + "build": "h1a28f6b_0" + }, + { + "name": "yaml-cpp", + "version": "0.7.0", + "build": "hc377ac9_1" + }, + { + "name": "yapf", + "version": "0.31.0", + "build": "pyhd3eb1b0_0" + }, + { + "name": "yarl", + "version": "1.8.1", + "build": "py311h80987f9_0" + }, + { + "name": "ypy-websocket", + "version": "0.8.2", + "build": "py311hca03da5_0" + }, + { + "name": "zeromq", + "version": "4.3.4", + "build": "hc377ac9_0" + }, + { + "name": "zfp", + "version": "0.5.5", + "build": "hc377ac9_6" + }, + { + "name": "zict", + "version": "2.2.0", + "build": "py311hca03da5_0" + }, + { + "name": "zipp", + "version": "3.11.0", + "build": "py311hca03da5_0" + }, + { + "name": "zlib", + "version": "1.2.13", + "build": "h5a0b063_0" + }, + { + "name": "zlib-ng", + "version": "2.0.7", + "build": "h80987f9_0" + }, + { + "name": "zope", + "version": "1.0", + "build": "py311hca03da5_1" + }, + { + "name": "zope.interface", + "version": "5.4.0", + "build": "py311h80987f9_0" + }, + { + "name": "zstandard", + "version": "0.19.0", + "build": "py311h80987f9_0" + }, + { + "name": "zstd", + "version": "1.5.5", + "build": "hd90d995_0" + } + ] +} diff --git a/results/info_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46.json b/results/info_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46.json new file mode 100644 index 0000000..4d243fb --- /dev/null +++ b/results/info_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46.json @@ -0,0 +1,101 @@ +{ + "custom": true, + "defaults": { + "arch": "arm64", + "cpu": "Apple M1 Max", + "machine": "Codys-MBP.attlocal.net", + "num_cpu": "10", + "os": "Darwin 23.2.0", + "ram": "68719476736" + }, + "machine": "e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46", + "os": { + "cpu_count": 10 + }, + "platform": { + "architecture": [ + "64bit", + "" + ], + "machine": "arm64", + "platform": "macOS-14.2.1-arm64-arm-64bit", + "processor": "arm", + "system": "Darwin" + }, + "psutil": { + "disk_partitions": [ + { + "device": "/dev/disk3s1s1", + "fstype": "apfs", + "maxfile": 255, + "maxpath": 1024, + "mountpoint": "/", + "opts": "ro,local,rootfs,dovolfs,journaled,multilabel" + }, + { + "device": "/dev/disk3s6", + "fstype": "apfs", + "maxfile": 255, + "maxpath": 1024, + "mountpoint": "/System/Volumes/VM", + "opts": "rw,noexec,local,dovolfs,dontbrowse,journaled,multilabel,noatime" + }, + { + "device": "/dev/disk3s2", + "fstype": "apfs", + "maxfile": 255, + "maxpath": 1024, + "mountpoint": "/System/Volumes/Preboot", + "opts": "rw,local,dovolfs,dontbrowse,journaled,multilabel" + }, + { + "device": "/dev/disk3s4", + "fstype": "apfs", + "maxfile": 255, + "maxpath": 1024, + "mountpoint": "/System/Volumes/Update", + "opts": "rw,local,dovolfs,dontbrowse,journaled,multilabel" + }, + { + "device": "/dev/disk1s2", + "fstype": "apfs", + "maxfile": 255, + "maxpath": 1024, + "mountpoint": "/System/Volumes/xarts", + "opts": "rw,noexec,local,dovolfs,dontbrowse,journaled,multilabel,noatime" + }, + { + "device": "/dev/disk1s1", + "fstype": "apfs", + "maxfile": 255, + "maxpath": 1024, + "mountpoint": "/System/Volumes/iSCPreboot", + "opts": "rw,local,dovolfs,dontbrowse,journaled,multilabel" + }, + { + "device": "/dev/disk1s3", + "fstype": "apfs", + "maxfile": 255, + "maxpath": 1024, + "mountpoint": "/System/Volumes/Hardware", + "opts": "rw,local,dovolfs,dontbrowse,journaled,multilabel" + }, + { + "device": "/dev/disk3s5", + "fstype": "apfs", + "maxfile": 255, + "maxpath": 1024, + "mountpoint": "/System/Volumes/Data", + "opts": "rw,local,dovolfs,dontbrowse,journaled,multilabel" + } + ], + "number_of_processes": 10, + "number_of_threads": 10, + "total_swap_memory": 0, + "total_virtual_memory": 68719476736 + }, + "sys": { + "platform": "darwin" + }, + "version": 1 +} diff --git a/results/results_timestamp-2024-02-18-22-41-00_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-18-22-41-00_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json new file mode 100644 index 0000000..fde1be3 --- /dev/null +++ b/results/results_timestamp-2024-02-18-22-41-00_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -0,0 +1,346 @@ +{ + "version": 2, + "timestamp": "2024-02-18-22-41-00", + "commit_hash": "0802e0b46f60721f7c7cfcad31ce7361ad99bed9", + "environment_hash": "2d1c68a2c7de02ab2c200e448b250bee5e46f212", + "machine_hash": "e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46", + "results": { + "network_tracking_remote_file_reading.FsspecNoCacheDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213354, + "amount_uploaded_in_bytes": 2678, + "total_transfer_in_bytes": 216032, + "amount_downloaded_in_number_of_packets": 157, + "amount_uploaded_in_number_of_packets": 30, + "total_transfer_in_number_of_packets": 187, + "total_traffic_in_number_of_web_packets": 30, + "total_transfer_time_in_seconds": 0.21126499999999998, + "network_total_time": 0.899406909942627 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 5640517, + "amount_uploaded_in_bytes": 20586, + "total_transfer_in_bytes": 5661103, + "amount_downloaded_in_number_of_packets": 3820, + "amount_uploaded_in_number_of_packets": 317, + "total_transfer_in_number_of_packets": 4137, + "total_traffic_in_number_of_web_packets": 317, + "total_transfer_time_in_seconds": 0.5295350000000001, + "network_total_time": 12.43470311164856 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192354, + "amount_uploaded_in_bytes": 2440, + "total_transfer_in_bytes": 194794, + "amount_downloaded_in_number_of_packets": 142, + "amount_uploaded_in_number_of_packets": 27, + "total_transfer_in_number_of_packets": 169, + "total_traffic_in_number_of_web_packets": 27, + "total_transfer_time_in_seconds": 0.231752, + "network_total_time": 0.8538448810577393 + } + }, + "network_tracking_remote_file_reading.RemfileDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 85452, + "amount_uploaded_in_bytes": 3441, + "total_transfer_in_bytes": 88893, + "amount_downloaded_in_number_of_packets": 68, + "amount_uploaded_in_number_of_packets": 48, + "total_transfer_in_number_of_packets": 116, + "total_traffic_in_number_of_web_packets": 48, + "total_transfer_time_in_seconds": 0.134261, + "network_total_time": 0.9113318920135498 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 321943, + "amount_uploaded_in_bytes": 6767, + "total_transfer_in_bytes": 328710, + "amount_downloaded_in_number_of_packets": 241, + "amount_uploaded_in_number_of_packets": 88, + "total_transfer_in_number_of_packets": 329, + "total_traffic_in_number_of_web_packets": 88, + "total_transfer_time_in_seconds": 0.477827, + "network_total_time": 1.348970890045166 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 87574, + "amount_uploaded_in_bytes": 3457, + "total_transfer_in_bytes": 91031, + "amount_downloaded_in_number_of_packets": 71, + "amount_uploaded_in_number_of_packets": 49, + "total_transfer_in_number_of_packets": 120, + "total_traffic_in_number_of_web_packets": 49, + "total_transfer_time_in_seconds": 0.166437, + "network_total_time": 0.7902591228485107 + } + }, + "network_tracking_remote_file_reading.RemfileNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 303780, + "amount_uploaded_in_bytes": 6139, + "total_transfer_in_bytes": 309919, + "amount_downloaded_in_number_of_packets": 231, + "amount_uploaded_in_number_of_packets": 77, + "total_transfer_in_number_of_packets": 308, + "total_traffic_in_number_of_web_packets": 77, + "total_transfer_time_in_seconds": 0.40484800000000004, + "network_total_time": 1.327265977859497 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 394514, + "amount_uploaded_in_bytes": 6581, + "total_transfer_in_bytes": 401095, + "amount_downloaded_in_number_of_packets": 298, + "amount_uploaded_in_number_of_packets": 80, + "total_transfer_in_number_of_packets": 378, + "total_traffic_in_number_of_web_packets": 80, + "total_transfer_time_in_seconds": 0.473356, + "network_total_time": 1.5639359951019287 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 244569, + "amount_uploaded_in_bytes": 5320, + "total_transfer_in_bytes": 249889, + "amount_downloaded_in_number_of_packets": 191, + "amount_uploaded_in_number_of_packets": 62, + "total_transfer_in_number_of_packets": 253, + "total_traffic_in_number_of_web_packets": 62, + "total_transfer_time_in_seconds": 0.365305, + "network_total_time": 1.145571231842041 + } + }, + "network_tracking_remote_file_reading.Ros3DirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.8091280460357666 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 29.734556913375854 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.7612967491149902 + } + }, + "network_tracking_remote_file_reading.Ros3NWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 487063, + "amount_uploaded_in_bytes": 6308, + "total_transfer_in_bytes": 493371, + "amount_downloaded_in_number_of_packets": 355, + "amount_uploaded_in_number_of_packets": 78, + "total_transfer_in_number_of_packets": 433, + "total_traffic_in_number_of_web_packets": 78, + "total_transfer_time_in_seconds": 0.5981620000000001, + "network_total_time": 1.8526577949523926 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 35315002, + "amount_uploaded_in_bytes": 94519, + "total_transfer_in_bytes": 35409521, + "amount_downloaded_in_number_of_packets": 24015, + "amount_uploaded_in_number_of_packets": 1058, + "total_transfer_in_number_of_packets": 25073, + "total_traffic_in_number_of_web_packets": 1058, + "total_transfer_time_in_seconds": 5.599829000000001, + "network_total_time": 72.8062891960144 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 386988, + "amount_uploaded_in_bytes": 5204, + "total_transfer_in_bytes": 392192, + "amount_downloaded_in_number_of_packets": 286, + "amount_uploaded_in_number_of_packets": 58, + "total_transfer_in_number_of_packets": 344, + "total_traffic_in_number_of_web_packets": 58, + "total_transfer_time_in_seconds": 0.449082, + "network_total_time": 1.5516750812530518 + } + }, + "network_tracking_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 43643408, + "amount_uploaded_in_bytes": 129957, + "total_transfer_in_bytes": 43773365, + "amount_downloaded_in_number_of_packets": 29379, + "amount_uploaded_in_number_of_packets": 2342, + "total_transfer_in_number_of_packets": 31721, + "total_traffic_in_number_of_web_packets": 2342, + "total_transfer_time_in_seconds": 19.544652, + "network_total_time": 98.39884209632874 + } + }, + "network_tracking_remote_slicing.RemfileContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 26649556, + "amount_uploaded_in_bytes": 40910, + "total_transfer_in_bytes": 26690466, + "amount_downloaded_in_number_of_packets": 18230, + "amount_uploaded_in_number_of_packets": 599, + "total_transfer_in_number_of_packets": 18829, + "total_traffic_in_number_of_web_packets": 599, + "total_transfer_time_in_seconds": 1.067944, + "network_total_time": 52.762423038482666 + } + }, + "network_tracking_remote_slicing.Ros3ContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 21839087, + "amount_uploaded_in_bytes": 73887, + "total_transfer_in_bytes": 21912974, + "amount_downloaded_in_number_of_packets": 14702, + "amount_uploaded_in_number_of_packets": 1335, + "total_transfer_in_number_of_packets": 16037, + "total_traffic_in_number_of_web_packets": 1335, + "total_transfer_time_in_seconds": 11.727111999999998, + "network_total_time": 53.144360065460205 + } + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.19989208300830796, + 0.21417250001104549, + 0.17405912495451048 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.3554070829995908, + 0.5155478750239126, + 0.3806707499898039 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.1808890420361422, + 0.17439649999141693, + 0.170739000022877 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.3148756250157021, + 0.2847250420018099, + 0.27867441601119936 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.33045591600239277, + 0.32284150004852563, + 0.31815612502396107 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.2627559579559602, + 0.2754818749963306, + 0.2832585000433028 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.24633191694738343, + 0.2588027500314638, + 0.21713104203809053 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 5.424143625015859, + 0.5454546670080163, + 0.5670053750509396 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.1502178329974413, + 0.16554116702172905, + 0.15327312500448897 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.2355401249951683, + 0.21851579198846593, + 0.2344549170229584 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 19.121092916990165, + 13.104883250023704, + 1.9237367499736138 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.2034971250104718, + 0.2016783329891041, + 0.18224275001557544 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.33891791600035504, + 0.31595862499671057, + 0.3122917920118198 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.40766366600291803, + 0.39572404202772304, + 0.38597333297366276 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.34101829200517386, + 0.3442599169793539, + 0.3462167080142535 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.427322749979794, + 0.3723275419906713, + 0.35274604201549664 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 8.119629624998197, + 1.4257672079838812, + 9.82429774996126 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.47090354101965204, + 0.4118334170198068, + 0.4113397910259664 + ] + }, + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.1199638330144808, + 1.2380580420140177, + 1.1288670840440318 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.060348041006364, + 0.9519652909948491, + 4.809384415973909 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.005515333032235503, + 0.0019109580316580832, + 0.0018897500121966004 + ] + } + } +} diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py index 436adf5..0348f22 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -27,72 +27,42 @@ class FsspecNoCacheDirectFileReadBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - - def network_activity_tracker_during_read(self, s3_url: str): + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) - return network_activity.network_statistics + return network_activity.asv_network_statistics class RemfileDirectFileReadBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - - def network_activity_tracker_during_read(self, s3_url: str): + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) - return network_activity.network_statistics + return network_activity.asv_network_statistics class Ros3DirectFileReadBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - - def network_activity_tracker_during_read(self, s3_url: str): + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file = read_hdf5_ros3(s3_url=s3_url) - return network_activity.network_statistics + return network_activity.asv_network_statistics class FsspecNoCacheNWBFileReadBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - def network_activity_tracker_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) - return network_activity.network_statistics + return network_activity.asv_network_statistics class RemfileNWBFileReadBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - - def network_activity_tracker_during_read(self, s3_url: str): + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) - return network_activity.network_statistics + return network_activity.asv_network_statistics class Ros3NWBFileReadBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - - def network_activity_tracker_during_read(self, s3_url: str): + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) - return network_activity.network_statistics + return network_activity.asv_network_statistics diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index 6c2fb49..6c07e62 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -3,6 +3,8 @@ import os from typing import Tuple +from asv.runner import BenchmarkResult + from nwb_benchmarks.core import ( get_object_by_name, get_s3_url, @@ -29,51 +31,36 @@ class FsspecNoCacheContinuousSliceBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def network_activity_tracker_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self._temp = self.data_to_slice[slice_range] - return network_tracker.network_statistics + return network_tracker.asv_network_statistics class RemfileContinuousSliceBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def network_activity_tracker_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self._temp = self.data_to_slice[slice_range] - return network_tracker.network_statistics + return network_tracker.asv_network_statistics class Ros3ContinuousSliceBenchmark: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - def network_activity_tracker_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: - self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=slice_range) - return network_tracker.network_statistics + self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) + return network_tracker.asv_network_statistics diff --git a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py index 4a183c1..545ae62 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py @@ -32,8 +32,6 @@ class DirectFileReadBenchmark: rounds = 1 repeat = 3 - param_names = param_names - params = params def time_read_hdf5_fsspec_no_cache(self, s3_url: str): self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) @@ -54,8 +52,6 @@ class NWBFileReadBenchmark: rounds = 1 repeat = 3 - param_names = param_names - params = params def time_read_hdf5_nwbfile_fsspec_no_cache(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py index 115a9c9..ff8dee6 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -29,8 +29,6 @@ class FsspecNoCacheContinuousSliceBenchmark: rounds = 1 repeat = 3 - param_names = param_names - params = params def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) @@ -45,8 +43,6 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): class RemfileContinuousSliceBenchmark: rounds = 1 repeat = 3 - param_names = param_names - params = params def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) @@ -61,8 +57,6 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): class Ros3ContinuousSliceBenchmark: rounds = 1 repeat = 3 - param_names = param_names - params = params def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index 0691ec5..a034f96 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -51,3 +51,6 @@ def stop_network_capture(self): self.pid_packets = self.network_profiler.get_packets_for_connections(self.pid_connections) # Compute all the network statistics self.network_statistics = NetworkStatistics.get_statistics(packets=self.pid_packets) + + # Very special structure required by ASV + self.asv_network_statistics = dict(samples=self.network_statistics, number=None) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index ea583e8..f2ce7de 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -42,7 +42,7 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil reduced_results = dict() for test_case, raw_results_list in raw_results_info["results"].items(): - if len(raw_results_list) != 12: # Only successful runs results in this length + if len(raw_results_list) not in [5, 12]: # Only successful runs results in these lengths continue flattened_joint_params = collections.defaultdict(list) From 595f02c1348345fd727efbdd3ad11447617dbf07 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Mon, 19 Feb 2024 01:54:03 -0500 Subject: [PATCH 30/72] remove implicit global behavior for params --- docs/running_benchmarks.rst | 2 ++ docs/writing_benchmarks.rst | 2 +- .../network_tracking_remote_file_reading.py | 18 ++++++++++++++++++ .../network_tracking_remote_slicing.py | 9 +++++++++ .../benchmarks/time_remote_file_reading.py | 4 ++++ .../benchmarks/time_remote_slicing.py | 6 ++++++ 6 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index d23fc6a..d5b0feb 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -47,6 +47,8 @@ Contributing Results To contribute your results back to the project, all you have to do is `git add` and `commit` the results in the `results` folder. +Due to the ``sudo`` requirement of the network tracking tests, if you ran ``sudo nwb_benchmarks run`` you will likely have to include ``sudo`` in the ``git commit`` as well; though this may also depend on whether or not you have ``pre-commit`` running locally. + Then, open a PR to merge the results to the `main` branch. .. note:: diff --git a/docs/writing_benchmarks.rst b/docs/writing_benchmarks.rst index ebd9c1f..74a246a 100644 --- a/docs/writing_benchmarks.rst +++ b/docs/writing_benchmarks.rst @@ -35,7 +35,7 @@ For timing functions, ``setup`` and ``teardown`` will be called before and after Be careful to assign objects fetched by operations within the tracking functions; otherwise, you may unintentionally track the garbage collection step triggered when the reference count of the return value reaches zero in the namespace. For relatively heavy I/O operations this can be non-negligible. -Finally, you can leverage ``params`` and ``param_names`` on all benchmark types to perform a structured iteration over many inputs to the operations. ``param_names`` is a list of length equal to the number of inputs you wish to pass to an operation. ``params`` is a list of lists; the outer list being of equal length to the number of inputs, and each inner list being equal in length to the number of different cases to iterate over. If defined at the global scope of a benchmark file, all classes in that file will inherit those ``param`` and ``param_names``. +Finally, you can leverage ``params`` and ``param_names`` on all benchmark types to perform a structured iteration over many inputs to the operations. ``param_names`` is a list of length equal to the number of inputs you wish to pass to an operation. ``params`` is a list of lists; the outer list being of equal length to the number of inputs, and each inner list being equal in length to the number of different cases to iterate over. .. note:: diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py index 0348f22..6e38f4c 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -27,6 +27,9 @@ class FsspecNoCacheDirectFileReadBenchmark: + param_names = param_names + params = params + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) @@ -34,6 +37,9 @@ def track_network_activity_during_read(self, s3_url: str): class RemfileDirectFileReadBenchmark: + param_names = param_names + params = params + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) @@ -41,6 +47,9 @@ def track_network_activity_during_read(self, s3_url: str): class Ros3DirectFileReadBenchmark: + param_names = param_names + params = params + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.file = read_hdf5_ros3(s3_url=s3_url) @@ -48,6 +57,9 @@ def track_network_activity_during_read(self, s3_url: str): class FsspecNoCacheNWBFileReadBenchmark: + param_names = param_names + params = params + def network_activity_tracker_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) @@ -55,6 +67,9 @@ def network_activity_tracker_during_read(self, s3_url: str): class RemfileNWBFileReadBenchmark: + param_names = param_names + params = params + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) @@ -62,6 +77,9 @@ def track_network_activity_during_read(self, s3_url: str): class Ros3NWBFileReadBenchmark: + param_names = param_names + params = params + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index 6c07e62..1278374 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -31,6 +31,9 @@ class FsspecNoCacheContinuousSliceBenchmark: + param_names = param_names + params = params + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) @@ -43,6 +46,9 @@ def track_network_activity_during_slice(self, s3_url: str, object_name: str, sli class RemfileContinuousSliceBenchmark: + param_names = param_names + params = params + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) @@ -55,6 +61,9 @@ def track_network_activity_during_slice(self, s3_url: str, object_name: str, sli class Ros3ContinuousSliceBenchmark: + param_names = param_names + params = params + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py index 545ae62..4a183c1 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py @@ -32,6 +32,8 @@ class DirectFileReadBenchmark: rounds = 1 repeat = 3 + param_names = param_names + params = params def time_read_hdf5_fsspec_no_cache(self, s3_url: str): self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) @@ -52,6 +54,8 @@ class NWBFileReadBenchmark: rounds = 1 repeat = 3 + param_names = param_names + params = params def time_read_hdf5_nwbfile_fsspec_no_cache(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py index ff8dee6..115a9c9 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -29,6 +29,8 @@ class FsspecNoCacheContinuousSliceBenchmark: rounds = 1 repeat = 3 + param_names = param_names + params = params def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) @@ -43,6 +45,8 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): class RemfileContinuousSliceBenchmark: rounds = 1 repeat = 3 + param_names = param_names + params = params def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) @@ -57,6 +61,8 @@ def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): class Ros3ContinuousSliceBenchmark: rounds = 1 repeat = 3 + param_names = param_names + params = params def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) From e4acb6e6bdea265d519a247b129966ed6659f7e8 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Mon, 19 Feb 2024 16:36:18 -0500 Subject: [PATCH 31/72] undo accidental deletion --- ...c211421a12aeede580b602fde9cda8143b0aa.json | 946 ++++++++++++++++++ ...05e6bc6fe1582a6a348d5cfddde3e86882815.json | 175 ++++ ...c211421a12aeede580b602fde9cda8143b0aa.json | 75 ++ 3 files changed, 1196 insertions(+) create mode 100644 results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json create mode 100644 results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json create mode 100644 results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json diff --git a/results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json new file mode 100644 index 0000000..c46ab88 --- /dev/null +++ b/results/info_environment-509c211421a12aeede580b602fde9cda8143b0aa.json @@ -0,0 +1,946 @@ +{ + "3.11.7 | packaged by Anaconda, Inc. | (main, Dec 15 2023, 18:05:47) [MSC v.1916 64 bit (AMD64)]": [ + { + "name": "aiobotocore", + "version": "2.11.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "aiohttp", + "version": "3.9.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "aioitertools", + "version": "0.11.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "aiosignal", + "version": "1.3.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "appdirs", + "version": "1.4.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "arrow", + "version": "1.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "asciitree", + "version": "0.3.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "asttokens", + "version": "2.4.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "asv", + "version": "0.6.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "asv-runner", + "version": "0.2.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "attrs", + "version": "23.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "bidsschematools", + "version": "0.7.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "blas", + "version": "1.0", + "build": "mkl" + }, + { + "name": "botocore", + "version": "1.34.34", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "bzip2", + "version": "1.0.8", + "build": "he774522_0" + }, + { + "name": "ca-certificates", + "version": "2023.12.12", + "build": "haa95532_0" + }, + { + "name": "cached-property", + "version": "1.5.2", + "build": "py_0" + }, + { + "name": "certifi", + "version": "2024.2.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "cfgv", + "version": "3.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "charset-normalizer", + "version": "3.3.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ci-info", + "version": "0.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "click", + "version": "8.1.7", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "click-didyoumean", + "version": "0.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "cloudpickle", + "version": "3.0.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "colorama", + "version": "0.4.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "comm", + "version": "0.2.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "dandi", + "version": "0.59.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "dandischema", + "version": "0.8.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "debugpy", + "version": "1.8.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "decorator", + "version": "5.1.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "dill", + "version": "0.3.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "distlib", + "version": "0.3.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "dnspython", + "version": "2.5.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "email-validator", + "version": "2.1.0.post1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "etelemetry", + "version": "0.3.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "executing", + "version": "2.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "fasteners", + "version": "0.19", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "filelock", + "version": "3.13.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "fqdn", + "version": "1.5.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "frozenlist", + "version": "1.4.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "fscacher", + "version": "0.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "fsspec", + "version": "2024.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "git", + "version": "2.40.1", + "build": "haa95532_1" + }, + { + "name": "h5py", + "version": "3.10.0", + "build": "nompi_py311h7195302_101", + "channel": "conda-forge" + }, + { + "name": "hdf5", + "version": "1.14.3", + "build": "nompi_h73e8ff5_100", + "channel": "conda-forge" + }, + { + "name": "hdmf", + "version": "3.12.1.dev7+g9b3282a", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "humanize", + "version": "4.9.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "identify", + "version": "2.5.34", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "idna", + "version": "3.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "importlib-metadata", + "version": "7.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "intel-openmp", + "version": "2023.1.0", + "build": "h59b6b97_46320" + }, + { + "name": "interleave", + "version": "0.2.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ipykernel", + "version": "6.29.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ipython", + "version": "8.21.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "isodate", + "version": "0.6.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "isoduration", + "version": "20.11.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jaraco-classes", + "version": "3.3.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jedi", + "version": "0.19.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jmespath", + "version": "1.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "joblib", + "version": "1.3.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "json5", + "version": "0.9.14", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jsonpointer", + "version": "2.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jsonschema", + "version": "4.21.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jsonschema-specifications", + "version": "2023.12.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jupyter-client", + "version": "8.6.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "jupyter-core", + "version": "5.7.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "keyring", + "version": "24.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "keyrings-alt", + "version": "5.0.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "libaec", + "version": "1.1.2", + "build": "h63175ca_1", + "channel": "conda-forge" + }, + { + "name": "libcurl", + "version": "8.5.0", + "build": "h86230a5_0" + }, + { + "name": "libffi", + "version": "3.4.4", + "build": "hd77b12b_0" + }, + { + "name": "libssh2", + "version": "1.10.0", + "build": "he2ea4bf_2" + }, + { + "name": "libzlib", + "version": "1.2.13", + "build": "hcfcfb64_5", + "channel": "conda-forge" + }, + { + "name": "llvmlite", + "version": "0.42.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "lxml", + "version": "5.1.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "matplotlib-inline", + "version": "0.1.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "mkl", + "version": "2023.1.0", + "build": "h6b88ed4_46358" + }, + { + "name": "mkl-service", + "version": "2.4.0", + "build": "py311h2bbff1b_1" + }, + { + "name": "mkl_fft", + "version": "1.3.8", + "build": "py311h2bbff1b_0" + }, + { + "name": "mkl_random", + "version": "1.2.4", + "build": "py311h59b6b97_0" + }, + { + "name": "more-itertools", + "version": "10.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "multidict", + "version": "6.0.5", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "natsort", + "version": "8.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "nest-asyncio", + "version": "1.6.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "nodeenv", + "version": "1.8.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "numba", + "version": "0.59.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "numcodecs", + "version": "0.12.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "numpy", + "version": "1.26.3", + "build": "py311hdab7c0b_0" + }, + { + "name": "numpy-base", + "version": "1.26.3", + "build": "py311hd01c5d8_0" + }, + { + "name": "nwb-benchmarks", + "version": "0.1.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "nwbinspector", + "version": "0.4.33", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "openssl", + "version": "3.2.1", + "build": "hcfcfb64_0", + "channel": "conda-forge" + }, + { + "name": "packaging", + "version": "23.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pandas", + "version": "2.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "parso", + "version": "0.8.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pip", + "version": "23.3.1", + "build": "py311haa95532_0" + }, + { + "name": "platformdirs", + "version": "4.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pre-commit", + "version": "3.6.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "prompt-toolkit", + "version": "3.0.43", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "psutil", + "version": "5.9.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pure-eval", + "version": "0.2.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pycryptodomex", + "version": "3.20.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pydantic", + "version": "1.10.14", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pygments", + "version": "2.17.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pympler", + "version": "1.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pynwb", + "version": "2.5.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pyout", + "version": "0.7.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pyshark", + "version": "0.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "python", + "version": "3.11.7", + "build": "he1021f5_0" + }, + { + "name": "python-dateutil", + "version": "2.8.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "python_abi", + "version": "3.11", + "build": "2_cp311", + "channel": "conda-forge" + }, + { + "name": "pytz", + "version": "2024.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pywin32", + "version": "306", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pywin32-ctypes", + "version": "0.2.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pyyaml", + "version": "6.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "pyzmq", + "version": "25.1.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "referencing", + "version": "0.33.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "remfile", + "version": "0.1.10", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "requests", + "version": "2.31.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "rfc3339-validator", + "version": "0.1.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "rfc3987", + "version": "1.3.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "rpds-py", + "version": "0.18.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ruamel-yaml", + "version": "0.18.6", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ruamel-yaml-clib", + "version": "0.2.8", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "s3fs", + "version": "2024.2.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "scipy", + "version": "1.12.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "semantic-version", + "version": "2.10.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "setuptools", + "version": "69.1.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "six", + "version": "1.16.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "spyder-kernels", + "version": "2.5.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "sqlite", + "version": "3.41.2", + "build": "h2bbff1b_0" + }, + { + "name": "stack-data", + "version": "0.6.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tabulate", + "version": "0.9.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tbb", + "version": "2021.8.0", + "build": "h59b6b97_0" + }, + { + "name": "tenacity", + "version": "8.2.3", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "termcolor", + "version": "2.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tk", + "version": "8.6.12", + "build": "h2bbff1b_0" + }, + { + "name": "tomli", + "version": "2.0.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tornado", + "version": "6.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tqdm", + "version": "4.66.2", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "traitlets", + "version": "5.14.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "types-python-dateutil", + "version": "2.8.19.20240106", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "typing-extensions", + "version": "4.9.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "tzdata", + "version": "2024.1", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "ucrt", + "version": "10.0.20348.0", + "build": "haa95532_0" + }, + { + "name": "uri-template", + "version": "1.3.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "urllib3", + "version": "2.0.7", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "vc", + "version": "14.2", + "build": "h21ff451_1" + }, + { + "name": "vc14_runtime", + "version": "14.38.33130", + "build": "h82b7239_18", + "channel": "conda-forge" + }, + { + "name": "virtualenv", + "version": "20.25.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "vs2015_runtime", + "version": "14.38.33130", + "build": "hcb4865c_18", + "channel": "conda-forge" + }, + { + "name": "wcwidth", + "version": "0.2.13", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "webcolors", + "version": "1.13", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "wheel", + "version": "0.41.2", + "build": "py311haa95532_0" + }, + { + "name": "wrapt", + "version": "1.16.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "xz", + "version": "5.4.5", + "build": "h8cc25b3_0" + }, + { + "name": "yarl", + "version": "1.9.4", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "zarr", + "version": "2.17.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "zarr-checksum", + "version": "0.4.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "zipp", + "version": "3.17.0", + "build": "pypi_0", + "channel": "pypi" + }, + { + "name": "zlib", + "version": "1.2.13", + "build": "hcfcfb64_5", + "channel": "conda-forge" + } + ] +} diff --git a/results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json b/results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json new file mode 100644 index 0000000..5645669 --- /dev/null +++ b/results/info_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815.json @@ -0,0 +1,175 @@ +{ + "cuda": { + "gpu_name": "NVIDIA GeForce RTX 2080", + "gpu_specifications": { + "ASYNC_ENGINE_COUNT": 6, + "CAN_MAP_HOST_MEMORY": 1, + "CAN_USE_HOST_POINTER_FOR_REGISTERED_MEM": 0, + "CLOCK_RATE": 1710000, + "COMPUTE_CAPABILITY_MAJOR": 7, + "COMPUTE_CAPABILITY_MINOR": 5, + "COMPUTE_MODE": 0, + "COMPUTE_PREEMPTION_SUPPORTED": 1, + "CONCURRENT_KERNELS": 1, + "CONCURRENT_MANAGED_ACCESS": 0, + "COOPERATIVE_LAUNCH": 1, + "COOPERATIVE_MULTI_DEVICE_LAUNCH": 0, + "ECC_ENABLED": 0, + "GLOBAL_L1_CACHE_SUPPORTED": 1, + "GLOBAL_MEMORY_BUS_WIDTH": 256, + "GPU_OVERLAP": 1, + "HOST_NATIVE_ATOMIC_SUPPORTED": 0, + "INTEGRATED": 0, + "IS_MULTI_GPU_BOARD": 0, + "KERNEL_EXEC_TIMEOUT": 1, + "L2_CACHE_SIZE": 4194304, + "LOCAL_L1_CACHE_SUPPORTED": 1, + "MANAGED_MEMORY": 1, + "MAX_BLOCK_DIM_X": 1024, + "MAX_BLOCK_DIM_Y": 1024, + "MAX_BLOCK_DIM_Z": 64, + "MAX_GRID_DIM_X": 2147483647, + "MAX_GRID_DIM_Y": 65535, + "MAX_GRID_DIM_Z": 65535, + "MAX_MAX_TEXTURE_2D_MIPMAPPED_HEIGHT": 32768, + "MAX_PITCH": 2147483647, + "MAX_REGISTERS_PER_BLOCK": 65536, + "MAX_REGISTERS_PER_MULTIPROCESSOR": 65536, + "MAX_SHARED_MEMORY_PER_BLOCK": 49152, + "MAX_SHARED_MEMORY_PER_BLOCK_OPTIN": 65536, + "MAX_SHARED_MEMORY_PER_MULTIPROCESSOR": 65536, + "MAX_SURFACE_1D_LAYERED_LAYERS": 2048, + "MAX_SURFACE_1D_LAYERED_WIDTH": 32768, + "MAX_SURFACE_1D_WIDTH": 32768, + "MAX_SURFACE_2D_HEIGHT": 65536, + "MAX_SURFACE_2D_LAYERED_HEIGHT": 32768, + "MAX_SURFACE_2D_LAYERED_LAYERS": 2048, + "MAX_SURFACE_2D_LAYERED_WIDTH": 32768, + "MAX_SURFACE_2D_WIDTH": 131072, + "MAX_SURFACE_3D_DEPTH": 16384, + "MAX_SURFACE_3D_HEIGHT": 16384, + "MAX_SURFACE_3D_WIDTH": 16384, + "MAX_SURFACE_CUBEMAP_LAYERED_LAYERS": 2046, + "MAX_SURFACE_CUBEMAP_LAYERED_WIDTH": 32768, + "MAX_SURFACE_CUBEMAP_WIDTH": 32768, + "MAX_TEXTURE_1D_LAYERED_LAYERS": 2048, + "MAX_TEXTURE_1D_LAYERED_WIDTH": 32768, + "MAX_TEXTURE_1D_LINEAR_WIDTH": 268435456, + "MAX_TEXTURE_1D_MIPMAPPED_WIDTH": 32768, + "MAX_TEXTURE_1D_WIDTH": 131072, + "MAX_TEXTURE_2D_GATHER_HEIGHT": 32768, + "MAX_TEXTURE_2D_GATHER_WIDTH": 32768, + "MAX_TEXTURE_2D_HEIGHT": 65536, + "MAX_TEXTURE_2D_LAYERED_HEIGHT": 32768, + "MAX_TEXTURE_2D_LAYERED_LAYERS": 2048, + "MAX_TEXTURE_2D_LAYERED_WIDTH": 32768, + "MAX_TEXTURE_2D_LINEAR_HEIGHT": 65000, + "MAX_TEXTURE_2D_LINEAR_PITCH": 2097120, + "MAX_TEXTURE_2D_LINEAR_WIDTH": 131072, + "MAX_TEXTURE_2D_MIPMAPPED_WIDTH": 32768, + "MAX_TEXTURE_2D_WIDTH": 131072, + "MAX_TEXTURE_3D_DEPTH": 16384, + "MAX_TEXTURE_3D_DEPTH_ALT": 32768, + "MAX_TEXTURE_3D_HEIGHT": 16384, + "MAX_TEXTURE_3D_HEIGHT_ALT": 8192, + "MAX_TEXTURE_3D_WIDTH": 16384, + "MAX_TEXTURE_3D_WIDTH_ALT": 8192, + "MAX_TEXTURE_CUBEMAP_LAYERED_LAYERS": 2046, + "MAX_TEXTURE_CUBEMAP_LAYERED_WIDTH": 32768, + "MAX_TEXTURE_CUBEMAP_WIDTH": 32768, + "MAX_THREADS_PER_BLOCK": 1024, + "MAX_THREADS_PER_MULTI_PROCESSOR": 1024, + "MEMORY_CLOCK_RATE": 7000000, + "MULTIPROCESSOR_COUNT": 46, + "MULTI_GPU_BOARD_GROUP_ID": 0, + "PAGEABLE_MEMORY_ACCESS": 0, + "PCI_BUS_ID": 1, + "PCI_DEVICE_ID": 0, + "PCI_DOMAIN_ID": 0, + "SINGLE_TO_DOUBLE_PRECISION_PERF_RATIO": 32, + "STREAM_PRIORITIES_SUPPORTED": 1, + "SURFACE_ALIGNMENT": 512, + "TCC_DRIVER": 0, + "TEXTURE_ALIGNMENT": 512, + "TEXTURE_PITCH_ALIGNMENT": 32, + "TOTAL_CONSTANT_MEMORY": 65536, + "UNIFIED_ADDRESSING": 1, + "WARP_SIZE": 32 + } + }, + "custom": true, + "defaults": { + "arch": "AMD64", + "cpu": "Intel(R) Core(TM) i7-9700K CPU @ 3.60GHz", + "machine": "DESKTOP-QJKETS8", + "num_cpu": "8", + "os": "Windows 10", + "ram": "" + }, + "machine": "9d805e6bc6fe1582a6a348d5cfddde3e86882815", + "os": { + "cpu_count": 8 + }, + "platform": { + "architecture": [ + "64bit", + "WindowsPE" + ], + "machine": "AMD64", + "platform": "Windows-10-10.0.19045-SP0", + "processor": "Intel64 Family 6 Model 158 Stepping 12, GenuineIntel", + "system": "Windows" + }, + "psutil": { + "disk_partitions": [ + { + "device": "C:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "C:\\", + "opts": "rw,fixed" + }, + { + "device": "D:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "D:\\", + "opts": "rw,fixed" + }, + { + "device": "E:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "E:\\", + "opts": "rw,fixed" + }, + { + "device": "F:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "F:\\", + "opts": "rw,fixed" + }, + { + "device": "G:\\", + "fstype": "NTFS", + "maxfile": 255, + "maxpath": 260, + "mountpoint": "G:\\", + "opts": "rw,fixed" + } + ], + "number_of_processes": 8, + "number_of_threads": 8, + "total_swap_memory": 17118638080, + "total_virtual_memory": 17118638080 + }, + "sys": { + "platform": "win32" + }, + "version": 1 +} diff --git a/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json new file mode 100644 index 0000000..8b652fa --- /dev/null +++ b/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json @@ -0,0 +1,75 @@ +{ + "version": 2, + "timestamp": "2024_02_17_00_59_37", + "commit_hash": "fcbc1ee93503008ba8f59846a44877d416592628", + "environment_hash": "509c211421a12aeede580b602fde9cda8143b0aa", + "machine_hash": "9d805e6bc6fe1582a6a348d5cfddde3e86882815", + "results": { + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.395349400001578, + 16.374563500052318, + 0.6053932000068016, + 0.6525553000392392, + 0.5618528000195511, + 0.5382418999797665, + 12.189247599977534, + 16.17910980002489, + 14.66157190001104, + 0.5886657999944873, + 0.746196600026451, + 12.95918340003118, + 0.6654667999828234, + 13.60833990003448, + 0.6666267000255175, + 0.5994374000001699, + 1.0131437000236474, + 0.9876768999965861 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 4.920633600035217, + 0.5284106999752112, + 0.9341197999892756, + 5.714047100045718, + 0.5253001000382937, + 0.5296251999679953, + 8.583959300012793, + 0.5409118999959901, + 0.540925600042101, + 8.369992599997204, + 9.031809899955988, + 0.5621228999807499, + 0.5253351999563165, + 0.514035000000149, + 0.5971826000022702, + 0.536596899968572, + 0.5279040000168607, + 6.630319699994288 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.006150400033220649, + 0.0063019999652169645, + 0.008725399966351688, + 0.0057545999879948795, + 0.0057687999797053635, + 0.005848100001458079, + 0.006170000007841736, + 0.005869700049515814, + 0.009840199956670403, + 0.0063875000341795385, + 0.0062386999488808215, + 0.007613099995069206, + 0.007296500029042363, + 0.006599900021683425, + 0.006746999977622181, + 0.007564299972727895, + 0.006594400038011372, + 0.007813100004568696 + ] + } + } +} From 46046fe0bd8b13ad00490926ed42f283d0f7d665 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 20 Feb 2024 00:12:56 -0500 Subject: [PATCH 32/72] purge faulty ros3 results --- ...c211421a12aeede580b602fde9cda8143b0aa.json | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json b/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json index 8b652fa..64d9a21 100644 --- a/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json +++ b/results/results_timestamp-2024-02-17-00-59-37_machine-9d805e6bc6fe1582a6a348d5cfddde3e86882815_environment-509c211421a12aeede580b602fde9cda8143b0aa.json @@ -48,28 +48,6 @@ 0.5279040000168607, 6.630319699994288 ] - }, - "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { - "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ - 0.006150400033220649, - 0.0063019999652169645, - 0.008725399966351688, - 0.0057545999879948795, - 0.0057687999797053635, - 0.005848100001458079, - 0.006170000007841736, - 0.005869700049515814, - 0.009840199956670403, - 0.0063875000341795385, - 0.0062386999488808215, - 0.007613099995069206, - 0.007296500029042363, - 0.006599900021683425, - 0.006746999977622181, - 0.007564299972727895, - 0.006594400038011372, - 0.007813100004568696 - ] } } } From b1e9f9f8e990c45807af889989942978ac59e5fe Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 20 Feb 2024 00:14:05 -0500 Subject: [PATCH 33/72] purge faulty ros3 results --- ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/results/results_timestamp-2024-02-18-22-41-00_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-18-22-41-00_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json index fde1be3..49da376 100644 --- a/results/results_timestamp-2024-02-18-22-41-00_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json +++ b/results/results_timestamp-2024-02-18-22-41-00_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -110,41 +110,6 @@ "network_total_time": 1.145571231842041 } }, - "network_tracking_remote_file_reading.Ros3DirectFileReadBenchmark.track_network_activity_during_read": { - "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { - "amount_downloaded_in_bytes": 0, - "amount_uploaded_in_bytes": 0, - "total_transfer_in_bytes": 0, - "amount_downloaded_in_number_of_packets": 0, - "amount_uploaded_in_number_of_packets": 0, - "total_transfer_in_number_of_packets": 0, - "total_traffic_in_number_of_web_packets": 0, - "total_transfer_time_in_seconds": 0.0, - "network_total_time": 0.8091280460357666 - }, - "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { - "amount_downloaded_in_bytes": 0, - "amount_uploaded_in_bytes": 0, - "total_transfer_in_bytes": 0, - "amount_downloaded_in_number_of_packets": 0, - "amount_uploaded_in_number_of_packets": 0, - "total_transfer_in_number_of_packets": 0, - "total_traffic_in_number_of_web_packets": 0, - "total_transfer_time_in_seconds": 0.0, - "network_total_time": 29.734556913375854 - }, - "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { - "amount_downloaded_in_bytes": 0, - "amount_uploaded_in_bytes": 0, - "total_transfer_in_bytes": 0, - "amount_downloaded_in_number_of_packets": 0, - "amount_uploaded_in_number_of_packets": 0, - "total_transfer_in_number_of_packets": 0, - "total_traffic_in_number_of_web_packets": 0, - "total_transfer_time_in_seconds": 0.0, - "network_total_time": 0.7612967491149902 - } - }, "network_tracking_remote_file_reading.Ros3NWBFileReadBenchmark.track_network_activity_during_read": { "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { "amount_downloaded_in_bytes": 487063, @@ -334,13 +299,6 @@ 0.9519652909948491, 4.809384415973909 ] - }, - "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { - "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ - 0.005515333032235503, - 0.0019109580316580832, - 0.0018897500121966004 - ] } } } From 23d2f70175648e63ea43dccf45500f5fa419cf6b Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 00:46:08 -0500 Subject: [PATCH 34/72] commit code changes; improve docs; make robust ros3 more readable --- docs/running_benchmarks.rst | 4 ++++ src/nwb_benchmarks/core/_streaming.py | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index d5b0feb..8616659 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -3,6 +3,10 @@ Running the Benchmarks Before running the benchmark suite, please ensure you are not running any additional heavy processes in the background to avoid interference or bottlenecks. +Also, please ensure prior to running the benchmark that all code changes have been committed to your local branch. + +For the most stable results, only run the benchmarks on the ``main`` branch. + To run the full benchmark suite, including network tracking tests (which require ``sudo`` on Mac or Linux), simply call... .. code-block:: diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py index 7ba6a73..cabb875 100644 --- a/src/nwb_benchmarks/core/_streaming.py +++ b/src/nwb_benchmarks/core/_streaming.py @@ -1,6 +1,6 @@ import time import warnings -from typing import Callable, Tuple, Union +from typing import Any, Callable, Tuple, Union import fsspec import h5py @@ -54,10 +54,10 @@ def read_hdf5_nwbfile_remfile(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5 def robust_ros3_read( command: Callable, - max_retries: int = 10, + max_retries: int = 20, command_args: Union[list, None] = None, command_kwargs: Union[dict, None] = None, -): +) -> Any: """ Attempt the command (usually acting on an S3 IO) up to the number of max_retries using exponential backoff. @@ -67,12 +67,14 @@ def robust_ros3_read( command_kwargs = command_kwargs or dict() for retry in range(max_retries): try: - return command(*command_args, **command_kwargs) - except Exception as exc: - if "curl" in str(exc): # 'cannot curl request' can show up in potentially many different error types - time.sleep(0.1 * 2**retry) - else: - raise exc + result = command(*command_args, **command_kwargs) + return result + except Exception as exception: + # 'cannot curl request' can show up in potentially many different error types + if "curl" not in str(exception): + raise exception + time.sleep(0.1 * 2**retry) + raise TimeoutError(f"Unable to complete the command ({command.__name__}) after {max_retries} attempts!") From 75355c8f57a96d19813e2a49e837ff8b0c7ba1ed Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 01:07:39 -0500 Subject: [PATCH 35/72] fix hanging name --- .../benchmarks/network_tracking_remote_file_reading.py | 2 +- src/nwb_benchmarks/core/_streaming.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py index 6e38f4c..79837ac 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -60,7 +60,7 @@ class FsspecNoCacheNWBFileReadBenchmark: param_names = param_names params = params - def network_activity_tracker_during_read(self, s3_url: str): + def track_network_activity_during_read(self, s3_url: str): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) return network_activity.asv_network_statistics diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py index cabb875..02bcea0 100644 --- a/src/nwb_benchmarks/core/_streaming.py +++ b/src/nwb_benchmarks/core/_streaming.py @@ -81,8 +81,9 @@ def robust_ros3_read( def read_hdf5_ros3(s3_url: str) -> h5py.File: """Load the raw HDF5 file from an S3 URL using ROS3 driver; does not formally read the NWB file.""" ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") + aws_region = bytes("us-east-2", "ascii") # TODO: generalize this as an argument if necessary file = robust_ros3_read( - command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3", aws_region=bytes("us-east-2", "ascii")) + command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3", aws_region=aws_region) ) return file From 5cfa550bb20bc5171a0e218c208b8c27e058d3b2 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 01:33:21 -0500 Subject: [PATCH 36/72] try adding additional timers for ros3 --- .../benchmarks/time_remote_slicing.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py index 115a9c9..da77564 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -1,5 +1,6 @@ """Basic benchmarks for stream NWB files and their contents.""" +import time from typing import Tuple from nwb_benchmarks.core import ( @@ -72,3 +73,37 @@ def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) + + +class Ros3ContinuousSliceBenchmarkProcessTime: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + timer = time.process_time # Default timer is timeit.default_timer + + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") + self.data_to_slice = self.neurodata_object.data + + def time_slice_clock(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + """Note: store as self._temp to avoid tracking garbage collection as well.""" + self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) + + +class Ros3ContinuousSliceBenchmarkRawTime: + rounds = 1 + repeat = 3 + param_names = param_names + params = params + timer = time.time + + def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") + self.data_to_slice = self.neurodata_object.data + + def time_slice_raw_time(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): + """Note: store as self._temp to avoid tracking garbage collection as well.""" + self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) From 2be781572524574f291af6a5250c981449cece60 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 01:35:26 -0500 Subject: [PATCH 37/72] add more results --- ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 346 ++++++++++++++++ ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 381 ++++++++++++++++++ ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 16 + 3 files changed, 743 insertions(+) create mode 100644 results/results_timestamp-2024-02-20-00-46-36_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json create mode 100644 results/results_timestamp-2024-02-20-01-07-39_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json create mode 100644 results/results_timestamp-2024-02-20-01-33-21_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json diff --git a/results/results_timestamp-2024-02-20-00-46-36_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-20-00-46-36_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json new file mode 100644 index 0000000..40cfa7e --- /dev/null +++ b/results/results_timestamp-2024-02-20-00-46-36_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -0,0 +1,346 @@ +{ + "version": 2, + "timestamp": "2024-02-20-00-46-36", + "commit_hash": "e80c2d932d85f4c15f282a8b84e818cefb7e78c6", + "environment_hash": "2d1c68a2c7de02ab2c200e448b250bee5e46f212", + "machine_hash": "e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46", + "results": { + "network_tracking_remote_file_reading.FsspecNoCacheDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213474, + "amount_uploaded_in_bytes": 2634, + "total_transfer_in_bytes": 216108, + "amount_downloaded_in_number_of_packets": 158, + "amount_uploaded_in_number_of_packets": 28, + "total_transfer_in_number_of_packets": 186, + "total_traffic_in_number_of_web_packets": 28, + "total_transfer_time_in_seconds": 0.19317900000000002, + "network_total_time": 0.7372112274169922 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 5643549, + "amount_uploaded_in_bytes": 7775, + "total_transfer_in_bytes": 5651324, + "amount_downloaded_in_number_of_packets": 3823, + "amount_uploaded_in_number_of_packets": 118, + "total_transfer_in_number_of_packets": 3941, + "total_traffic_in_number_of_web_packets": 118, + "total_transfer_time_in_seconds": 0.385904, + "network_total_time": 10.110046863555908 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192382, + "amount_uploaded_in_bytes": 2852, + "total_transfer_in_bytes": 195234, + "amount_downloaded_in_number_of_packets": 141, + "amount_uploaded_in_number_of_packets": 33, + "total_transfer_in_number_of_packets": 174, + "total_traffic_in_number_of_web_packets": 33, + "total_transfer_time_in_seconds": 0.18170399999999998, + "network_total_time": 0.714885950088501 + } + }, + "network_tracking_remote_file_reading.RemfileDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 88699, + "amount_uploaded_in_bytes": 3495, + "total_transfer_in_bytes": 92194, + "amount_downloaded_in_number_of_packets": 70, + "amount_uploaded_in_number_of_packets": 49, + "total_transfer_in_number_of_packets": 119, + "total_traffic_in_number_of_web_packets": 49, + "total_transfer_time_in_seconds": 0.12304400000000004, + "network_total_time": 0.8128571510314941 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 86105, + "amount_uploaded_in_bytes": 5069, + "total_transfer_in_bytes": 91174, + "amount_downloaded_in_number_of_packets": 79, + "amount_uploaded_in_number_of_packets": 59, + "total_transfer_in_number_of_packets": 138, + "total_traffic_in_number_of_web_packets": 59, + "total_transfer_time_in_seconds": 0.22937200000000002, + "network_total_time": 0.7552578449249268 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 38203, + "amount_uploaded_in_bytes": 2674, + "total_transfer_in_bytes": 40877, + "amount_downloaded_in_number_of_packets": 37, + "amount_uploaded_in_number_of_packets": 33, + "total_transfer_in_number_of_packets": 70, + "total_traffic_in_number_of_web_packets": 33, + "total_transfer_time_in_seconds": 0.13973299999999997, + "network_total_time": 0.7740828990936279 + } + }, + "network_tracking_remote_file_reading.RemfileNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 302828, + "amount_uploaded_in_bytes": 6493, + "total_transfer_in_bytes": 309321, + "amount_downloaded_in_number_of_packets": 229, + "amount_uploaded_in_number_of_packets": 82, + "total_transfer_in_number_of_packets": 311, + "total_traffic_in_number_of_web_packets": 82, + "total_transfer_time_in_seconds": 0.302652, + "network_total_time": 1.1026599407196045 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 336962, + "amount_uploaded_in_bytes": 5373, + "total_transfer_in_bytes": 342335, + "amount_downloaded_in_number_of_packets": 257, + "amount_uploaded_in_number_of_packets": 54, + "total_transfer_in_number_of_packets": 311, + "total_traffic_in_number_of_web_packets": 54, + "total_transfer_time_in_seconds": 0.413829, + "network_total_time": 1.236659049987793 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 200658, + "amount_uploaded_in_bytes": 3192, + "total_transfer_in_bytes": 203850, + "amount_downloaded_in_number_of_packets": 149, + "amount_uploaded_in_number_of_packets": 38, + "total_transfer_in_number_of_packets": 187, + "total_traffic_in_number_of_web_packets": 38, + "total_transfer_time_in_seconds": 0.22162999999999994, + "network_total_time": 1.0691828727722168 + } + }, + "network_tracking_remote_file_reading.Ros3DirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.3520839214324951 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 16011513, + "amount_uploaded_in_bytes": 26683, + "total_transfer_in_bytes": 16038196, + "amount_downloaded_in_number_of_packets": 10858, + "amount_uploaded_in_number_of_packets": 374, + "total_transfer_in_number_of_packets": 11232, + "total_traffic_in_number_of_web_packets": 374, + "total_transfer_time_in_seconds": 0.4006, + "network_total_time": 26.902956008911133 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.3080167770385742 + } + }, + "network_tracking_remote_file_reading.Ros3NWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 479345, + "amount_uploaded_in_bytes": 6162, + "total_transfer_in_bytes": 485507, + "amount_downloaded_in_number_of_packets": 352, + "amount_uploaded_in_number_of_packets": 72, + "total_transfer_in_number_of_packets": 424, + "total_traffic_in_number_of_web_packets": 72, + "total_transfer_time_in_seconds": 0.42814700000000006, + "network_total_time": 1.5349469184875488 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 35187774, + "amount_uploaded_in_bytes": 93439, + "total_transfer_in_bytes": 35281213, + "amount_downloaded_in_number_of_packets": 23883, + "amount_uploaded_in_number_of_packets": 1054, + "total_transfer_in_number_of_packets": 24937, + "total_traffic_in_number_of_web_packets": 1054, + "total_transfer_time_in_seconds": 5.284585999999999, + "network_total_time": 58.232357025146484 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 387088, + "amount_uploaded_in_bytes": 5360, + "total_transfer_in_bytes": 392448, + "amount_downloaded_in_number_of_packets": 291, + "amount_uploaded_in_number_of_packets": 63, + "total_transfer_in_number_of_packets": 354, + "total_traffic_in_number_of_web_packets": 63, + "total_transfer_time_in_seconds": 0.414528, + "network_total_time": 1.3121049404144287 + } + }, + "network_tracking_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 43657390, + "amount_uploaded_in_bytes": 44026, + "total_transfer_in_bytes": 43701416, + "amount_downloaded_in_number_of_packets": 29579, + "amount_uploaded_in_number_of_packets": 769, + "total_transfer_in_number_of_packets": 30348, + "total_traffic_in_number_of_web_packets": 769, + "total_transfer_time_in_seconds": 1.1056540000000001, + "network_total_time": 71.27475690841675 + } + }, + "network_tracking_remote_slicing.RemfileContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 26751191, + "amount_uploaded_in_bytes": 38520, + "total_transfer_in_bytes": 26789711, + "amount_downloaded_in_number_of_packets": 18587, + "amount_uploaded_in_number_of_packets": 520, + "total_transfer_in_number_of_packets": 19107, + "total_traffic_in_number_of_web_packets": 520, + "total_transfer_time_in_seconds": 0.758081, + "network_total_time": 45.405421018600464 + } + }, + "network_tracking_remote_slicing.Ros3ContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 21812949, + "amount_uploaded_in_bytes": 27590, + "total_transfer_in_bytes": 21840539, + "amount_downloaded_in_number_of_packets": 14756, + "amount_uploaded_in_number_of_packets": 465, + "total_transfer_in_number_of_packets": 15221, + "total_traffic_in_number_of_web_packets": 465, + "total_transfer_time_in_seconds": 0.5304300000000002, + "network_total_time": 35.06142997741699 + } + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.16877824999392033, + 0.19543575000716373, + 0.1770209579844959 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.3509329170337878, + 0.3309523750212975, + 0.31621412496315315 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.1973942500189878, + 0.14922070800093934, + 0.14971779199549928 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.24883179203607142, + 0.23245162499370053, + 0.26791058300295845 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.2916938749840483, + 0.26400566700613126, + 0.2940791250439361 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.2628347919671796, + 0.24597983300918713, + 0.2678367500193417 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.1865590840461664, + 0.20669766695937142, + 0.17177899996750057 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.5326967919827439, + 6.851267166028265, + 0.4476575829903595 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.15867208398412913, + 0.1610627499758266, + 0.1714515839703381 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.21281491598347202, + 0.22454887500498444, + 0.2026822079787962 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 17.405665375001263, + 13.09660079202149, + 20.35388500004774 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.225040125020314, + 0.2166273749899119, + 0.2371447499608621 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.29837250005220994, + 0.31618850002996624, + 0.30228229198837653 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.3879139579948969, + 0.579778624989558, + 0.3599985000328161 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.3506162500125356, + 0.3050851660082117, + 0.28935916704358533 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.43795741599751636, + 0.4272256250260398, + 0.4215052080107853 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 2.916308999992907, + 1.0988173329969868, + 3.279229292005766 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.4365296670002863, + 0.3908755839802325, + 0.4189352499670349 + ] + }, + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.1624044169439003, + 0.8959497499745339, + 0.6047527919872664 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.6832950829993933, + 8.449490583967417, + 6.970342791988514 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.006103624997194856, + 0.0019077499746344984, + 0.001912957988679409 + ] + } + } +} diff --git a/results/results_timestamp-2024-02-20-01-07-39_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-20-01-07-39_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json new file mode 100644 index 0000000..303ab11 --- /dev/null +++ b/results/results_timestamp-2024-02-20-01-07-39_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -0,0 +1,381 @@ +{ + "version": 2, + "timestamp": "2024-02-20-01-07-39", + "commit_hash": "75355c8f57a96d19813e2a49e837ff8b0c7ba1ed", + "environment_hash": "2d1c68a2c7de02ab2c200e448b250bee5e46f212", + "machine_hash": "e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46", + "results": { + "network_tracking_remote_file_reading.FsspecNoCacheDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213320, + "amount_uploaded_in_bytes": 2678, + "total_transfer_in_bytes": 215998, + "amount_downloaded_in_number_of_packets": 156, + "amount_uploaded_in_number_of_packets": 30, + "total_transfer_in_number_of_packets": 186, + "total_traffic_in_number_of_web_packets": 30, + "total_transfer_time_in_seconds": 0.179358, + "network_total_time": 0.7478981018066406 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 5640003, + "amount_uploaded_in_bytes": 8004, + "total_transfer_in_bytes": 5648007, + "amount_downloaded_in_number_of_packets": 3817, + "amount_uploaded_in_number_of_packets": 124, + "total_transfer_in_number_of_packets": 3941, + "total_traffic_in_number_of_web_packets": 124, + "total_transfer_time_in_seconds": 0.46628099999999995, + "network_total_time": 10.288937091827393 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192432, + "amount_uploaded_in_bytes": 2450, + "total_transfer_in_bytes": 194882, + "amount_downloaded_in_number_of_packets": 143, + "amount_uploaded_in_number_of_packets": 26, + "total_transfer_in_number_of_packets": 169, + "total_traffic_in_number_of_web_packets": 26, + "total_transfer_time_in_seconds": 0.187216, + "network_total_time": 0.6843838691711426 + } + }, + "network_tracking_remote_file_reading.FsspecNoCacheNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213454, + "amount_uploaded_in_bytes": 2958, + "total_transfer_in_bytes": 216412, + "amount_downloaded_in_number_of_packets": 158, + "amount_uploaded_in_number_of_packets": 34, + "total_transfer_in_number_of_packets": 192, + "total_traffic_in_number_of_web_packets": 34, + "total_transfer_time_in_seconds": 0.175066, + "network_total_time": 0.751417875289917 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 44980742, + "amount_uploaded_in_bytes": 52847, + "total_transfer_in_bytes": 45033589, + "amount_downloaded_in_number_of_packets": 30987, + "amount_uploaded_in_number_of_packets": 808, + "total_transfer_in_number_of_packets": 31795, + "total_traffic_in_number_of_web_packets": 808, + "total_transfer_time_in_seconds": 1.675869, + "network_total_time": 77.90417504310608 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192579, + "amount_uploaded_in_bytes": 2798, + "total_transfer_in_bytes": 195377, + "amount_downloaded_in_number_of_packets": 143, + "amount_uploaded_in_number_of_packets": 32, + "total_transfer_in_number_of_packets": 175, + "total_traffic_in_number_of_web_packets": 32, + "total_transfer_time_in_seconds": 0.22213600000000003, + "network_total_time": 0.7634181976318359 + } + }, + "network_tracking_remote_file_reading.RemfileDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 87959, + "amount_uploaded_in_bytes": 3495, + "total_transfer_in_bytes": 91454, + "amount_downloaded_in_number_of_packets": 71, + "amount_uploaded_in_number_of_packets": 49, + "total_transfer_in_number_of_packets": 120, + "total_traffic_in_number_of_web_packets": 49, + "total_transfer_time_in_seconds": 0.146259, + "network_total_time": 0.9239380359649658 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 283430, + "amount_uploaded_in_bytes": 5135, + "total_transfer_in_bytes": 288565, + "amount_downloaded_in_number_of_packets": 217, + "amount_uploaded_in_number_of_packets": 58, + "total_transfer_in_number_of_packets": 275, + "total_traffic_in_number_of_web_packets": 58, + "total_transfer_time_in_seconds": 0.357353, + "network_total_time": 1.066493034362793 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 56701, + "amount_uploaded_in_bytes": 2269, + "total_transfer_in_bytes": 58970, + "amount_downloaded_in_number_of_packets": 49, + "amount_uploaded_in_number_of_packets": 27, + "total_transfer_in_number_of_packets": 76, + "total_traffic_in_number_of_web_packets": 27, + "total_transfer_time_in_seconds": 0.12480999999999999, + "network_total_time": 0.7339892387390137 + } + }, + "network_tracking_remote_file_reading.RemfileNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 276952, + "amount_uploaded_in_bytes": 5791, + "total_transfer_in_bytes": 282743, + "amount_downloaded_in_number_of_packets": 216, + "amount_uploaded_in_number_of_packets": 69, + "total_transfer_in_number_of_packets": 285, + "total_traffic_in_number_of_web_packets": 69, + "total_transfer_time_in_seconds": 0.29779799999999995, + "network_total_time": 1.0031211376190186 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 394381, + "amount_uploaded_in_bytes": 5771, + "total_transfer_in_bytes": 400152, + "amount_downloaded_in_number_of_packets": 294, + "amount_uploaded_in_number_of_packets": 65, + "total_transfer_in_number_of_packets": 359, + "total_traffic_in_number_of_web_packets": 65, + "total_transfer_time_in_seconds": 0.37571000000000004, + "network_total_time": 1.2605290412902832 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 275022, + "amount_uploaded_in_bytes": 5917, + "total_transfer_in_bytes": 280939, + "amount_downloaded_in_number_of_packets": 209, + "amount_uploaded_in_number_of_packets": 75, + "total_transfer_in_number_of_packets": 284, + "total_traffic_in_number_of_web_packets": 75, + "total_transfer_time_in_seconds": 0.329709, + "network_total_time": 1.0414228439331055 + } + }, + "network_tracking_remote_file_reading.Ros3DirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.3325979709625244 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 29.524303913116455 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.3150928020477295 + } + }, + "network_tracking_remote_file_reading.Ros3NWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 479461, + "amount_uploaded_in_bytes": 6108, + "total_transfer_in_bytes": 485569, + "amount_downloaded_in_number_of_packets": 352, + "amount_uploaded_in_number_of_packets": 71, + "total_transfer_in_number_of_packets": 423, + "total_traffic_in_number_of_web_packets": 71, + "total_transfer_time_in_seconds": 0.42768800000000007, + "network_total_time": 1.4909050464630127 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 35301640, + "amount_uploaded_in_bytes": 121662, + "total_transfer_in_bytes": 35423302, + "amount_downloaded_in_number_of_packets": 24162, + "amount_uploaded_in_number_of_packets": 1354, + "total_transfer_in_number_of_packets": 25516, + "total_traffic_in_number_of_web_packets": 1354, + "total_transfer_time_in_seconds": 5.102728, + "network_total_time": 60.182926177978516 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 386704, + "amount_uploaded_in_bytes": 5566, + "total_transfer_in_bytes": 392270, + "amount_downloaded_in_number_of_packets": 286, + "amount_uploaded_in_number_of_packets": 68, + "total_transfer_in_number_of_packets": 354, + "total_traffic_in_number_of_web_packets": 68, + "total_transfer_time_in_seconds": 0.37023700000000004, + "network_total_time": 1.2854137420654297 + } + }, + "network_tracking_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 43625762, + "amount_uploaded_in_bytes": 45165, + "total_transfer_in_bytes": 43670927, + "amount_downloaded_in_number_of_packets": 29469, + "amount_uploaded_in_number_of_packets": 814, + "total_transfer_in_number_of_packets": 30283, + "total_traffic_in_number_of_web_packets": 814, + "total_transfer_time_in_seconds": 1.2660369999999999, + "network_total_time": 75.88961505889893 + } + }, + "network_tracking_remote_slicing.RemfileContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 26660433, + "amount_uploaded_in_bytes": 84260, + "total_transfer_in_bytes": 26744693, + "amount_downloaded_in_number_of_packets": 17975, + "amount_uploaded_in_number_of_packets": 1494, + "total_transfer_in_number_of_packets": 19469, + "total_traffic_in_number_of_web_packets": 1494, + "total_transfer_time_in_seconds": 13.038521, + "network_total_time": 56.507325887680054 + } + }, + "network_tracking_remote_slicing.Ros3ContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 24317005, + "amount_uploaded_in_bytes": 42027, + "total_transfer_in_bytes": 24359032, + "amount_downloaded_in_number_of_packets": 16800, + "amount_uploaded_in_number_of_packets": 561, + "total_transfer_in_number_of_packets": 17361, + "total_traffic_in_number_of_web_packets": 561, + "total_transfer_time_in_seconds": 0.49266400000000005, + "network_total_time": 38.03533697128296 + } + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.1975440419628285, + 0.19764095800928771, + 0.20242683397373185 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.4294463749974966, + 0.731981834047474, + 0.46631766599603 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.22334979102015495, + 0.16376287501771003, + 0.16609179199440405 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.2511703330092132, + 0.26317758398363367, + 0.29316137498244643 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.39911620900966227, + 0.3771514170221053, + 0.35551079199649394 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.2508740830235183, + 0.23205150000285357, + 0.29161675000796095 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.19018558401148766, + 0.20751762500731274, + 0.18415612500393763 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 4.269924750027712, + 0.48960999998962507, + 0.49908387497998774 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.14857987500727177, + 0.14313845802098513, + 0.16894533298909664 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.19136233301833272, + 0.1975544579909183, + 0.20394166698679328 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 10.169489874970168, + 1.7298543329816312, + 2.1313127920147963 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.23817525000777096, + 0.21016737498575822, + 0.20800362498266622 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.29083804198307917, + 0.3688720000209287, + 0.35467358300229535 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.32692958402913064, + 0.33379520900780335, + 0.4219536670134403 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.31507154199061915, + 0.32252858398715034, + 0.3305945419706404 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.4108242079964839, + 0.4222325829905458, + 0.38277679099701345 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 7.772533749986906, + 5.544184999947902, + 1.147499541984871 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.5286890419665724, + 0.31896750000305474, + 0.3440035830135457 + ] + }, + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.179418749990873, + 0.778133915970102, + 0.7576090420479886 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.8728850409970619, + 0.7532737079891376, + 0.7387993339798413 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.006911083008162677, + 0.0018965830095112324, + 0.0018801669939421117 + ] + } + } +} diff --git a/results/results_timestamp-2024-02-20-01-33-21_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-20-01-33-21_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json new file mode 100644 index 0000000..27f591c --- /dev/null +++ b/results/results_timestamp-2024-02-20-01-33-21_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -0,0 +1,16 @@ +{ + "version": 2, + "timestamp": "2024-02-20-01-33-21", + "commit_hash": "5cfa550bb20bc5171a0e218c208b8c27e058d3b2", + "environment_hash": "2d1c68a2c7de02ab2c200e448b250bee5e46f212", + "machine_hash": "e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46", + "results": { + "time_remote_slicing.Ros3ContinuousSliceBenchmarkRawTime.time_slice_raw_time": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.0060291290283203125, + 0.002125978469848633, + 0.0018842220306396484 + ] + } + } +} From babddf0a8c656868a19992ba6496a093525a5804 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Feb 2024 14:51:54 -0800 Subject: [PATCH 38/72] Update docs/running_benchmarks.rst --- docs/running_benchmarks.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index 8616659..c0520e6 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -7,7 +7,8 @@ Also, please ensure prior to running the benchmark that all code changes have be For the most stable results, only run the benchmarks on the ``main`` branch. -To run the full benchmark suite, including network tracking tests (which require ``sudo`` on Mac or Linux), simply call... +To run the full benchmark suite, including network tracking tests (which require ``sudo`` on Mac and AIX platforms due to the +use `psutil net_connections `_), simply call... .. code-block:: From b6b8c4976a6fb1b0a4f4af144bc0b7ea87d01b98 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Feb 2024 22:52:00 +0000 Subject: [PATCH 39/72] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/running_benchmarks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index c0520e6..974e6a9 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -7,7 +7,7 @@ Also, please ensure prior to running the benchmark that all code changes have be For the most stable results, only run the benchmarks on the ``main`` branch. -To run the full benchmark suite, including network tracking tests (which require ``sudo`` on Mac and AIX platforms due to the +To run the full benchmark suite, including network tracking tests (which require ``sudo`` on Mac and AIX platforms due to the use `psutil net_connections `_), simply call... .. code-block:: From 22907c9a07d5d70487e520515025bf7fac604a7c Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 22:05:13 -0500 Subject: [PATCH 40/72] add automatic chdmod to all final file creations --- docs/running_benchmarks.rst | 9 +++++++-- src/nwb_benchmarks/command_line_interface.py | 1 + src/nwb_benchmarks/setup/_reduce_results.py | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index 974e6a9..ea59247 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -50,9 +50,14 @@ If you want to get a full traceback to examine why a new test might be failing, Contributing Results -------------------- -To contribute your results back to the project, all you have to do is `git add` and `commit` the results in the `results` folder. +To contribute your results back to the project, please use the following workflow...AIX -Due to the ``sudo`` requirement of the network tracking tests, if you ran ``sudo nwb_benchmarks run`` you will likely have to include ``sudo`` in the ``git commit`` as well; though this may also depend on whether or not you have ``pre-commit`` running locally. +.. code-block:: + + git checkout -b new_results_from_<...> + git add results/ + git commit -m "New results from ...." . + git push Then, open a PR to merge the results to the `main` branch. diff --git a/src/nwb_benchmarks/command_line_interface.py b/src/nwb_benchmarks/command_line_interface.py index f072d6b..17efb12 100644 --- a/src/nwb_benchmarks/command_line_interface.py +++ b/src/nwb_benchmarks/command_line_interface.py @@ -86,6 +86,7 @@ def main() -> None: "the results! Please manually remove these." ) raw_results_file_path = globbed_json_file_paths[0] + reduce_results( raw_results_file_path=raw_results_file_path, raw_environment_info_file_path=raw_environment_info_file_path ) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index f2ce7de..909859a 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -4,6 +4,7 @@ import datetime import hashlib import json +import os import pathlib import shutil import sys @@ -107,4 +108,9 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil with open(file=parsed_environment_file_path, mode="w") as io: json.dump(obj=parsed_environment_info, fp=io, indent=4) + # Network tests require admin permissions, which can alter write permissions of any files created + os.chmod(path=parsed_results_file, mode=os.W_OK) + os.chmod(path=machine_info_file_path, mode=os.W_OK) + os.chmod(path=parsed_environment_file_path, mode=os.W_OK) + raw_results_file_path.unlink() From e9bfccd2428a0ddd753d318d640f875e3c12f8d0 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Feb 2024 19:20:12 -0800 Subject: [PATCH 41/72] Update docs/running_benchmarks.rst --- docs/running_benchmarks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index ea59247..023725d 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -50,7 +50,7 @@ If you want to get a full traceback to examine why a new test might be failing, Contributing Results -------------------- -To contribute your results back to the project, please use the following workflow...AIX +To contribute your results back to the project, please use the following workflow... .. code-block:: From 27d7a4344287909678cb722b94114233bf3f518c Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Feb 2024 19:30:48 -0800 Subject: [PATCH 42/72] Update src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py --- .../benchmarks/network_tracking_remote_slicing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index 1278374..a7e0508 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -1,4 +1,4 @@ -"""Basic benchmarks for stream NWB files and their contents.""" +"""Basic benchmarks for streaming access to NWB files and their contents.""" import os from typing import Tuple From daee3d143586694059d63c4a8aa30b074d7f62ec Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Feb 2024 19:33:38 -0800 Subject: [PATCH 43/72] Update src/nwb_benchmarks/benchmarks/time_remote_file_reading.py --- src/nwb_benchmarks/benchmarks/time_remote_file_reading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py index 4a183c1..59cc799 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py @@ -1,4 +1,4 @@ -"""Basic benchmarks for stream NWB files and their contents.""" +"""Basic benchmarks for timing streaming access of NWB files and their contents.""" from nwb_benchmarks.core import ( get_s3_url, From be9711ddbba3c6f5557e95897dad5258a7de0223 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Feb 2024 19:34:28 -0800 Subject: [PATCH 44/72] Update src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py --- .../benchmarks/network_tracking_remote_file_reading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py index 79837ac..de016bf 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -1,4 +1,4 @@ -"""Basic benchmarks for stream NWB files and their contents.""" +"""Basic benchmarks for profiling network statistics for streaming access to NWB files and their contents.""" import os From 726300ff18b943256ba6320be9c1dab811679ca3 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Feb 2024 19:35:47 -0800 Subject: [PATCH 45/72] Update src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py --- .../benchmarks/network_tracking_remote_slicing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index a7e0508..12b623b 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -1,4 +1,4 @@ -"""Basic benchmarks for streaming access to NWB files and their contents.""" +"""Basic benchmarks for profiling network statistics for streaming access to slices of data stored in NWB files.""" import os from typing import Tuple From 9d2d91d7bf77b05c82a2c7b19661e125119e9e80 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Feb 2024 19:37:11 -0800 Subject: [PATCH 46/72] Update src/nwb_benchmarks/benchmarks/time_remote_slicing.py --- src/nwb_benchmarks/benchmarks/time_remote_slicing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py index da77564..b589ffd 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -1,4 +1,4 @@ -"""Basic benchmarks for stream NWB files and their contents.""" +"""Basic benchmarks for timing streaming access to slices of data stored in NWB files.""" import time from typing import Tuple From ad388b9af42cbf58b61069ce5ccd5ad2085e5cd4 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Feb 2024 19:50:59 -0800 Subject: [PATCH 47/72] Update src/nwb_benchmarks/core/_network_tracker.py --- src/nwb_benchmarks/core/_network_tracker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index a034f96..35ad3b8 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -1,4 +1,7 @@ -"""Base class for implementing benchmarks for evaluating network performance metrics for streaming read of NWB files.""" +""" +Network tracker for capturing network traffic and performance metrics during the execution of particular +methods or code snippets. +""" import contextlib import os From 111f8537536f3be736f905a3e55cc9f58c46484d Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 22:51:01 -0500 Subject: [PATCH 48/72] fix the proper chmod mode --- src/nwb_benchmarks/core/_network_tracker.py | 3 +++ src/nwb_benchmarks/setup/_reduce_results.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index a034f96..8c0820c 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -53,4 +53,7 @@ def stop_network_capture(self): self.network_statistics = NetworkStatistics.get_statistics(packets=self.pid_packets) # Very special structure required by ASV + # 'result' is the display value in console + # 'samples' is the value tracked in our results + # 'number' is simply required, but needs to be None for custom track_ functions self.asv_network_statistics = dict(samples=self.network_statistics, number=None) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 909859a..5e2473d 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -8,6 +8,7 @@ import pathlib import shutil import sys +import stat from typing import Dict, List @@ -109,8 +110,8 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil json.dump(obj=parsed_environment_info, fp=io, indent=4) # Network tests require admin permissions, which can alter write permissions of any files created - os.chmod(path=parsed_results_file, mode=os.W_OK) - os.chmod(path=machine_info_file_path, mode=os.W_OK) - os.chmod(path=parsed_environment_file_path, mode=os.W_OK) + os.chmod(path=parsed_results_file, mode=stat.S_IROTH | stat.S_IWOTH) + os.chmod(path=machine_info_file_path, mode=stat.S_IROTH | stat.S_IWOTH) + os.chmod(path=parsed_environment_file_path, mode=stat.S_IROTH | stat.S_IWOTH) raw_results_file_path.unlink() From 732ea9dea2cc21d632719af5a148d4ae530c53ba Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 22:53:26 -0500 Subject: [PATCH 49/72] improve display --- src/nwb_benchmarks/core/_network_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index 5de58ff..7842b18 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -59,4 +59,4 @@ def stop_network_capture(self): # 'result' is the display value in console # 'samples' is the value tracked in our results # 'number' is simply required, but needs to be None for custom track_ functions - self.asv_network_statistics = dict(samples=self.network_statistics, number=None) + self.asv_network_statistics = dict(results="success", samples=self.network_statistics, number=None) From 0d4bb924e086e187c1138c6003344c791a1357a8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 03:53:35 +0000 Subject: [PATCH 50/72] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/nwb_benchmarks/setup/_reduce_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 5e2473d..9ee85ee 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -7,8 +7,8 @@ import os import pathlib import shutil -import sys import stat +import sys from typing import Dict, List From 1b0118f35f47e3af204b159ebe8261ceee57e0c7 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 23:16:29 -0500 Subject: [PATCH 51/72] remove display --- src/nwb_benchmarks/core/_network_tracker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index 7842b18..278855c 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -56,7 +56,6 @@ def stop_network_capture(self): self.network_statistics = NetworkStatistics.get_statistics(packets=self.pid_packets) # Very special structure required by ASV - # 'result' is the display value in console # 'samples' is the value tracked in our results # 'number' is simply required, but needs to be None for custom track_ functions - self.asv_network_statistics = dict(results="success", samples=self.network_statistics, number=None) + self.asv_network_statistics = dict(samples=self.network_statistics, number=None) From 70cb5ed055cd2ad5e1db9697ff9d47c626ab41ec Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 23:19:37 -0500 Subject: [PATCH 52/72] try direct chmod in subprocess --- src/nwb_benchmarks/setup/_reduce_results.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 5e2473d..ad29bc7 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -4,11 +4,14 @@ import datetime import hashlib import json -import os + +# import os import pathlib import shutil import sys -import stat + +# import stat +import subprocess from typing import Dict, List @@ -110,8 +113,11 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil json.dump(obj=parsed_environment_info, fp=io, indent=4) # Network tests require admin permissions, which can alter write permissions of any files created - os.chmod(path=parsed_results_file, mode=stat.S_IROTH | stat.S_IWOTH) - os.chmod(path=machine_info_file_path, mode=stat.S_IROTH | stat.S_IWOTH) - os.chmod(path=parsed_environment_file_path, mode=stat.S_IROTH | stat.S_IWOTH) + # os.chmod(path=parsed_results_file, mode=stat.S_IROTH | stat.S_IWOTH) + # os.chmod(path=machine_info_file_path, mode=stat.S_IROTH | stat.S_IWOTH) + # os.chmod(path=parsed_environment_file_path, mode=stat.S_IROTH | stat.S_IWOTH) + subprocess.run(["chmod", "-R", "+rw", parsed_results_file]) + subprocess.run(["chmod", "-R", "+rw", machine_info_file_path]) + subprocess.run(["chmod", "-R", "+rw", parsed_environment_file_path]) raw_results_file_path.unlink() From 5f4bfdf43bc09e82933af238e726a8dd87977ad3 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 23:36:54 -0500 Subject: [PATCH 53/72] commit results without needing sudo --- ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 395 ++++++++++++++++++ 1 file changed, 395 insertions(+) create mode 100644 results/results_timestamp-2024-02-20-23-19-37_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json diff --git a/results/results_timestamp-2024-02-20-23-19-37_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-20-23-19-37_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json new file mode 100644 index 0000000..89f6343 --- /dev/null +++ b/results/results_timestamp-2024-02-20-23-19-37_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -0,0 +1,395 @@ +{ + "version": 2, + "timestamp": "2024-02-20-23-19-37", + "commit_hash": "70cb5ed055cd2ad5e1db9697ff9d47c626ab41ec", + "environment_hash": "2d1c68a2c7de02ab2c200e448b250bee5e46f212", + "machine_hash": "e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46", + "results": { + "network_tracking_remote_file_reading.FsspecNoCacheDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.3955998420715332 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 5648279, + "amount_uploaded_in_bytes": 22577, + "total_transfer_in_bytes": 5670856, + "amount_downloaded_in_number_of_packets": 3816, + "amount_uploaded_in_number_of_packets": 385, + "total_transfer_in_number_of_packets": 4201, + "total_traffic_in_number_of_web_packets": 385, + "total_transfer_time_in_seconds": 3.2794689999999997, + "network_total_time": 12.811427116394043 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192494, + "amount_uploaded_in_bytes": 2636, + "total_transfer_in_bytes": 195130, + "amount_downloaded_in_number_of_packets": 142, + "amount_uploaded_in_number_of_packets": 29, + "total_transfer_in_number_of_packets": 171, + "total_traffic_in_number_of_web_packets": 29, + "total_transfer_time_in_seconds": 0.22898600000000002, + "network_total_time": 0.8273327350616455 + } + }, + "network_tracking_remote_file_reading.FsspecNoCacheNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213266, + "amount_uploaded_in_bytes": 2570, + "total_transfer_in_bytes": 215836, + "amount_downloaded_in_number_of_packets": 155, + "amount_uploaded_in_number_of_packets": 28, + "total_transfer_in_number_of_packets": 183, + "total_traffic_in_number_of_web_packets": 28, + "total_transfer_time_in_seconds": 0.20920899999999998, + "network_total_time": 0.8600068092346191 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 44966840, + "amount_uploaded_in_bytes": 56467, + "total_transfer_in_bytes": 45023307, + "amount_downloaded_in_number_of_packets": 30508, + "amount_uploaded_in_number_of_packets": 916, + "total_transfer_in_number_of_packets": 31424, + "total_traffic_in_number_of_web_packets": 916, + "total_transfer_time_in_seconds": 2.3420110000000003, + "network_total_time": 92.00857973098755 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192508, + "amount_uploaded_in_bytes": 2558, + "total_transfer_in_bytes": 195066, + "amount_downloaded_in_number_of_packets": 144, + "amount_uploaded_in_number_of_packets": 28, + "total_transfer_in_number_of_packets": 172, + "total_traffic_in_number_of_web_packets": 28, + "total_transfer_time_in_seconds": 0.22030000000000002, + "network_total_time": 0.8312501907348633 + } + }, + "network_tracking_remote_file_reading.RemfileDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 205021, + "amount_uploaded_in_bytes": 5609, + "total_transfer_in_bytes": 210630, + "amount_downloaded_in_number_of_packets": 161, + "amount_uploaded_in_number_of_packets": 72, + "total_transfer_in_number_of_packets": 233, + "total_traffic_in_number_of_web_packets": 72, + "total_transfer_time_in_seconds": 0.322973, + "network_total_time": 0.9783899784088135 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 309386, + "amount_uploaded_in_bytes": 6041, + "total_transfer_in_bytes": 315427, + "amount_downloaded_in_number_of_packets": 237, + "amount_uploaded_in_number_of_packets": 75, + "total_transfer_in_number_of_packets": 312, + "total_traffic_in_number_of_web_packets": 75, + "total_transfer_time_in_seconds": 0.360857, + "network_total_time": 1.2275950908660889 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 88700, + "amount_uploaded_in_bytes": 2269, + "total_transfer_in_bytes": 90969, + "amount_downloaded_in_number_of_packets": 70, + "amount_uploaded_in_number_of_packets": 27, + "total_transfer_in_number_of_packets": 97, + "total_traffic_in_number_of_web_packets": 27, + "total_transfer_time_in_seconds": 0.12125099999999998, + "network_total_time": 0.8713409900665283 + } + }, + "network_tracking_remote_file_reading.RemfileNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 302371, + "amount_uploaded_in_bytes": 6355, + "total_transfer_in_bytes": 308726, + "amount_downloaded_in_number_of_packets": 230, + "amount_uploaded_in_number_of_packets": 81, + "total_transfer_in_number_of_packets": 311, + "total_traffic_in_number_of_web_packets": 81, + "total_transfer_time_in_seconds": 0.287581, + "network_total_time": 1.1958057880401611 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 394673, + "amount_uploaded_in_bytes": 6051, + "total_transfer_in_bytes": 400724, + "amount_downloaded_in_number_of_packets": 301, + "amount_uploaded_in_number_of_packets": 69, + "total_transfer_in_number_of_packets": 370, + "total_traffic_in_number_of_web_packets": 69, + "total_transfer_time_in_seconds": 0.352166, + "network_total_time": 1.3695590496063232 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 83062, + "amount_uploaded_in_bytes": 3403, + "total_transfer_in_bytes": 86465, + "amount_downloaded_in_number_of_packets": 67, + "amount_uploaded_in_number_of_packets": 48, + "total_transfer_in_number_of_packets": 115, + "total_traffic_in_number_of_web_packets": 48, + "total_transfer_time_in_seconds": 0.12606099999999995, + "network_total_time": 1.0965180397033691 + } + }, + "network_tracking_remote_file_reading.Ros3DirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213816, + "amount_uploaded_in_bytes": 2399, + "total_transfer_in_bytes": 216215, + "amount_downloaded_in_number_of_packets": 155, + "amount_uploaded_in_number_of_packets": 27, + "total_transfer_in_number_of_packets": 182, + "total_traffic_in_number_of_web_packets": 27, + "total_transfer_time_in_seconds": 0.17862999999999998, + "network_total_time": 0.8210649490356445 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 31.338226079940796 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192932, + "amount_uploaded_in_bytes": 2465, + "total_transfer_in_bytes": 195397, + "amount_downloaded_in_number_of_packets": 141, + "amount_uploaded_in_number_of_packets": 28, + "total_transfer_in_number_of_packets": 169, + "total_traffic_in_number_of_web_packets": 28, + "total_transfer_time_in_seconds": 0.186069, + "network_total_time": 0.8099350929260254 + } + }, + "network_tracking_remote_file_reading.Ros3NWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 479453, + "amount_uploaded_in_bytes": 5886, + "total_transfer_in_bytes": 485339, + "amount_downloaded_in_number_of_packets": 358, + "amount_uploaded_in_number_of_packets": 69, + "total_transfer_in_number_of_packets": 427, + "total_traffic_in_number_of_web_packets": 69, + "total_transfer_time_in_seconds": 0.455936, + "network_total_time": 1.676819086074829 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 35155935, + "amount_uploaded_in_bytes": 131339, + "total_transfer_in_bytes": 35287274, + "amount_downloaded_in_number_of_packets": 23901, + "amount_uploaded_in_number_of_packets": 1640, + "total_transfer_in_number_of_packets": 25541, + "total_traffic_in_number_of_web_packets": 1640, + "total_transfer_time_in_seconds": 7.366164, + "network_total_time": 77.98207998275757 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 386882, + "amount_uploaded_in_bytes": 4972, + "total_transfer_in_bytes": 391854, + "amount_downloaded_in_number_of_packets": 288, + "amount_uploaded_in_number_of_packets": 57, + "total_transfer_in_number_of_packets": 345, + "total_traffic_in_number_of_web_packets": 57, + "total_transfer_time_in_seconds": 0.40694500000000006, + "network_total_time": 1.4816248416900635 + } + }, + "network_tracking_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 43636242, + "amount_uploaded_in_bytes": 45165, + "total_transfer_in_bytes": 43681407, + "amount_downloaded_in_number_of_packets": 29582, + "amount_uploaded_in_number_of_packets": 814, + "total_transfer_in_number_of_packets": 30396, + "total_traffic_in_number_of_web_packets": 814, + "total_transfer_time_in_seconds": 1.2273760000000002, + "network_total_time": 88.51429462432861 + } + }, + "network_tracking_remote_slicing.RemfileContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 26642362, + "amount_uploaded_in_bytes": 44394, + "total_transfer_in_bytes": 26686756, + "amount_downloaded_in_number_of_packets": 18076, + "amount_uploaded_in_number_of_packets": 751, + "total_transfer_in_number_of_packets": 18827, + "total_traffic_in_number_of_web_packets": 751, + "total_transfer_time_in_seconds": 3.540335, + "network_total_time": 56.298409938812256 + } + }, + "network_tracking_remote_slicing.Ros3ContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 21815543, + "amount_uploaded_in_bytes": 24016, + "total_transfer_in_bytes": 21839559, + "amount_downloaded_in_number_of_packets": 14806, + "amount_uploaded_in_number_of_packets": 394, + "total_transfer_in_number_of_packets": 15200, + "total_traffic_in_number_of_web_packets": 394, + "total_transfer_time_in_seconds": 0.466329, + "network_total_time": 43.48882031440735 + } + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.17715879203751683, + 0.176556000020355, + 0.18595758301671594 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.3508900419692509, + 0.4155230830074288, + 0.3198263749945909 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.17845275002764538, + 0.18187874997965991, + 0.1963200840400532 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.25080033304402605, + 0.2608719579875469, + 0.2644805000163615 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.34410829102853313, + 0.3690198750118725, + 0.41711995797231793 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.3083925420069136, + 0.29030449996935204, + 0.24265133298467845 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.17758229101309553, + 0.22222658403916284, + 0.17699337500380352 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.4846240420010872, + 0.4898677910096012, + 0.5192736249882728 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.16609825001796708, + 0.14171562500996515, + 0.19643812504364178 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.2392821250250563, + 0.23176312498981133, + 0.23621750000165775 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 1.7832427499815822, + 1.8307567500160076, + 1.655011959024705 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.19989087502472103, + 0.18279033299768344, + 0.18984599999384955 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.3017706669634208, + 0.3020462919957936, + 0.35886416700668633 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.35299637500429526, + 0.3754637090023607, + 0.3578742920071818 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.3394672920112498, + 0.3541580840246752, + 0.3476929999887943 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.48499466601060703, + 0.42012891598278657, + 0.4118519159965217 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 3.672362665995024, + 10.271374667005148, + 10.809336708975025 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.42910233297152445, + 0.3605557080009021, + 0.3629299160093069 + ] + }, + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.3956505000242032, + 0.7865626669954509, + 0.9384660830255598 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.7423827919992618, + 0.813982457970269, + 0.6474876659922302 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.005855499999597669, + 0.0019187919679097831, + 0.0018848750041797757 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmarkProcessTime.time_slice_clock": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.005695999999999479, + 0.001799000000000106, + 0.0015710000000002111 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmarkRawTime.time_slice_raw_time": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.005246877670288086, + 0.001558065414428711, + 0.0019178390502929688 + ] + } + } +} \ No newline at end of file From 71eeac13025776a13d82ca46d778f1795670d902 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 04:37:45 +0000 Subject: [PATCH 54/72] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ...46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json | 2 +- src/nwb_benchmarks/setup/_reduce_results.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/results/results_timestamp-2024-02-20-23-19-37_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-20-23-19-37_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json index 89f6343..dbcb3df 100644 --- a/results/results_timestamp-2024-02-20-23-19-37_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json +++ b/results/results_timestamp-2024-02-20-23-19-37_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -392,4 +392,4 @@ ] } } -} \ No newline at end of file +} diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 1c618e8..f6b5f6d 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -6,7 +6,6 @@ import json import pathlib import shutil -import sys import subprocess import sys from typing import Dict, List From c151aedfda1b930485b75e9d06e539b3fa02a5c7 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 23:38:10 -0500 Subject: [PATCH 55/72] remove comments --- src/nwb_benchmarks/setup/_reduce_results.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 1c618e8..81d7183 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -110,9 +110,6 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil json.dump(obj=parsed_environment_info, fp=io, indent=4) # Network tests require admin permissions, which can alter write permissions of any files created - # os.chmod(path=parsed_results_file, mode=stat.S_IROTH | stat.S_IWOTH) - # os.chmod(path=machine_info_file_path, mode=stat.S_IROTH | stat.S_IWOTH) - # os.chmod(path=parsed_environment_file_path, mode=stat.S_IROTH | stat.S_IWOTH) subprocess.run(["chmod", "-R", "+rw", parsed_results_file]) subprocess.run(["chmod", "-R", "+rw", machine_info_file_path]) subprocess.run(["chmod", "-R", "+rw", parsed_environment_file_path]) From 6a15904888f73569ab6d33836623c1cfe9ab2462 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 23:43:36 -0500 Subject: [PATCH 56/72] nest in platform check --- src/nwb_benchmarks/setup/_reduce_results.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 60903bd..71c39ae 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -109,8 +109,9 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil json.dump(obj=parsed_environment_info, fp=io, indent=4) # Network tests require admin permissions, which can alter write permissions of any files created - subprocess.run(["chmod", "-R", "+rw", parsed_results_file]) - subprocess.run(["chmod", "-R", "+rw", machine_info_file_path]) - subprocess.run(["chmod", "-R", "+rw", parsed_environment_file_path]) + if sys.platform in ["darwin", "linux"]: + subprocess.run(["chmod", "-R", "+rw", parsed_results_file]) + subprocess.run(["chmod", "-R", "+rw", machine_info_file_path]) + subprocess.run(["chmod", "-R", "+rw", parsed_environment_file_path]) raw_results_file_path.unlink() From 6488e4cf1eae324d8e17e9033c5b3b7f9ee9670d Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 23:48:22 -0500 Subject: [PATCH 57/72] add tshak install to setup --- docs/setup.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/setup.rst b/docs/setup.rst index 47e990d..b0c1f01 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -20,3 +20,5 @@ Setup initial machine configuration values with .. code-block:: nwb_benchmarks setup + +You will also need to install the custom network tracking software ``tshark`` using `their instructions `_. From e100a022e35a3dc7ecfc75deab8b511829be36d4 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 20 Feb 2024 23:49:26 -0500 Subject: [PATCH 58/72] add tshak install to setup --- docs/running_benchmarks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index 023725d..3c0801b 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -14,7 +14,7 @@ use `psutil net_connections Date: Tue, 20 Feb 2024 23:55:15 -0500 Subject: [PATCH 59/72] move TSHARK path to top level --- src/nwb_benchmarks/__init__.py | 6 ++++++ .../benchmarks/network_tracking_remote_file_reading.py | 3 +-- .../benchmarks/network_tracking_remote_slicing.py | 6 +----- src/nwb_benchmarks/core/_network_tracker.py | 5 +---- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/nwb_benchmarks/__init__.py b/src/nwb_benchmarks/__init__.py index 9f6e6ba..96ac8e2 100644 --- a/src/nwb_benchmarks/__init__.py +++ b/src/nwb_benchmarks/__init__.py @@ -1 +1,7 @@ +"""Outermost exposed imports; including global environment variables.""" + +import os + from .command_line_interface import main + +TSHARK_PATH = os.environ.get("TSHARK_PATH", None) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py index de016bf..a5772fd 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -2,6 +2,7 @@ import os +from nwb_benchmarks import TSHARK_PATH from nwb_benchmarks.core import ( get_s3_url, network_activity_tracker, @@ -13,8 +14,6 @@ read_hdf5_ros3, ) -TSHARK_PATH = os.environ.get("TSHARK_PATH", None) - param_names = ["s3_url"] params = [ get_s3_url(dandiset_id="000717", dandi_path="sub-mock/sub-mock_ses-ecephys1.nwb"), diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index 12b623b..71df8d9 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -1,10 +1,8 @@ """Basic benchmarks for profiling network statistics for streaming access to slices of data stored in NWB files.""" -import os from typing import Tuple -from asv.runner import BenchmarkResult - +from nwb_benchmarks import TSHARK_PATH from nwb_benchmarks.core import ( get_object_by_name, get_s3_url, @@ -15,8 +13,6 @@ robust_ros3_read, ) -TSHARK_PATH = os.environ.get("TSHARK_PATH", None) - param_names = ["s3_url", "object_name", "slice_range"] params = ( [ diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index 278855c..92644cd 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -1,7 +1,4 @@ -""" -Network tracker for capturing network traffic and performance metrics during the execution of particular -methods or code snippets. -""" +"""Network tracker for capturing traffic and performance metrics during the execution of methods or code snippets.""" import contextlib import os From 00b6fa99ec23dfd9d6c276c5d284f378869d018f Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 21 Feb 2024 00:20:42 -0500 Subject: [PATCH 60/72] add retry counter; remove extraneous ros3 timers --- ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 395 ++++++++++++++++++ .../benchmarks/time_remote_slicing.py | 37 +- src/nwb_benchmarks/core/_network_tracker.py | 2 +- src/nwb_benchmarks/core/_nwb_helpers.py | 6 +- src/nwb_benchmarks/core/_streaming.py | 63 ++- 5 files changed, 452 insertions(+), 51 deletions(-) create mode 100644 results/results_timestamp-2024-02-20-23-55-15_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json diff --git a/results/results_timestamp-2024-02-20-23-55-15_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-20-23-55-15_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json new file mode 100644 index 0000000..7a31d1d --- /dev/null +++ b/results/results_timestamp-2024-02-20-23-55-15_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -0,0 +1,395 @@ +{ + "version": 2, + "timestamp": "2024-02-20-23-55-15", + "commit_hash": "e2896c110214db26030708ca60b5d79ecc342108", + "environment_hash": "2d1c68a2c7de02ab2c200e448b250bee5e46f212", + "machine_hash": "e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46", + "results": { + "network_tracking_remote_file_reading.FsspecNoCacheDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213465, + "amount_uploaded_in_bytes": 2808, + "total_transfer_in_bytes": 216273, + "amount_downloaded_in_number_of_packets": 157, + "amount_uploaded_in_number_of_packets": 31, + "total_transfer_in_number_of_packets": 188, + "total_traffic_in_number_of_web_packets": 31, + "total_transfer_time_in_seconds": 0.23523000000000002, + "network_total_time": 0.9398150444030762 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 5640155, + "amount_uploaded_in_bytes": 8598, + "total_transfer_in_bytes": 5648753, + "amount_downloaded_in_number_of_packets": 3819, + "amount_uploaded_in_number_of_packets": 135, + "total_transfer_in_number_of_packets": 3954, + "total_traffic_in_number_of_web_packets": 135, + "total_transfer_time_in_seconds": 0.47888500000000006, + "network_total_time": 12.147579193115234 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192322, + "amount_uploaded_in_bytes": 2278, + "total_transfer_in_bytes": 194600, + "amount_downloaded_in_number_of_packets": 141, + "amount_uploaded_in_number_of_packets": 24, + "total_transfer_in_number_of_packets": 165, + "total_traffic_in_number_of_web_packets": 24, + "total_transfer_time_in_seconds": 0.210496, + "network_total_time": 0.8345870971679688 + } + }, + "network_tracking_remote_file_reading.FsspecNoCacheNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213376, + "amount_uploaded_in_bytes": 2624, + "total_transfer_in_bytes": 216000, + "amount_downloaded_in_number_of_packets": 157, + "amount_uploaded_in_number_of_packets": 29, + "total_transfer_in_number_of_packets": 186, + "total_traffic_in_number_of_web_packets": 29, + "total_transfer_time_in_seconds": 0.24099199999999998, + "network_total_time": 0.9193868637084961 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 44979311, + "amount_uploaded_in_bytes": 125914, + "total_transfer_in_bytes": 45105225, + "amount_downloaded_in_number_of_packets": 30336, + "amount_uploaded_in_number_of_packets": 2120, + "total_transfer_in_number_of_packets": 32456, + "total_traffic_in_number_of_web_packets": 2120, + "total_transfer_time_in_seconds": 16.123603999999997, + "network_total_time": 104.36652302742004 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192376, + "amount_uploaded_in_bytes": 2548, + "total_transfer_in_bytes": 194924, + "amount_downloaded_in_number_of_packets": 142, + "amount_uploaded_in_number_of_packets": 29, + "total_transfer_in_number_of_packets": 171, + "total_traffic_in_number_of_web_packets": 29, + "total_transfer_time_in_seconds": 0.199737, + "network_total_time": 0.8429298400878906 + } + }, + "network_tracking_remote_file_reading.RemfileDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 88699, + "amount_uploaded_in_bytes": 3495, + "total_transfer_in_bytes": 92194, + "amount_downloaded_in_number_of_packets": 70, + "amount_uploaded_in_number_of_packets": 49, + "total_transfer_in_number_of_packets": 119, + "total_traffic_in_number_of_web_packets": 49, + "total_transfer_time_in_seconds": 0.15773200000000004, + "network_total_time": 1.0658478736877441 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 308913, + "amount_uploaded_in_bytes": 4367, + "total_transfer_in_bytes": 313280, + "amount_downloaded_in_number_of_packets": 232, + "amount_uploaded_in_number_of_packets": 44, + "total_transfer_in_number_of_packets": 276, + "total_traffic_in_number_of_web_packets": 44, + "total_transfer_time_in_seconds": 0.413258, + "network_total_time": 1.3433141708374023 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 79044, + "amount_uploaded_in_bytes": 3133, + "total_transfer_in_bytes": 82177, + "amount_downloaded_in_number_of_packets": 65, + "amount_uploaded_in_number_of_packets": 43, + "total_transfer_in_number_of_packets": 108, + "total_traffic_in_number_of_web_packets": 43, + "total_transfer_time_in_seconds": 0.13951299999999997, + "network_total_time": 0.9029419422149658 + } + }, + "network_tracking_remote_file_reading.RemfileNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 293867, + "amount_uploaded_in_bytes": 6308, + "total_transfer_in_bytes": 300175, + "amount_downloaded_in_number_of_packets": 224, + "amount_uploaded_in_number_of_packets": 77, + "total_transfer_in_number_of_packets": 301, + "total_traffic_in_number_of_web_packets": 77, + "total_transfer_time_in_seconds": 0.393575, + "network_total_time": 1.353158950805664 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 374948, + "amount_uploaded_in_bytes": 5987, + "total_transfer_in_bytes": 380935, + "amount_downloaded_in_number_of_packets": 282, + "amount_uploaded_in_number_of_packets": 69, + "total_transfer_in_number_of_packets": 351, + "total_traffic_in_number_of_web_packets": 69, + "total_transfer_time_in_seconds": 0.40197499999999997, + "network_total_time": 1.4983501434326172 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 273868, + "amount_uploaded_in_bytes": 6032, + "total_transfer_in_bytes": 279900, + "amount_downloaded_in_number_of_packets": 210, + "amount_uploaded_in_number_of_packets": 74, + "total_transfer_in_number_of_packets": 284, + "total_traffic_in_number_of_web_packets": 74, + "total_transfer_time_in_seconds": 0.35297199999999995, + "network_total_time": 1.2297568321228027 + } + }, + "network_tracking_remote_file_reading.Ros3DirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 78, + "total_transfer_in_bytes": 78, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 1, + "total_transfer_in_number_of_packets": 1, + "total_traffic_in_number_of_web_packets": 1, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.39023590087890625 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 37.39911913871765 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.35997891426086426 + } + }, + "network_tracking_remote_file_reading.Ros3NWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 478927, + "amount_uploaded_in_bytes": 5702, + "total_transfer_in_bytes": 484629, + "amount_downloaded_in_number_of_packets": 350, + "amount_uploaded_in_number_of_packets": 67, + "total_transfer_in_number_of_packets": 417, + "total_traffic_in_number_of_web_packets": 67, + "total_transfer_time_in_seconds": 0.41292199999999996, + "network_total_time": 1.784958839416504 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 35066801, + "amount_uploaded_in_bytes": 113848, + "total_transfer_in_bytes": 35180649, + "amount_downloaded_in_number_of_packets": 23795, + "amount_uploaded_in_number_of_packets": 1523, + "total_transfer_in_number_of_packets": 25318, + "total_traffic_in_number_of_web_packets": 1523, + "total_transfer_time_in_seconds": 10.843134000000001, + "network_total_time": 73.67179703712463 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 386694, + "amount_uploaded_in_bytes": 5242, + "total_transfer_in_bytes": 391936, + "amount_downloaded_in_number_of_packets": 285, + "amount_uploaded_in_number_of_packets": 62, + "total_transfer_in_number_of_packets": 347, + "total_traffic_in_number_of_web_packets": 62, + "total_transfer_time_in_seconds": 0.381419, + "network_total_time": 1.5194900035858154 + } + }, + "network_tracking_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 43643298, + "amount_uploaded_in_bytes": 119431, + "total_transfer_in_bytes": 43762729, + "amount_downloaded_in_number_of_packets": 29379, + "amount_uploaded_in_number_of_packets": 2155, + "total_transfer_in_number_of_packets": 31534, + "total_traffic_in_number_of_web_packets": 2155, + "total_transfer_time_in_seconds": 17.965873999999996, + "network_total_time": 102.67956304550171 + } + }, + "network_tracking_remote_slicing.RemfileContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 26703165, + "amount_uploaded_in_bytes": 23390, + "total_transfer_in_bytes": 26726555, + "amount_downloaded_in_number_of_packets": 18769, + "amount_uploaded_in_number_of_packets": 347, + "total_transfer_in_number_of_packets": 19116, + "total_traffic_in_number_of_web_packets": 347, + "total_transfer_time_in_seconds": 1.017185, + "network_total_time": 55.94029116630554 + } + }, + "network_tracking_remote_slicing.Ros3ContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 21835664, + "amount_uploaded_in_bytes": 17957, + "total_transfer_in_bytes": 21853621, + "amount_downloaded_in_number_of_packets": 14869, + "amount_uploaded_in_number_of_packets": 322, + "total_transfer_in_number_of_packets": 15191, + "total_traffic_in_number_of_web_packets": 322, + "total_transfer_time_in_seconds": 0.5984780000000001, + "network_total_time": 45.13759112358093 + } + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.18535041698487476, + 0.1558114580111578, + 0.1751023749820888 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 1.76780225004768, + 0.34658087498974055, + 2.3738495410070755 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.19912083400413394, + 0.20313024998176843, + 0.1708520830143243 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.25597366702277213, + 0.2717441249988042, + 0.306006666040048 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.2946432909811847, + 0.30996066599618644, + 0.317540124990046 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.27507241698913276, + 0.27276237501064315, + 0.29841270798351616 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.3639707920374349, + 0.1841727080172859, + 0.18496062501799315 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.48805195902241394, + 0.4446351250517182, + 0.4324715419788845 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.18401004100451246, + 0.17305520799709484, + 0.17850824998458847 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.2768035840126686, + 0.25942616595420986, + 0.24880333297187462 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 2.3120491669978946, + 1.7823654999956489, + 18.306173916964326 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.219203665968962, + 0.23996062501100823, + 0.20788229198660702 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.33586708398070186, + 0.35359166603302583, + 0.641282832948491 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.40897937497356907, + 0.41631520801456645, + 0.41940587502904236 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.37571987498085946, + 0.3541067090118304, + 0.360793832980562 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.44721474999096245, + 0.48484758398262784, + 0.5628422090085223 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 2.8786464169970714, + 1.065437707991805, + 4.620046124968212 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.38613225001608953, + 0.32727620902005583, + 0.37022366700693965 + ] + }, + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 1.071321709023323, + 0.7801771659869701, + 0.7377782499534078 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.7397622079588473, + 0.7443573750206269, + 0.7445429999497719 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.005742750014178455, + 0.001033790991641581, + 0.002084582985844463 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmarkProcessTime.time_slice_clock": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.0061109999999997555, + 0.001930999999999905, + 0.0019439999999999458 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmarkRawTime.time_slice_raw_time": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.0023369789123535156, + 0.0012688636779785156, + 0.0012538433074951172 + ] + } + } +} diff --git a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py index b589ffd..d2a83bf 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -9,7 +9,6 @@ read_hdf5_nwbfile_fsspec_no_cache, read_hdf5_nwbfile_remfile, read_hdf5_nwbfile_ros3, - robust_ros3_read, ) # TODO: add the others @@ -72,38 +71,4 @@ def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): def time_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): """Note: store as self._temp to avoid tracking garbage collection as well.""" - self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) - - -class Ros3ContinuousSliceBenchmarkProcessTime: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - timer = time.process_time # Default timer is timeit.default_timer - - def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) - self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") - self.data_to_slice = self.neurodata_object.data - - def time_slice_clock(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - """Note: store as self._temp to avoid tracking garbage collection as well.""" - self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) - - -class Ros3ContinuousSliceBenchmarkRawTime: - rounds = 1 - repeat = 3 - param_names = param_names - params = params - timer = time.time - - def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) - self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") - self.data_to_slice = self.neurodata_object.data - - def time_slice_raw_time(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - """Note: store as self._temp to avoid tracking garbage collection as well.""" - self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) + self.data_to_slice[slice_range] diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index 92644cd..7e4021b 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -1,4 +1,4 @@ -"""Network tracker for capturing traffic and performance metrics during the execution of methods or code snippets.""" +"""Network tracker for capturing traffic and performance metrics during the execution of methods or code snippets asdasdadad.""" import contextlib import os diff --git a/src/nwb_benchmarks/core/_nwb_helpers.py b/src/nwb_benchmarks/core/_nwb_helpers.py index dd8d30f..ecf3fb7 100644 --- a/src/nwb_benchmarks/core/_nwb_helpers.py +++ b/src/nwb_benchmarks/core/_nwb_helpers.py @@ -4,7 +4,11 @@ def get_object_by_name(nwbfile: pynwb.NWBFile, object_name: str) -> Any: - """Simple helper function to retrieve a neurodata object by its name, if it is unique.""" + """ + Simple helper function to retrieve a neurodata object by its name, if it is unique. + + This method should only be used in the `setup` method of a benchmark class. + """ object_names = [neurodata_object.name for neurodata_object in nwbfile.objects.values()] assert len(object_names) == len( set(object_names) diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py index 02bcea0..a765a22 100644 --- a/src/nwb_benchmarks/core/_streaming.py +++ b/src/nwb_benchmarks/core/_streaming.py @@ -57,18 +57,25 @@ def robust_ros3_read( max_retries: int = 20, command_args: Union[list, None] = None, command_kwargs: Union[dict, None] = None, -) -> Any: +) -> Tuple[Any, int]: """ Attempt the command (usually acting on an S3 IO) up to the number of max_retries using exponential backoff. Usually a good idea to use this to wrap any operations performed using the ROS3 driver. + + Returns + ------- + result : Any + The object returned by running the comand. + retries : int + The number of retries. """ command_args = command_args or [] command_kwargs = command_kwargs or dict() - for retry in range(max_retries): + for retries, retry in enumerate(range(max_retries)): try: result = command(*command_args, **command_kwargs) - return result + return (retries, result) except Exception as exception: # 'cannot curl request' can show up in potentially many different error types if "curl" not in str(exception): @@ -78,19 +85,49 @@ def robust_ros3_read( raise TimeoutError(f"Unable to complete the command ({command.__name__}) after {max_retries} attempts!") -def read_hdf5_ros3(s3_url: str) -> h5py.File: - """Load the raw HDF5 file from an S3 URL using ROS3 driver; does not formally read the NWB file.""" +def read_hdf5_ros3(s3_url: str, retry: bool = True) -> Tuple[h5py.File, Union[int, None]]: + """ + Load the raw HDF5 file from an S3 URL using ROS3 driver; does not formally read the NWB file. + + Returns + ------- + file : h5py.File + The remote HDF5 file object. + retries : int + The number of retries, if `retry` is `True`. + """ ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") aws_region = bytes("us-east-2", "ascii") # TODO: generalize this as an argument if necessary - file = robust_ros3_read( - command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3", aws_region=aws_region) - ) - return file + if retry: + file, retries = robust_ros3_read( + command=h5py.File, command_kwargs=dict(name=ros3_form, driver="ros3", aws_region=aws_region) + ) + else: + retries = None + file = h5py.File(name=ros3_form, driver="ros3", aws_region=aws_region) + return (file, retries) -def read_hdf5_nwbfile_ros3(s3_url: str) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO]: - """Read an HDF5 NWB file from an S3 URL using the ROS3 driver from h5py.""" +def read_hdf5_nwbfile_ros3(s3_url: str, retry: bool = True) -> Tuple[pynwb.NWBFile, pynwb.NWBHDF5IO, Union[int, None]]: + """ + Read an HDF5 NWB file from an S3 URL using the ROS3 driver from h5py. + + Returns + ------- + NWBFile : pynwb.NWBFile + The remote NWBFile object. + NWBHDF5IO : pynwb.NWBHDF5IO + The open IO object used to open the file. Must be kept to ensure garbage collection does not prematurely close + the IO. + retries : int + The number of retries, if `retry` is `True`. + """ ros3_form = s3_url.replace("https://dandiarchive.s3.amazonaws.com", "s3://dandiarchive") io = pynwb.NWBHDF5IO(path=ros3_form, mode="r", load_namespaces=True, driver="ros3") - nwbfile = robust_ros3_read(command=io.read) - return (nwbfile, io) + + if retry: + nwbfile, retries = robust_ros3_read(command=io.read) + else: + retries = None + nwbfile = io.read() + return (nwbfile, io, retries) From 7dd894558a2ddab2ffe3b704fe07810c01c2537b Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 21 Feb 2024 00:25:44 -0500 Subject: [PATCH 61/72] disable retry in timing; track retries in network --- .../network_tracking_remote_file_reading.py | 30 ++++++++++--------- .../network_tracking_remote_slicing.py | 3 +- .../benchmarks/time_remote_file_reading.py | 4 +-- src/nwb_benchmarks/setup/_reduce_results.py | 6 ++-- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py index a5772fd..12a4c0c 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_file_reading.py @@ -30,9 +30,9 @@ class FsspecNoCacheDirectFileReadBenchmark: params = params def track_network_activity_during_read(self, s3_url: str): - with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self.file, self.bytestream = read_hdf5_fsspec_no_cache(s3_url=s3_url) - return network_activity.asv_network_statistics + return network_tracker.asv_network_statistics class RemfileDirectFileReadBenchmark: @@ -40,9 +40,9 @@ class RemfileDirectFileReadBenchmark: params = params def track_network_activity_during_read(self, s3_url: str): - with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) - return network_activity.asv_network_statistics + return network_tracker.asv_network_statistics class Ros3DirectFileReadBenchmark: @@ -50,9 +50,10 @@ class Ros3DirectFileReadBenchmark: params = params def track_network_activity_during_read(self, s3_url: str): - with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: - self.file = read_hdf5_ros3(s3_url=s3_url) - return network_activity.asv_network_statistics + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: + self.file, retries = read_hdf5_ros3(s3_url=s3_url) + network_tracker.asv_network_statistics.update(retries=retries) + return network_tracker.asv_network_statistics class FsspecNoCacheNWBFileReadBenchmark: @@ -60,9 +61,9 @@ class FsspecNoCacheNWBFileReadBenchmark: params = params def track_network_activity_during_read(self, s3_url: str): - with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_fsspec_no_cache(s3_url=s3_url) - return network_activity.asv_network_statistics + return network_tracker.asv_network_statistics class RemfileNWBFileReadBenchmark: @@ -70,9 +71,9 @@ class RemfileNWBFileReadBenchmark: params = params def track_network_activity_during_read(self, s3_url: str): - with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) - return network_activity.asv_network_statistics + return network_tracker.asv_network_statistics class Ros3NWBFileReadBenchmark: @@ -80,6 +81,7 @@ class Ros3NWBFileReadBenchmark: params = params def track_network_activity_during_read(self, s3_url: str): - with network_activity_tracker(tshark_path=TSHARK_PATH) as network_activity: - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) - return network_activity.asv_network_statistics + with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: + self.nwbfile, self.io, retries = read_hdf5_nwbfile_ros3(s3_url=s3_url) + network_tracker.asv_network_statistics.update(retries=retries) + return network_tracker.asv_network_statistics diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index 71df8d9..a98f9a1 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -67,5 +67,6 @@ def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): def track_network_activity_during_slice(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): with network_activity_tracker(tshark_path=TSHARK_PATH) as network_tracker: - self._temp = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) + self._temp, retries = robust_ros3_read(command=self.data_to_slice.__getitem__, command_args=(slice_range,)) + network_tracker.asv_network_statistics.update(retries=retries) return network_tracker.asv_network_statistics diff --git a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py index 59cc799..f55ab13 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py @@ -42,7 +42,7 @@ def time_read_hdf5_remfile(self, s3_url: str): self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) def time_read_hdf5_ros3(self, s3_url: str): - self.file = read_hdf5_ros3(s3_url=s3_url) + self.file = read_hdf5_ros3(s3_url=s3_url, retry=False) class NWBFileReadBenchmark: @@ -64,4 +64,4 @@ def time_read_hdf5_nwbfile_remfile(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) def time_read_hdf5_nwbfile_ros3(self, s3_url: str): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) + self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url, retry=False) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 71c39ae..41b482d 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -110,8 +110,8 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil # Network tests require admin permissions, which can alter write permissions of any files created if sys.platform in ["darwin", "linux"]: - subprocess.run(["chmod", "-R", "+rw", parsed_results_file]) - subprocess.run(["chmod", "-R", "+rw", machine_info_file_path]) - subprocess.run(["chmod", "-R", "+rw", parsed_environment_file_path]) + subprocess.run(["chmod", "-R", "+rw", parsed_results_file.absolute()]) + subprocess.run(["chmod", "-R", "+rw", machine_info_file_path.absolute()]) + subprocess.run(["chmod", "-R", "+rw", parsed_environment_file_path.absolute()]) raw_results_file_path.unlink() From 742eb6aeb0b6f2eb07082a58bfa1e775ca1cd6bd Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 21 Feb 2024 00:27:09 -0500 Subject: [PATCH 62/72] Update src/nwb_benchmarks/core/_network_tracker.py Co-authored-by: Oliver Ruebel --- src/nwb_benchmarks/core/_network_tracker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index 7e4021b..2637d7a 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -13,6 +13,7 @@ @contextlib.contextmanager def network_activity_tracker(tshark_path: Union[pathlib.Path, None] = None): + """Context manager for tracking network activity and statistics for the code executed in the context""" network_tracker = NetworkTracker() try: From f7b946047f878e982a342e8b3d352051d4c3b773 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 21 Feb 2024 00:37:27 -0500 Subject: [PATCH 63/72] debug --- src/nwb_benchmarks/core/_streaming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/core/_streaming.py b/src/nwb_benchmarks/core/_streaming.py index a765a22..5205745 100644 --- a/src/nwb_benchmarks/core/_streaming.py +++ b/src/nwb_benchmarks/core/_streaming.py @@ -75,7 +75,7 @@ def robust_ros3_read( for retries, retry in enumerate(range(max_retries)): try: result = command(*command_args, **command_kwargs) - return (retries, result) + return (result, retries) except Exception as exception: # 'cannot curl request' can show up in potentially many different error types if "curl" not in str(exception): From f8d22146ba6b3a6e35496ee2da69515e372f0c18 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 21 Feb 2024 00:46:11 -0500 Subject: [PATCH 64/72] debug --- .../benchmarks/network_tracking_remote_slicing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py index a98f9a1..52479a3 100644 --- a/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/network_tracking_remote_slicing.py @@ -61,7 +61,7 @@ class Ros3ContinuousSliceBenchmark: params = params def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) + self.nwbfile, self.io, _ = read_hdf5_nwbfile_ros3(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data From d888ad839226c15009deae1732eb40e4d3967f03 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 21 Feb 2024 01:58:50 -0500 Subject: [PATCH 65/72] debug --- src/nwb_benchmarks/benchmarks/time_remote_file_reading.py | 4 ++-- src/nwb_benchmarks/benchmarks/time_remote_slicing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py index f55ab13..3f61583 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_file_reading.py @@ -42,7 +42,7 @@ def time_read_hdf5_remfile(self, s3_url: str): self.file, self.bytestream = read_hdf5_remfile(s3_url=s3_url) def time_read_hdf5_ros3(self, s3_url: str): - self.file = read_hdf5_ros3(s3_url=s3_url, retry=False) + self.file, _ = read_hdf5_ros3(s3_url=s3_url, retry=False) class NWBFileReadBenchmark: @@ -64,4 +64,4 @@ def time_read_hdf5_nwbfile_remfile(self, s3_url: str): self.nwbfile, self.io, self.file, self.bytestream = read_hdf5_nwbfile_remfile(s3_url=s3_url) def time_read_hdf5_nwbfile_ros3(self, s3_url: str): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url, retry=False) + self.nwbfile, self.io, _ = read_hdf5_nwbfile_ros3(s3_url=s3_url, retry=False) diff --git a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py index d2a83bf..3df896c 100644 --- a/src/nwb_benchmarks/benchmarks/time_remote_slicing.py +++ b/src/nwb_benchmarks/benchmarks/time_remote_slicing.py @@ -65,7 +65,7 @@ class Ros3ContinuousSliceBenchmark: params = params def setup(self, s3_url: str, object_name: str, slice_range: Tuple[slice]): - self.nwbfile, self.io = read_hdf5_nwbfile_ros3(s3_url=s3_url) + self.nwbfile, self.io, _ = read_hdf5_nwbfile_ros3(s3_url=s3_url) self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name="ElectricalSeriesAp") self.data_to_slice = self.neurodata_object.data From 6d49c50e0c16f50ad942b1be3f72cc9638014b88 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Wed, 21 Feb 2024 01:59:47 -0500 Subject: [PATCH 66/72] debug and new results --- ...c68a2c7de02ab2c200e448b250bee5e46f212.json | 381 ++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 results/results_timestamp-2024-02-21-00-46-11_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json diff --git a/results/results_timestamp-2024-02-21-00-46-11_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json b/results/results_timestamp-2024-02-21-00-46-11_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json new file mode 100644 index 0000000..8a00c5b --- /dev/null +++ b/results/results_timestamp-2024-02-21-00-46-11_machine-e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46_environment-2d1c68a2c7de02ab2c200e448b250bee5e46f212.json @@ -0,0 +1,381 @@ +{ + "version": 2, + "timestamp": "2024-02-21-00-46-11", + "commit_hash": "f8d22146ba6b3a6e35496ee2da69515e372f0c18", + "environment_hash": "2d1c68a2c7de02ab2c200e448b250bee5e46f212", + "machine_hash": "e9c0f1370ec8dc3640d91c7d0f381a6d61bd9a46", + "results": { + "network_tracking_remote_file_reading.FsspecNoCacheDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213430, + "amount_uploaded_in_bytes": 2894, + "total_transfer_in_bytes": 216324, + "amount_downloaded_in_number_of_packets": 158, + "amount_uploaded_in_number_of_packets": 34, + "total_transfer_in_number_of_packets": 192, + "total_traffic_in_number_of_web_packets": 34, + "total_transfer_time_in_seconds": 0.24709400000000004, + "network_total_time": 0.8842370510101318 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 5642361, + "amount_uploaded_in_bytes": 10380, + "total_transfer_in_bytes": 5652741, + "amount_downloaded_in_number_of_packets": 3824, + "amount_uploaded_in_number_of_packets": 168, + "total_transfer_in_number_of_packets": 3992, + "total_traffic_in_number_of_web_packets": 168, + "total_transfer_time_in_seconds": 0.42717200000000005, + "network_total_time": 11.959815979003906 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192465, + "amount_uploaded_in_bytes": 2690, + "total_transfer_in_bytes": 195155, + "amount_downloaded_in_number_of_packets": 142, + "amount_uploaded_in_number_of_packets": 30, + "total_transfer_in_number_of_packets": 172, + "total_traffic_in_number_of_web_packets": 30, + "total_transfer_time_in_seconds": 0.21733200000000003, + "network_total_time": 0.8138318061828613 + } + }, + "network_tracking_remote_file_reading.FsspecNoCacheNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213376, + "amount_uploaded_in_bytes": 2580, + "total_transfer_in_bytes": 215956, + "amount_downloaded_in_number_of_packets": 157, + "amount_uploaded_in_number_of_packets": 27, + "total_transfer_in_number_of_packets": 184, + "total_traffic_in_number_of_web_packets": 27, + "total_transfer_time_in_seconds": 0.19700200000000007, + "network_total_time": 0.8540201187133789 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 44951282, + "amount_uploaded_in_bytes": 90717, + "total_transfer_in_bytes": 45041999, + "amount_downloaded_in_number_of_packets": 30731, + "amount_uploaded_in_number_of_packets": 1459, + "total_transfer_in_number_of_packets": 32190, + "total_traffic_in_number_of_web_packets": 1459, + "total_transfer_time_in_seconds": 1.778902, + "network_total_time": 91.48940324783325 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 192432, + "amount_uploaded_in_bytes": 2720, + "total_transfer_in_bytes": 195152, + "amount_downloaded_in_number_of_packets": 143, + "amount_uploaded_in_number_of_packets": 31, + "total_transfer_in_number_of_packets": 174, + "total_traffic_in_number_of_web_packets": 31, + "total_transfer_time_in_seconds": 0.20254, + "network_total_time": 0.8090779781341553 + } + }, + "network_tracking_remote_file_reading.RemfileDirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 88424, + "amount_uploaded_in_bytes": 3495, + "total_transfer_in_bytes": 91919, + "amount_downloaded_in_number_of_packets": 70, + "amount_uploaded_in_number_of_packets": 49, + "total_transfer_in_number_of_packets": 119, + "total_traffic_in_number_of_web_packets": 49, + "total_transfer_time_in_seconds": 0.14394099999999996, + "network_total_time": 0.9140260219573975 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 299776, + "amount_uploaded_in_bytes": 6690, + "total_transfer_in_bytes": 306466, + "amount_downloaded_in_number_of_packets": 232, + "amount_uploaded_in_number_of_packets": 83, + "total_transfer_in_number_of_packets": 315, + "total_traffic_in_number_of_web_packets": 83, + "total_transfer_time_in_seconds": 0.40701300000000007, + "network_total_time": 1.2917938232421875 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 202789, + "amount_uploaded_in_bytes": 4129, + "total_transfer_in_bytes": 206918, + "amount_downloaded_in_number_of_packets": 159, + "amount_uploaded_in_number_of_packets": 46, + "total_transfer_in_number_of_packets": 205, + "total_traffic_in_number_of_web_packets": 46, + "total_transfer_time_in_seconds": 0.328635, + "network_total_time": 0.9448418617248535 + } + }, + "network_tracking_remote_file_reading.RemfileNWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 270274, + "amount_uploaded_in_bytes": 5483, + "total_transfer_in_bytes": 275757, + "amount_downloaded_in_number_of_packets": 211, + "amount_uploaded_in_number_of_packets": 63, + "total_transfer_in_number_of_packets": 274, + "total_traffic_in_number_of_web_packets": 63, + "total_transfer_time_in_seconds": 0.29056000000000004, + "network_total_time": 1.1204347610473633 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 387059, + "amount_uploaded_in_bytes": 5569, + "total_transfer_in_bytes": 392628, + "amount_downloaded_in_number_of_packets": 290, + "amount_uploaded_in_number_of_packets": 58, + "total_transfer_in_number_of_packets": 348, + "total_traffic_in_number_of_web_packets": 58, + "total_transfer_time_in_seconds": 0.405108, + "network_total_time": 1.4558892250061035 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 272399, + "amount_uploaded_in_bytes": 5816, + "total_transfer_in_bytes": 278215, + "amount_downloaded_in_number_of_packets": 208, + "amount_uploaded_in_number_of_packets": 70, + "total_transfer_in_number_of_packets": 278, + "total_traffic_in_number_of_web_packets": 70, + "total_transfer_time_in_seconds": 0.33613400000000004, + "network_total_time": 1.176321029663086 + } + }, + "network_tracking_remote_file_reading.Ros3DirectFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 213921, + "amount_uploaded_in_bytes": 2649, + "total_transfer_in_bytes": 216570, + "amount_downloaded_in_number_of_packets": 155, + "amount_uploaded_in_number_of_packets": 30, + "total_transfer_in_number_of_packets": 185, + "total_traffic_in_number_of_web_packets": 30, + "total_transfer_time_in_seconds": 0.177023, + "network_total_time": 0.8193769454956055 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 17544886, + "amount_uploaded_in_bytes": 44017, + "total_transfer_in_bytes": 17588903, + "amount_downloaded_in_number_of_packets": 11898, + "amount_uploaded_in_number_of_packets": 561, + "total_transfer_in_number_of_packets": 12459, + "total_traffic_in_number_of_web_packets": 561, + "total_transfer_time_in_seconds": 0.5545960000000001, + "network_total_time": 32.607885122299194 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 0, + "amount_uploaded_in_bytes": 0, + "total_transfer_in_bytes": 0, + "amount_downloaded_in_number_of_packets": 0, + "amount_uploaded_in_number_of_packets": 0, + "total_transfer_in_number_of_packets": 0, + "total_traffic_in_number_of_web_packets": 0, + "total_transfer_time_in_seconds": 0.0, + "network_total_time": 0.8448257446289062 + } + }, + "network_tracking_remote_file_reading.Ros3NWBFileReadBenchmark.track_network_activity_during_read": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": { + "amount_downloaded_in_bytes": 478981, + "amount_uploaded_in_bytes": 5756, + "total_transfer_in_bytes": 484737, + "amount_downloaded_in_number_of_packets": 351, + "amount_uploaded_in_number_of_packets": 68, + "total_transfer_in_number_of_packets": 419, + "total_traffic_in_number_of_web_packets": 68, + "total_transfer_time_in_seconds": 0.41176599999999997, + "network_total_time": 1.6515042781829834 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": { + "amount_downloaded_in_bytes": 35450039, + "amount_uploaded_in_bytes": 111812, + "total_transfer_in_bytes": 35561851, + "amount_downloaded_in_number_of_packets": 24136, + "amount_uploaded_in_number_of_packets": 1231, + "total_transfer_in_number_of_packets": 25367, + "total_traffic_in_number_of_web_packets": 1231, + "total_transfer_time_in_seconds": 5.304318, + "network_total_time": 70.38967871665955 + }, + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": { + "amount_downloaded_in_bytes": 387085, + "amount_uploaded_in_bytes": 5324, + "total_transfer_in_bytes": 392409, + "amount_downloaded_in_number_of_packets": 287, + "amount_uploaded_in_number_of_packets": 60, + "total_transfer_in_number_of_packets": 347, + "total_traffic_in_number_of_web_packets": 60, + "total_transfer_time_in_seconds": 0.46406800000000004, + "network_total_time": 1.5739588737487793 + } + }, + "network_tracking_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 43655066, + "amount_uploaded_in_bytes": 43766, + "total_transfer_in_bytes": 43698832, + "amount_downloaded_in_number_of_packets": 29581, + "amount_uploaded_in_number_of_packets": 769, + "total_transfer_in_number_of_packets": 30350, + "total_traffic_in_number_of_web_packets": 769, + "total_transfer_time_in_seconds": 1.1397160000000002, + "network_total_time": 88.5408878326416 + } + }, + "network_tracking_remote_slicing.RemfileContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 26725746, + "amount_uploaded_in_bytes": 31124, + "total_transfer_in_bytes": 26756870, + "amount_downloaded_in_number_of_packets": 18328, + "amount_uploaded_in_number_of_packets": 442, + "total_transfer_in_number_of_packets": 18770, + "total_traffic_in_number_of_web_packets": 442, + "total_transfer_time_in_seconds": 0.884508, + "network_total_time": 54.08054709434509 + } + }, + "network_tracking_remote_slicing.Ros3ContinuousSliceBenchmark.track_network_activity_during_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": { + "amount_downloaded_in_bytes": 21817605, + "amount_uploaded_in_bytes": 17126, + "total_transfer_in_bytes": 21834731, + "amount_downloaded_in_number_of_packets": 14790, + "amount_uploaded_in_number_of_packets": 307, + "total_transfer_in_number_of_packets": 15097, + "total_traffic_in_number_of_web_packets": 307, + "total_transfer_time_in_seconds": 0.4851080000000001, + "network_total_time": 44.72109293937683 + } + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.2212292089825496, + 0.21823545801453292, + 0.1996501250541769 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.37255916598951444, + 0.36288745899219066, + 0.4583110410021618 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.19372841698350385, + 0.17198204097803682, + 0.2010872500250116 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.3017203750205226, + 0.2855189169640653, + 0.3239751660148613 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.3139351250138134, + 0.27876750001451, + 0.3089893329888582 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.27953795896610245, + 0.29537116701249033, + 0.31544358300743625 + ] + }, + "time_remote_file_reading.DirectFileReadBenchmark.time_read_hdf5_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.19201925001107156, + 0.2206440840382129, + 0.18069725000532344 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.5366601249552332, + 6.783211790956557, + 0.5053167910082266 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.15501166600733995, + 0.16271379200043157, + 0.1557736670365557 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_fsspec_no_cache": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.2543450419907458, + 0.32001166697591543, + 0.22787766699912027 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 14.781256792019121, + 17.860192250052933, + 2.1884840829879977 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.2372769580106251, + 0.2547845409717411, + 0.21445054101059213 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_remfile": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.40089979203185067, + 0.37968241702765226, + 0.37375612498726696 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 0.3845805829623714, + 0.41974262497387826, + 0.41953700000885874 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.3822036249912344, + 0.3541829170426354, + 0.3704506250214763 + ] + }, + "time_remote_file_reading.NWBFileReadBenchmark.time_read_hdf5_nwbfile_ros3": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/dc3/028/dc302863-52b3-4675-9b5e-cf5f8c7636a7'\",)": [ + 0.44693970802472904, + 0.3840806250227615, + 0.4541446669609286 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\",)": [ + 7.882018208038062, + 1.17685808299575, + 1.0427757499855943 + ], + "(\"'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'\",)": [ + 0.3935280420118943, + 0.3732153750024736, + 0.3740567500353791 + ] + }, + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 16.80059095803881, + 15.545239707978908, + 0.9837783330003731 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 7.968111458001658, + 0.6783055830164813, + 0.6636589999543503 + ] + }, + "time_remote_slicing.Ros3ContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.0019529589917510748, + 0.0019359170109964907, + 0.0019236249499954283 + ] + } + } +} From ee3c985d8acf4539fb41b015e85c07ceb928c71d Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Wed, 21 Feb 2024 12:33:50 -0500 Subject: [PATCH 67/72] add in_seconds --- src/nwb_benchmarks/core/_network_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index 2637d7a..3a7c1fe 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -27,7 +27,7 @@ def network_activity_tracker(tshark_path: Union[pathlib.Path, None] = None): t1 = time.time() network_total_time = t1 - t0 - network_tracker.network_statistics["network_total_time"] = network_total_time + network_tracker.network_statistics["network_total_time_in_seconds"] = network_total_time class NetworkTracker: From c1b73f934a26321e21ab1418261b70c82bcf5837 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 21 Feb 2024 13:51:30 -0800 Subject: [PATCH 68/72] Add funding acknowledgment to docs/index.rst --- docs/index.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 963072a..8629206 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,7 +3,14 @@ NWB Benchmarks This project is an effort to understand, in a robust and reproducible manner, the principles underlying optimized file storage patterns for reading and writing NWB files from both local filesystems and remotely from the cloud (in particular, AWS S3 buckets). -Funding is provided by NOSI ... +Development of the NWB cloud benchmarks is supported by the National Institute Of Neurological Disorders +And Stroke of the National Institutes of Health under Award Number +`U24NS120057 `_ +as part of supplement award +`3U24NS120057-03S1 `_ +on *Evaluation and optimization of NWB neurophysiology software and data in the cloud* . +The content is solely the responsibility of the authors and does not necessarily represent the official +views of the National Institutes of Health.” .. toctree:: :maxdepth: 2 From f7e2ee3d5d599fcde739a81c024b075f71bbbeae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:51:53 +0000 Subject: [PATCH 69/72] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/index.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8629206..5af1347 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,13 +3,13 @@ NWB Benchmarks This project is an effort to understand, in a robust and reproducible manner, the principles underlying optimized file storage patterns for reading and writing NWB files from both local filesystems and remotely from the cloud (in particular, AWS S3 buckets). -Development of the NWB cloud benchmarks is supported by the National Institute Of Neurological Disorders -And Stroke of the National Institutes of Health under Award Number -`U24NS120057 `_ -as part of supplement award -`3U24NS120057-03S1 `_ -on *Evaluation and optimization of NWB neurophysiology software and data in the cloud* . -The content is solely the responsibility of the authors and does not necessarily represent the official +Development of the NWB cloud benchmarks is supported by the National Institute Of Neurological Disorders +And Stroke of the National Institutes of Health under Award Number +`U24NS120057 `_ +as part of supplement award +`3U24NS120057-03S1 `_ +on *Evaluation and optimization of NWB neurophysiology software and data in the cloud* . +The content is solely the responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health.” .. toctree:: From 24b515c0319b51b0da2e8265a271de8b70e4aeef Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Wed, 21 Feb 2024 17:41:51 -0500 Subject: [PATCH 70/72] decrease indent to save more space; add docs on result parsing --- docs/development.rst | 36 +++++++++++++++++++ .../setup/_configure_machine.py | 2 +- src/nwb_benchmarks/setup/_reduce_results.py | 6 ++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 38d2f9d..be486d4 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -67,8 +67,44 @@ Customized Parsing of Results Since our first approach to simplifying the sharing of results is to just commit them to the common GitHub repo, it was noticed that the default results files stored a lot of extraneous information. +For example, here is an abridged example of the raw ASV output file... + +.. code-block:: json + + {"commit_hash": "ee3c985d8acf4539fb41b015e85c07ceb928c71d", "env_name": "existing-pyD__anaconda3_envs_nwb_benchmarks_3_11_created_on_2_17_2024_python.exe", "date": 1708536830000, "params": , "python": "3.11", "requirements": {}, "env_vars": {}, "result_columns": ["result", "params", "version", "started_at", "duration", "stats_ci_99_a", "stats_ci_99_b", "stats_q_25", "stats_q_75", "stats_number", "stats_repeat", "samples", "profile"], "results": {"time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": [[12.422975199995562], [["'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'"], ["'ElectricalSeriesAp'"], ["(slice(0, 30000, None), slice(0, 384, None))"]], "bb6fdd6142015840e188d19b7e06b38dfab294af60a25c67711404eeb0bc815f", 1708552612283, 59.726, [-22.415], [40.359], [6.5921], [13.078], [1], [3], [[0.8071024999953806, 0.9324163000565022, 0.5638924000086263]]], "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": [[0.5849523999495432], [["'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'"], ["'ElectricalSeriesAp'"], ["(slice(0, 30000, None), slice(0, 384, None))"]], "f9c77e937b6e41c5a75803e962cc9a6f08cb830f97b04f7a68627a07fd324c11", 1708552672010, 10.689, [0.56549], [0.60256], [0.58225], [0.58626], [1], [3], [[0.5476778000593185, 8.321383600006811, 9.654714399948716]]]}, "durations": {}, "version": 2} + Since all we're really after here is the raw tracking output, some custom reduction of the original results files is performed so that only the minimal amount of information needed is actually stored in the final results files. These parsed results follow the dandi-esque name pattern ``result_timestamp-%Y-%M-%D-%H-%M-%S_machine-_environment-.json`` and are stored in the outer level ``results`` folder along with some ``info_machine-`` and ``info_environment-`` header files that are not regenerated whenever the hashes are the same. +The same file reduced then appears as... + +.. code-block:: json + + { + "version": 2, + "timestamp": "2024-02-21-12-33-50", + "commit_hash": "ee3c985d8acf4539fb41b015e85c07ceb928c71d", + "environment_hash": "246cf6a886d9a66a9b593d52cb681998fab55adf", + "machine_hash": "e109d91eb8c6806274a5a7909c735869415384e9", + "results": { + "time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.8071024999953806, + 0.9324163000565022, + 0.5638924000086263 + ] + }, + "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": { + "(\"'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'\", \"'ElectricalSeriesAp'\", '(slice(0, 30000, None), slice(0, 384, None))')": [ + 0.5476778000593185, + 8.321383600006811, + 9.654714399948716 + ] + } + } + } + +which is also indented for improved human readability and line-by-line GitHub tracking. This indentation adds about 50 bytes per kilobyte compared to no indentation. + .. note:: If this ``results`` folder eventually becomes too large for Git to reasonably handle, we will explore options to share via other data storage services. diff --git a/src/nwb_benchmarks/setup/_configure_machine.py b/src/nwb_benchmarks/setup/_configure_machine.py index 9d6911c..639bbd8 100644 --- a/src/nwb_benchmarks/setup/_configure_machine.py +++ b/src/nwb_benchmarks/setup/_configure_machine.py @@ -89,7 +89,7 @@ def customize_asv_machine_file(file_path: pathlib.Path, overwrite: bool = False) } with open(file=file_path, mode="w") as io: - json.dump(fp=io, obj=custom_file_info, indent=4) + json.dump(fp=io, obj=custom_file_info, indent=1) def ensure_machine_info_current(file_path: pathlib.Path): diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 41b482d..311d425 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -43,7 +43,7 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil reduced_results = dict() for test_case, raw_results_list in raw_results_info["results"].items(): - if len(raw_results_list) not in [5, 12]: # Only successful runs results in these lengths + if len(raw_results_list) != 12: # Only successful runs results in this length continue flattened_joint_params = collections.defaultdict(list) @@ -94,7 +94,7 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil main_results_folder.mkdir(parents=True, exist_ok=True) with open(file=parsed_results_file, mode="w") as io: - json.dump(obj=reduced_results_info, fp=io, indent=4) + json.dump(obj=reduced_results_info, fp=io, indent=1) # At least one level of indent makes it easier to read # Copy machine file to main results machine_info_file_path = raw_results_file_path.parent / "machine.json" @@ -106,7 +106,7 @@ def reduce_results(raw_results_file_path: pathlib.Path, raw_environment_info_fil parsed_environment_file_path = main_results_folder / f"info_environment-{environment_hash}.json" if not parsed_environment_file_path.exists(): with open(file=parsed_environment_file_path, mode="w") as io: - json.dump(obj=parsed_environment_info, fp=io, indent=4) + json.dump(obj=parsed_environment_info, fp=io, indent=1) # Network tests require admin permissions, which can alter write permissions of any files created if sys.platform in ["darwin", "linux"]: From 7a2f7a526b143292586aa9a0e402cf924b62ff98 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Wed, 21 Feb 2024 17:44:09 -0500 Subject: [PATCH 71/72] add more docs --- docs/development.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/development.rst b/docs/development.rst index be486d4..93f48ba 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -73,7 +73,9 @@ For example, here is an abridged example of the raw ASV output file... {"commit_hash": "ee3c985d8acf4539fb41b015e85c07ceb928c71d", "env_name": "existing-pyD__anaconda3_envs_nwb_benchmarks_3_11_created_on_2_17_2024_python.exe", "date": 1708536830000, "params": , "python": "3.11", "requirements": {}, "env_vars": {}, "result_columns": ["result", "params", "version", "started_at", "duration", "stats_ci_99_a", "stats_ci_99_b", "stats_q_25", "stats_q_75", "stats_number", "stats_repeat", "samples", "profile"], "results": {"time_remote_slicing.FsspecNoCacheContinuousSliceBenchmark.time_slice": [[12.422975199995562], [["'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'"], ["'ElectricalSeriesAp'"], ["(slice(0, 30000, None), slice(0, 384, None))"]], "bb6fdd6142015840e188d19b7e06b38dfab294af60a25c67711404eeb0bc815f", 1708552612283, 59.726, [-22.415], [40.359], [6.5921], [13.078], [1], [3], [[0.8071024999953806, 0.9324163000565022, 0.5638924000086263]]], "time_remote_slicing.RemfileContinuousSliceBenchmark.time_slice": [[0.5849523999495432], [["'https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a'"], ["'ElectricalSeriesAp'"], ["(slice(0, 30000, None), slice(0, 384, None))"]], "f9c77e937b6e41c5a75803e962cc9a6f08cb830f97b04f7a68627a07fd324c11", 1708552672010, 10.689, [0.56549], [0.60256], [0.58225], [0.58626], [1], [3], [[0.5476778000593185, 8.321383600006811, 9.654714399948716]]]}, "durations": {}, "version": 2} -Since all we're really after here is the raw tracking output, some custom reduction of the original results files is performed so that only the minimal amount of information needed is actually stored in the final results files. These parsed results follow the dandi-esque name pattern ``result_timestamp-%Y-%M-%D-%H-%M-%S_machine-_environment-.json`` and are stored in the outer level ``results`` folder along with some ``info_machine-`` and ``info_environment-`` header files that are not regenerated whenever the hashes are the same. +This structure is both hard to read due to no indentation, poorly self-annotated due to everything being JSON arrays instead of objects with representative names, and there are a large number of values here that we don't really care about. + +Since all we're after here is the raw tracking output, some custom reduction of the original results files is performed so that only the minimal amount of information needed is actually stored in the final results files. These parsed results follow the dandi-esque name pattern ``result_timestamp-%Y-%M-%D-%H-%M-%S_machine-_environment-.json`` and are stored in the outer level ``results`` folder along with some ``info_machine-`` and ``info_environment-`` header files that are not regenerated whenever the hashes are the same. The same file reduced then appears as... From 9d2c462dec245427f219c825e283dab091d37b77 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 21 Feb 2024 14:59:09 -0800 Subject: [PATCH 72/72] Add __init__ and docstrings for NetworkTracker --- src/nwb_benchmarks/core/_network_tracker.py | 44 +++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/nwb_benchmarks/core/_network_tracker.py b/src/nwb_benchmarks/core/_network_tracker.py index 3a7c1fe..381deb9 100644 --- a/src/nwb_benchmarks/core/_network_tracker.py +++ b/src/nwb_benchmarks/core/_network_tracker.py @@ -1,4 +1,4 @@ -"""Network tracker for capturing traffic and performance metrics during the execution of methods or code snippets asdasdadad.""" +"""Network tracker for capturing traffic and performance metrics during the execution of methods or code snippets.""" import contextlib import os @@ -31,8 +31,36 @@ def network_activity_tracker(tshark_path: Union[pathlib.Path, None] = None): class NetworkTracker: + """ + Track network traffic and compute network statistics for operations performed by the current + Python process during the active period of the tracker, which is started by `start_network_capture` + and ended by `stop_network_capture`. + + :ivar connections_thread: Instance of `CaptureConnections` used to relate network connections to process IDs + :ivar network_profiler: Instance of `NetworkProfiler` used to capture network traffic with TShark + :ivar pid_connections: connections for the PID of this process + :ivar self.pid_packets: Network packets associated with this PID (i.e., `os.getpid()`) + :ivar self.network_statistics: Network statistics calculated via `NetworkStatistics.get_statistics` + :ivar self.asv_network_statistics: The network statistics wrapped in a dict for compliance with ASV + + """ + + def __init__(self): + self.connections_thread = None + self.network_profiler = None + self.pid_connections = None + self.pid_packets = None + self.network_statistics = None + self.asv_network_statistics = None + def start_network_capture(self, tshark_path: Union[pathlib.Path, None] = None): - # start the capture_connections() function to update the current connections of this machine + """ + Start capturing the connections on this machine as well as all network packets + + Side effects: This functions sets the following instance variables: + * self.connections_thread + * self.network_profile + """ self.connections_thread = CaptureConnections() self.connections_thread.start() time.sleep(0.2) # not sure if this is needed but just to be safe @@ -42,7 +70,17 @@ def start_network_capture(self, tshark_path: Union[pathlib.Path, None] = None): self.network_profiler.start_capture(tshark_path=tshark_path) def stop_network_capture(self): - # Stop capturing packets and connections + """ + Stop capturing network packets and connections. + + Note: This function will fail if `start_network_capture` was not called first. + + Side effects: This functions sets the following instance variables: + * self.pid_connections + * self.pid_packets + * self.network_statistics + * self.asv_network_statistics + """ self.network_profiler.stop_capture() self.connections_thread.stop()