From 01ca77e57b8ec94d0ff3bdbb8322f8e44bccdd3f Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 14 Jun 2024 01:01:36 -0500 Subject: [PATCH 1/3] fix: bugfix in Block Detection --- aeon/dj_pipeline/analysis/block_analysis.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/aeon/dj_pipeline/analysis/block_analysis.py b/aeon/dj_pipeline/analysis/block_analysis.py index 43d71f84..c92a2024 100644 --- a/aeon/dj_pipeline/analysis/block_analysis.py +++ b/aeon/dj_pipeline/analysis/block_analysis.py @@ -35,7 +35,7 @@ def make(self, key): """On a per-chunk basis, check for the presence of new block, insert into Block table.""" # find the 0s # that would mark the start of a new block - # if the 0 is the first index - look back at the previous chunk + # In the BlockState data - if the 0 is the first index - look back at the previous chunk # if the previous timestamp belongs to a previous epoch -> block_end is the previous timestamp # else block_end is the timestamp of this 0 chunk_start, chunk_end = (acquisition.Chunk & key).fetch1("chunk_start", "chunk_end") @@ -56,16 +56,26 @@ def make(self, key): ) block_state_query = acquisition.Environment.BlockState & exp_key & chunk_restriction - block_state_df = fetch_stream(block_state_query)[previous_block_start:chunk_end] + block_state_df = fetch_stream(block_state_query) + block_state_df.index = block_state_df.index.round("us") # timestamp precision in DJ is only at microseconds + block_state_df = block_state_df.loc[(block_state_df.index > previous_block_start) + & (block_state_df.index <= chunk_end)] - block_ends = block_state_df[block_state_df.pellet_ct.diff() < 0] + # detecting block end times + # find the 0s + block_ends = block_state_df[block_state_df.pellet_ct == 0] + # account for the double 0s - find any 0s that are within 1 second of each other, remove the 2nd one + double_0s = block_ends.index.to_series().diff().dt.total_seconds() < 1 + # find the indices of the 2nd 0s and remove + double_0s = double_0s.shift(-1).fillna(False) + block_ends = block_ends[~double_0s] block_entries = [] for idx, block_end in enumerate(block_ends.index): if idx == 0: if previous_block_key: # if there is a previous block - insert "block_end" for the previous block - previous_pellet_time = block_state_df[:block_end].index[-2] + previous_pellet_time = block_state_df[:block_end].index[-1] previous_epoch = ( acquisition.Epoch.join(acquisition.EpochEnd, left=True) & exp_key From 3bb31406e3cfcddc61a52e822356c86bbb13442e Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Thu, 20 Jun 2024 17:03:32 -0500 Subject: [PATCH 2/3] feat(block_analysis): update "block_end" --- aeon/dj_pipeline/analysis/block_analysis.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/aeon/dj_pipeline/analysis/block_analysis.py b/aeon/dj_pipeline/analysis/block_analysis.py index c92a2024..6dd772d5 100644 --- a/aeon/dj_pipeline/analysis/block_analysis.py +++ b/aeon/dj_pipeline/analysis/block_analysis.py @@ -55,14 +55,15 @@ def make(self, key): key["experiment_name"], previous_block_start, chunk_end ) + # detecting block end times + # pellet count reset - find 0s in BlockState + block_state_query = acquisition.Environment.BlockState & exp_key & chunk_restriction block_state_df = fetch_stream(block_state_query) block_state_df.index = block_state_df.index.round("us") # timestamp precision in DJ is only at microseconds block_state_df = block_state_df.loc[(block_state_df.index > previous_block_start) & (block_state_df.index <= chunk_end)] - # detecting block end times - # find the 0s block_ends = block_state_df[block_state_df.pellet_ct == 0] # account for the double 0s - find any 0s that are within 1 second of each other, remove the 2nd one double_0s = block_ends.index.to_series().diff().dt.total_seconds() < 1 @@ -243,6 +244,10 @@ def make(self, key): } ) + # update block_end if last timestamp of encoder_df is before the current block_end + if encoder_df.index[-1] < block_end: + block_end = encoder_df.index[-1] + # Subject data # Get all unique subjects that visited the environment over the entire exp; # For each subject, see 'type' of visit most recent to start of block @@ -258,6 +263,7 @@ def make(self, key): _df = subject_visits_df[subject_visits_df.id == subject_name] if _df.type[-1] != "Exit": subject_names.append(subject_name) + for subject_name in subject_names: # positions - query for CameraTop, identity_name matches subject_name, pos_query = ( @@ -301,6 +307,14 @@ def make(self, key): } ) + # update block_end if last timestamp of pos_df is before the current block_end + if pos_df.index[-1] < block_end: + block_end = pos_df.index[-1] + + if block_end != (Block & key).fetch1("block_end"): + Block.update1({**key, "block_end": block_end}) + self.update1({**key, "block_duration": (block_end - block_start).total_seconds() / 3600}) + @schema class BlockSubjectAnalysis(dj.Computed): @@ -511,7 +525,7 @@ def make(self, key): @schema class BlockPlots(dj.Computed): - definition = """ + definition = """ -> BlockAnalysis --- subject_positions_plot: longblob From e3e51f2e8086337f6c98446ca3f55596e3b8b8d7 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Thu, 20 Jun 2024 17:04:29 -0500 Subject: [PATCH 3/3] format: black formatting --- aeon/dj_pipeline/analysis/block_analysis.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/aeon/dj_pipeline/analysis/block_analysis.py b/aeon/dj_pipeline/analysis/block_analysis.py index 6dd772d5..ef50138f 100644 --- a/aeon/dj_pipeline/analysis/block_analysis.py +++ b/aeon/dj_pipeline/analysis/block_analysis.py @@ -60,9 +60,12 @@ def make(self, key): block_state_query = acquisition.Environment.BlockState & exp_key & chunk_restriction block_state_df = fetch_stream(block_state_query) - block_state_df.index = block_state_df.index.round("us") # timestamp precision in DJ is only at microseconds - block_state_df = block_state_df.loc[(block_state_df.index > previous_block_start) - & (block_state_df.index <= chunk_end)] + block_state_df.index = block_state_df.index.round( + "us" + ) # timestamp precision in DJ is only at microseconds + block_state_df = block_state_df.loc[ + (block_state_df.index > previous_block_start) & (block_state_df.index <= chunk_end) + ] block_ends = block_state_df[block_state_df.pellet_ct == 0] # account for the double 0s - find any 0s that are within 1 second of each other, remove the 2nd one