From 04b5c30eae5de16346dbbd301880533fc1cee7d1 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 8 Mar 2024 22:33:51 +0000 Subject: [PATCH 1/8] feat: :sparkles: add SocialExperiment page in sciviz --- .../dj_pipeline/webapps/sciviz/specsheet.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml b/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml index c7d2229b..38cfaaf4 100644 --- a/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml +++ b/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml @@ -541,6 +541,50 @@ SciViz: aeon_report = aeon_archived_exp02_report return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['region_time_fraction_hourly_plotly']) + SocialExperiment: + route: /social_experiment + grids: + grid3: + type: fixed + columns: 1 + row_height: 700 + components: + SocialExperiment: + route: /social_experiment_grid + link: /per_social_experiment + x: 0 + y: 0 + height: 1 + width: 1 + type: antd-table + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_acquisition, aeon_block_analysis, aeon_tracking): + + import pandas as pd + acquisition = aeon_acquisition + block_analysis = aeon_block_analysis + tracking = aeon_tracking + + query = acquisition.Experiment.aggr(block_analysis.Block, block_count="COUNT(experiment_name)") + acquisition.Experiment.aggr(acquisition.Chunk, chunk_count="COUNT(experiment_name)", latest_chunk_start="MAX(chunk_start)") + + df = tracking.SLEAPTracking.PoseIdentity.proj("identity_name").fetch(format="frame") + df = df.groupby('experiment_name')['identity_name'].unique().reset_index() + + subject_query = None + + for exp in query.fetch("experiment_name"): + # get identity names for each experiment + identities = df[df['experiment_name'] == exp]['identity_name'].values[0] + if not subject_query: + subject_query = dj.U("experiment_name", "subject") & (query & f"experiment_name = '{exp}'").proj(subject=f"CONCAT('{', '.join(identities)}')") + else: + subject_query += dj.U("experiment_name", "subject") & (query & f"experiment_name = '{exp}'").proj(subject=f"CONCAT('{', '.join(identities)}')") + + query.join(subject_query, left=True).proj(..., participants="subject") + BlockAnalysis: route: /block_analysis grids: From edff5012b2332cb67c63fa25ecc7368bd34b1f58 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 8 Mar 2024 22:49:27 +0000 Subject: [PATCH 2/8] fix: :bug: add return statement --- aeon/dj_pipeline/webapps/sciviz/specsheet.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml b/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml index 38cfaaf4..5ce11497 100644 --- a/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml +++ b/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml @@ -544,7 +544,7 @@ SciViz: SocialExperiment: route: /social_experiment grids: - grid3: + grid1: type: fixed columns: 1 row_height: 700 @@ -583,7 +583,7 @@ SciViz: else: subject_query += dj.U("experiment_name", "subject") & (query & f"experiment_name = '{exp}'").proj(subject=f"CONCAT('{', '.join(identities)}')") - query.join(subject_query, left=True).proj(..., participants="subject") + return {'query': query.join(subject_query, left=True).proj(..., participants="subject"), 'fetch_args': []} BlockAnalysis: route: /block_analysis From bdd6e05fdf8b6a22daacbf994db8fc9828e85306 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 11 Mar 2024 19:11:04 +0000 Subject: [PATCH 3/8] feat: :sparkles: simplify with GROUP CONCAT --- .../dj_pipeline/webapps/sciviz/specsheet.yaml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml b/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml index 5ce11497..ea2ef70b 100644 --- a/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml +++ b/aeon/dj_pipeline/webapps/sciviz/specsheet.yaml @@ -563,27 +563,18 @@ SciViz: dj_query: > def dj_query(aeon_acquisition, aeon_block_analysis, aeon_tracking): - import pandas as pd acquisition = aeon_acquisition block_analysis = aeon_block_analysis tracking = aeon_tracking query = acquisition.Experiment.aggr(block_analysis.Block, block_count="COUNT(experiment_name)") + acquisition.Experiment.aggr(acquisition.Chunk, chunk_count="COUNT(experiment_name)", latest_chunk_start="MAX(chunk_start)") - df = tracking.SLEAPTracking.PoseIdentity.proj("identity_name").fetch(format="frame") - df = df.groupby('experiment_name')['identity_name'].unique().reset_index() + query = query.join(acquisition.Experiment.aggr( + tracking.SLEAPTracking.PoseIdentity, + participants="GROUP_CONCAT(DISTINCT identity_name SEPARATOR ', ')" + ), left=True) - subject_query = None - - for exp in query.fetch("experiment_name"): - # get identity names for each experiment - identities = df[df['experiment_name'] == exp]['identity_name'].values[0] - if not subject_query: - subject_query = dj.U("experiment_name", "subject") & (query & f"experiment_name = '{exp}'").proj(subject=f"CONCAT('{', '.join(identities)}')") - else: - subject_query += dj.U("experiment_name", "subject") & (query & f"experiment_name = '{exp}'").proj(subject=f"CONCAT('{', '.join(identities)}')") - - return {'query': query.join(subject_query, left=True).proj(..., participants="subject"), 'fetch_args': []} + return {'query': query, 'fetch_args': []} BlockAnalysis: route: /block_analysis From e6768a9b5f3cf6877c261b8aa9d6b1bd6782d620 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Tue, 19 Mar 2024 23:14:58 +0000 Subject: [PATCH 4/8] fix: :bug: fix KeyError in BlockDetection --- aeon/dj_pipeline/analysis/block_analysis.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/aeon/dj_pipeline/analysis/block_analysis.py b/aeon/dj_pipeline/analysis/block_analysis.py index 08cab6e4..ebed430b 100644 --- a/aeon/dj_pipeline/analysis/block_analysis.py +++ b/aeon/dj_pipeline/analysis/block_analysis.py @@ -8,8 +8,10 @@ from matplotlib import path as mpl_path from aeon.analysis import utils as analysis_utils -from aeon.dj_pipeline import acquisition, fetch_stream, get_schema_name, streams, tracking -from aeon.dj_pipeline.analysis.visit import filter_out_maintenance_periods, get_maintenance_periods +from aeon.dj_pipeline import (acquisition, fetch_stream, get_schema_name, + streams, tracking) +from aeon.dj_pipeline.analysis.visit import (filter_out_maintenance_periods, + get_maintenance_periods) schema = dj.schema(get_schema_name("block_analysis")) logger = dj.logger @@ -516,7 +518,8 @@ def make(self, key): ) block_query = acquisition.Environment.BlockState & chunk_restriction - block_df = fetch_stream(block_query)[previous_block_start:chunk_end] + block_df = fetch_stream(block_query) + block_df = block_df[block_df.index.to_series().between(previous_block_start, chunk_end)] block_ends = block_df[block_df.pellet_ct.diff() < 0] From 6fd1d84632fefeba6e8cb36e597a89a2372bca7a Mon Sep 17 00:00:00 2001 From: JaerongA Date: Wed, 20 Mar 2024 00:55:39 +0000 Subject: [PATCH 5/8] apply .sort_index() inside BlockDetection make function --- aeon/dj_pipeline/analysis/block_analysis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aeon/dj_pipeline/analysis/block_analysis.py b/aeon/dj_pipeline/analysis/block_analysis.py index ebed430b..d7c134b8 100644 --- a/aeon/dj_pipeline/analysis/block_analysis.py +++ b/aeon/dj_pipeline/analysis/block_analysis.py @@ -518,8 +518,7 @@ def make(self, key): ) block_query = acquisition.Environment.BlockState & chunk_restriction - block_df = fetch_stream(block_query) - block_df = block_df[block_df.index.to_series().between(previous_block_start, chunk_end)] + block_df = fetch_stream(block_query).sort_index()[previous_block_start:chunk_end] block_ends = block_df[block_df.pellet_ct.diff() < 0] From fd2a704ac329216deb97a2b6ff526f243a1a11e4 Mon Sep 17 00:00:00 2001 From: lochhh Date: Wed, 27 Mar 2024 18:24:46 +0000 Subject: [PATCH 6/8] Add sciviz local specsheet --- .../webapps/sciviz/specsheet-local.yaml | 667 ++++++++++++++++++ 1 file changed, 667 insertions(+) create mode 100644 aeon/dj_pipeline/webapps/sciviz/specsheet-local.yaml diff --git a/aeon/dj_pipeline/webapps/sciviz/specsheet-local.yaml b/aeon/dj_pipeline/webapps/sciviz/specsheet-local.yaml new file mode 100644 index 00000000..3fa293aa --- /dev/null +++ b/aeon/dj_pipeline/webapps/sciviz/specsheet-local.yaml @@ -0,0 +1,667 @@ +version: "v0.1.0" +LabBook: null +SciViz: + auth: + mode: "database" + hostname: host.docker.internal + pages: + Colony: + route: /colony_page + grids: + grid1: + type: fixed + columns: 1 + row_height: 1000 + components: + Pyrat Subjects: + route: /colony_page_pyrat_subjects + x: 0 + y: 0 + height: 1 + width: 1 + type: antd-table + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_subject): + query = aeon_subject.Subject * aeon_subject.SubjectDetail * aeon_subject.SubjectReferenceWeight.proj('reference_weight', min_since_last_update='TIMESTAMPDIFF(MINUTE, last_updated_time, UTC_TIMESTAMP())') & 'available = 1' + return {'query': query.proj(..., '-available'), 'fetch_args': []} + + Pyrat User Entry: + route: /colony_page_pyrat_user_entry + x: 0 + y: 1 + height: 0.3 + width: 1 + type: form + tables: + - aeon_lab.User + map: + - type: attribute + input: SWC Username + destination: user + - type: attribute + input: Pyrat Responsible Owner + destination: responsible_owner + - type: attribute + input: Pyrat Responsible ID + destination: responsible_id + + Pyrat Sync Task: + route: /colony_page_pyrat_sync_task + x: 0 + y: 1.3 + height: 0.3 + width: 1 + type: form + tables: + - aeon_subject.PyratIngestionTask + map: + - type: attribute + input: Task Scheduled Time + destination: pyrat_task_scheduled_time + + ExperimentEntry: + route: /experiment_entry + grids: + grid5: + type: fixed + columns: 1 + row_height: 1000 + components: + New Experiment: + route: /exp_form + x: 0 + y: 0 + height: 0.4 + width: 1 + type: form + tables: + - aeon_acquisition.Experiment + - aeon_acquisition.Experiment.DevicesSchema + map: + - type: attribute + input: Experiment Name + destination: experiment_name + - type: attribute + input: Start Time + destination: experiment_start_time + - type: attribute + input: Description + destination: experiment_description + - type: table + input: Lab Arena + destination: aeon_lab.Arena + - type: table + input: Lab Location + destination: aeon_lab.Location + - type: attribute + input: Experiment Type + destination: aeon_acquisition.ExperimentType + - type: table + input: Devices Schema Name + destination: aeon_acquisition.DevicesSchema + + New Experiment Subject: + route: /exp_subject_form + x: 0 + y: 0.4 + height: 0.3 + width: 1 + type: form + tables: + - aeon_acquisition.Experiment.Subject + map: + - type: table + input: Experiment Name + destination: aeon_acquisition.Experiment + - type: table + input: Subject in the experiment + destination: aeon_subject.Subject + + New Experiment Note: + route: /exp_note_form + x: 0 + y: 0.7 + height: 0.3 + width: 1 + type: form + tables: + - aeon_acquisition.Experiment.Note + map: + - type: table + input: Experiment Name + destination: aeon_acquisition.Experiment + - type: attribute + input: Note Time + destination: note_timestamp + - type: attribute + input: Note Type + destination: note_type + - type: attribute + input: Note + destination: note + + New Experiment Directory: + route: /exp_directory_form + x: 0 + y: 1.0 + height: 0.3 + width: 1 + type: form + tables: + - aeon_acquisition.Experiment.Directory + map: + - type: table + input: Experiment Name + destination: aeon_acquisition.Experiment + - type: table + input: Directory Type + destination: aeon_acquisition.DirectoryType + - type: table + input: Pipeline Repository + destination: aeon_acquisition.PipelineRepository + - type: attribute + input: Directory Path + destination: directory_path + + New Experiment Type: + route: /exp_type_form + x: 0 + y: 1.3 + height: 0.3 + width: 1 + type: form + tables: + - aeon_acquisition.ExperimentType + map: + - type: attribute + input: Experiment Type + destination: experiment_type + + Subjects: + route: /subjects + grids: + grid1: + type: fixed + columns: 1 + row_height: 700 + components: + Animals: + route: /allsubjects + link: /per_subject_report + x: 0 + y: 0 + height: 1 + width: 1 + type: antd-table + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_acquisition, aeon_archived_exp02_analysis): + acquisition = aeon_archived_exp02_acquisition + visit_analysis = aeon_archived_exp02_analysis + query = acquisition.Experiment.Subject.aggr(visit_analysis.VisitEnd.join(visit_analysis.Visit, left=True), first_visit_start='MIN(visit_start)', last_visit_end='MAX(visit_end)', total_visit_count='COUNT(visit_start)', total_visit_duration='SUM(visit_duration)') + query = query.proj("first_visit_start", "last_visit_end", "total_visit_count", total_visit_duration="CAST(total_visit_duration AS DOUBLE(10, 3))") + return {'query': query, 'fetch_args': {'order_by': 'last_visit_end DESC'}} + + VisitSummary: + route: /visit_summary + grids: + grid3: + type: fixed + columns: 1 + row_height: 700 + components: + VisitSummary: + route: /visit_summary_grid3_1 + link: /per_visit_report + x: 0 + y: 0 + height: 1 + width: 1 + type: antd-table + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_analysis): + aeon_analysis = aeon_archived_exp02_analysis + query = aeon_analysis.Visit.aggr(aeon_analysis.VisitSummary, ..., duration="SUM(day_duration)", total_distance_travelled="SUM(total_distance_travelled)", total_pellet_count="SUM(total_pellet_count)", total_wheel_distance_travelled="SUM(total_wheel_distance_travelled)", keep_all_rows=True) + query = query.join(aeon_analysis.VisitEnd, left=True) + query = query.proj("visit_end", total_pellet_count="CAST(total_pellet_count AS DOUBLE)", duration="CAST(duration AS DOUBLE(10, 3))", total_distance_travelled="CAST(total_distance_travelled AS DOUBLE(10, 3))", total_wheel_distance_travelled="CAST(total_wheel_distance_travelled AS DOUBLE(10, 3))") + return {'query': query, 'fetch_args': {'order_by': 'visit_end DESC'}} + + ExperimentReport: + route: /experiment_report + grids: + experiment_report: + route: /experiment_report + type: dynamic + columns: 1 + row_height: 1000 + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_acquisition): + acquisition = aeon_archived_exp01_acquisition + return {'query': acquisition.Experiment(), 'fetch_args': []} + component_templates: + comp3: + route: /avg_time_distribution + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_report): + report = aeon_archived_exp01_report + return {'query': report.ExperimentTimeDistribution(), 'fetch_args': ['time_distribution_plotly']} + + SubjectReport: + route: /subject_report + grids: + subject_report: + route: /subject_report + type: dynamic + columns: 2 + row_height: 1000 + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_acquisition): + acquisition = aeon_archived_exp01_acquisition + return {'query': acquisition.Experiment.Subject & {'experiment_name': 'exp0.1-r0'}, 'fetch_args': []} + component_templates: + comp1: + route: /subject_meta + type: metadata + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_acquisition): + return dict(query=aeon_archived_exp01_acquisition.Experiment.Subject(), fetch_args=[]) + comp2: + route: /reward_diff_plot + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_report): + report = aeon_archived_exp01_report + return {'query': report.SubjectRewardRateDifference(), 'fetch_args': ['reward_rate_difference_plotly']} + comp3: + route: /wheel_distance_travelled + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_report): + report = aeon_archived_exp01_report + return {'query': report.SubjectWheelTravelledDistance(), 'fetch_args': ['wheel_travelled_distance_plotly']} + + PerSubjectReport: + hidden: true + route: /per_subject_report + grids: + per_subject_report: + type: fixed + route: /per_subject_report + columns: 1 + row_height: 400 + components: + comp1: + route: /per_subject_meta + x: 0 + y: 0 + height: 1 + width: 1 + type: metadata + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_acquisition): + return dict(query=aeon_archived_exp01_acquisition.Experiment.Subject(), fetch_args=[]) + comp2: + route: /per_subject_reward_diff_plot + x: 0 + y: 1 + height: 1 + width: 1 + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_report): + report = aeon_archived_exp01_report + return {'query': report.SubjectRewardRateDifference(), 'fetch_args': ['reward_rate_difference_plotly']} + comp3: + route: /per_subject_wheel_distance_travelled + x: 0 + y: 2 + height: 1 + width: 1 + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp01_report): + report = aeon_archived_exp01_report + return {'query': report.SubjectWheelTravelledDistance(), 'fetch_args': ['wheel_travelled_distance_plotly']} + + PerVisitReport: + hidden: true + route: /per_visit_report + grids: + per_visit_report: + type: fixed + route: /per_visit_report + columns: 1 + row_height: 400 + components: + comp1: + route: /per_visit_meta + x: 0 + y: 0 + height: 1 + width: 1 + type: metadata + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_analysis): + aeon_analysis = aeon_archived_exp02_analysis + query = aeon_analysis.Visit.aggr(aeon_analysis.VisitSummary, ..., duration="SUM(day_duration)", total_distance_travelled="SUM(total_distance_travelled)", total_pellet_count="SUM(total_pellet_count)", total_wheel_distance_travelled="SUM(total_wheel_distance_travelled)", keep_all_rows=True) + query = query.join(aeon_analysis.VisitEnd, left=True) + query = query.proj("visit_end", total_pellet_count="CAST(total_pellet_count AS DOUBLE)", duration="CAST(duration AS DOUBLE(10, 3))", total_distance_travelled="CAST(total_distance_travelled AS DOUBLE(10, 3))", total_wheel_distance_travelled="CAST(total_wheel_distance_travelled AS DOUBLE(10, 3))") + return {'query': query, 'fetch_args': []} + comp2: + route: /per_visit_summary_plot + x: 0 + y: 1 + height: 1 + width: 1 + type: file:image:attach + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + report = aeon_archived_exp02_report + return {'query': report.VisitDailySummaryPlot(), 'fetch_args': ['summary_plot_png']} + + Visits247: + route: /visits247 + grids: + visit_daily_summary: + route: /visit_daily_summary_grid1 + type: dynamic + columns: 1 + row_height: 4000 + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return {'query': aeon_report.VisitDailySummaryPlot.proj(), 'fetch_args': []} + component_templates: + comp1: + route: /visit_daily_summary_pellet_count + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['pellet_count_plotly']) + comp2: + route: /visit_daily_summary_wheel_distance_travelled + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['wheel_distance_travelled_plotly']) + comp3: + route: /visit_daily_summary_total_distance_travelled + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['total_distance_travelled_plotly']) + comp4: + route: /visit_daily_summary_weight_patch + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['weight_patch_plotly']) + comp5: + route: /visit_daily_summary_foraging_bouts + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['foraging_bouts_plotly']) + comp6: + route: /visit_daily_summary_foraging_bouts_pellet_count + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['foraging_bouts_pellet_count_plotly']) + comp7: + route: /visit_daily_summary_foraging_bouts_duration + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['foraging_bouts_duration_plotly']) + comp8: + route: /visit_daily_summary_region_time_fraction_daily + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['region_time_fraction_daily_plotly']) + comp9: + route: /visit_daily_summary_region_time_fraction_hourly + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_archived_exp02_report): + aeon_report = aeon_archived_exp02_report + return dict(query=aeon_report.VisitDailySummaryPlot(), fetch_args=['region_time_fraction_hourly_plotly']) + + SocialExperiment: + route: /social_experiment + grids: + grid1: + type: fixed + columns: 1 + row_height: 700 + components: + SocialExperiment: + route: /social_experiment_grid + link: /per_social_experiment + x: 0 + y: 0 + height: 1 + width: 1 + type: antd-table + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_acquisition, aeon_block_analysis, aeon_tracking): + + acquisition = aeon_acquisition + block_analysis = aeon_block_analysis + tracking = aeon_tracking + + query = acquisition.Experiment.aggr(block_analysis.Block, block_count="COUNT(experiment_name)") + acquisition.Experiment.aggr(acquisition.Chunk, chunk_count="COUNT(experiment_name)", latest_chunk_start="MAX(chunk_start)") + + query = query.join(acquisition.Experiment.aggr( + tracking.SLEAPTracking.PoseIdentity, + participants="GROUP_CONCAT(DISTINCT identity_name SEPARATOR ', ')" + ), left=True) + + return {'query': query, 'fetch_args': []} + + BlockAnalysis: + route: /block_analysis + grids: + grid3: + type: fixed + columns: 1 + row_height: 700 + components: + BlockAnalysis: + route: /block_analysis_grid + link: /per_block_report + x: 0 + y: 0 + height: 1 + width: 1 + type: antd-table + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_block_analysis): + aeon_analysis = aeon_block_analysis + return {'query': aeon_analysis.Block * aeon_analysis.BlockAnalysis, 'fetch_args': {'order_by': 'block_end DESC'}} + + PerBlockReport: + hidden: true + route: /per_block_report + grids: + per_block_report: + type: fixed + route: /per_block_report + columns: 1 + row_height: 1500 + components: + comp1: + route: /per_block_meta + x: 0 + y: 0 + height: 0.2 + width: 0.8 + type: metadata + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_block_analysis): + aeon_analysis = aeon_block_analysis + query = aeon_analysis.Block * aeon_analysis.BlockAnalysis + return dict(query=query, fetch_args=[]) + comp2: + route: /subject_positions_plot + x: 0 + y: 0.2 + height: 0.5 + width: 0.8 + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_block_analysis): + aeon_analysis = aeon_block_analysis + return {'query': aeon_block_analysis.BlockPlots(), 'fetch_args': ['subject_positions_plot']} + comp3: + route: /subject_weights_plot + x: 0 + y: 0.7 + height: 0.5 + width: 0.8 + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_block_analysis): + aeon_analysis = aeon_block_analysis + return {'query': aeon_block_analysis.BlockPlots(), 'fetch_args': ['subject_weights_plot']} + + comp4: + route: /patch_distance_travelled_plot + x: 0 + y: 1.2 + height: 0.5 + width: 0.8 + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_block_analysis): + aeon_analysis = aeon_block_analysis + return {'query': aeon_block_analysis.BlockPlots(), 'fetch_args': ['patch_distance_travelled_plot']} + + comp5: + route: /patch_rate_plot + x: 0 + y: 1.7 + height: 0.4 + width: 0.6 + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_block_analysis): + aeon_analysis = aeon_block_analysis + return {'query': aeon_block_analysis.BlockPlots(), 'fetch_args': ['patch_rate_plot']} + + comp6: + route: /cumulative_pellet_plot + x: 0 + y: 2.1 + height: 0.4 + width: 0.6 + type: plot:plotly:stored_json + restriction: > + def restriction(**kwargs): + return dict(**kwargs) + dj_query: > + def dj_query(aeon_block_analysis): + aeon_analysis = aeon_block_analysis + return {'query': aeon_block_analysis.BlockPlots(), 'fetch_args': ['cumulative_pellet_plot']} From b139d8a6790bb9f839b9c4410c777ce2cf390a91 Mon Sep 17 00:00:00 2001 From: lochhh Date: Wed, 27 Mar 2024 18:26:09 +0000 Subject: [PATCH 7/8] Update docker-compose-local.yaml --- .../webapps/sciviz/docker-compose-local.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/aeon/dj_pipeline/webapps/sciviz/docker-compose-local.yaml b/aeon/dj_pipeline/webapps/sciviz/docker-compose-local.yaml index 2ac3f1ac..7703a795 100644 --- a/aeon/dj_pipeline/webapps/sciviz/docker-compose-local.yaml +++ b/aeon/dj_pipeline/webapps/sciviz/docker-compose-local.yaml @@ -7,14 +7,14 @@ services: pharus: cpus: 2.0 mem_limit: 4g - image: jverswijver/pharus:0.8.5-PY_VER-3.9 + image: datajoint/pharus:0.8.10-py3.9 environment: # - FLASK_ENV=development # enables logging to console from Flask - - PHARUS_SPEC_PATH=/main/specsheet.yaml # for dynamic utils spec + - PHARUS_SPEC_PATH=/main/specsheet-local.yaml # for dynamic utils spec - PHARUS_MODE=DEV user: root volumes: - - ./specsheet.yaml:/main/specsheet.yaml #copy the spec over to /main/specs/YOUR_SPEC_NAME + - ./specsheet-local.yaml:/main/specsheet-local.yaml #copy the spec over to /main/specs/YOUR_SPEC_NAME - ./apk_requirements.txt:/tmp/apk_requirements.txt - /ceph/aeon/aeon:/ceph/aeon/aeon command: @@ -23,12 +23,12 @@ services: - | apk add --update git g++ && git clone -b datajoint_pipeline https://github.com/SainsburyWellcomeCentre/aeon_mecha.git && - pip install -e ./aeon_mecha && + pip install -e ./aeon_mecha --ignore-requires-python && pharus_update() { [ -z "$$GUNICORN_PID" ] || kill $$GUNICORN_PID gunicorn --bind 0.0.0.0:$${PHARUS_PORT} pharus.server:app & GUNICORN_PID=$$! - } + } && pharus_update echo "[$$(date -u '+%Y-%m-%d %H:%M:%S')][DataJoint]: Monitoring Pharus updates..." INIT_TIME=$$(date +%s) @@ -52,16 +52,16 @@ services: sci-viz: cpus: 2.0 mem_limit: 16g - image: jverswijver/sci-viz:2.3.3-hotfix3 + image: jverswijver/sci-viz:2.3.5-beta environment: - CHOKIDAR_USEPOLLING=true - REACT_APP_DJSCIVIZ_BACKEND_PREFIX=/api - - DJSCIVIZ_SPEC_PATH=/main/specsheet.yaml + - DJSCIVIZ_SPEC_PATH=/main/specsheet-local.yaml - DJSCIVIZ_MODE=DEV - NODE_OPTIONS="--max-old-space-size=12000" user: root volumes: - - ./specsheet.yaml:/main/specsheet.yaml + - ./specsheet-local.yaml:/main/specsheet-local.yaml # ports: # - "3000:3000" command: From 264ca9425d5020b4418fb3f8b623f54e95998a85 Mon Sep 17 00:00:00 2001 From: lochhh Date: Wed, 27 Mar 2024 18:26:26 +0000 Subject: [PATCH 8/8] Update sciviz local setup readme --- aeon/dj_pipeline/webapps/sciviz/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aeon/dj_pipeline/webapps/sciviz/README.md b/aeon/dj_pipeline/webapps/sciviz/README.md index 6f1cef44..a43a5a04 100644 --- a/aeon/dj_pipeline/webapps/sciviz/README.md +++ b/aeon/dj_pipeline/webapps/sciviz/README.md @@ -26,7 +26,7 @@ HOST_UID=$(id -u) docker-compose -f docker-compose-local.yaml up ``` #### Verify the deployment -- In your web browser, navigate to: [https://localhost/login](https://localhost/login) +- In your web browser, navigate to: [https://localhost/](https://localhost/) - Set `Username` and `Password` to your own database user account (if you need one, please contact Project Aeon admin team). - Click `Connect`. - To stop the application, docker compose down with the following: @@ -52,7 +52,8 @@ HOST_UID=$(id -u) docker-compose -f docker-compose-remote.yaml up -d --- ## Dynamic spec sheet -Sci-Viz is used to build visualization dashboards, this is done through a single spec sheet. The one for this deployment is called `specsheet.yaml` +Sci-Viz is used to build visualization dashboards, this is done through a single spec sheet. The one used in production is called `specsheet.yaml`, +and the one used locally is called `specsheet-local.yaml`. Some notes about the spec sheet if you plan to tweak the website yourself: - Page names under pages must have a unique name without spaces