diff --git a/software/management/dashboard/parse.py b/software/management/dashboard/parse.py index 293c956b..959d93b7 100644 --- a/software/management/dashboard/parse.py +++ b/software/management/dashboard/parse.py @@ -44,11 +44,10 @@ def process_tottag_data(data, experiment_start_time): else: i += 1 elif data[i] == STORAGE_TYPE_IMU: - #TODO: Uncomment the following after all in-field TotTags have been updated: - #if data[i+5] <= MAX_IMU_DATA_LENGTH: - # i += 6 + data[i+5] - #else: - i += 1 + if data[i+5] <= MAX_IMU_DATA_LENGTH: + i += 6 + data[i+5] + else: + i += 1 elif data[i] == STORAGE_TYPE_BLE_SCAN: if data[i+5] < MAX_NUM_DEVICES: log_data[timestamp]['b'] = [] diff --git a/software/management/dashboard/tottag.py b/software/management/dashboard/tottag.py index 7fa728f6..7c7065fc 100755 --- a/software/management/dashboard/tottag.py +++ b/software/management/dashboard/tottag.py @@ -177,11 +177,10 @@ def process_tottag_data(from_uid, storage_directory, details, data, save_raw_fil else: i += 1 elif data[i] == STORAGE_TYPE_IMU: - #TODO: Uncomment the following after all in-field TotTags have been updated: - #if data[i+5] <= MAX_IMU_DATA_LENGTH: - # i += 6 + data[i+5] - #else: - i += 1 + if data[i+5] <= MAX_IMU_DATA_LENGTH: + i += 6 + data[i+5] + else: + i += 1 elif data[i] == STORAGE_TYPE_BLE_SCAN: if data[i+5] < MAX_NUM_DEVICES: log_data[timestamp]['b'] = [] @@ -438,6 +437,8 @@ async def download_logs(self): params = await self.command_queue.get() self.storage_directory = params['dir'] self.download_raw_logs = params['raw'] + if params['full']: + params['start'] = params['end'] = 0 try: self.data_length = 0 await self.connected_device.start_notify(MAINTENANCE_DATA_SERVICE_UUID, partial(self.data_callback)) @@ -497,6 +498,7 @@ def __init__(self, mode_switch_visibility=False): self.failed_devices = [] self.use_daily_times = tk.IntVar() self.download_raw_data = tk.IntVar() + self.download_full_log = tk.IntVar() self.ble_command_queue = asyncio.Queue() self.ble_result_queue = queue.Queue() self.tottag_selection = tk.StringVar(self.master, 'Press "Scan for TotTags" to begin...') @@ -588,6 +590,7 @@ def _date_entry_changed(self, event): def _download_logs(self): self._clear_canvas() self.download_raw_data.set(0) + self.download_full_log.set(0) prompt_area = tk.Frame(self.canvas) prompt_area.place(relx=0.5, rely=0.5, anchor=tk.CENTER) tk.Label(prompt_area, text="Download Deployment Log Files").grid(column=0, row=0, columnspan=4, sticky=tk.W+tk.E+tk.N+tk.S) @@ -608,29 +611,34 @@ def _download_logs(self): end_time_controls = tk.Frame(prompt_area) end_time_controls.grid(column=2, row=7, columnspan=2, sticky=tk.W+tk.E+tk.N+tk.S) ttk.Label(start_time_controls, text="Start Date: ").pack(side=tk.LEFT) - date_entry = DateEntry(start_time_controls, textvariable=self.start_date, selectmode='day', firstweekday='sunday', showweeknumbers=False, date_pattern='mm/dd/yyyy') - date_entry.pack(side=tk.LEFT) - date_entry.bind('', self._focus_in) - date_entry.bind('', self._date_entry_clicked) - date_entry.bind("<>", self._date_entry_changed) - date_entry = DateEntry(end_time_controls, textvariable=self.end_date, selectmode='day', firstweekday='sunday', showweeknumbers=False, date_pattern='mm/dd/yyyy') - date_entry.pack(side=tk.RIGHT) - date_entry.bind('', self._focus_in) - date_entry.bind('', self._date_entry_clicked) - date_entry.bind("<>", self._date_entry_changed) + date_entry_start = DateEntry(start_time_controls, textvariable=self.start_date, selectmode='day', firstweekday='sunday', showweeknumbers=False, date_pattern='mm/dd/yyyy') + date_entry_start.pack(side=tk.LEFT) + date_entry_start.bind('', self._focus_in) + date_entry_start.bind('', self._date_entry_clicked) + date_entry_start.bind("<>", self._date_entry_changed) + date_entry_end = DateEntry(end_time_controls, textvariable=self.end_date, selectmode='day', firstweekday='sunday', showweeknumbers=False, date_pattern='mm/dd/yyyy') + date_entry_end.pack(side=tk.RIGHT) + date_entry_end.bind('', self._focus_in) + date_entry_end.bind('', self._date_entry_clicked) + date_entry_end.bind("<>", self._date_entry_changed) ttk.Label(end_time_controls, text="End Date: ").pack(side=tk.RIGHT) - ttk.Checkbutton(prompt_area, text="Download Raw Unprocessed Data", variable=self.download_raw_data).grid(column=0, columnspan=4, row=8, pady=5, sticky=tk.W+tk.N) + def change_dl_times_state(self): + date_entry_start['state'] = ['disabled' if self.download_full_log.get() else 'normal'] + date_entry_end['state'] = ['disabled' if self.download_full_log.get() else 'normal'] + ttk.Checkbutton(prompt_area, text="Download Full Logs", variable=self.download_full_log, command=partial(change_dl_times_state, self)).grid(column=0, columnspan=4, row=8, pady=(5, 0), sticky=tk.W+tk.N) + ttk.Checkbutton(prompt_area, text="Download Raw Unprocessed Data", variable=self.download_raw_data).grid(column=0, columnspan=4, row=9, pady=5, sticky=tk.W+tk.N) def begin_download(self): self.data_length = 0 ble_issue_command(self.event_loop, self.ble_command_queue, 'DOWNLOAD') ble_issue_command(self.event_loop, self.ble_command_queue, { 'dir': self.save_directory.get(), 'raw': self.download_raw_data.get(), + 'full': self.download_full_log.get(), 'start': pack_datetime(str(tzlocal.get_localzone()), self.start_date.get(), "00:00", False), 'end': pack_datetime(str(tzlocal.get_localzone()), self.end_date.get(), "00:00", False) + 86400 }) - ttk.Button(prompt_area, text="Begin", command=partial(begin_download, self)).grid(column=1, row=9) - ttk.Button(prompt_area, text="Cancel", command=partial(self._clear_canvas_with_prompt)).grid(column=2, row=9) + ttk.Button(prompt_area, text="Begin", command=partial(begin_download, self)).grid(column=1, row=10) + ttk.Button(prompt_area, text="Cancel", command=partial(self._clear_canvas_with_prompt)).grid(column=2, row=10) def _create_new_experiment(self): self._clear_canvas() @@ -726,6 +734,12 @@ def construct_details(self): tk.messagebox.showerror('TotTag Error', 'ERROR: You have chosen the same label for multiple TotTags!') else: chosen_labels.append(labels[i]) + if not errors: + start_timestamp = int(datetime.datetime.strptime(self.start_date.get(), '%m/%d/%Y').astimezone(pytz.utc).timestamp()) + end_timestamp = int(datetime.datetime.strptime(self.end_date.get(), '%m/%d/%Y').astimezone(pytz.utc).timestamp()) + if end_timestamp - start_timestamp > 1814400: + tk.messagebox.showerror('TotTag Error', 'ERROR: Deployment duration cannot last longer than 21 days') + errors = True if not errors: details = { 'start_time': pack_datetime(self.tottag_timezone.get(), self.start_date.get(), self.start_time.get(), False), diff --git a/software/management/dashboard/trim_pkl.py b/software/management/dashboard/trim_pkl.py new file mode 100644 index 00000000..a8c1ea94 --- /dev/null +++ b/software/management/dashboard/trim_pkl.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import datetime, pytz, tzlocal +import os, pickle, sys + +def trim_data(filename, start_timestamp, end_timestamp): + with open(filename, 'rb') as file: + data = pickle.load(file) + return [item for item in data if item['t'] >= start_timestamp and item['t'] <= end_timestamp] + + +if __name__ == "__main__": + + if len(sys.argv) != 4: + print('Usage: ./trim_pkl.py [LOGFILE] [EXPERIMENT_START_TIME MM/DD/YYYY] [EXPERIMENT_END_TIME MM/DD/YYYY]') + sys.exit(0) + + experiment_start_time = pytz.timezone(str(tzlocal.get_localzone())).localize(datetime.datetime.strptime(sys.argv[2], '%m/%d/%Y')).astimezone(pytz.utc).timestamp() + experiment_end_time = pytz.timezone(str(tzlocal.get_localzone())).localize(datetime.datetime.strptime(sys.argv[3], '%m/%d/%Y')).astimezone(pytz.utc).timestamp() + trimmed = trim_data(sys.argv[1], experiment_start_time, experiment_end_time) + with open(os.path.join(os.path.dirname(sys.argv[1]), 'trimmed_' + os.path.basename(sys.argv[1])), 'wb') as file: + pickle.dump(trimmed, file, protocol=pickle.HIGHEST_PROTOCOL) + print(trimmed) diff --git a/software/management/setup.py b/software/management/setup.py index 1cd7a982..fd360f6c 100644 --- a/software/management/setup.py +++ b/software/management/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name='tottag', - version='1.0.2', + version='1.0.4', author='Will Hedgecock', author_email='ronald.w.hedgecock@vanderbilt.edu', description='TotTag Management Dashboard',