Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added google drive upload capabilities, removed unneeded functions #14738

Merged
merged 2 commits into from
Mar 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion abr-testing/Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,30 @@ name = "pypi"

[packages]
abr-testing = { editable = true, path = "." }
google-api-python-client = "==2.41.0"
httplib2 = "==0.22.0"
types-httplib2 = "*"
oauth2client = "==4.1.3"
gspread = "==6.0.2"
hardware-testing = {editable = true, path = "../hardware-testing"}
opentrons-shared-data = {editable = true, path = "./../shared-data/python"}
opentrons-hardware = {editable = true, path = "./../hardware", extras=['FLEX']}
opentrons = {editable = true, path = "./../api", extras=['flex-hardware']}

[dev-packages]
atomicwrites = "==1.4.1"
colorama = "==0.4.4"
pytest = "==7.1.1"
pytest-cov = "==2.10.1"
mypy = "==0.981"
mypy = "==1.8.0"
black = "==22.3.0"
flake8 = "~=3.9.0"
flake8-annotations = "~=2.6.2"
flake8-docstrings = "~=1.6.0"
flake8-noqa = "~=1.2.1"
requests = "==2.27.1"
types-requests = "==2.25.6"
google-api-python-client-stubs = "*"

[requires]
python_version = "3.10"
916 changes: 896 additions & 20 deletions abr-testing/Pipfile.lock

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions abr-testing/abr_testing/data_collection/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Collect run logs and upload."""
212 changes: 212 additions & 0 deletions abr-testing/abr_testing/data_collection/abr_google_drive.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
"""Read ABR run logs from google drive."""
import argparse
import os
import sys
import json
import gspread # type: ignore[import]
from datetime import datetime, timedelta
from abr_testing.data_collection import read_robot_logs
from typing import Set, Dict, Any
from abr_testing.google_automation import google_drive_tool, google_sheets_tool


def get_modules(file_results: Dict[str, str]) -> Dict[str, Any]:
"""Get module IPs and models from run log."""
modList = (
"heaterShakerModuleV1",
"temperatureModuleV2",
"magneticBlockV1",
"thermocyclerModuleV2",
)
all_modules = {key: "" for key in modList}
for module in file_results.get("modules", []):
if isinstance(module, dict) and module.get("model") in modList:
try:
all_modules[module["model"]] = module["serialNumber"]
except KeyError:
all_modules[module["model"]] = "EMPTYSN"

return all_modules


def create_data_dictionary(
runs_to_save: Set[str], storage_directory: str
) -> Dict[Any, Dict[str, Any]]:
"""Pull data from run files and format into a dictionary."""
runs_and_robots = {}
for filename in os.listdir(storage_directory):
file_path = os.path.join(storage_directory, filename)
if file_path.endswith(".json"):
with open(file_path) as file:
file_results = json.load(file)
else:
continue
run_id = file_results.get("run_id")
if run_id in runs_to_save:
robot = file_results.get("robot_name")
protocol_name = file_results["protocol"]["metadata"].get("protocolName", "")
software_version = file_results.get("API_Version", "")
left_pipette = file_results.get("left", "")
right_pipette = file_results.get("right", "")
extension = file_results.get("extension", "")
(
num_of_errors,
error_type,
error_code,
error_instrument,
error_level,
) = read_robot_logs.get_error_info(file_results)
all_modules = get_modules(file_results)

start_time_str, complete_time_str, start_date, run_time_min = (
"",
"",
"",
0.0,
)
try:
start_time = datetime.strptime(
file_results.get("startedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z"
)
adjusted_start_time = start_time - timedelta(hours=5)
start_date = str(adjusted_start_time.date())
start_time_str = str(adjusted_start_time).split("+")[0]
complete_time = datetime.strptime(
file_results.get("completedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z"
)
adjusted_complete_time = complete_time - timedelta(hours=5)
complete_time_str = str(adjusted_complete_time).split("+")[0]
run_time = complete_time - start_time
run_time_min = run_time.total_seconds() / 60
except ValueError:
pass # Handle datetime parsing errors if necessary

if run_time_min > 0:
row = {
"Robot": robot,
"Run_ID": run_id,
"Protocol_Name": protocol_name,
"Software Version": software_version,
"Date": start_date,
"Start_Time": start_time_str,
"End_Time": complete_time_str,
"Run_Time (min)": run_time_min,
"Errors": num_of_errors,
"Error_Code": error_code,
"Error_Type": error_type,
"Error_Instrument": error_instrument,
"Error_Level": error_level,
"Left Mount": left_pipette,
"Right Mount": right_pipette,
"Extension": extension,
}
row_2 = {**row, **all_modules}
runs_and_robots[run_id] = row_2
else:
os.remove(file_path)
print(f"Run ID: {run_id} has a run time of 0 minutes. Run removed.")
return runs_and_robots


if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Read run logs on google drive.")
parser.add_argument(
"storage_directory",
metavar="STORAGE_DIRECTORY",
type=str,
nargs=1,
help="Path to long term storage directory for run logs.",
)
parser.add_argument(
"folder_name",
metavar="FOLDER_NAME",
type=str,
nargs=1,
help="Google Drive folder name.",
)
parser.add_argument(
"google_sheet_name",
metavar="GOOGLE_SHEET_NAME",
type=str,
nargs=1,
help="Google sheet name.",
)
args = parser.parse_args()
folder_name = args.folder_name[0]
storage_directory = args.storage_directory[0]
google_sheet_name = args.google_sheet_name[0]
parent_folder = False
try:
credentials_path = os.path.join(storage_directory, "credentials.json")
except FileNotFoundError:
print(f"Add credentials.json file to: {storage_directory}.")
sys.exit()
try:
google_drive = google_drive_tool.google_drive(
credentials_path, folder_name, parent_folder
)
print("Connected to google drive.")
except json.decoder.JSONDecodeError:
print(
"Credential file is damaged. Get from https://console.cloud.google.com/apis/credentials"
)
sys.exit()
# Get run ids on google sheet
try:
google_sheet = google_sheets_tool.google_sheet(
credentials_path, google_sheet_name, 0
)
print(f"Connected to google sheet: {google_sheet_name}")
except gspread.exceptions.APIError:
print("ERROR: Check google sheet name. Check credentials file.")
sys.exit()
run_ids_on_gs = google_sheet.get_column(2)
run_ids_on_gs = set(run_ids_on_gs)
# Read Google Drive .json files
google_drive_files = google_drive.list_folder()
google_drive_files_json = [
file for file in google_drive_files if file.endswith(".json")
]
# read local directory
list_of_files = os.listdir(storage_directory)
local_files_json = set(
file for file in os.listdir(storage_directory) if file.endswith(".json")
)
missing_files = local_files_json - set(google_drive_files_json)
print(f"Missing files: {len(missing_files)}")

# Uploads files that are not in google drive directory
google_drive.upload_missing_files(storage_directory, missing_files)

# Run ids in google_drive_folder
run_ids_on_gd = read_robot_logs.get_run_ids_from_google_drive(google_drive)
missing_runs_from_gs = read_robot_logs.get_unseen_run_ids(
run_ids_on_gd, run_ids_on_gs
)
# Add missing runs to google sheet
runs_and_robots = create_data_dictionary(missing_runs_from_gs, storage_directory)
headers = [
"Robot",
"Run_ID",
"Protocol_Name",
"Software Version",
"Date",
"Start_Time",
"End_Time",
"Run_Time (min)",
"Errors",
"Error_Code",
"Error_Type",
"Error_Instrument",
"Error_Level",
"Left Mount",
"Right Mount",
"Extension",
"heaterShakerModuleV1",
"temperatureModuleV2",
"magneticBlockV1",
"thermocyclerModuleV2",
]
read_robot_logs.write_to_local_and_google_sheet(
runs_and_robots, storage_directory, google_sheet_name, google_sheet, headers
)
52 changes: 52 additions & 0 deletions abr-testing/abr_testing/data_collection/error_levels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
Prefix,Error Code,Description,Categories,Level of Failure,
1,1000,Communication Error,Hardware communication failed,2,
1,1001,Canbus Communication Error,Hardware communication failed,2,
1,1002,Internal USB Communication Error,Hardware communication failed,2,
1,1003,Module Communication Error,Hardware communication failed,2,
1,1004,Command Timed Out,Hardware communication failed,3,
1,1005,Firmware Update Failed,Hardware communication failed,2,
1,1006,Internal Message Format Error,Hardware communication failed,2,
1,1007,Canbus Configuration Error,Hardware communication failed,2,
1,1008,Canbus Bus Error,Hardware communication failed,2,
2,2000,Robotics Control Error,A Robot Action Failed,3,
2,2001,Motion Failed,A Robot Action Failed,4,
2,2002,Homing Failed,A Robot Action Failed,4,
2,2003,Stall or Collision Detected,A Robot Action Failed,3-4,
2,2004,Motion Planning Failed,A Robot Action Failed,3,
2,2005,Position Estimation Invalid,A Robot Action Failed,3,
2,2006,Move Condition Not Met,A Robot Action Failed,3,
2,2007,Calibration Structure Not Found,A Robot Action Failed,4,
2,2008,Edge Not Found,A Robot Action Failed,3,
2,2009,Early Capactivive Sense Trigger,A Robot Action Failed,4,
2,2010,Innacrruate Non Contact Sweep,A Robot Action Failed,3,
2,2011,Misaligned Gantry,A Robot Action Failed,3,
2,2012,Unmatched Tip Presence States,A Robot Action Failed,3-4,
2,2013,Position Unknown,A Robot Action Failed,4,
2,2014,Execution Cancelled,A Robot Action Failed,3-4,
2,2015,Failed Gripper Pickup Error,A Robot Action Failed,3-4,
3,3000,Robotics Interaction Error,A Robot Interaction Failed,3,
3,3001,Labware Dropped,A Robot Interaction Failed,3-4,
3,3002,Labware Not Picked Up,A Robot Interaction Failed,3-4,
3,3003,Tip Pickup Failed,A Robot Interaction Failed,4,
3,3004,Tip Drop Failed,A Robot Interaction Failed,4,
3,3005,Unexpeted Tip Removal,A Robot Interaction Failed,4,
3,3006,Pipette Overpressure,A Robot Interaction Failed,3-4,
3,3008,E-Stop Activated,A Robot Interaction Failed,Not an error,
3,3009,E-Stop Not Present,A Robot Interaction Failed,5,
3,3010,Pipette Not Present,A Robot Interaction Failed,5,
3,3011,Gripper Not Present,A Robot Interaction Failed,5,
3,3012,Unexpected Tip Attach,A Robot Interaction Failed,4,
3,3013,Firmware Update Required,A Robot Interaction Failed,Not an error,
3,3014,Invalid ID Actuator,A Robot Interaction Failed,3,
3,3015,Module Not Pesent,A Robot Interaction Failed,5,Not an error
3,3016,Invalid Instrument Data,A Robot Interaction Failed,3,
3,3017,Invalid Liquid Class Name,A Robot Interaction Failed,5,Not an error
3,3018,Tip Detector Not Found,A Robot Interaction Failed,3,
4,4000,General Error,A Software Error Occured,2-4,How severe does a general error get
4,4001,Robot In Use,A Software Error Occured,5,Not an error
4,4002,API Removed,A Software Error Occured,5,used an old app on a new robot
4,4003,Not Supported On Robot Type,A Software Error Occured,5,Not an error
4,4004,Command Precondition Violated,A Software Error Occured,5,Not an error
4,4005,Command Parameter Limit Violated,A Software Error Occured,5,Not an error
4,4006,Invalid Protocol Data,A Software Error Occured,5,Not an error
4,4007,API Misconfiguration,A Software Error Occured,5,Not an error
8 changes: 8 additions & 0 deletions abr-testing/abr_testing/data_collection/error_levels.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
"""ABR Error Levels.
This reads a csv file that has error level categorization.
"""

import os

ERROR_LEVELS_PATH = os.path.join(os.path.dirname(__file__), "error_levels.csv")
Loading
Loading