added google drive upload capabilities, removed unneeded functions (#…

…14738)

<!--
Thanks for taking the time to open a pull request! Please make sure
you've read the "Opening Pull Requests" section of our Contributing
Guide:


https://github.com/Opentrons/opentrons/blob/edge/CONTRIBUTING.md#opening-pull-requests

To ensure your code is reviewed quickly and thoroughly, please fill out
the sections below to the best of your ability!
-->

# Overview

Connects ABR with Google Drive.

# Test Plan

<!--
Use this section to describe the steps that you took to test your Pull
Request.
If you did not perform any testing provide justification why.

OT-3 Developers: You should default to testing on actual physical
hardware.
Once again, if you did not perform testing against hardware, justify
why.

Note: It can be helpful to write a test plan before doing development

Example Test Plan (HTTP API Change)

- Verified that new optional argument `dance-party` causes the robot to
flash its lights, move the pipettes,
then home.
- Verified that when you omit the `dance-party` option the robot homes
normally
- Added protocol that uses `dance-party` argument to G-Code Testing
Suite
- Ran protocol that did not use `dance-party` argument and everything
was successful
- Added unit tests to validate that changes to pydantic model are
correct

-->

# Changelog

Added modules for google_sheet writing and google_drive uploading
Changed ABR data uploading to check google_sheet for run ids rather than
the local csv
Moved ABR files out of hardware testing

# Review requests

<!--
Describe any requests for your reviewers here.
-->

# Risk assessment

ignoring typing errors in google_drive_tool due to issues with
google-api-client-typing
ignoring import errors with hardware_testing due to conflicting
dependencies with google-api-client-typing

abr-testing/Pipfile

@@ -5,20 +5,30 @@ name = "pypi"
 
 [packages]
 abr-testing = { editable = true, path = "." }
+google-api-python-client = "==2.41.0"
+httplib2 = "==0.22.0"
+types-httplib2 = "*"
+oauth2client = "==4.1.3"
+gspread = "==6.0.2"
+hardware-testing = {editable = true, path = "../hardware-testing"}
+opentrons-shared-data = {editable = true, path = "./../shared-data/python"}
+opentrons-hardware = {editable = true, path = "./../hardware", extras=['FLEX']}
+opentrons = {editable = true, path = "./../api", extras=['flex-hardware']}
 
 [dev-packages]
 atomicwrites = "==1.4.1"
 colorama = "==0.4.4"
 pytest = "==7.1.1"
 pytest-cov = "==2.10.1"
-mypy = "==0.981"
+mypy = "==1.8.0"
 black = "==22.3.0"
 flake8 = "~=3.9.0"
 flake8-annotations = "~=2.6.2"
 flake8-docstrings = "~=1.6.0"
 flake8-noqa = "~=1.2.1"
 requests = "==2.27.1"
 types-requests = "==2.25.6"
+google-api-python-client-stubs = "*"
 
 [requires]
 python_version = "3.10"

abr-testing/abr_testing/data_collection/__init__.py

@@ -0,0 +1 @@
+"""Collect run logs and upload."""

abr-testing/abr_testing/data_collection/abr_google_drive.py

@@ -0,0 +1,212 @@
+"""Read ABR run logs from google drive."""
+import argparse
+import os
+import sys
+import json
+import gspread  # type: ignore[import]
+from datetime import datetime, timedelta
+from abr_testing.data_collection import read_robot_logs
+from typing import Set, Dict, Any
+from abr_testing.google_automation import google_drive_tool, google_sheets_tool
+
+
+def get_modules(file_results: Dict[str, str]) -> Dict[str, Any]:
+    """Get module IPs and models from run log."""
+    modList = (
+        "heaterShakerModuleV1",
+        "temperatureModuleV2",
+        "magneticBlockV1",
+        "thermocyclerModuleV2",
+    )
+    all_modules = {key: "" for key in modList}
+    for module in file_results.get("modules", []):
+        if isinstance(module, dict) and module.get("model") in modList:
+            try:
+                all_modules[module["model"]] = module["serialNumber"]
+            except KeyError:
+                all_modules[module["model"]] = "EMPTYSN"
+
+    return all_modules
+
+
+def create_data_dictionary(
+    runs_to_save: Set[str], storage_directory: str
+) -> Dict[Any, Dict[str, Any]]:
+    """Pull data from run files and format into a dictionary."""
+    runs_and_robots = {}
+    for filename in os.listdir(storage_directory):
+        file_path = os.path.join(storage_directory, filename)
+        if file_path.endswith(".json"):
+            with open(file_path) as file:
+                file_results = json.load(file)
+        else:
+            continue
+        run_id = file_results.get("run_id")
+        if run_id in runs_to_save:
+            robot = file_results.get("robot_name")
+            protocol_name = file_results["protocol"]["metadata"].get("protocolName", "")
+            software_version = file_results.get("API_Version", "")
+            left_pipette = file_results.get("left", "")
+            right_pipette = file_results.get("right", "")
+            extension = file_results.get("extension", "")
+            (
+                num_of_errors,
+                error_type,
+                error_code,
+                error_instrument,
+                error_level,
+            ) = read_robot_logs.get_error_info(file_results)
+            all_modules = get_modules(file_results)
+
+            start_time_str, complete_time_str, start_date, run_time_min = (
+                "",
+                "",
+                "",
+                0.0,
+            )
+            try:
+                start_time = datetime.strptime(
+                    file_results.get("startedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z"
+                )
+                adjusted_start_time = start_time - timedelta(hours=5)
+                start_date = str(adjusted_start_time.date())
+                start_time_str = str(adjusted_start_time).split("+")[0]
+                complete_time = datetime.strptime(
+                    file_results.get("completedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z"
+                )
+                adjusted_complete_time = complete_time - timedelta(hours=5)
+                complete_time_str = str(adjusted_complete_time).split("+")[0]
+                run_time = complete_time - start_time
+                run_time_min = run_time.total_seconds() / 60
+            except ValueError:
+                pass  # Handle datetime parsing errors if necessary
+
+            if run_time_min > 0:
+                row = {
+                    "Robot": robot,
+                    "Run_ID": run_id,
+                    "Protocol_Name": protocol_name,
+                    "Software Version": software_version,
+                    "Date": start_date,
+                    "Start_Time": start_time_str,
+                    "End_Time": complete_time_str,
+                    "Run_Time (min)": run_time_min,
+                    "Errors": num_of_errors,
+                    "Error_Code": error_code,
+                    "Error_Type": error_type,
+                    "Error_Instrument": error_instrument,
+                    "Error_Level": error_level,
+                    "Left Mount": left_pipette,
+                    "Right Mount": right_pipette,
+                    "Extension": extension,
+                }
+                row_2 = {**row, **all_modules}
+                runs_and_robots[run_id] = row_2
+            else:
+                os.remove(file_path)
+                print(f"Run ID: {run_id} has a run time of 0 minutes. Run removed.")
+    return runs_and_robots
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Read run logs on google drive.")
+    parser.add_argument(
+        "storage_directory",
+        metavar="STORAGE_DIRECTORY",
+        type=str,
+        nargs=1,
+        help="Path to long term storage directory for run logs.",
+    )
+    parser.add_argument(
+        "folder_name",
+        metavar="FOLDER_NAME",
+        type=str,
+        nargs=1,
+        help="Google Drive folder name.",
+    )
+    parser.add_argument(
+        "google_sheet_name",
+        metavar="GOOGLE_SHEET_NAME",
+        type=str,
+        nargs=1,
+        help="Google sheet name.",
+    )
+    args = parser.parse_args()
+    folder_name = args.folder_name[0]
+    storage_directory = args.storage_directory[0]
+    google_sheet_name = args.google_sheet_name[0]
+    parent_folder = False
+    try:
+        credentials_path = os.path.join(storage_directory, "credentials.json")
+    except FileNotFoundError:
+        print(f"Add credentials.json file to: {storage_directory}.")
+        sys.exit()
+    try:
+        google_drive = google_drive_tool.google_drive(
+            credentials_path, folder_name, parent_folder
+        )
+        print("Connected to google drive.")
+    except json.decoder.JSONDecodeError:
+        print(
+            "Credential file is damaged. Get from https://console.cloud.google.com/apis/credentials"
+        )
+        sys.exit()
+    # Get run ids on google sheet
+    try:
+        google_sheet = google_sheets_tool.google_sheet(
+            credentials_path, google_sheet_name, 0
+        )
+        print(f"Connected to google sheet: {google_sheet_name}")
+    except gspread.exceptions.APIError:
+        print("ERROR: Check google sheet name. Check credentials file.")
+        sys.exit()
+    run_ids_on_gs = google_sheet.get_column(2)
+    run_ids_on_gs = set(run_ids_on_gs)
+    # Read Google Drive .json files
+    google_drive_files = google_drive.list_folder()
+    google_drive_files_json = [
+        file for file in google_drive_files if file.endswith(".json")
+    ]
+    # read local directory
+    list_of_files = os.listdir(storage_directory)
+    local_files_json = set(
+        file for file in os.listdir(storage_directory) if file.endswith(".json")
+    )
+    missing_files = local_files_json - set(google_drive_files_json)
+    print(f"Missing files: {len(missing_files)}")
+
+    # Uploads files that are not in google drive directory
+    google_drive.upload_missing_files(storage_directory, missing_files)
+
+    # Run ids in google_drive_folder
+    run_ids_on_gd = read_robot_logs.get_run_ids_from_google_drive(google_drive)
+    missing_runs_from_gs = read_robot_logs.get_unseen_run_ids(
+        run_ids_on_gd, run_ids_on_gs
+    )
+    # Add missing runs to google sheet
+    runs_and_robots = create_data_dictionary(missing_runs_from_gs, storage_directory)
+    headers = [
+        "Robot",
+        "Run_ID",
+        "Protocol_Name",
+        "Software Version",
+        "Date",
+        "Start_Time",
+        "End_Time",
+        "Run_Time (min)",
+        "Errors",
+        "Error_Code",
+        "Error_Type",
+        "Error_Instrument",
+        "Error_Level",
+        "Left Mount",
+        "Right Mount",
+        "Extension",
+        "heaterShakerModuleV1",
+        "temperatureModuleV2",
+        "magneticBlockV1",
+        "thermocyclerModuleV2",
+    ]
+    read_robot_logs.write_to_local_and_google_sheet(
+        runs_and_robots, storage_directory, google_sheet_name, google_sheet, headers
+    )

abr-testing/abr_testing/data_collection/error_levels.csv

@@ -0,0 +1,52 @@
+Prefix,Error Code,Description,Categories,Level of Failure,
+1,1000,Communication Error,Hardware communication failed,2,
+1,1001,Canbus Communication Error,Hardware communication failed,2,
+1,1002,Internal USB Communication Error,Hardware communication failed,2,
+1,1003,Module Communication Error,Hardware communication failed,2,
+1,1004,Command Timed Out,Hardware communication failed,3,
+1,1005,Firmware Update Failed,Hardware communication failed,2,
+1,1006,Internal Message Format Error,Hardware communication failed,2,
+1,1007,Canbus Configuration Error,Hardware communication failed,2,
+1,1008,Canbus Bus Error,Hardware communication failed,2,
+2,2000,Robotics Control Error,A Robot Action Failed,3,
+2,2001,Motion Failed,A Robot Action Failed,4,
+2,2002,Homing Failed,A Robot Action Failed,4,
+2,2003,Stall or Collision Detected,A Robot Action Failed,3-4,
+2,2004,Motion Planning Failed,A Robot Action Failed,3,
+2,2005,Position Estimation Invalid,A Robot Action Failed,3,
+2,2006,Move Condition Not Met,A Robot Action Failed,3,
+2,2007,Calibration Structure Not Found,A Robot Action Failed,4,
+2,2008,Edge Not Found,A Robot Action Failed,3,
+2,2009,Early Capactivive Sense Trigger,A Robot Action Failed,4,
+2,2010,Innacrruate Non Contact Sweep,A Robot Action Failed,3,
+2,2011,Misaligned Gantry,A Robot Action Failed,3,
+2,2012,Unmatched Tip Presence States,A Robot Action Failed,3-4,
+2,2013,Position Unknown,A Robot Action Failed,4,
+2,2014,Execution Cancelled,A Robot Action Failed,3-4,
+2,2015,Failed Gripper Pickup Error,A Robot Action Failed,3-4,
+3,3000,Robotics Interaction Error,A Robot Interaction Failed,3,
+3,3001,Labware Dropped,A Robot Interaction Failed,3-4,
+3,3002,Labware Not Picked Up,A Robot Interaction Failed,3-4,
+3,3003,Tip Pickup Failed,A Robot Interaction Failed,4,
+3,3004,Tip Drop Failed,A Robot Interaction Failed,4,
+3,3005,Unexpeted Tip Removal,A Robot Interaction Failed,4,
+3,3006,Pipette Overpressure,A Robot Interaction Failed,3-4,
+3,3008,E-Stop Activated,A Robot Interaction Failed,Not an error,
+3,3009,E-Stop Not Present,A Robot Interaction Failed,5,
+3,3010,Pipette Not Present,A Robot Interaction Failed,5,
+3,3011,Gripper Not Present,A Robot Interaction Failed,5,
+3,3012,Unexpected Tip Attach,A Robot Interaction Failed,4,
+3,3013,Firmware Update Required,A Robot Interaction Failed,Not an error,
+3,3014,Invalid ID Actuator,A Robot Interaction Failed,3,
+3,3015,Module Not Pesent,A Robot Interaction Failed,5,Not an error
+3,3016,Invalid Instrument Data,A Robot Interaction Failed,3,
+3,3017,Invalid Liquid Class Name,A Robot Interaction Failed,5,Not an error
+3,3018,Tip Detector Not Found,A Robot Interaction Failed,3,
+4,4000,General Error,A Software Error Occured,2-4,How severe does a general error get
+4,4001,Robot In Use,A Software Error Occured,5,Not an error
+4,4002,API Removed,A Software Error Occured,5,used an old app on a new robot
+4,4003,Not Supported On Robot Type,A Software Error Occured,5,Not an error
+4,4004,Command Precondition Violated,A Software Error Occured,5,Not an error
+4,4005,Command Parameter Limit Violated,A Software Error Occured,5,Not an error
+4,4006,Invalid Protocol Data,A Software Error Occured,5,Not an error
+4,4007,API Misconfiguration,A Software Error Occured,5,Not an error

abr-testing/abr_testing/data_collection/error_levels.py

@@ -0,0 +1,8 @@
+"""ABR Error Levels.
+
+This reads a csv file that has error level categorization.
+"""
+
+import os
+
+ERROR_LEVELS_PATH = os.path.join(os.path.dirname(__file__), "error_levels.csv")