From 560b01ed73a9f5efe5027721142c7f9087258cb8 Mon Sep 17 00:00:00 2001 From: Ruge Li <91452427+rugeli@users.noreply.github.com> Date: Mon, 13 May 2024 14:01:26 -0700 Subject: [PATCH] Skip staging db and aws on missing credentials (#243) * draft * formatting * revise return message * update documentation * remove logger from upload script * add a function to retrieve readme link * provide readme url when db not initialized --- README.md | 8 +++- cellpack/autopack/AWSHandler.py | 17 ++++--- cellpack/autopack/DBRecipeHandler.py | 6 +++ cellpack/autopack/FirebaseHandler.py | 26 +++++++---- cellpack/autopack/__init__.py | 5 +++ .../upy/simularium/simularium_helper.py | 45 ++++++++++--------- cellpack/bin/upload.py | 20 ++++++--- 7 files changed, 82 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index cbbea9878..a41d1bcc0 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,12 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for information related to developing the ### Firebase Firestore 1. Step-by-step Guide - * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore) - * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard. + * For dev database: + * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore) + * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard. + * For staging database: + * Reach out to the code owner for the necessary credentials. + * Set up an `.env` file as instructed. **MIT license** diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py index b4c6397ec..977b06085 100644 --- a/cellpack/autopack/AWSHandler.py +++ b/cellpack/autopack/AWSHandler.py @@ -3,7 +3,7 @@ from urllib.parse import parse_qs, urlparse, urlunparse import boto3 -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError, NoCredentialsError class AWSHandler(object): @@ -115,8 +115,13 @@ def save_file_and_get_url(self, file_path): """ Uploads a file to S3 and returns the base url """ - file_name = self.upload_file(file_path) - base_url = self.create_presigned_url(file_name) - if file_name and base_url: - if self.is_url_valid(base_url): - return file_name, base_url + try: + file_name = self.upload_file(file_path) + base_url = self.create_presigned_url(file_name) + if file_name and base_url: + if self.is_url_valid(base_url): + return file_name, base_url + except NoCredentialsError as e: + print(f"AWS credentials are not configured, details:{e}") + return None, None + return None, None diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index d166621d0..c5b9bf225 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -771,3 +771,9 @@ def cleanup_results(self): Check if the results in the database are expired and delete them if the linked object expired. """ self.result_doc.handle_expired_results() + + def readme_url(self): + """ + Return the URL to the README file for the database setup section. + """ + return "https://github.com/mesoscope/cellpack?tab=readme-ov-file#introduction-to-remote-databases" diff --git a/cellpack/autopack/FirebaseHandler.py b/cellpack/autopack/FirebaseHandler.py index b8d64624d..a4c60f222 100644 --- a/cellpack/autopack/FirebaseHandler.py +++ b/cellpack/autopack/FirebaseHandler.py @@ -24,14 +24,12 @@ def __init__(self, default_db=None): # check if firebase is already initialized if not FirebaseHandler._initialized: db_choice = FirebaseHandler.which_db(default_db=default_db) - if db_choice == "staging": - cred = FirebaseHandler.get_staging_creds() - else: - cred = FirebaseHandler.get_dev_creds() - login = credentials.Certificate(cred) - firebase_admin.initialize_app(login) - FirebaseHandler._initialized = True - FirebaseHandler._db = firestore.client() + cred = FirebaseHandler.get_creds(db_choice) + if cred: + login = credentials.Certificate(cred) + firebase_admin.initialize_app(login) + FirebaseHandler._db = firestore.client() + FirebaseHandler._initialized = True self.db = FirebaseHandler._db self.name = "firebase" @@ -49,6 +47,14 @@ def which_db(default_db=None): print(f"Using {options.get(choice, 'dev')} database -------------") return options.get(choice, "dev") # default to dev db for recipe uploads + @staticmethod + def get_creds(db_choice): + if db_choice == "staging": + cred = FirebaseHandler.get_staging_creds() + else: + cred = FirebaseHandler.get_dev_creds() + return cred + @staticmethod def doc_to_dict(doc): return doc.to_dict() @@ -115,8 +121,10 @@ def get_staging_creds(): # set override=True to refresh the .env file if softwares or tokens updated load_dotenv(dotenv_path="./.env", override=False) FIREBASE_TOKEN = os.getenv("FIREBASE_TOKEN") - firebase_key = FIREBASE_TOKEN.replace("\\n", "\n") FIREBASE_EMAIL = os.getenv("FIREBASE_EMAIL") + if not FIREBASE_TOKEN or not FIREBASE_EMAIL: + return + firebase_key = FIREBASE_TOKEN.replace("\\n", "\n") return { "type": "service_account", "project_id": "cell-pack-database", diff --git a/cellpack/autopack/__init__.py b/cellpack/autopack/__init__.py index 143544f0e..12de9315c 100755 --- a/cellpack/autopack/__init__.py +++ b/cellpack/autopack/__init__.py @@ -387,6 +387,11 @@ def load_file(filename, destination="", cache="geometries", force=None): if database_name == "firebase": db = DATABASE_IDS.handlers().get(database_name) initialize_db = db() + if not initialize_db._initialized: + readme_url = "https://github.com/mesoscope/cellpack?tab=readme-ov-file#introduction-to-remote-databases" + sys.exit( + f"The selected database is not initialized. Please set up Firebase credentials to pack remote recipes. Refer to the instructions at {readme_url}" + ) db_handler = DBRecipeLoader(initialize_db) recipe_id = file_path.split("/")[-1] db_doc, _ = db_handler.collect_docs_by_id( diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index 1e047e560..457e0c854 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -7,7 +7,6 @@ import matplotlib import numpy as np import trimesh -from botocore.exceptions import NoCredentialsError from simulariumio import ( TrajectoryConverter, @@ -23,7 +22,7 @@ from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE from cellpack.autopack.upy import hostHelper -from cellpack.autopack.DBRecipeHandler import DBUploader +from cellpack.autopack.DBRecipeHandler import DBUploader, DBMaintenance from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS import collada @@ -1390,22 +1389,13 @@ def raycast_test(self, obj, start, end, length, **kw): def post_and_open_file(self, file_name, open_results_in_browser=True): simularium_file = Path(f"{file_name}.simularium") url = None - try: - _, url = simulariumHelper.store_result_file(simularium_file, storage="aws") - except Exception as e: - aws_readme_url = ( - "https://github.com/mesoscope/cellpack/blob/main/README.md#aws-s3" - ) - if isinstance(e, NoCredentialsError): - print( - f"need to configure your aws account, find instructions here: {aws_readme_url}" - ) - else: - print( - f"An error occurred while storing the file {simularium_file} to S3: {e}" - ) - if url is not None and open_results_in_browser: - simulariumHelper.open_in_simularium(url) + file_name, url = simulariumHelper.store_result_file( + simularium_file, storage="aws" + ) + if file_name and url: + simulariumHelper.store_metadata(file_name, url, db="firebase") + if open_results_in_browser: + simulariumHelper.open_in_simularium(url) @staticmethod def store_result_file(file_path, storage=None): @@ -1416,8 +1406,12 @@ def store_result_file(file_path, storage=None): sub_folder_name="simularium", region_name="us-west-2", ) - file_name, url = initialized_handler.save_file_and_get_url(file_path) - simulariumHelper.store_metadata(file_name, url, db="firebase") + file_name, url = initialized_handler.save_file_and_get_url(file_path) + if not file_name or not url: + db_maintainer = DBMaintenance(initialized_handler) + print( + f"If AWS access needed, please refer to the instructions at {db_maintainer.readme_url()}. \nSkipping the opening of new browser tabs -------------" + ) return file_name, url @staticmethod @@ -1427,8 +1421,15 @@ def store_metadata(file_name, url, db=None): initialized_db = handler( default_db="staging" ) # default to staging for metadata uploads - db_uploader = DBUploader(initialized_db) - db_uploader.upload_result_metadata(file_name, url) + if initialized_db._initialized: + db_uploader = DBUploader(initialized_db) + db_uploader.upload_result_metadata(file_name, url) + else: + db_maintainer = DBMaintenance(initialized_db) + print( + f"Firebase credentials are not found. If needed, please refer to the instructions at {db_maintainer.readme_url()}. \nSkipping firebase staging database -------------" + ) + return @staticmethod def open_in_simularium(aws_url): diff --git a/cellpack/bin/upload.py b/cellpack/bin/upload.py index d7deb061d..62f1b9f18 100644 --- a/cellpack/bin/upload.py +++ b/cellpack/bin/upload.py @@ -1,6 +1,8 @@ +import sys import fire + from cellpack.autopack.FirebaseHandler import FirebaseHandler -from cellpack.autopack.DBRecipeHandler import DBUploader +from cellpack.autopack.DBRecipeHandler import DBUploader, DBMaintenance from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS from cellpack.autopack.loaders.recipe_loader import RecipeLoader @@ -18,11 +20,17 @@ def upload( if db_id == DATABASE_IDS.FIREBASE: # fetch the service key json file db_handler = FirebaseHandler() - recipe_loader = RecipeLoader(recipe_path) - recipe_full_data = recipe_loader._read(resolve_inheritance=False) - recipe_meta_data = recipe_loader.get_only_recipe_metadata() - recipe_db_handler = DBUploader(db_handler) - recipe_db_handler.upload_recipe(recipe_meta_data, recipe_full_data) + if FirebaseHandler._initialized: + recipe_loader = RecipeLoader(recipe_path) + recipe_full_data = recipe_loader._read(resolve_inheritance=False) + recipe_meta_data = recipe_loader.get_only_recipe_metadata() + recipe_db_handler = DBUploader(db_handler) + recipe_db_handler.upload_recipe(recipe_meta_data, recipe_full_data) + else: + db_maintainer = DBMaintenance(db_handler) + sys.exit( + f"The selected database is not initialized. Please set up Firebase credentials to upload recipes. Refer to the instructions at {db_maintainer.readme_url()} " + ) def main():