Skip staging db and aws on missing credentials (#243)

* draft

* formatting

* revise return message

* update documentation

* remove logger from upload script

* add a function to retrieve readme link

* provide readme url when db not initialized

README.md

@@ -85,8 +85,12 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for information related to developing the
 
 ### Firebase Firestore
 1. Step-by-step Guide
-   * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore)
-   * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard.
+   * For dev database:
+     * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore)
+     * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard.
+   * For staging database:
+       * Reach out to the code owner for the necessary credentials.
+       * Set up an `.env` file as instructed.
 
 **MIT license**
 

cellpack/autopack/AWSHandler.py

@@ -3,7 +3,7 @@
 from urllib.parse import parse_qs, urlparse, urlunparse
 
 import boto3
-from botocore.exceptions import ClientError
+from botocore.exceptions import ClientError, NoCredentialsError
 
 
 class AWSHandler(object):
@@ -115,8 +115,13 @@ def save_file_and_get_url(self, file_path):
         """
         Uploads a file to S3 and returns the base url
         """
-        file_name = self.upload_file(file_path)
-        base_url = self.create_presigned_url(file_name)
-        if file_name and base_url:
-            if self.is_url_valid(base_url):
-                return file_name, base_url
+        try:
+            file_name = self.upload_file(file_path)
+            base_url = self.create_presigned_url(file_name)
+            if file_name and base_url:
+                if self.is_url_valid(base_url):
+                    return file_name, base_url
+        except NoCredentialsError as e:
+            print(f"AWS credentials are not configured, details:{e}")
+            return None, None
+        return None, None

cellpack/autopack/DBRecipeHandler.py

@@ -771,3 +771,9 @@ def cleanup_results(self):
         Check if the results in the database are expired and delete them if the linked object expired.
         """
         self.result_doc.handle_expired_results()
+
+    def readme_url(self):
+        """
+        Return the URL to the README file for the database setup section.
+        """
+        return "https://github.com/mesoscope/cellpack?tab=readme-ov-file#introduction-to-remote-databases"

cellpack/autopack/FirebaseHandler.py

@@ -24,14 +24,12 @@ def __init__(self, default_db=None):
         # check if firebase is already initialized
         if not FirebaseHandler._initialized:
             db_choice = FirebaseHandler.which_db(default_db=default_db)
-            if db_choice == "staging":
-                cred = FirebaseHandler.get_staging_creds()
-            else:
-                cred = FirebaseHandler.get_dev_creds()
-            login = credentials.Certificate(cred)
-            firebase_admin.initialize_app(login)
-            FirebaseHandler._initialized = True
-            FirebaseHandler._db = firestore.client()
+            cred = FirebaseHandler.get_creds(db_choice)
+            if cred:
+                login = credentials.Certificate(cred)
+                firebase_admin.initialize_app(login)
+                FirebaseHandler._db = firestore.client()
+                FirebaseHandler._initialized = True
 
         self.db = FirebaseHandler._db
         self.name = "firebase"
@@ -49,6 +47,14 @@ def which_db(default_db=None):
         print(f"Using {options.get(choice, 'dev')} database -------------")
         return options.get(choice, "dev")  # default to dev db for recipe uploads
 
+    @staticmethod
+    def get_creds(db_choice):
+        if db_choice == "staging":
+            cred = FirebaseHandler.get_staging_creds()
+        else:
+            cred = FirebaseHandler.get_dev_creds()
+        return cred
+
     @staticmethod
     def doc_to_dict(doc):
         return doc.to_dict()
@@ -115,8 +121,10 @@ def get_staging_creds():
         # set override=True to refresh the .env file if softwares or tokens updated
         load_dotenv(dotenv_path="./.env", override=False)
         FIREBASE_TOKEN = os.getenv("FIREBASE_TOKEN")
-        firebase_key = FIREBASE_TOKEN.replace("\\n", "\n")
         FIREBASE_EMAIL = os.getenv("FIREBASE_EMAIL")
+        if not FIREBASE_TOKEN or not FIREBASE_EMAIL:
+            return
+        firebase_key = FIREBASE_TOKEN.replace("\\n", "\n")
         return {
             "type": "service_account",
             "project_id": "cell-pack-database",

cellpack/autopack/__init__.py

@@ -387,6 +387,11 @@ def load_file(filename, destination="", cache="geometries", force=None):
         if database_name == "firebase":
             db = DATABASE_IDS.handlers().get(database_name)
             initialize_db = db()
+            if not initialize_db._initialized:
+                readme_url = "https://github.com/mesoscope/cellpack?tab=readme-ov-file#introduction-to-remote-databases"
+                sys.exit(
+                    f"The selected database is not initialized. Please set up Firebase credentials to pack remote recipes. Refer to the instructions at {readme_url}"
+                )
             db_handler = DBRecipeLoader(initialize_db)
             recipe_id = file_path.split("/")[-1]
             db_doc, _ = db_handler.collect_docs_by_id(

cellpack/autopack/upy/simularium/simularium_helper.py

@@ -7,7 +7,6 @@
 import matplotlib
 import numpy as np
 import trimesh
-from botocore.exceptions import NoCredentialsError
 
 from simulariumio import (
     TrajectoryConverter,
@@ -23,7 +22,7 @@
 from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE
 
 from cellpack.autopack.upy import hostHelper
-from cellpack.autopack.DBRecipeHandler import DBUploader
+from cellpack.autopack.DBRecipeHandler import DBUploader, DBMaintenance
 from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS
 import collada
 
@@ -1390,22 +1389,13 @@ def raycast_test(self, obj, start, end, length, **kw):
     def post_and_open_file(self, file_name, open_results_in_browser=True):
         simularium_file = Path(f"{file_name}.simularium")
         url = None
-        try:
-            _, url = simulariumHelper.store_result_file(simularium_file, storage="aws")
-        except Exception as e:
-            aws_readme_url = (
-                "https://github.com/mesoscope/cellpack/blob/main/README.md#aws-s3"
-            )
-            if isinstance(e, NoCredentialsError):
-                print(
-                    f"need to configure your aws account, find instructions here: {aws_readme_url}"
-                )
-            else:
-                print(
-                    f"An error occurred while storing the file {simularium_file} to S3: {e}"
-                )
-        if url is not None and open_results_in_browser:
-            simulariumHelper.open_in_simularium(url)
+        file_name, url = simulariumHelper.store_result_file(
+            simularium_file, storage="aws"
+        )
+        if file_name and url:
+            simulariumHelper.store_metadata(file_name, url, db="firebase")
+            if open_results_in_browser:
+                simulariumHelper.open_in_simularium(url)
 
     @staticmethod
     def store_result_file(file_path, storage=None):
@@ -1416,8 +1406,12 @@ def store_result_file(file_path, storage=None):
                 sub_folder_name="simularium",
                 region_name="us-west-2",
             )
-        file_name, url = initialized_handler.save_file_and_get_url(file_path)
-        simulariumHelper.store_metadata(file_name, url, db="firebase")
+            file_name, url = initialized_handler.save_file_and_get_url(file_path)
+            if not file_name or not url:
+                db_maintainer = DBMaintenance(initialized_handler)
+                print(
+                    f"If AWS access needed, please refer to the instructions at {db_maintainer.readme_url()}. \nSkipping the opening of new browser tabs  -------------"
+                )
         return file_name, url
 
     @staticmethod
@@ -1427,8 +1421,15 @@ def store_metadata(file_name, url, db=None):
             initialized_db = handler(
                 default_db="staging"
             )  # default to staging for metadata uploads
-            db_uploader = DBUploader(initialized_db)
-            db_uploader.upload_result_metadata(file_name, url)
+            if initialized_db._initialized:
+                db_uploader = DBUploader(initialized_db)
+                db_uploader.upload_result_metadata(file_name, url)
+            else:
+                db_maintainer = DBMaintenance(initialized_db)
+                print(
+                    f"Firebase credentials are not found. If needed, please refer to the instructions at {db_maintainer.readme_url()}. \nSkipping firebase staging database -------------"
+                )
+        return
 
     @staticmethod
     def open_in_simularium(aws_url):

cellpack/bin/upload.py

@@ -1,6 +1,8 @@
+import sys
 import fire
+
 from cellpack.autopack.FirebaseHandler import FirebaseHandler
-from cellpack.autopack.DBRecipeHandler import DBUploader
+from cellpack.autopack.DBRecipeHandler import DBUploader, DBMaintenance
 
 from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS
 from cellpack.autopack.loaders.recipe_loader import RecipeLoader
@@ -18,11 +20,17 @@ def upload(
     if db_id == DATABASE_IDS.FIREBASE:
         # fetch the service key json file
         db_handler = FirebaseHandler()
-        recipe_loader = RecipeLoader(recipe_path)
-        recipe_full_data = recipe_loader._read(resolve_inheritance=False)
-        recipe_meta_data = recipe_loader.get_only_recipe_metadata()
-        recipe_db_handler = DBUploader(db_handler)
-        recipe_db_handler.upload_recipe(recipe_meta_data, recipe_full_data)
+        if FirebaseHandler._initialized:
+            recipe_loader = RecipeLoader(recipe_path)
+            recipe_full_data = recipe_loader._read(resolve_inheritance=False)
+            recipe_meta_data = recipe_loader.get_only_recipe_metadata()
+            recipe_db_handler = DBUploader(db_handler)
+            recipe_db_handler.upload_recipe(recipe_meta_data, recipe_full_data)
+        else:
+            db_maintainer = DBMaintenance(db_handler)
+            sys.exit(
+                f"The selected database is not initialized. Please set up Firebase credentials to upload recipes. Refer to the instructions at {db_maintainer.readme_url()} "
+            )
 
 
 def main():