staging: run recipes from firebase (#179)

* [wip] prep recipe data for packing

* get creds from local file

* save firebase creds to a .creds file

* remove cred arg

* check for already handled values in remote recipes

* can pack one sphere

* * adding username to .creds
* formatting

* move `write_username_to_creds`

* download recipe testing

* edit comment

* code refactor

* lint

* format tests

* add prep_db_doc

* changed class name in DBRecipeHandler

* fix lint and test errors

* initialize firebase handler only once

* refactor message

* add remote db options in `pack`

* remove a print statement

* rename and reorg DB handler

* fix tests

* move database_ids enum to interface_objects

* remove db_handler in pack and recipe_loader

* send db_handler in to autopack

* rename functions

* integrate DATABASE_NAMES into interface_objects

* lint

* Feature/run inherited objects (#198)

* turn off resolving inheritance while uploading

* able to upload recipes having "inherit" key

* get download and pack to work, refactors needed

* refactors

* formatting

* testing and refactor

* Feature/save metadata to firebase (#206)

* refactor AWS and firebase handler

* databases initiation handling

* refactor

* Update .gitignore

Co-authored-by: Saurabh Mogre <[email protected]>

* add file existence check

* refactor is_nested_list method

* revert write_json_file

* formatting

---------

Co-authored-by: meganrm <[email protected]>
Co-authored-by: Saurabh Mogre <[email protected]>

cellpack/autopack/AWSHandler.py

@@ -10,6 +10,10 @@ class AWSHandler(object):
     Handles all the AWS S3 operations
     """
 
+    # class attributes
+    _session_created = False
+    _s3_client = None
+
     def __init__(
         self,
         bucket_name,
@@ -18,12 +22,21 @@ def __init__(
     ):
         self.bucket_name = bucket_name
         self.folder_name = sub_folder_name
-        session = boto3.Session()
-        self.s3_client = session.client(
+        # Create a session if one does not exist
+        if not AWSHandler._session_created:
+            self._create_session(region_name)
+            AWSHandler._session_created = True
+        else:
+            # use the existing session
+            self.s3_client = AWSHandler._s3_client
+
+    def _create_session(self, region_name):
+        AWSHandler._s3_client = boto3.client(
             "s3",
             endpoint_url=f"https://s3.{region_name}.amazonaws.com",
             region_name=region_name,
         )
+        self.s3_client = AWSHandler._s3_client
 
     def get_aws_object_key(self, object_name):
         if self.folder_name is not None:
@@ -82,4 +95,4 @@ def save_file(self, file_path):
         """
         file_name = self.upload_file(file_path)
         if file_name:
-            return self.create_presigned_url(file_name)
+            return file_name, self.create_presigned_url(file_name)

cellpack/autopack/FirebaseHandler.py

@@ -1,25 +1,36 @@
+import ast
 import firebase_admin
 from firebase_admin import credentials, firestore
+from google.cloud.exceptions import NotFound
+from cellpack.autopack.loaders.utils import read_json_file, write_json_file
 
 
 class FirebaseHandler(object):
     """
     Retrieve data and perform common tasks when working with firebase.
     """
 
-    def __init__(self, cred_path):
-        login = credentials.Certificate(cred_path)
-        firebase_admin.initialize_app(login)
-        self.db = firestore.client()
+    # use class attributes to maintain a consistent state across all instances
+    _initialized = False
+    _db = None
+
+    def __init__(self):
+        # check if firebase is already initialized
+        if not FirebaseHandler._initialized:
+            cred_path = FirebaseHandler.get_creds()
+            login = credentials.Certificate(cred_path)
+            firebase_admin.initialize_app(login)
+            FirebaseHandler._initialized = True
+            FirebaseHandler._db = firestore.client()
+
+        self.db = FirebaseHandler._db
         self.name = "firebase"
 
+    # common utility methods
     @staticmethod
     def doc_to_dict(doc):
         return doc.to_dict()
 
-    def db_name(self):
-        return self.name
-
     @staticmethod
     def doc_id(doc):
         return doc.id
@@ -28,6 +39,10 @@ def doc_id(doc):
     def create_path(collection, doc_id):
         return f"firebase:{collection}/{doc_id}"
 
+    @staticmethod
+    def create_timestamp():
+        return firestore.SERVER_TIMESTAMP
+
     @staticmethod
     def get_path_from_ref(doc):
         return doc.path
@@ -40,34 +55,51 @@ def get_collection_id_from_path(path):
         id = components[1]
         return collection, id
 
-    @staticmethod
-    def update_reference_on_doc(doc_ref, index, new_item_ref):
-        doc_ref.update({index: new_item_ref})
+    # Create methods
+    def set_doc(self, collection, id, data):
+        doc, doc_ref = self.get_doc_by_id(collection, id)
+        if not doc:
+            doc_ref = self.db.collection(collection).document(id)
+            doc_ref.set(data)
+            print(f"successfully uploaded to path: {doc_ref.path}")
+            return doc_ref
+        else:
+            print(
+                f"ERROR: {doc_ref.path} already exists. If uploading new data, provide a unique recipe name."
+            )
+            return
+
+    def upload_doc(self, collection, data):
+        return self.db.collection(collection).add(data)
 
+    # Read methods
     @staticmethod
-    def update_elements_in_array(doc_ref, index, new_item_ref, remove_item):
-        doc_ref.update({index: firestore.ArrayRemove([remove_item])})
-        doc_ref.update({index: firestore.ArrayUnion([new_item_ref])})
+    def get_creds():
+        creds = read_json_file("./.creds")
+        if creds is None or "firebase" not in creds:
+            creds = FirebaseHandler.write_creds_path()
+        return creds["firebase"]
 
     @staticmethod
-    def is_reference(path):
-        if not isinstance(path, str):
-            return False
-        if path is None:
-            return False
-        if path.startswith("firebase:"):
-            return True
-        return False
+    def get_username():
+        creds = read_json_file("./.creds")
+        try:
+            return creds["username"]
+        except KeyError:
+            raise ValueError("No username found in .creds file")
+
+    def db_name(self):
+        return self.name
 
     def get_doc_by_name(self, collection, name):
         db = self.db
         data_ref = db.collection(collection)
         docs = data_ref.where("name", "==", name).get()  # docs is an array
         return docs
 
-    # `doc` is a DocumentSnapshot object
-    # `doc_ref` is a DocumentReference object to perform operations on the doc
     def get_doc_by_id(self, collection, id):
+        # `doc` is a DocumentSnapshot object
+        # `doc_ref` is a DocumentReference object to perform operations on the doc
         doc_ref = self.db.collection(collection).document(id)
         doc = doc_ref.get()
         if doc.exists:
@@ -79,19 +111,56 @@ def get_doc_by_ref(self, path):
         collection, id = FirebaseHandler.get_collection_id_from_path(path)
         return self.get_doc_by_id(collection, id)
 
-    def set_doc(self, collection, id, data):
-        doc, doc_ref = self.get_doc_by_id(collection, id)
-        if not doc:
-            doc_ref = self.db.collection(collection).document(id)
-            doc_ref.set(data)
-            print(f"successfully uploaded to path: {doc_ref.path}")
-            return doc_ref
+    # Update methods
+    def update_doc(self, collection, id, data):
+        doc_ref = self.db.collection(collection).document(id)
+        doc_ref.update(data)
+        print(f"successfully updated to path: {doc_ref.path}")
+        return doc_ref
+
+    @staticmethod
+    def update_reference_on_doc(doc_ref, index, new_item_ref):
+        doc_ref.update({index: new_item_ref})
+
+    @staticmethod
+    def update_elements_in_array(doc_ref, index, new_item_ref, remove_item):
+        doc_ref.update({index: firestore.ArrayRemove([remove_item])})
+        doc_ref.update({index: firestore.ArrayUnion([new_item_ref])})
+
+    def update_or_create(self, collection, id, data):
+        """
+        If the input id exists, update the doc. If not, create a new file.
+        """
+        try:
+            self.update_doc(collection, id, data)
+        except NotFound:
+            self.set_doc(collection, id, data)
+
+    # other utils
+    @staticmethod
+    def write_creds_path():
+        path = ast.literal_eval(input("provide path to firebase credentials: "))
+        data = read_json_file(path)
+        if data is None:
+            raise ValueError("The path to your credentials doesn't exist")
+        firebase_cred = {"firebase": data}
+        creds = read_json_file("./.creds")
+        if creds is None:
+            write_json_file("./.creds", firebase_cred)
         else:
-            print(f"ERROR, already data at this path:{collection}/{id}")
-            return
+            creds["firebase"] = data
+            write_json_file("./.creds", creds)
+        return firebase_cred
 
-    def upload_doc(self, collection, data):
-        return self.db.collection(collection).add(data)
+    @staticmethod
+    def is_reference(path):
+        if not isinstance(path, str):
+            return False
+        if path is None:
+            return False
+        if path.startswith("firebase:"):
+            return True
+        return False
 
     @staticmethod
     def is_firebase_obj(obj):

cellpack/autopack/__init__.py

@@ -41,13 +41,16 @@
 import re
 import shutil
 from os import path, environ
+import getpass
 from pathlib import Path
 import urllib.request as urllib
 from collections import OrderedDict
 import ssl
 import json
+from cellpack.autopack.DBRecipeHandler import DBRecipeLoader
+from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS
 
-from cellpack.autopack.interface_objects.meta_enum import MetaEnum
+from cellpack.autopack.loaders.utils import read_json_file, write_json_file
 
 
 packageContainsVFCommands = 1
@@ -192,19 +195,15 @@ def checkPath():
                 autopackdir = pref_path["autopackdir"]
 
 
-class DATABASE_NAME(MetaEnum):
-    GITHUB = "github:"
-    FIREBASE = "firebase:"
-
-
 REPLACE_PATH = {
     "autoPACKserver": autoPACKserver,
     "autopackdir": autopackdir,
     "autopackdata": appdata,
-    DATABASE_NAME.GITHUB: autoPACKserver,
-    DATABASE_NAME.FIREBASE: None,
+    f"{DATABASE_IDS.GITHUB}:": autoPACKserver,
+    f"{DATABASE_IDS.FIREBASE}:": None,
 }
 
+
 global CURRENT_RECIPE_PATH
 CURRENT_RECIPE_PATH = appdata
 # we keep the file here, it come with the distribution
@@ -280,7 +279,7 @@ def is_remote_path(file_path):
     """
     @param file_path: str
     """
-    for ele in DATABASE_NAME:
+    for ele in DATABASE_IDS.with_colon():
         if ele in file_path:
             return True
 
@@ -384,10 +383,16 @@ def read_text_file(filename, destination="", cache="collisionTrees", force=None)
 def load_file(filename, destination="", cache="geometries", force=None):
     if is_remote_path(filename):
         database_name, file_path = convert_db_shortname_to_url(filename)
+        # command example: `pack -r firebase:recipes/[FIREBASE-RECIPE-ID] -c [CONFIG-FILE-PATH]`
         if database_name == "firebase":
-            # TODO: read from firebase
-            # return data
-            pass
+            db = DATABASE_IDS.handlers().get(database_name)
+            db_handler = DBRecipeLoader(db)
+            recipe_id = file_path.split("/")[-1]
+            db_doc, _ = db_handler.collect_docs_by_id(
+                collection="recipes", id=recipe_id
+            )
+            downloaded_recipe_data = db_handler.prep_db_doc_for_download(db_doc)
+            return downloaded_recipe_data, database_name
         else:
             local_file_path = get_local_file_location(
                 file_path, destination=destination, cache=cache, force=force
@@ -396,7 +401,7 @@ def load_file(filename, destination="", cache="geometries", force=None):
         local_file_path = get_local_file_location(
             filename, destination=destination, cache=cache, force=force
         )
-    return json.load(open(local_file_path, "r"))
+    return json.load(open(local_file_path, "r")), None
 
 
 def fixPath(adict):  # , k, v):
@@ -532,16 +537,27 @@ def clearCaches(*args):
             print("problem cleaning ", cache_dir[k])
 
 
+def write_username_to_creds():
+    username = getpass.getuser()
+    creds = read_json_file("./.creds")
+    if creds is None or "username" not in creds:
+        creds = {}
+        creds["username"] = username
+        write_json_file("./.creds", creds)
+
+
 # we should read a file to fill the RECIPE Dictionary
 # so we can add some and write/save setup
 # afdir  or user_pref
-
 if checkAtstartup:
     checkPath()
     # updatePathJSON()
     # checkRecipeAvailable()
     log.info("path are updated ")
 
+# write username to creds
+write_username_to_creds()
+
 log.info(f"currently number recipes is {len(RECIPES)}")
 # check cache directory create if doesnt exit.abs//should be in user pref?
 # ?

cellpack/autopack/interface_objects/database_ids.py

@@ -0,0 +1,28 @@
+from .meta_enum import MetaEnum
+from cellpack.autopack.AWSHandler import AWSHandler
+from cellpack.autopack.FirebaseHandler import FirebaseHandler
+
+
+class DATABASE_IDS(MetaEnum):
+    FIREBASE = "firebase"
+    GITHUB = "github"
+    AWS = "aws"
+
+    @classmethod
+    def with_colon(cls):
+        return [f"{ele}:" for ele in cls.values()]
+
+    @classmethod
+    def handlers(cls):
+        def create_aws_handler(bucket_name, sub_folder_name, region_name):
+            return AWSHandler(
+                bucket_name=bucket_name,
+                sub_folder_name=sub_folder_name,
+                region_name=region_name,
+            )
+
+        handlers_dict = {
+            cls.FIREBASE: FirebaseHandler(),
+            cls.AWS: create_aws_handler,
+        }
+        return handlers_dict