*rename DBHandler and update/upload metadata to DB

cellpack/autopack/DBRecipeHandler.py → cellpack/autopack/DBHandler.py

@@ -4,6 +4,8 @@
 
 from cellpack.autopack.utils import deep_merge
 
+from google.cloud.exceptions import NotFound
+
 
 class DataDoc(object):
     def __init__(
@@ -124,7 +126,7 @@ def resolve_local_regions(self, local_data, recipe_data, db):
         Recursively resolves the regions of a composition from local data.
         Restructure the local data to match the db data.
         """
-        unpack_recipe_data = DBRecipeHandler.prep_data_for_db(recipe_data)
+        unpack_recipe_data = DBHandler.prep_data_for_db(recipe_data)
         prep_recipe_data = ObjectDoc.convert_representation(unpack_recipe_data, db)
         # `gradients` is a list, convert it to dict for easy access and replace
         CompositionDoc.gradient_list_to_dict(prep_recipe_data)
@@ -326,7 +328,7 @@ def should_write(self, db):
                 # if there is repr in the obj doc from db
                 full_doc_data = ObjectDoc.convert_representation(doc, db)
                 # unpack objects to dicts in local data for comparison
-                local_data = DBRecipeHandler.prep_data_for_db(self.as_dict())
+                local_data = DBHandler.prep_data_for_db(self.as_dict())
                 difference = DeepDiff(full_doc_data, local_data, ignore_order=True)
                 if not difference:
                     return doc, db.doc_id(doc)
@@ -342,15 +344,15 @@ def should_write(self, db, grad_name):
         docs = db.get_doc_by_name("gradients", grad_name)
         if docs and len(docs) >= 1:
             for doc in docs:
-                local_data = DBRecipeHandler.prep_data_for_db(db.doc_to_dict(doc))
+                local_data = DBHandler.prep_data_for_db(db.doc_to_dict(doc))
                 db_data = db.doc_to_dict(doc)
                 difference = DeepDiff(db_data, local_data, ignore_order=True)
                 if not difference:
                     return doc, db.doc_id(doc)
         return None, None
 
 
-class DBRecipeHandler(object):
+class DBHandler(object):
     def __init__(self, db_handler):
         self.db = db_handler
         self.objects_to_path_map = {}
@@ -381,20 +383,20 @@ def prep_data_for_db(data):
         modified_data = {}
         for key, value in data.items():
             # convert 2d array to dict
-            if DBRecipeHandler.is_nested_list(value):
+            if DBHandler.is_nested_list(value):
                 flatten_dict = dict(zip([str(i) for i in range(len(value))], value))
-                modified_data[key] = DBRecipeHandler.prep_data_for_db(flatten_dict)
+                modified_data[key] = DBHandler.prep_data_for_db(flatten_dict)
             # If the value is an object, we want to convert it to dict
             elif isinstance(value, object) and "__dict__" in dir(value):
                 unpacked_value = vars(value)
                 modified_data[key] = unpacked_value
                 if isinstance(unpacked_value, dict):
-                    modified_data[key] = DBRecipeHandler.prep_data_for_db(
+                    modified_data[key] = DBHandler.prep_data_for_db(
                         unpacked_value
                     )
             # If the value is a dictionary, recursively convert its nested lists to dictionaries
             elif isinstance(value, dict):
-                modified_data[key] = DBRecipeHandler.prep_data_for_db(value)
+                modified_data[key] = DBHandler.prep_data_for_db(value)
             else:
                 modified_data[key] = value
         return modified_data
@@ -404,7 +406,7 @@ def upload_data(self, collection, data, id=None):
         If should_write is true, upload the data to the database
         """
         # check if we need to convert part of the data(2d arrays and objs to dict)
-        modified_data = DBRecipeHandler.prep_data_for_db(data)
+        modified_data = DBHandler.prep_data_for_db(data)
         if id is None:
             name = modified_data["name"]
             doc = self.db.upload_doc(collection, modified_data)
@@ -546,6 +548,17 @@ def upload_recipe(self, recipe_meta_data, recipe_data):
         key = self.get_recipe_id(recipe_to_save)
         self.upload_data("recipes", recipe_to_save, key)
 
+    def update_or_create_metadata(self, collection, id, data):
+        """
+        If the input id exists, update the metadata. If not, create a new file.
+        """
+        try:
+            self.db.update_doc(collection, id, data)
+        except NotFound:
+            self.db.set_doc(collection, id, data)
+
+
+
     def prep_db_doc_for_download(self, db_doc):
         """
         convert data from db and resolve references.

cellpack/autopack/FirebaseHandler.py

@@ -81,7 +81,7 @@ def get_collection_id_from_path(path):
     def update_doc(self, collection, id, data):
         doc_ref = self.db.collection(collection).document(id)
         doc_ref.update(data)
-        print(f"successfully uploaded to path: {doc_ref.path}")
+        print(f"successfully updated to path: {doc_ref.path}")
         return doc_ref
 
     @staticmethod

cellpack/autopack/__init__.py

@@ -50,7 +50,7 @@
 
 from cellpack.autopack.interface_objects.meta_enum import MetaEnum
 from cellpack.autopack.FirebaseHandler import FirebaseHandler
-from cellpack.autopack.DBRecipeHandler import DBRecipeHandler
+from cellpack.autopack.DBHandler import DBHandler
 from cellpack.autopack.loaders.utils import read_json_file, write_json_file
 
 
@@ -392,9 +392,9 @@ def load_file(filename, destination="", cache="geometries", force=None):
         # command example: `pack -r firebase:recipes/peroxisomes_surface_gradient_v-linear -c examples/packing-configs/peroxisome_packing_config.json`
         if database_name == "firebase":
             recipe_id = file_path.split("/")[-1]
-            db = FirebaseHandler()
-            db_doc, _ = db.get_doc_by_id(collection="recipes", id=recipe_id)
-            db_handler = DBRecipeHandler(db)
+            # TODO: do we need to include the case where the recipe is downloaded from firebase?
+            db_handler = DBHandler(FirebaseHandler())
+            db_doc, _ = db_handler.db.get_doc_by_id(collection="recipes", id=recipe_id)
             downloaded_recipe_data = db_handler.prep_db_doc_for_download(db_doc)
             return downloaded_recipe_data, database_name
         else:

cellpack/autopack/upy/simularium/simularium_helper.py

@@ -20,6 +20,7 @@
 )
 from simulariumio.cellpack import CellpackConverter, HAND_TYPE
 from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE
+from cellpack.autopack.DBHandler import DBHandler
 from cellpack.autopack.FirebaseHandler import FirebaseHandler
 
 from cellpack.autopack.upy import hostHelper
@@ -1352,6 +1353,7 @@ def post_and_open_file(self, file_name):
                 # TODO: add info on AWS installation and setup to our documetation
                 # TODO: link to documentation section on setting up AWS CLI and boto3 authentation
                 print(f"need to configure your aws account (link to readme here)")
+            # TODO: an ValueError is raised (in autopack/__init__ L396) when firebase app is already initialized, i.e. storing the recipe that downloaded from firebase
             else:
                 print(
                     f"An error occurred while storing the file {simularium_file} to S3:",
@@ -1384,19 +1386,15 @@ def store_results_to_s3(file_path):
         )
         file_name = handler.upload_file(file_path)
         url = handler.create_presigned_url(file_name)
-        # simulariumHelper.upload_metadata_to_firebase(file_name, url)
+        simulariumHelper.upload_metadata_to_firebase(file_name, url)
         return url
 
     @staticmethod
     def upload_metadata_to_firebase(result_file_name, aws_url):
-        # TODO: use db handler (rename to not have recipe in it)
-        # write a update_doc function that in the firebase handler
-        # class update doc instead of set doc
-        # check to make sure it writes a doc if non exists
-        db = FirebaseHandler()
-        username = db.get_username()
-        timestamp = db.create_timestamp()
-        db.update_doc(
+        db_handler = DBHandler(FirebaseHandler())
+        username = db_handler.db.get_username()
+        timestamp = db_handler.db.create_timestamp()
+        db_handler.update_or_create_metadata(
             "results",
             result_file_name,
             {"user": username, "created": timestamp, "aws_url": aws_url},

cellpack/bin/upload.py

@@ -1,7 +1,7 @@
 from enum import Enum
 import fire
 from cellpack.autopack.FirebaseHandler import FirebaseHandler
-from cellpack.autopack.DBRecipeHandler import DBRecipeHandler
+from cellpack.autopack.DBHandler import DBHandler
 
 from cellpack.autopack.loaders.recipe_loader import RecipeLoader
 
@@ -26,7 +26,7 @@ def upload(
         recipe_loader = RecipeLoader(recipe_path)
         recipe_full_data = recipe_loader.recipe_data
         recipe_meta_data = recipe_loader.get_only_recipe_metadata()
-        recipe_db_handler = DBRecipeHandler(db_handler)
+        recipe_db_handler = DBHandler(db_handler)
         recipe_db_handler.upload_recipe(recipe_meta_data, recipe_full_data)
 
 

cellpack/tests/test_comp_doc.py

@@ -1,4 +1,4 @@
-from cellpack.autopack.DBRecipeHandler import CompositionDoc
+from cellpack.autopack.DBHandler import CompositionDoc
 from cellpack.tests.mocks.mock_db import MockDB
 
 mock_db = MockDB({})

cellpack/tests/test_db_recipe_handler.py

@@ -1,13 +1,13 @@
-from cellpack.autopack.DBRecipeHandler import DBRecipeHandler
+from cellpack.autopack.DBHandler import DBHandler
 from cellpack.tests.mocks.mock_db import MockDB
 
 mock_db = MockDB({})
 
 
 def test_is_nested_list():
-    assert DBRecipeHandler.is_nested_list([]) is False
-    assert DBRecipeHandler.is_nested_list([[], []]) is True
-    assert DBRecipeHandler.is_nested_list([[1, 2], [3, 4]]) is True
+    assert DBHandler.is_nested_list([]) is False
+    assert DBHandler.is_nested_list([[], []]) is True
+    assert DBHandler.is_nested_list([[1, 2], [3, 4]]) is True
 
 
 def test_prep_data_for_db():
@@ -24,7 +24,7 @@ def test_prep_data_for_db():
         },
         "max_jitter": [1, 1, 0],
     }
-    new_data = DBRecipeHandler.prep_data_for_db(input_data)
+    new_data = DBHandler.prep_data_for_db(input_data)
     assert new_data == converted_data
 
 
@@ -37,7 +37,7 @@ def test_upload_data_with_recipe_and_id():
         "composition": {"test": {"inherit": "firebase:test_collection/test_id"}},
     }
     id = "test_id"
-    recipe_doc = DBRecipeHandler(mock_db)
+    recipe_doc = DBHandler(mock_db)
     expected_result = recipe_doc.upload_data(collection, data, id)
 
     assert expected_result[0] == "test_id"
@@ -50,7 +50,7 @@ def test_upload_data_with_object():
         "name": "test",
         "test_key": "test_value",
     }
-    object_doc = DBRecipeHandler(mock_db)
+    object_doc = DBHandler(mock_db)
     expected_result = object_doc.upload_data(collection, data)
 
     assert expected_result[0] == "test_id"
@@ -59,7 +59,7 @@ def test_upload_data_with_object():
 
 def test_upload_objects():
     data = {"test": {"test_key": "test_value"}}
-    object_doc = DBRecipeHandler(mock_db)
+    object_doc = DBHandler(mock_db)
     object_doc.upload_objects(data)
 
     assert object_doc.objects_to_path_map == {"test": "firebase:objects/test_id"}
@@ -83,7 +83,7 @@ def test_upload_compositions():
         },
     }
 
-    composition_doc = DBRecipeHandler(mock_db)
+    composition_doc = DBHandler(mock_db)
     references_to_update = composition_doc.upload_compositions(
         composition, recipe_to_save, recipe_data
     )
@@ -97,7 +97,7 @@ def test_upload_compositions():
 
 def test_upload_gradients():
     data = [{"name": "test_grad_name", "test_key": "test_value"}]
-    gradient_doc = DBRecipeHandler(mock_db)
+    gradient_doc = DBHandler(mock_db)
     gradient_doc.upload_gradients(data)
 
 
@@ -108,7 +108,7 @@ def test_get_recipe_id():
         "objects": None,
         "composition": {},
     }
-    recipe_doc = DBRecipeHandler(mock_db)
+    recipe_doc = DBHandler(mock_db)
     assert recipe_doc.get_recipe_id(recipe_data) == "test_v-1.0.0"
 
 
@@ -132,7 +132,7 @@ def test_upload_collections():
         },
     }
 
-    recipe_doc = DBRecipeHandler(mock_db)
+    recipe_doc = DBHandler(mock_db)
     expected_result = {
         "name": "one_sphere",
         "version": "1.0.0",
@@ -166,7 +166,7 @@ def test_upload_recipe():
         },
     }
 
-    recipe_doc = DBRecipeHandler(mock_db)
+    recipe_doc = DBHandler(mock_db)
     recipe_doc.upload_recipe(recipe_meta_data, recipe_data)
     assert recipe_doc.comp_to_path_map == {
         "space": {"path": "firebase:composition/test_id", "id": "test_id"},

cellpack/tests/test_gradient_doc.py

@@ -1,4 +1,4 @@
-from cellpack.autopack.DBRecipeHandler import GradientDoc
+from cellpack.autopack.DBHandler import GradientDoc
 from cellpack.tests.mocks.mock_db import MockDB
 
 mock_db = MockDB({})

cellpack/tests/test_object_doc.py

@@ -1,4 +1,4 @@
-from cellpack.autopack.DBRecipeHandler import ObjectDoc
+from cellpack.autopack.DBHandler import ObjectDoc
 from cellpack.tests.mocks.mock_db import MockDB
 
 mock_db = MockDB({})