Merge pull request #161 from mesoscope/feature/firebase-upload-test

Feature/firebase upload test

cellpack/autopack/DBRecipeHandler.py

@@ -56,7 +56,8 @@ def as_dict(self):
         data["regions"] = self.regions
         return data
 
-    def get_reference_data(self, key_or_dict, db):
+    @staticmethod
+    def get_reference_data(key_or_dict, db):
         """
         Returns the db data for a reference, and the key if it exists.
         Key --> the name of a composition
@@ -66,14 +67,13 @@ def get_reference_data(self, key_or_dict, db):
             key = key_or_dict
             downloaded_data, _ = db.get_doc_by_ref(key)
             return downloaded_data, None
-        else:
+        elif key_or_dict and isinstance(key_or_dict, dict):
             object_dict = key_or_dict
             if "object" in object_dict and db.is_reference(object_dict["object"]):
                 key = object_dict["object"]
                 downloaded_data, _ = db.get_doc_by_ref(key)
                 return downloaded_data, key
-            else:
-                return {}, None
+        return {}, None
 
     def resolve_db_regions(self, db_data, db):
         """
@@ -164,11 +164,34 @@ def check_and_replace_references(
                                     "index": update_field_path,
                                     "name": region_item,
                                 }
-
                         elif not db.is_reference(region_item["object"]):
                             obj_name = region_item["object"]
                             region_item["object"] = objects_to_path_map.get(obj_name)
 
+    @staticmethod
+    def update_reference(
+        db,
+        composition_id,
+        referring_comp_id,
+        index,
+        remove_comp_name,
+        update_in_array=False,
+    ):
+        """
+        Update comp references in the recipe
+        """
+        doc, doc_ref = db.get_doc_by_id("composition", composition_id)
+        if doc is None:
+            return
+        _, new_item_ref = db.get_doc_by_id("composition", referring_comp_id)
+        update_ref_path = f"{db.db_name()}:{db.get_path_from_ref(new_item_ref)}"
+        if update_in_array:
+            db.update_elements_in_array(
+                doc_ref, index, update_ref_path, remove_comp_name
+            )
+        else:
+            db.update_reference_on_doc(doc_ref, index, update_ref_path)
+
     def should_write(self, db, recipe_data):
         """
         Compares the resolved local composition to the resolved db composition.
@@ -186,9 +209,14 @@ def should_write(self, db, recipe_data):
         if db_docs and len(db_docs) >= 1:
             for doc in db_docs:
                 db_data = db.doc_to_dict(doc)
+                shallow_match = True
                 for item in CompositionDoc.SHALLOW_MATCH:
                     if db_data[item] != local_data[item]:
+                        print(db_data[item], local_data[item])
+                        shallow_match = False
                         break
+                if not shallow_match:
+                    continue
                 if local_data["regions"] is None and db_data["regions"] is None:
                     # found a match, so shouldn't write
                     return False, db.doc_id(doc)
@@ -202,8 +230,8 @@ def should_write(self, db, recipe_data):
                         ignore_type_in_groups=[tuple, list],
                     )
                     if not difference:
-                        return False, db.doc_id(doc)
-        return True, None
+                        return doc, db.doc_id(doc)
+        return None, None
 
 
 class ObjectDoc(DataDoc):
@@ -212,6 +240,13 @@ def __init__(self, name, settings):
         self.name = name
         self.settings = settings
 
+    def as_dict(self):
+        data = dict()
+        data["name"] = self.name
+        for key in self.settings:
+            data[key] = self.settings[key]
+        return data
+
     @staticmethod
     def convert_positions_in_representation(data):
         convert_data = {}
@@ -247,13 +282,6 @@ def convert_representation(doc, db):
                 ] = ObjectDoc.convert_positions_in_representation(position_value)
         return convert_doc
 
-    def as_dict(self):
-        data = dict()
-        data["name"] = self.name
-        for key in self.settings:
-            data[key] = self.settings[key]
-        return data
-
     def should_write(self, db):
         docs = db.get_doc_by_name("objects", self.name)
         if docs and len(docs) >= 1:
@@ -308,7 +336,6 @@ def prep_data_for_db(data):
                 modified_data[key] = value
         return modified_data
 
-    #
     def upload_data(self, collection, data, id=None):
         """
         If should_write is true, upload the data to the database
@@ -319,7 +346,7 @@ def upload_data(self, collection, data, id=None):
             name = modified_data["name"]
             doc = self.db.upload_doc(collection, modified_data)
             # doc is a tuple, e.g (DatetimeWithNanoseconds, data_obj)
-            doc_path = doc[1].path
+            doc_path = self.db.get_path_from_ref(doc[1])
             doc_id = self.db.doc_id(doc[1])
             print(f"successfully uploaded {name} to path: {doc_path}")
             return doc_id, self.db.create_path(collection, doc_id)
@@ -334,7 +361,7 @@ def upload_objects(self, objects):
             object_doc = ObjectDoc(name=obj_name, settings=objects[obj_name])
             _, doc_id = object_doc.should_write(self.db)
             if doc_id:
-                print(f"objects/{object_doc.name} is already exists in firestore")
+                print(f"objects/{object_doc.name} is already in firestore")
             else:
                 _, obj_path = self.upload_data("objects", object_doc.as_dict())
                 self.objects_to_path_map[obj_name] = obj_path
@@ -361,7 +388,7 @@ def upload_compositions(self, compositions, recipe_to_save, recipe_data):
                 path = self.db.create_path("composition", doc_id)
                 self.comp_to_path_map[comp_name]["path"] = path
                 self.comp_to_path_map[comp_name]["id"] = doc_id
-                print(f"composition/{comp_name} is already exists in firestore")
+                print(f"composition/{comp_name} is already in firestore")
             else:
                 # replace with paths for outer objs in comp, then upload
                 comp_doc.check_and_replace_references(
@@ -392,30 +419,6 @@ def get_recipe_id(self, recipe_data):
         key = f"{recipe_name}_v{recipe_version}"
         return key
 
-    def update_reference(
-        self,
-        composition_id,
-        referring_comp_id,
-        index,
-        remove_comp_name,
-        update_in_array=False,
-    ):
-        """
-        Update comp references in the recipe
-        """
-        doc, doc_ref = self.db.get_doc_by_id("composition", composition_id)
-        if doc is None:
-            return
-        else:
-            _, new_item_ref = self.db.get_doc_by_id("composition", referring_comp_id)
-            update_ref_path = f"{self.db.name}:{new_item_ref.path}"
-            if update_in_array:
-                self.db.update_elements_in_array(
-                    doc_ref, index, update_ref_path, remove_comp_name
-                )
-            else:
-                self.db.update_reference_on_doc(doc_ref, index, update_ref_path)
-
     def upload_collections(self, recipe_meta_data, recipe_data):
         """
         Separate collections from recipe data and upload them to db
@@ -440,8 +443,8 @@ def upload_collections(self, recipe_meta_data, recipe_data):
                 name = inner_data["name"]
 
                 item_id = self.comp_to_path_map[name]["id"]
-                self.update_reference(
-                    comp_id, item_id, index, name, update_in_array=True
+                CompositionDoc.update_reference(
+                    self.db, comp_id, item_id, index, name, update_in_array=True
                 )
         return recipe_to_save
 

cellpack/autopack/FirebaseHandler.py

@@ -17,6 +17,9 @@ def __init__(self, cred_path):
     def doc_to_dict(doc):
         return doc.to_dict()
 
+    def db_name(self):
+        return self.name
+
     @staticmethod
     def doc_id(doc):
         return doc.id
@@ -25,6 +28,10 @@ def doc_id(doc):
     def create_path(collection, doc_id):
         return f"firebase:{collection}/{doc_id}"
 
+    @staticmethod
+    def get_path_from_ref(doc):
+        return doc.path
+
     @staticmethod
     def get_collection_id_from_path(path):
         # path example = firebase:composition/uid_1

cellpack/tests/mocks/mock_db.py

@@ -1,23 +1,71 @@
+from unittest.mock import Mock
+
+
 class MockDB(object):
     def __init__(self, data) -> None:
         for index, name in enumerate(data):
-            obj = data[name]
-            obj["id"] = index
+            self.obj = data[name]
+            self.obj["id"] = index
         self.data = data
+        self.name = "test_db"
 
     @staticmethod
     def is_firebase_obj(obj):
         return True
 
+    def db_name(self):
+        return "test_db"
+
     @staticmethod
     def doc_to_dict(doc):
         return doc
 
     def get_doc_by_name(self, _, name):
-        if name in self.data:
-            return self.data[name]
+        if len(self.data) >= 1:
+            if name in self.data["name"]:
+                return [self.data]
         else:
             return None
 
     def doc_id(self, doc):
+        if doc:
+            doc["id"] = "test_id"
         return doc["id"]
+
+    @staticmethod
+    def is_reference(path):
+        if path is None:
+            return False
+        if path.startswith("firebase:"):
+            return True
+        return False
+
+    @staticmethod
+    def get_doc_by_ref(key):
+        if key:
+            return {"test": "downloaded_data"}, key
+        else:
+            return {}, None
+
+    def upload_doc(self, collection, doc):
+        return ("test_datetime", doc)
+
+    def set_doc(self, collection, id, data):
+        doc_ref = Mock()
+        return doc_ref
+
+    def get_path_from_ref(self, doc):
+        return "firebase:test_collection/test_id"
+
+    def create_path(self, collection, doc_id):
+        return f"firebase:{collection}/{doc_id}"
+
+    def get_doc_by_id(self, collection, id):
+        doc_ref = Mock()
+        return self.data, doc_ref
+
+    def update_reference_on_doc(self, doc_ref, index, new_item_ref):
+        return True
+
+    def update_elements_in_array(self, doc_ref, index, new_item_ref, remove_item):
+        return True