diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBHandler.py similarity index 96% rename from cellpack/autopack/DBRecipeHandler.py rename to cellpack/autopack/DBHandler.py index 94e03717d..b643bdbc7 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBHandler.py @@ -4,6 +4,8 @@ from cellpack.autopack.utils import deep_merge +from google.cloud.exceptions import NotFound + class DataDoc(object): def __init__( @@ -124,7 +126,7 @@ def resolve_local_regions(self, local_data, recipe_data, db): Recursively resolves the regions of a composition from local data. Restructure the local data to match the db data. """ - unpack_recipe_data = DBRecipeHandler.prep_data_for_db(recipe_data) + unpack_recipe_data = DBHandler.prep_data_for_db(recipe_data) prep_recipe_data = ObjectDoc.convert_representation(unpack_recipe_data, db) # `gradients` is a list, convert it to dict for easy access and replace CompositionDoc.gradient_list_to_dict(prep_recipe_data) @@ -326,7 +328,7 @@ def should_write(self, db): # if there is repr in the obj doc from db full_doc_data = ObjectDoc.convert_representation(doc, db) # unpack objects to dicts in local data for comparison - local_data = DBRecipeHandler.prep_data_for_db(self.as_dict()) + local_data = DBHandler.prep_data_for_db(self.as_dict()) difference = DeepDiff(full_doc_data, local_data, ignore_order=True) if not difference: return doc, db.doc_id(doc) @@ -342,7 +344,7 @@ def should_write(self, db, grad_name): docs = db.get_doc_by_name("gradients", grad_name) if docs and len(docs) >= 1: for doc in docs: - local_data = DBRecipeHandler.prep_data_for_db(db.doc_to_dict(doc)) + local_data = DBHandler.prep_data_for_db(db.doc_to_dict(doc)) db_data = db.doc_to_dict(doc) difference = DeepDiff(db_data, local_data, ignore_order=True) if not difference: @@ -350,7 +352,7 @@ def should_write(self, db, grad_name): return None, None -class DBRecipeHandler(object): +class DBHandler(object): def __init__(self, db_handler): self.db = db_handler self.objects_to_path_map = {} @@ -381,20 +383,20 @@ def prep_data_for_db(data): modified_data = {} for key, value in data.items(): # convert 2d array to dict - if DBRecipeHandler.is_nested_list(value): + if DBHandler.is_nested_list(value): flatten_dict = dict(zip([str(i) for i in range(len(value))], value)) - modified_data[key] = DBRecipeHandler.prep_data_for_db(flatten_dict) + modified_data[key] = DBHandler.prep_data_for_db(flatten_dict) # If the value is an object, we want to convert it to dict elif isinstance(value, object) and "__dict__" in dir(value): unpacked_value = vars(value) modified_data[key] = unpacked_value if isinstance(unpacked_value, dict): - modified_data[key] = DBRecipeHandler.prep_data_for_db( + modified_data[key] = DBHandler.prep_data_for_db( unpacked_value ) # If the value is a dictionary, recursively convert its nested lists to dictionaries elif isinstance(value, dict): - modified_data[key] = DBRecipeHandler.prep_data_for_db(value) + modified_data[key] = DBHandler.prep_data_for_db(value) else: modified_data[key] = value return modified_data @@ -404,7 +406,7 @@ def upload_data(self, collection, data, id=None): If should_write is true, upload the data to the database """ # check if we need to convert part of the data(2d arrays and objs to dict) - modified_data = DBRecipeHandler.prep_data_for_db(data) + modified_data = DBHandler.prep_data_for_db(data) if id is None: name = modified_data["name"] doc = self.db.upload_doc(collection, modified_data) @@ -546,6 +548,17 @@ def upload_recipe(self, recipe_meta_data, recipe_data): key = self.get_recipe_id(recipe_to_save) self.upload_data("recipes", recipe_to_save, key) + def update_or_create_metadata(self, collection, id, data): + """ + If the input id exists, update the metadata. If not, create a new file. + """ + try: + self.db.update_doc(collection, id, data) + except NotFound: + self.db.set_doc(collection, id, data) + + + def prep_db_doc_for_download(self, db_doc): """ convert data from db and resolve references. diff --git a/cellpack/autopack/FirebaseHandler.py b/cellpack/autopack/FirebaseHandler.py index 320e06843..a0b067605 100644 --- a/cellpack/autopack/FirebaseHandler.py +++ b/cellpack/autopack/FirebaseHandler.py @@ -81,7 +81,7 @@ def get_collection_id_from_path(path): def update_doc(self, collection, id, data): doc_ref = self.db.collection(collection).document(id) doc_ref.update(data) - print(f"successfully uploaded to path: {doc_ref.path}") + print(f"successfully updated to path: {doc_ref.path}") return doc_ref @staticmethod diff --git a/cellpack/autopack/__init__.py b/cellpack/autopack/__init__.py index f2b12ab5a..76e92790c 100755 --- a/cellpack/autopack/__init__.py +++ b/cellpack/autopack/__init__.py @@ -50,7 +50,7 @@ from cellpack.autopack.interface_objects.meta_enum import MetaEnum from cellpack.autopack.FirebaseHandler import FirebaseHandler -from cellpack.autopack.DBRecipeHandler import DBRecipeHandler +from cellpack.autopack.DBHandler import DBHandler from cellpack.autopack.loaders.utils import read_json_file, write_json_file @@ -392,9 +392,9 @@ def load_file(filename, destination="", cache="geometries", force=None): # command example: `pack -r firebase:recipes/peroxisomes_surface_gradient_v-linear -c examples/packing-configs/peroxisome_packing_config.json` if database_name == "firebase": recipe_id = file_path.split("/")[-1] - db = FirebaseHandler() - db_doc, _ = db.get_doc_by_id(collection="recipes", id=recipe_id) - db_handler = DBRecipeHandler(db) + # TODO: do we need to include the case where the recipe is downloaded from firebase? + db_handler = DBHandler(FirebaseHandler()) + db_doc, _ = db_handler.db.get_doc_by_id(collection="recipes", id=recipe_id) downloaded_recipe_data = db_handler.prep_db_doc_for_download(db_doc) return downloaded_recipe_data, database_name else: diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index 2e395b75a..59510e580 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -20,6 +20,7 @@ ) from simulariumio.cellpack import CellpackConverter, HAND_TYPE from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE +from cellpack.autopack.DBHandler import DBHandler from cellpack.autopack.FirebaseHandler import FirebaseHandler from cellpack.autopack.upy import hostHelper @@ -1352,6 +1353,7 @@ def post_and_open_file(self, file_name): # TODO: add info on AWS installation and setup to our documetation # TODO: link to documentation section on setting up AWS CLI and boto3 authentation print(f"need to configure your aws account (link to readme here)") + # TODO: an ValueError is raised (in autopack/__init__ L396) when firebase app is already initialized, i.e. storing the recipe that downloaded from firebase else: print( f"An error occurred while storing the file {simularium_file} to S3:", @@ -1384,19 +1386,15 @@ def store_results_to_s3(file_path): ) file_name = handler.upload_file(file_path) url = handler.create_presigned_url(file_name) - # simulariumHelper.upload_metadata_to_firebase(file_name, url) + simulariumHelper.upload_metadata_to_firebase(file_name, url) return url @staticmethod def upload_metadata_to_firebase(result_file_name, aws_url): - # TODO: use db handler (rename to not have recipe in it) - # write a update_doc function that in the firebase handler - # class update doc instead of set doc - # check to make sure it writes a doc if non exists - db = FirebaseHandler() - username = db.get_username() - timestamp = db.create_timestamp() - db.update_doc( + db_handler = DBHandler(FirebaseHandler()) + username = db_handler.db.get_username() + timestamp = db_handler.db.create_timestamp() + db_handler.update_or_create_metadata( "results", result_file_name, {"user": username, "created": timestamp, "aws_url": aws_url}, diff --git a/cellpack/bin/upload.py b/cellpack/bin/upload.py index 0a823f28a..fde2bff09 100644 --- a/cellpack/bin/upload.py +++ b/cellpack/bin/upload.py @@ -1,7 +1,7 @@ from enum import Enum import fire from cellpack.autopack.FirebaseHandler import FirebaseHandler -from cellpack.autopack.DBRecipeHandler import DBRecipeHandler +from cellpack.autopack.DBHandler import DBHandler from cellpack.autopack.loaders.recipe_loader import RecipeLoader @@ -26,7 +26,7 @@ def upload( recipe_loader = RecipeLoader(recipe_path) recipe_full_data = recipe_loader.recipe_data recipe_meta_data = recipe_loader.get_only_recipe_metadata() - recipe_db_handler = DBRecipeHandler(db_handler) + recipe_db_handler = DBHandler(db_handler) recipe_db_handler.upload_recipe(recipe_meta_data, recipe_full_data) diff --git a/cellpack/tests/test_comp_doc.py b/cellpack/tests/test_comp_doc.py index 29e429f24..38b5862bc 100644 --- a/cellpack/tests/test_comp_doc.py +++ b/cellpack/tests/test_comp_doc.py @@ -1,4 +1,4 @@ -from cellpack.autopack.DBRecipeHandler import CompositionDoc +from cellpack.autopack.DBHandler import CompositionDoc from cellpack.tests.mocks.mock_db import MockDB mock_db = MockDB({}) diff --git a/cellpack/tests/test_db_recipe_handler.py b/cellpack/tests/test_db_recipe_handler.py index 3a6a11621..6230fc8d3 100644 --- a/cellpack/tests/test_db_recipe_handler.py +++ b/cellpack/tests/test_db_recipe_handler.py @@ -1,13 +1,13 @@ -from cellpack.autopack.DBRecipeHandler import DBRecipeHandler +from cellpack.autopack.DBHandler import DBHandler from cellpack.tests.mocks.mock_db import MockDB mock_db = MockDB({}) def test_is_nested_list(): - assert DBRecipeHandler.is_nested_list([]) is False - assert DBRecipeHandler.is_nested_list([[], []]) is True - assert DBRecipeHandler.is_nested_list([[1, 2], [3, 4]]) is True + assert DBHandler.is_nested_list([]) is False + assert DBHandler.is_nested_list([[], []]) is True + assert DBHandler.is_nested_list([[1, 2], [3, 4]]) is True def test_prep_data_for_db(): @@ -24,7 +24,7 @@ def test_prep_data_for_db(): }, "max_jitter": [1, 1, 0], } - new_data = DBRecipeHandler.prep_data_for_db(input_data) + new_data = DBHandler.prep_data_for_db(input_data) assert new_data == converted_data @@ -37,7 +37,7 @@ def test_upload_data_with_recipe_and_id(): "composition": {"test": {"inherit": "firebase:test_collection/test_id"}}, } id = "test_id" - recipe_doc = DBRecipeHandler(mock_db) + recipe_doc = DBHandler(mock_db) expected_result = recipe_doc.upload_data(collection, data, id) assert expected_result[0] == "test_id" @@ -50,7 +50,7 @@ def test_upload_data_with_object(): "name": "test", "test_key": "test_value", } - object_doc = DBRecipeHandler(mock_db) + object_doc = DBHandler(mock_db) expected_result = object_doc.upload_data(collection, data) assert expected_result[0] == "test_id" @@ -59,7 +59,7 @@ def test_upload_data_with_object(): def test_upload_objects(): data = {"test": {"test_key": "test_value"}} - object_doc = DBRecipeHandler(mock_db) + object_doc = DBHandler(mock_db) object_doc.upload_objects(data) assert object_doc.objects_to_path_map == {"test": "firebase:objects/test_id"} @@ -83,7 +83,7 @@ def test_upload_compositions(): }, } - composition_doc = DBRecipeHandler(mock_db) + composition_doc = DBHandler(mock_db) references_to_update = composition_doc.upload_compositions( composition, recipe_to_save, recipe_data ) @@ -97,7 +97,7 @@ def test_upload_compositions(): def test_upload_gradients(): data = [{"name": "test_grad_name", "test_key": "test_value"}] - gradient_doc = DBRecipeHandler(mock_db) + gradient_doc = DBHandler(mock_db) gradient_doc.upload_gradients(data) @@ -108,7 +108,7 @@ def test_get_recipe_id(): "objects": None, "composition": {}, } - recipe_doc = DBRecipeHandler(mock_db) + recipe_doc = DBHandler(mock_db) assert recipe_doc.get_recipe_id(recipe_data) == "test_v-1.0.0" @@ -132,7 +132,7 @@ def test_upload_collections(): }, } - recipe_doc = DBRecipeHandler(mock_db) + recipe_doc = DBHandler(mock_db) expected_result = { "name": "one_sphere", "version": "1.0.0", @@ -166,7 +166,7 @@ def test_upload_recipe(): }, } - recipe_doc = DBRecipeHandler(mock_db) + recipe_doc = DBHandler(mock_db) recipe_doc.upload_recipe(recipe_meta_data, recipe_data) assert recipe_doc.comp_to_path_map == { "space": {"path": "firebase:composition/test_id", "id": "test_id"}, diff --git a/cellpack/tests/test_gradient_doc.py b/cellpack/tests/test_gradient_doc.py index 34b093a95..430474add 100644 --- a/cellpack/tests/test_gradient_doc.py +++ b/cellpack/tests/test_gradient_doc.py @@ -1,4 +1,4 @@ -from cellpack.autopack.DBRecipeHandler import GradientDoc +from cellpack.autopack.DBHandler import GradientDoc from cellpack.tests.mocks.mock_db import MockDB mock_db = MockDB({}) diff --git a/cellpack/tests/test_object_doc.py b/cellpack/tests/test_object_doc.py index 6b82d2eac..94d1a60b8 100644 --- a/cellpack/tests/test_object_doc.py +++ b/cellpack/tests/test_object_doc.py @@ -1,4 +1,4 @@ -from cellpack.autopack.DBRecipeHandler import ObjectDoc +from cellpack.autopack.DBHandler import ObjectDoc from cellpack.tests.mocks.mock_db import MockDB mock_db = MockDB({})