From 40da3bea555c5e76531cf0d4ff7122ba59a3fa3e Mon Sep 17 00:00:00 2001 From: Saurabh Mogre Date: Mon, 26 Jun 2023 16:21:58 -0700 Subject: [PATCH] * save and load grids and mesh store to pickle * clear cache for saved meshes during loading --- cellpack/autopack/BaseGrid.py | 3 +- cellpack/autopack/Environment.py | 112 +++++++++++++----- cellpack/autopack/MeshStore.py | 2 +- cellpack/autopack/ingredient/Ingredient.py | 4 +- cellpack/autopack/utils.py | 12 ++ ...> peroxisome_parallel_packing_config.json} | 19 +-- 6 files changed, 108 insertions(+), 44 deletions(-) rename examples/packing-configs/{parallel_packing_config.json => peroxisome_parallel_packing_config.json} (59%) diff --git a/cellpack/autopack/BaseGrid.py b/cellpack/autopack/BaseGrid.py index 16c4bcfca..efcab8ab4 100644 --- a/cellpack/autopack/BaseGrid.py +++ b/cellpack/autopack/BaseGrid.py @@ -208,8 +208,7 @@ def setup(self, boundingBox): self.free_points = list(range(self.gridVolume)) self.nbFreePoints = len(self.free_points) self.log.info( - "Lookup: %d, bounding box: %r, gridSpacing %r, length compartment_ids %r", - boundingBox, + "gridSpacing %r, length compartment_ids %r", self.gridSpacing, len(self.compartment_ids), ) diff --git a/cellpack/autopack/Environment.py b/cellpack/autopack/Environment.py index 3c1d45f65..a123e8055 100644 --- a/cellpack/autopack/Environment.py +++ b/cellpack/autopack/Environment.py @@ -74,6 +74,7 @@ ingredient_compare0, ingredient_compare1, ingredient_compare2, + load_object_from_pickle, ) from cellpack.autopack.writers import Writer from .Compartment import CompartmentList, Compartment @@ -99,6 +100,9 @@ encoder.FLOAT_REPR = lambda o: format(o, ".8g") +# set default pickle protocol to highest level +pickle.DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL + SEED = 15 LOG = False verbose = 0 @@ -1049,6 +1053,44 @@ def saveGridLogsAsJson(self, gridFileOut): json.dump(data, fp=f) f.close() + def restore_grids_from_pickle(self, grid_file_path): + """ + Read and setup the grid from the given filename. (pickle) + """ + print(f"Loading grid from {grid_file_path}") + with open(grid_file_path, "rb") as file_obj: + # load env grid + self.grid = load_object_from_pickle(self.grid, file_obj) + + # load compartment grids + for ct, compartment in enumerate(self.compartments): + self.compartments[ct] = load_object_from_pickle(compartment, file_obj) + + # load mesh store + self.mesh_store = load_object_from_pickle(self.mesh_store, file_obj) + + # clear the triangles_tree cache + for _, geom in self.mesh_store.scene.geometry.items(): + geom._cache.delete("triangles_tree") + + def save_grids_to_pickle(self, grid_file_path): + """ + Save the current grid and compartment grids to file. (pickle) + """ + print(f"Saving grid to {grid_file_path}") + if not os.path.exists(os.path.dirname(grid_file_path)): + raise ValueError(f"Check grid file path: {grid_file_path}") + with open(grid_file_path, "wb") as file_obj: + # dump env grid + pickle.dump(self.grid, file_obj) + + # dump compartment grids + for compartment in self.compartments: + pickle.dump(compartment, file_obj) + + # dump mesh store + pickle.dump(self.mesh_store, file_obj) + def restoreGridFromFile(self, gridFileName): """ Read and setup the grid from the given filename. (pickle) @@ -1058,8 +1100,9 @@ def restoreGridFromFile(self, gridFileName): aSurfaceGrids = [] f = open(gridFileName, "rb") self.readArraysFromFile(f) - for compartment in self.compartments: - compartment.readGridFromFile(f) + for ct, compartment in enumerate(self.compartments): + # compartment.readGridFromFile(f) + compartment = compartment.load_compartment_object(f) aInteriorGrids.append(compartment.insidePoints) aSurfaceGrids.append(compartment.surfacePoints) compartment.OGsrfPtsBht = spatial.cKDTree( @@ -1068,6 +1111,7 @@ def restoreGridFromFile(self, gridFileName): compartment.compute_volume_and_set_count( self, compartment.surfacePoints, compartment.insidePoints, areas=None ) + self.compartments[ct] = compartment f.close() # TODO: restore surface distances on loading from grid self.grid.aInteriorGrids = aInteriorGrids @@ -1368,30 +1412,27 @@ def buildGrid(self, rebuild=True): self.grid.filename = self.previous_grid_file if self.nFill == 0: # first fill, after we can just reset self.log.info("restore from file") - self.restoreGridFromFile(self.previous_grid_file) + # self.restoreGridFromFile(self.previous_grid_file) + self.restore_grids_from_pickle(self.previous_grid_file) else: self.build_compartment_grids() - self.exteriorVolume = self.grid.computeExteriorVolume( - compartments=self.compartments, - space=self.smallestProteinSize, - fbox_bb=self.fbox_bb, - ) - if self.previous_grid_file is None: - self.saveGridToFile(self.grid_file_out) - self.grid.filename = self.grid_file_out - self.previous_grid_file = self.grid_file_out + self.exteriorVolume = self.grid.computeExteriorVolume( + compartments=self.compartments, + space=self.smallestProteinSize, + fbox_bb=self.fbox_bb, + ) - r = self.exteriorRecipe - if r: - r.setCount(self.exteriorVolume) # should actually use the fillBB - if not rebuild: - for c in self.compartments: - c.setCount() - else: - self.grid.distToClosestSurf_store = self.grid.distToClosestSurf[:] + r = self.exteriorRecipe + if r: + r.setCount(self.exteriorVolume) # should actually use the fillBB + + if not rebuild: + for c in self.compartments: + c.setCount() + else: + self.grid.distToClosestSurf_store = self.grid.distToClosestSurf[:] - if self.previous_grid_file is not None: distance = self.grid.distToClosestSurf # [:] nbFreePoints = nbPoints # -1 for i, mingrs in enumerate( @@ -1409,6 +1450,13 @@ def buildGrid(self, rebuild=True): ) self.grid.nbFreePoints = nbFreePoints + # save grids to pickle + # self.saveGridToFile(self.grid_file_out) + self.grid.filename = self.grid_file_out + self.previous_grid_file = self.grid_file_out + + self.save_grids_to_pickle(self.grid_file_out) + if self.use_gradient and len(self.gradients) and rebuild: for g in self.gradients: gradient = self.gradients[g] @@ -2251,6 +2299,7 @@ def pack_grid( if ingr.encapsulating_radius > self.largestProteinSize: self.largestProteinSize = ingr.encapsulating_radius + self.log.info( "attempting to place near %d: %r", ptInd, @@ -2930,17 +2979,16 @@ def setupPanda(self): loadPrcFileData( "", """ - load-display p3tinydisplay # to force CPU only rendering - (to make it available as an option if everything else fail, use aux-display p3tinydisplay) - audio-library-name null # Prevent ALSA errors - show-frame-rate-meter 0 - sync-video 0 - bullet-max-objects 10240 - bullet-broadphase-algorithm sap - bullet-sap-extents 10000.0 - textures-power-2 up - textures-auto-power-2 #t -""", + load-display p3tinydisplay # to force CPU only rendering (to make it available as an option if everything else fail, use aux-display p3tinydisplay) + audio-library-name null # Prevent ALSA errors + show-frame-rate-meter 0 + sync-video 0 + bullet-max-objects 10240 + bullet-broadphase-algorithm sap + bullet-sap-extents 10000.0 + textures-power-2 up + textures-auto-power-2 #t + """, ) # loadPrcFileData("", "window-type none" ) # Make sure we don't need a graphics engine diff --git a/cellpack/autopack/MeshStore.py b/cellpack/autopack/MeshStore.py index 375682154..3c7a987ed 100644 --- a/cellpack/autopack/MeshStore.py +++ b/cellpack/autopack/MeshStore.py @@ -268,7 +268,7 @@ def get_smallest_radius(self, geomname, center): mesh = self.get_object(geomname) if mesh is not None: query = trimesh.proximity.ProximityQuery(mesh) - (closet_point, distance, triangle_id) = query.on_surface([center]) + (_, distance, _) = query.on_surface([center]) return distance[0] return 1.0 diff --git a/cellpack/autopack/ingredient/Ingredient.py b/cellpack/autopack/ingredient/Ingredient.py index e02a881bc..37b6c1721 100644 --- a/cellpack/autopack/ingredient/Ingredient.py +++ b/cellpack/autopack/ingredient/Ingredient.py @@ -1085,9 +1085,9 @@ def far_enough_from_surfaces(self, point, cutoff): ): continue # checking compartments I don't belong to - res = compartment.OGsrfPtsBht.query(tuple(numpy.array([point]))) + res = compartment.OGsrfPtsBht.query(point) if len(res) == 2: - d = res[0][0] + d = res[0] if d < cutoff: # too close to a surface return False diff --git a/cellpack/autopack/utils.py b/cellpack/autopack/utils.py index def3b3fad..3de6b26d5 100644 --- a/cellpack/autopack/utils.py +++ b/cellpack/autopack/utils.py @@ -1,6 +1,7 @@ import collections import copy import numpy +import pickle def get_distance(pt1, pt2): @@ -169,3 +170,14 @@ def get_paired_key(val_dict, key1=None, key2=None): for key in val_dict: if (key1 in key) and (key2 in key): return key + + +def load_object_from_pickle(input_object, pickle_file_object): + """ + Update an object from a pickle file + """ + try: + input_object = pickle.load(pickle_file_object) + except Exception as e: + raise ValueError(f"Error loading saved object: {e}") + return input_object diff --git a/examples/packing-configs/parallel_packing_config.json b/examples/packing-configs/peroxisome_parallel_packing_config.json similarity index 59% rename from examples/packing-configs/parallel_packing_config.json rename to examples/packing-configs/peroxisome_parallel_packing_config.json index a59dd6801..43145cd49 100644 --- a/examples/packing-configs/parallel_packing_config.json +++ b/examples/packing-configs/peroxisome_parallel_packing_config.json @@ -1,19 +1,24 @@ { - "name": "parallel_packing", + "name": "analyze_parallel", "format": "simularium", "inner_grid_method": "trimesh", "live_packing": false, "ordered_packing": false, - "number_of_packings": 100, "out": "out/parallel", "overwrite_place_method": true, "place_method": "spheresSST", "save_analyze_result": true, - "show_grid_plot": true, - "save_plot_figures": true, - "spacing": 1, + "show_grid_plot": false, + "save_plot_figures": false, + "parallel": true, + "load_from_grid_file": true, + "number_of_packings": 100, + "spacing": null, "use_periodicity": false, "show_sphere_trees": false, - "load_from_grid_file": true, - "parallel": true + "image_export_options": { + "hollow": false, + "voxel_size": [1,1,1], + "projection_axis": "z" + } } \ No newline at end of file