diff --git a/hdf5_io_tests.py b/hdf5_io_tests.py deleted file mode 100644 index fe36b329..00000000 --- a/hdf5_io_tests.py +++ /dev/null @@ -1,117 +0,0 @@ -import smash -import numpy as np - - -setup, mesh = smash.load_dataset("cance") -model = smash.Model(setup, mesh) -model.run(inplace=True) - -#save a single dictionary to hdf5 -smash.tools.hdf5_handler.save_dict_to_hdf5("saved_dictionary.hdf5",mesh) - -#generate the structure of the object: it is a dict of key:data to save: typeofstructure={light,medium,full} -keys_data=smash.io.hdf5_io.generate_smash_object_structure(model,typeofstructure="medium") -print(keys_data) -#add a new data to save: -keys_data["parameters"].append('ci') - -#Save a single smash model -smash.save_smash_model_to_hdf5("./model_light.hdf5", model, content="light", replace=True) -smash.save_smash_model_to_hdf5("./model_medium.hdf5", model, content="medium", replace=True) -smash.save_smash_model_to_hdf5("./model_full.hdf5", model, content="full", replace=True) -smash.save_smash_model_to_hdf5("./model_user.hdf5", model, keys_data=keys_data, replace=True) - -#adding subdata -sub_data={"sub_data1":"mydata"} -sub_data.update({"sub_data2":2.5}) -sub_data.update({"sub_data3":{"sub_sub_data1":2.5,"sub_sub_data2":np.zeros(10)}}) - -smash.save_smash_model_to_hdf5("./model_sub_data.hdf5", model, content="medium",sub_data=sub_data, replace=True) - - -#view the hdf5 file -hdf5=smash.tools.hdf5_handler.open_hdf5("./model_user.hdf5") -hdf5.keys() -hdf5["mesh"].keys() -hdf5["parameters"].keys() -hdf5["output"].keys() -hdf5["output"].attrs.keys() -hdf5["output/fstates"].keys() -hdf5["setup"].attrs.keys() -hdf5.close() - -#view the hdf5 file with sub_data -hdf5=smash.tools.hdf5_handler.open_hdf5("./model_sub_data.hdf5") -hdf5.keys() -hdf5.attrs.keys() -hdf5.close() - - -#save multi smash model at different place -smash.save_smash_model_to_hdf5("./multi_model.hdf5", model,location="model1",replace=True) -smash.save_smash_model_to_hdf5("./multi_model.hdf5", model,location="model2",replace=False) - - -hdf5=smash.tools.hdf5_handler.open_hdf5("./multi_model.hdf5") -hdf5.keys() -hdf5["model2"]["setup"].attrs.keys() -hdf5["model2"]["mesh"].keys() -hdf5["model2"]["output"].keys() -hdf5["model2"]["output"].attrs.keys() -hdf5.close() - -#manually group different object in an hdf5 -hdf5=smash.tools.hdf5_handler.open_hdf5("./model_subgroup.hdf5", replace=True) -hdf5=smash.tools.hdf5_handler.add_hdf5_sub_group(hdf5, subgroup="model1") -hdf5=smash.tools.hdf5_handler.add_hdf5_sub_group(hdf5, subgroup="model2") -keys_data=smash.io.hdf5_io.generate_smash_object_structure(model,typeofstructure="medium") -keys_data_2=smash.tools.object_handler.generate_object_structure(model) -smash.tools.hdf5_handler._dump_object_to_hdf5_from_iteratable(hdf5["model1"], model, keys_data) -smash.tools.hdf5_handler._dump_object_to_hdf5_from_iteratable(hdf5["model2"], model, keys_data_2) - -hdf5=smash.tools.hdf5_handler.open_hdf5("./model_subgroup.hdf5", replace=False) -hdf5=smash.tools.hdf5_handler.add_hdf5_sub_group(hdf5, subgroup="model3") -keys_data=smash.io.hdf5_io.generate_smash_object_structure(model,typeofstructure="medium") -smash.tools.hdf5_handler._dump_object_to_hdf5_from_iteratable(hdf5["model3"], model, keys_data) - -hdf5.keys() -hdf5["model1"].keys() -hdf5["model2"].keys() -hdf5["model3"].keys() -hdf5.close() - - -#read model object to a dictionnay -dictionary=smash.tools.object_handler.read_object_as_dict(model) -dictionary.keys() -dictionary["mesh"]["code"] - -######### Reading HDF5 - -#load an hdf5 file to a dictionary -dictionary=smash.load_hdf5_file("./multi_model.hdf5") -dictionary["model1"].keys() -dictionary["model1"]["mesh"].keys() - -#load a hdf5 file with any sub_data -dictionary=smash.load_hdf5_file("./model_sub_data.hdf5") -dictionary.keys() - -#read only a part of an hdf5 file -hdf5=smash.tools.hdf5_handler.open_hdf5("./multi_model.hdf5") -dictionary=smash.tools.hdf5_handler.read_hdf5_as_dict(hdf5["model1"]) -dictionary.keys() - -#reload a full model object -model_reloaded=smash.load_hdf5_file("./model_medium.hdf5",as_model=True) #get error -model_reloaded=smash.load_hdf5_file("./model_full.hdf5",as_model=True) -model_reloaded -model_reloaded.run() - -#TODO : - -# compile documentation -# tests failed -# remove hdf5_io_test.py -# black *.py - diff --git a/smash/io/hdf5_io.py b/smash/io/hdf5_io.py index 14a73d8b..d9436a7e 100644 --- a/smash/io/hdf5_io.py +++ b/smash/io/hdf5_io.py @@ -32,8 +32,11 @@ __all__ = ["save_smash_model_to_hdf5", "load_hdf5_file"] - -def _generate_light_smash_object_structure(structure: str,structure_parameters=STRUCTURE_PARAMETERS,structure_states=STRUCTURE_STATES): +def _generate_light_smash_object_structure( + structure: str, + structure_parameters=STRUCTURE_PARAMETERS, + structure_states=STRUCTURE_STATES, +): """ this function create a light dictionnary containing the required data-structure to save a smash model object to an hdf5 file @@ -42,9 +45,9 @@ def _generate_light_smash_object_structure(structure: str,structure_parameters=S structure : str the smash model structure used {gr-a, gr-b, gr-c, gr-d} structure_parameters: dict - the dict containing the parameter to be saved for each model structure + the dict containing the parameter to be saved for each model structure structure_states: dict - the dict containing the states to be saved for each model structure + the dict containing the states to be saved for each model structure Returns ------- @@ -53,7 +56,19 @@ def _generate_light_smash_object_structure(structure: str,structure_parameters=S """ return { "setup": ["dt", "end_time", "start_time"], - "mesh": ["active_cell", "area", "code", "dx", "ng", "ymax", "xmin", "nrow", "ncol", "gauge_pos", "flwacc"], + "mesh": [ + "active_cell", + "area", + "code", + "dx", + "ng", + "ymax", + "xmin", + "nrow", + "ncol", + "gauge_pos", + "flwacc", + ], "input_data": ["qobs"], "parameters": structure_parameters[ structure @@ -67,8 +82,11 @@ def _generate_light_smash_object_structure(structure: str,structure_parameters=S } - -def _generate_medium_smash_object_structure(structure: str,structure_parameters=STRUCTURE_PARAMETERS,structure_states=STRUCTURE_STATES): +def _generate_medium_smash_object_structure( + structure: str, + structure_parameters=STRUCTURE_PARAMETERS, + structure_states=STRUCTURE_STATES, +): """ this function create a medium dictionnary containing the required data-structure to save a smash model object to an hdf5 file @@ -77,9 +95,9 @@ def _generate_medium_smash_object_structure(structure: str,structure_parameters= structure : str the smash model structure used {gr-a, gr-b, gr-c, gr-d} structure_parameters: dict - the dict containing the parameter to be saved for each model structure + the dict containing the parameter to be saved for each model structure structure_states: dict - the dict containing the states to be saved for each model structure + the dict containing the states to be saved for each model structure Returns ------- @@ -88,7 +106,22 @@ def _generate_medium_smash_object_structure(structure: str,structure_parameters= """ return { "setup": ["dt", "end_time", "start_time", "structure", "_ntime_step"], - "mesh": ["active_cell", "area", "code", "dx", "flwdir", "nac", "ng", "path", "ymax", "xmin", "nrow", "ncol", "gauge_pos", "flwacc"], + "mesh": [ + "active_cell", + "area", + "code", + "dx", + "flwdir", + "nac", + "ng", + "path", + "ymax", + "xmin", + "nrow", + "ncol", + "gauge_pos", + "flwacc", + ], "input_data": ["mean_prcp", "mean_pet", "qobs"], "parameters": structure_parameters[ structure @@ -103,7 +136,7 @@ def _generate_medium_smash_object_structure(structure: str,structure_parameters= "qsim", "cost", "cost_jobs", - "cost_jreg" + "cost_jreg", ], } @@ -116,23 +149,22 @@ def _generate_full_smash_object_structure(instance): ---------- instance : object a custom python object. - + Returns ------- list : A list containing keys and dictionary matching the structure of the python object. """ - key_data=smash.tools.object_handler.generate_object_structure(instance) - - key_list=list() + key_data = smash.tools.object_handler.generate_object_structure(instance) + + key_list = list() key_list.append(key_data) key_list.append("_last_update") - - return key_list + return key_list -def generate_smash_object_structure(instance,typeofstructure="medium"): +def generate_smash_object_structure(instance, typeofstructure="medium"): """ this function create a dictionnary containing a complete ar partial structure of an object in order to save it to an hdf5. This functions is a conveninet way to generate the key_data as a dictionary. Then personnal keys can be added to the key_data dict. @@ -142,33 +174,37 @@ def generate_smash_object_structure(instance,typeofstructure="medium"): a custom python object. typeofstructure : str the structure type : light, medium, full - + Returns ------- dict : A list or dictionary matching the structure of the python object. """ - structure=instance.setup.structure - - if typeofstructure=="light": - - key_data=_generate_light_smash_object_structure(structure) - - elif typeofstructure=="medium": - - key_data=_generate_medium_smash_object_structure(structure) - - elif typeofstructure=="full": - - key_data=_generate_full_smash_object_structure(instance) - - return key_data + structure = instance.setup.structure + + if typeofstructure == "light": + key_data = _generate_light_smash_object_structure(structure) + elif typeofstructure == "medium": + key_data = _generate_medium_smash_object_structure(structure) + + elif typeofstructure == "full": + key_data = _generate_full_smash_object_structure(instance) + + return key_data -def save_smash_model_to_hdf5(path_to_hdf5, instance, keys_data=None, content="medium", location="./", sub_data=None, replace=True): +def save_smash_model_to_hdf5( + path_to_hdf5, + instance, + keys_data=None, + content="medium", + location="./", + sub_data=None, + replace=True, +): """ - Save an instance of smash.Model to an hdf5 file. + Save an instance of smash.Model to an hdf5 file. Parameters ---------- @@ -186,17 +222,17 @@ def save_smash_model_to_hdf5(path_to_hdf5, instance, keys_data=None, content="me a dictionary containing extra-data to be saved replace : Boolean replace an existing hdf5 file. Default is False - + Examples -------- setup, mesh = smash.load_dataset("cance") model = smash.Model(setup, mesh) model.run(inplace=True) - + keys_data=smash.io.hdf5_io.generate_smash_object_structure(model,typeofstructure="medium") #add a new data to save: keys_data["parameters"].append('ci') - + #Save a single smash model smash.save_smash_model_to_hdf5("./model_light.hdf5", model, content="light", replace=True) smash.save_smash_model_to_hdf5("./model_medium.hdf5", model, content="medium", replace=True) @@ -211,29 +247,29 @@ def save_smash_model_to_hdf5(path_to_hdf5, instance, keys_data=None, content="me smash.save_smash_model_to_hdf5("./model_sub_data.hdf5", model, content="medium",sub_data=sub_data, replace=True) """ if content == "light": - - keys_data=_generate_light_smash_object_structure(instance.setup.structure) - + keys_data = _generate_light_smash_object_structure(instance.setup.structure) + elif content == "medium": - - keys_data=_generate_medium_smash_object_structure(instance.setup.structure) - + keys_data = _generate_medium_smash_object_structure(instance.setup.structure) + elif content == "full": - - keys_data=_generate_full_smash_object_structure(instance) - - if isinstance(keys_data,(dict,list)): - - smash.tools.hdf5_handler.save_object_to_hdf5(path_to_hdf5, instance, keys_data, location=location, sub_data=sub_data,replace=replace) - - else: - - raise ValueError( - f"{keys_data} must be a instance of list or dict." - ) + keys_data = _generate_full_smash_object_structure(instance) + + if isinstance(keys_data, (dict, list)): + smash.tools.hdf5_handler.save_object_to_hdf5( + path_to_hdf5, + instance, + keys_data, + location=location, + sub_data=sub_data, + replace=replace, + ) + + else: + raise ValueError(f"{keys_data} must be a instance of list or dict.") -def load_hdf5_file(f_hdf5,as_model=False): +def load_hdf5_file(f_hdf5, as_model=False): """ Load an hdf5 file to a dictionary or to an instance of smash.Model. @@ -243,32 +279,30 @@ def load_hdf5_file(f_hdf5,as_model=False): path to the hdf5 file as_model : Boolean load the hdf5 as a smash model. Default is False - + Return -------- instance : an instance of the smash model or a dictionary - + Examples -------- #load an hdf5 file to a dictionary dictionary=smash.load_hdf5_file("./multi_model.hdf5") dictionary["model1"].keys() dictionary["model1"]["mesh"].keys() - + #reload a full model object model_reloaded=smash.load_hdf5_file("./model_full.hdf5",as_model=True) model_reloaded model_reloaded.run() """ if as_model: - - instance=read_hdf5_to_model_object(f_hdf5) + instance = read_hdf5_to_model_object(f_hdf5) return instance - + else: - - hdf5=smash.tools.hdf5_handler.open_hdf5(f_hdf5, read_only=True, replace=False) - dictionary=smash.tools.hdf5_handler.read_hdf5_as_dict(hdf5) + hdf5 = smash.tools.hdf5_handler.open_hdf5(f_hdf5, read_only=True, replace=False) + dictionary = smash.tools.hdf5_handler.read_hdf5_as_dict(hdf5) hdf5.close() return dictionary @@ -284,7 +318,6 @@ def _parse_hdf5_to_derived_type(hdf5_ins, derived_type): setattr(derived_type, ds, hdf5_ins[ds][:]) for attr in hdf5_ins.attrs.keys(): - # check if value is equal to "_None_" (None string because hdf5 does not supported) if hdf5_ins.attrs[attr] == "_None_": setattr(derived_type, attr, None) @@ -342,12 +375,11 @@ def read_hdf5_to_model_object(path: str) -> Model: if os.path.isfile(path): with h5py.File(path, "r") as f: - - if not f.attrs.__contains__('_last_update'): + if not f.attrs.__contains__("_last_update"): raise ValueError( f'The hdf5 file {path} does not contain the full smash object structure and therefore cannot be loaded as a smash model object. The full structure of a smash model object can be saved using smash.save_smash_model_to_hdf5(filename, smash_model, content="full").' ) - + instance = smash.Model(None, None) if "descriptor_name" in f["setup"].keys(): @@ -364,9 +396,7 @@ def read_hdf5_to_model_object(path: str) -> Model: et = pd.Timestamp(instance.setup.end_time) - instance.setup._ntime_step = ( - et - st - ).total_seconds() / instance.setup.dt + instance.setup._ntime_step = (et - st).total_seconds() / instance.setup.dt instance.mesh = MeshDT( instance.setup, @@ -401,6 +431,5 @@ def read_hdf5_to_model_object(path: str) -> Model: return instance - else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) diff --git a/smash/tools/hdf5_handler.py b/smash/tools/hdf5_handler.py index 122daefc..ff0f856c 100644 --- a/smash/tools/hdf5_handler.py +++ b/smash/tools/hdf5_handler.py @@ -6,6 +6,7 @@ from smash.tools import object_handler + def open_hdf5(path, read_only=False, replace=False): """ Open or create an HDF5 file. @@ -31,39 +32,27 @@ def open_hdf5(path, read_only=False, replace=False): >>> hdf5.attrs.keys() """ if not path.endswith(".hdf5"): - path = path + ".hdf5" - + if read_only: - if os.path.isfile(path): - - f= h5py.File(path, "r") - + f = h5py.File(path, "r") + else: - - raise ValueError( - f"File {path} does not exist." - ) - + raise ValueError(f"File {path} does not exist.") + else: - if replace: - - f= h5py.File(path, "w") - + f = h5py.File(path, "w") + else: - if os.path.isfile(path): - - f= h5py.File(path, "a") - + f = h5py.File(path, "a") + else: - - f= h5py.File(path, "w") - - return f + f = h5py.File(path, "w") + return f def add_hdf5_sub_group(hdf5, subgroup=None): @@ -90,18 +79,15 @@ def add_hdf5_sub_group(hdf5, subgroup=None): >>> hdf5.attrs.keys() """ if subgroup is not None: - - if subgroup=="": - - subgroup="./" - + if subgroup == "": + subgroup = "./" + hdf5.require_group(subgroup) - - return hdf5 + return hdf5 -def _dump_object_to_hdf5_from_list_attribute(hdf5,instance,list_attr): +def _dump_object_to_hdf5_from_list_attribute(hdf5, instance, list_attr): """ dump a object to a hdf5 file from a list of attributes @@ -114,37 +100,27 @@ def _dump_object_to_hdf5_from_list_attribute(hdf5,instance,list_attr): list_attr : list a list of attribute """ - if isinstance(list_attr,list): - + if isinstance(list_attr, list): for attr in list_attr: - if isinstance(attr, str): - _dump_object_to_hdf5_from_str_attribute(hdf5, instance, attr) - - elif isinstance(attr,list): - + + elif isinstance(attr, list): _dump_object_to_hdf5_from_list_attribute(hdf5, instance, attr) - - elif isinstance(attr,dict): - + + elif isinstance(attr, dict): _dump_object_to_hdf5_from_dict_attribute(hdf5, instance, attr) - + else: - raise ValueError( f"inconsistent {attr} in {list_attr}. {attr} must be a an instance of dict, list or str" ) - - else: - - raise ValueError( - f"{list_attr} must be a instance of list." - ) + else: + raise ValueError(f"{list_attr} must be a instance of list.") -def _dump_object_to_hdf5_from_dict_attribute(hdf5,instance,dict_attr): +def _dump_object_to_hdf5_from_dict_attribute(hdf5, instance, dict_attr): """ dump a object to a hdf5 file from a dictionary of attributes @@ -157,47 +133,39 @@ def _dump_object_to_hdf5_from_dict_attribute(hdf5,instance,dict_attr): dict_attr : dict a dictionary of attribute """ - if isinstance(dict_attr,dict): - + if isinstance(dict_attr, dict): for attr, value in dict_attr.items(): - - hdf5=add_hdf5_sub_group(hdf5, subgroup=attr) - + hdf5 = add_hdf5_sub_group(hdf5, subgroup=attr) + try: - - sub_instance=getattr(instance, attr) - + sub_instance = getattr(instance, attr) + except: - - sub_instance=instance - - if isinstance(value,dict): - - _dump_object_to_hdf5_from_dict_attribute(hdf5[attr], sub_instance, value) - - if isinstance(value,list): - - _dump_object_to_hdf5_from_list_attribute(hdf5[attr], sub_instance, value) - - elif isinstance(value,str): - + sub_instance = instance + + if isinstance(value, dict): + _dump_object_to_hdf5_from_dict_attribute( + hdf5[attr], sub_instance, value + ) + + if isinstance(value, list): + _dump_object_to_hdf5_from_list_attribute( + hdf5[attr], sub_instance, value + ) + + elif isinstance(value, str): _dump_object_to_hdf5_from_str_attribute(hdf5[attr], sub_instance, value) - - else : - + + else: raise ValueError( f"inconsistent '{attr}' in '{dict_attr}'. Dict({attr}) must be a instance of dict, list or str" ) - - else: - - raise ValueError( - f"{dict_attr} must be a instance of dict." - ) + else: + raise ValueError(f"{dict_attr} must be a instance of dict.") -def _dump_object_to_hdf5_from_str_attribute(hdf5,instance,str_attr): +def _dump_object_to_hdf5_from_str_attribute(hdf5, instance, str_attr): """ dump a object to a hdf5 file from a string attribute @@ -211,19 +179,16 @@ def _dump_object_to_hdf5_from_str_attribute(hdf5,instance,str_attr): a string attribute """ if isinstance(str_attr, str): - try: - value = getattr(instance, str_attr) - - if isinstance(value, (np.ndarray,list)): - - if isinstance(value,list): - value=np.array(value) - + + if isinstance(value, (np.ndarray, list)): + if isinstance(value, list): + value = np.array(value) + if value.dtype == "object" or value.dtype.char == "U": value = value.astype("S") - + hdf5.create_dataset( str_attr, shape=value.shape, @@ -232,27 +197,20 @@ def _dump_object_to_hdf5_from_str_attribute(hdf5,instance,str_attr): compression="gzip", chunks=True, ) - + elif value is None: - - hdf5.attrs[str_attr] = "_None_" - + hdf5.attrs[str_attr] = "_None_" + else: - hdf5.attrs[str_attr] = value - + except: - raise ValueError( f"Unable to dump attribute {str_attr} with value {value} from {instance}" ) - - else: - - raise ValueError( - f"{str_attr} must be a instance of str." - ) + else: + raise ValueError(f"{str_attr} must be a instance of str.") def _dump_object_to_hdf5_from_iteratable(hdf5, instance, iteratable=None): @@ -267,13 +225,13 @@ def _dump_object_to_hdf5_from_iteratable(hdf5, instance, iteratable=None): a custom python object. iteratable : list | dict a list or a dict of attribute - + Examples -------- setup, mesh = smash.load_dataset("cance") model = smash.Model(setup, mesh) model.run(inplace=True) - + hdf5=smash.tools.hdf5_handler.open_hdf5("./model.hdf5", replace=True) hdf5=smash.tools.hdf5_handler.add_hdf5_sub_group(hdf5, subgroup="model1") keys_data=smash.io.hdf5_io.generate_smash_object_structure(model,typeofstructure="medium") @@ -284,23 +242,17 @@ def _dump_object_to_hdf5_from_iteratable(hdf5, instance, iteratable=None): keys_data=smash.io.hdf5_io.generate_smash_object_structure(model,typeofstructure="light") smash.tools.hdf5_handler._dump_object_to_hdf5_from_iteratable(hdf5["model2"], model, keys_data) """ - if isinstance(iteratable,list): - - _dump_object_to_hdf5_from_list_attribute(hdf5,instance,iteratable) - - elif isinstance(iteratable,dict): - - _dump_object_to_hdf5_from_dict_attribute(hdf5,instance,iteratable) - - else : - - raise ValueError( - f"{iteratable} must be a instance of list or dict." - ) + if isinstance(iteratable, list): + _dump_object_to_hdf5_from_list_attribute(hdf5, instance, iteratable) + + elif isinstance(iteratable, dict): + _dump_object_to_hdf5_from_dict_attribute(hdf5, instance, iteratable) + else: + raise ValueError(f"{iteratable} must be a instance of list or dict.") -def _dump_dict_to_hdf5(hdf5,dictionary): +def _dump_dict_to_hdf5(hdf5, dictionary): """ dump a dictionary to an hdf5 file @@ -311,29 +263,24 @@ def _dump_dict_to_hdf5(hdf5,dictionary): dictionary : dict a custom python dictionary """ - if isinstance(dictionary,dict): - + if isinstance(dictionary, dict): for attr, value in dictionary.items(): - try: - - if isinstance(value,(dict)): - - hdf5=add_hdf5_sub_group(hdf5, subgroup=attr) - _dump_dict_to_hdf5(hdf5[attr],value) - - elif isinstance(value, (np.ndarray,list)): - - if isinstance(value,(list)): - value=np.array(value) - + if isinstance(value, (dict)): + hdf5 = add_hdf5_sub_group(hdf5, subgroup=attr) + _dump_dict_to_hdf5(hdf5[attr], value) + + elif isinstance(value, (np.ndarray, list)): + if isinstance(value, (list)): + value = np.array(value) + if value.dtype == "object" or value.dtype.char == "U": value = value.astype("S") - - #remove dataset if exist + + # remove dataset if exist if attr in hdf5.keys(): del hdf5[attr] - + hdf5.create_dataset( attr, shape=value.shape, @@ -342,30 +289,21 @@ def _dump_dict_to_hdf5(hdf5,dictionary): compression="gzip", chunks=True, ) - + elif value is None: - hdf5.attrs[attr] = "_None_" - + else: - hdf5.attrs[attr] = value - + except: - - raise ValueError( - f"Unable to save attribute {attr} with value {value}" - ) - - else: - - raise ValueError( - f"{dictionary} must be a instance of dict." - ) + raise ValueError(f"Unable to save attribute {attr} with value {value}") + else: + raise ValueError(f"{dictionary} must be a instance of dict.") -def save_dict_to_hdf5(path_to_hdf5,dictionary=None,location="./",replace=False): +def save_dict_to_hdf5(path_to_hdf5, dictionary=None, location="./", replace=False): """ dump a dictionary to an hdf5 file @@ -379,30 +317,27 @@ def save_dict_to_hdf5(path_to_hdf5,dictionary=None,location="./",replace=False): path location or subgroup where to write data in the hdf5 file replace : Boolean replace an existing hdf5 file. Default is False - + Examples -------- setup, mesh = smash.load_dataset("cance") model = smash.Model(setup, mesh) model.run(inplace=True) - + smash.tools.hdf5_handler.save_dict_to_hdf5("saved_dictionary.hdf5",mesh) """ - if isinstance(dictionary,dict): - - hdf5=open_hdf5(path_to_hdf5, replace=replace) - hdf5=add_hdf5_sub_group(hdf5, subgroup=location) + if isinstance(dictionary, dict): + hdf5 = open_hdf5(path_to_hdf5, replace=replace) + hdf5 = add_hdf5_sub_group(hdf5, subgroup=location) _dump_dict_to_hdf5(hdf5[location], dictionary) - - else: - - raise ValueError( - f"The input {dictionary} must be a instance of dict." - ) + else: + raise ValueError(f"The input {dictionary} must be a instance of dict.") -def save_object_to_hdf5(f_hdf5, instance, keys_data=None, location="./", sub_data=None, replace=False): +def save_object_to_hdf5( + f_hdf5, instance, keys_data=None, location="./", sub_data=None, replace=False +): """ dump an object to an hdf5 file @@ -421,20 +356,18 @@ def save_object_to_hdf5(f_hdf5, instance, keys_data=None, location="./", sub_dat replace : Boolean replace an existing hdf5 file. Default is False """ - + if keys_data is None: - keys_data=smash.tools.object_handler.generate_object_structure(instance) - - hdf5=open_hdf5(f_hdf5, replace=replace) - hdf5=add_hdf5_sub_group(hdf5, subgroup=location) + keys_data = smash.tools.object_handler.generate_object_structure(instance) + + hdf5 = open_hdf5(f_hdf5, replace=replace) + hdf5 = add_hdf5_sub_group(hdf5, subgroup=location) _dump_object_to_hdf5_from_iteratable(hdf5[location], instance, keys_data) - - if isinstance(sub_data,dict): - + + if isinstance(sub_data, dict): _dump_dict_to_hdf5(hdf5[location], sub_data) - - hdf5.close() + hdf5.close() def read_hdf5_as_dict(hdf5): @@ -445,11 +378,11 @@ def read_hdf5_as_dict(hdf5): ---------- hdf5 : str path to the hdf5 file - + Return -------- dictionary : dict, a dictionary of all keys and attribute included in the hdf5 file - + Examples -------- #read only a part of an hdf5 file @@ -457,58 +390,47 @@ def read_hdf5_as_dict(hdf5): dictionary=smash.tools.hdf5_handler.read_hdf5_as_dict(hdf5["model1"]) dictionary.keys() """ - dictionary={} - - for key,item in hdf5.items(): - + dictionary = {} + + for key, item in hdf5.items(): if str(type(item)).find("group") != -1: - - dictionary.update({key:read_hdf5_as_dict(item)}) - - list_attr=list(item.attrs.keys()) - + dictionary.update({key: read_hdf5_as_dict(item)}) + + list_attr = list(item.attrs.keys()) + for key_attr in list_attr: - # check if value is equal to "_None_" (None string because hdf5 does not supported) if item.attrs[key_attr] == "_None_": - - dictionary[key].update({key_attr:None}) - + dictionary[key].update({key_attr: None}) + else: - - dictionary[key].update({key_attr:item.attrs[key_attr]}) - + dictionary[key].update({key_attr: item.attrs[key_attr]}) + if str(type(item)).find("dataset") != -1: - if item[:].dtype.char == "S": - - values=item[:].astype("U") - + values = item[:].astype("U") + else: - - values=item[:] - - dictionary.update({key:values}) - - list_attr=list(item.attrs.keys()) - + values = item[:] + + dictionary.update({key: values}) + + list_attr = list(item.attrs.keys()) + for key_attr in list_attr: - # check if value is equal to "_None_" (None string because hdf5 does not supported) if item.attrs[key_attr] == "_None_": - dictionary[key].update({key_attr:None}) + dictionary[key].update({key_attr: None}) else: - dictionary.update({key_attr:item.attrs[key_attr]}) - - list_attr=list(hdf5.attrs.keys()) - + dictionary.update({key_attr: item.attrs[key_attr]}) + + list_attr = list(hdf5.attrs.keys()) + for key_attr in list_attr: - # check if value is equal to "_None_" (None string because hdf5 does not supported) if hdf5.attrs[key_attr] == "_None_": - dictionary.update({key_attr:None}) + dictionary.update({key_attr: None}) else: - dictionary.update({key_attr:hdf5.attrs[key_attr]}) - - return dictionary + dictionary.update({key_attr: hdf5.attrs[key_attr]}) + return dictionary diff --git a/smash/tools/object_handler.py b/smash/tools/object_handler.py index 7e23d1ca..e3d885b5 100644 --- a/smash/tools/object_handler.py +++ b/smash/tools/object_handler.py @@ -12,66 +12,56 @@ def generate_object_structure(instance): ---------- instance : object a custom python object. - + Returns ------- list or dict : A list or dictionary matching the structure of the python object. """ - key_data={} - key_list=list() - return_list=False - + key_data = {} + key_list = list() + return_list = False + for attr in dir(instance): - if not attr.startswith("_") and not attr in ["from_handle", "copy"]: - try: - value = getattr(instance, attr) - - if isinstance(value, (np.ndarray,list)): - - if isinstance(value,list): - value=np.array(value) - + + if isinstance(value, (np.ndarray, list)): + if isinstance(value, list): + value = np.array(value) + if value.dtype == "object" or value.dtype.char == "U": value = value.astype("S") - - #key_data.update({attr:value}) + + # key_data.update({attr:value}) key_list.append(attr) - return_list=True - - elif isinstance(value,(str,float,int)): - - #key_data.update({attr:value}) + return_list = True + + elif isinstance(value, (str, float, int)): + # key_data.update({attr:value}) key_list.append(attr) - return_list=True - - else: - - depp_key_data=generate_object_structure(value) - - if (len(depp_key_data)>0): - key_data.update({attr:depp_key_data}) - + return_list = True + + else: + depp_key_data = generate_object_structure(value) + + if len(depp_key_data) > 0: + key_data.update({attr: depp_key_data}) + except: - pass - + if return_list: - for attr, value in key_data.items(): - key_list.append({attr:value}) - + key_list.append({attr: value}) + return key_list - + else: - return key_data - def read_object_as_dict(instance): """ create a dictionary from a custom python object @@ -80,48 +70,40 @@ def read_object_as_dict(instance): ---------- instance : object an custom python object - + Return ---------- key_data: dict an dictionary containing all keys and atributes of the object """ - key_data={} - key_list=list() - return_list=False - + key_data = {} + key_list = list() + return_list = False + for attr in dir(instance): - if not attr.startswith("_") and not attr in ["from_handle", "copy"]: - try: - value = getattr(instance, attr) - - if isinstance(value, (np.ndarray,list)): - - if isinstance(value,list): - value=np.array(value) - + + if isinstance(value, (np.ndarray, list)): + if isinstance(value, list): + value = np.array(value) + if value.dtype == "object" or value.dtype.char == "U": value = value.astype("S") - - key_data.update({attr:value}) - - elif isinstance(value,(str,float,int)): - - key_data.update({attr:value}) - - else: - - depp_key_data=read_object_as_dict(value) - - if (len(depp_key_data)>0): - key_data.update({attr:depp_key_data}) - + + key_data.update({attr: value}) + + elif isinstance(value, (str, float, int)): + key_data.update({attr: value}) + + else: + depp_key_data = read_object_as_dict(value) + + if len(depp_key_data) > 0: + key_data.update({attr: depp_key_data}) + except: - pass - - return key_data + return key_data