diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index f9a04caa1..ffa2b2d50 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -18,28 +18,28 @@ jobs: os: [ubuntu-latest] mpi-version: [mpich] python-version: [3.9, "3.10", "3.11", "3.12"] - pydantic-version: ["2.6.4"] + pydantic-version: ["2.8.2"] comms-type: [m, l] include: - os: macos-latest python-version: "3.11" mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: m - os: macos-latest python-version: "3.11" mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: l - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.13" + pydantic-version: "1.10.17" comms-type: m - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.13" + pydantic-version: "1.10.17" comms-type: l env: @@ -163,4 +163,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.23.6 + - uses: crate-ci/typos@v1.24.1 diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 2b49aebfa..e05fc5032 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -12,38 +12,38 @@ jobs: os: [ubuntu-latest] mpi-version: [mpich] python-version: [3.9, "3.10", "3.11", "3.12"] - pydantic-version: ["2.6.4"] + pydantic-version: ["2.8.2"] comms-type: [m, l] include: - os: macos-latest python-version: 3.11 mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: m - os: macos-latest python-version: 3.11 mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: l - os: ubuntu-latest python-version: "3.10" mpi-version: mpich - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" comms-type: t - os: ubuntu-latest mpi-version: "openmpi" - pydantic-version: "2.6.4" + pydantic-version: "2.8.2" python-version: "3.12" comms-type: l - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.13" + pydantic-version: "1.10.17" comms-type: m - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.13" + pydantic-version: "1.10.17" comms-type: l env: @@ -157,16 +157,20 @@ jobs: pip install git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject pip install scikit-build packaging Tasmanian --user - - name: Install other testing dependencies + - name: Install Balsam on Pydantic 1 + if: matrix.pydantic-version == '1.10.17' run: | - conda install octave conda install pyzmq - pip install -r install/testing_requirements.txt - pip install -r install/misc_feature_requirements.txt git clone https://github.com/argonne-lcf/balsam.git sed -i -e "s/pyzmq>=22.1.0,<23.0.0/pyzmq>=23.0.0,<24.0.0/" ./balsam/setup.cfg cd balsam; pip install -e .; cd .. + - name: Install other testing dependencies + run: | + conda install octave + pip install -r install/testing_requirements.txt + pip install -r install/misc_feature_requirements.txt + git clone --recurse-submodules -b develop https://github.com/POptUS/IBCDFO.git pushd IBCDFO/minq/py/minq5/ export PYTHONPATH="$PYTHONPATH:$(pwd)" @@ -224,15 +228,26 @@ jobs: rm ./libensemble/tests/regression_tests/test_persistent_tasmanian.py rm ./libensemble/tests/regression_tests/test_persistent_tasmanian_async.py - - name: Remove Balsam/Globus-compute tests on Pydantic 2 + - name: Install redis/proxystore on Pydantic 2 if: matrix.pydantic-version == '2.6.4' + run: | + pip install redis + pip install proxystore==0.7.0 + + - name: Remove proxystore test on Pydantic 1 + if: matrix.pydantic-version == '1.10.13' + run: | + rm ./libensemble/tests/regression_tests/test_proxystore_integration.py + + - name: Remove Balsam/Globus-compute tests on Pydantic 2 + if: matrix.pydantic-version == '2.8.2' run: | rm ./libensemble/tests/unit_tests/test_ufunc_runners.py rm ./libensemble/tests/unit_tests/test_executor_balsam.py - name: Start Redis if: matrix.os == 'ubuntu-latest' - uses: supercharge/redis-github-action@1.7.0 + uses: supercharge/redis-github-action@1.8.0 with: redis-version: 7 @@ -261,4 +276,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.23.6 + - uses: crate-ci/typos@v1.24.1 diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 37631333d..2397da52f 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -305,7 +305,7 @@ macOS and Windows Errors .. _Installing PETSc On Microsoft Windows: https://petsc.org/release/install/windows/#recommended-installation-methods .. _option to srun: https://docs.nersc.gov/systems/perlmutter/running-jobs/#single-gpu-tasks-in-parallel -.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter +.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/architecture/ .. _Python multiprocessing docs: https://docs.python.org/3/library/multiprocessing.html .. _SDF: https://sdf.slac.stanford.edu/public/doc/#/?id=what-is-the-sdf .. _Support: https://libensemble.readthedocs.io/en/main/introduction.html#resources diff --git a/docs/examples/surmise.rst b/docs/examples/surmise.rst index 72fa87413..69d0f068e 100644 --- a/docs/examples/surmise.rst +++ b/docs/examples/surmise.rst @@ -2,15 +2,11 @@ persistent_surmise ------------------ Required: Surmise_ - -Note that currently the github fork https://github.com/mosesyhc/surmise should be used:: - - pip install --upgrade git+https://github.com/bandframework/surmise.git@develop - -The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` uses this generator as an example of the capability to cancel pending simulations. +The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` uses this generator as an +example of the capability to cancel pending simulations. .. automodule:: persistent_surmise_calib :members: :no-undoc-members: -.. _Surmise: https://surmise.readthedocs.io/en/latest/index.html +.. _Surmise: https://github.com/bandframework/surmise diff --git a/docs/nitpicky b/docs/nitpicky index 20a0f851d..8315471a1 100644 --- a/docs/nitpicky +++ b/docs/nitpicky @@ -43,6 +43,7 @@ py:class libensemble.resources.platforms.Aurora py:class libensemble.resources.platforms.GenericROCm py:class libensemble.resources.platforms.Crusher py:class libensemble.resources.platforms.Frontier +py:class libensemble.resources.platforms.Perlmutter py:class libensemble.resources.platforms.PerlmutterCPU py:class libensemble.resources.platforms.PerlmutterGPU py:class libensemble.resources.platforms.Polaris diff --git a/docs/platforms/perlmutter.rst b/docs/platforms/perlmutter.rst index a8737cd68..915a647af 100644 --- a/docs/platforms/perlmutter.rst +++ b/docs/platforms/perlmutter.rst @@ -190,7 +190,7 @@ See the NERSC Perlmutter_ docs for more information about Perlmutter. .. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ .. _NERSC: https://www.nersc.gov/ .. _option to srun: https://docs.nersc.gov/systems/perlmutter/running-jobs/#single-gpu-tasks-in-parallel -.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/ +.. _Perlmutter: https://docs.nersc.gov/systems/perlmutter/architecture/ .. _Python on Perlmutter: https://docs.nersc.gov/development/languages/python/using-python-perlmutter/ .. _Slurm: https://slurm.schedmd.com/ .. _video: https://www.youtube.com/watch?v=Av8ctYph7-Y diff --git a/install/misc_feature_requirements.txt b/install/misc_feature_requirements.txt index 05416fde6..b72c73c16 100644 --- a/install/misc_feature_requirements.txt +++ b/install/misc_feature_requirements.txt @@ -1,2 +1 @@ -globus-compute-sdk==2.26.0 -proxystore==0.7.0 +globus-compute-sdk==2.27.0 diff --git a/libensemble/gen_classes/aposmm.py b/libensemble/gen_classes/aposmm.py index 8e8fb47f0..108282e07 100644 --- a/libensemble/gen_classes/aposmm.py +++ b/libensemble/gen_classes/aposmm.py @@ -14,13 +14,11 @@ class APOSMM(LibensembleGenThreadInterfacer): """ def __init__( - self, gen_specs: dict = {}, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {}, **kwargs + self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs ) -> None: from libensemble.gen_funcs.persistent_aposmm import aposmm gen_specs["gen_f"] = aposmm - if len(kwargs) > 0: # so user can specify aposmm-specific parameters as kwargs to constructor - gen_specs["user"] = kwargs if not gen_specs.get("out"): # gen_specs never especially changes for aposmm even as the problem varies n = len(kwargs["lb"]) or len(kwargs["ub"]) gen_specs["out"] = [ @@ -32,9 +30,8 @@ def __init__( ] gen_specs["persis_in"] = ["x", "f", "local_pt", "sim_id", "sim_ended", "x_on_cube", "local_min"] if not persis_info: - persis_info = add_unique_random_streams({}, 4, seed=4321)[1] - persis_info["nworkers"] = 4 - super().__init__(gen_specs, History, persis_info, libE_info) + persis_info = add_unique_random_streams({}, 2, seed=4321)[1] + super().__init__(History, persis_info, gen_specs, libE_info, **kwargs) self.all_local_minima = [] self.results_idx = 0 self.last_ask = None diff --git a/libensemble/gen_classes/sampling.py b/libensemble/gen_classes/sampling.py index 275624bb9..166286482 100644 --- a/libensemble/gen_classes/sampling.py +++ b/libensemble/gen_classes/sampling.py @@ -31,10 +31,8 @@ class UniformSample(SampleBase): mode by adjusting the allocation function. """ - def __init__(self, _, persis_info, gen_specs, libE_info=None): - self.persis_info = persis_info - self.gen_specs = gen_specs - self.libE_info = libE_info + def __init__(self, _=[], persis_info={}, gen_specs={}, libE_info=None, **kwargs): + super().__init__(_, persis_info, gen_specs, libE_info, **kwargs) self._get_user_params(self.gen_specs["user"]) def ask_numpy(self, n_trials): @@ -57,10 +55,9 @@ class UniformSampleDicts(Generator): mode by adjusting the allocation function. """ - def __init__(self, _, persis_info, gen_specs, libE_info=None): - self.persis_info = persis_info + def __init__(self, _, persis_info, gen_specs, libE_info=None, **kwargs): self.gen_specs = gen_specs - self.libE_info = libE_info + self.persis_info = persis_info self._get_user_params(self.gen_specs["user"]) def ask(self, n_trials): diff --git a/libensemble/gen_classes/surmise.py b/libensemble/gen_classes/surmise.py index 3e1810f98..b62cd20dc 100644 --- a/libensemble/gen_classes/surmise.py +++ b/libensemble/gen_classes/surmise.py @@ -14,14 +14,14 @@ class Surmise(LibensembleGenThreadInterfacer): """ def __init__( - self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {} + self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {} ) -> None: from libensemble.gen_funcs.persistent_surmise_calib import surmise_calib gen_specs["gen_f"] = surmise_calib if ("sim_id", int) not in gen_specs["out"]: gen_specs["out"].append(("sim_id", int)) - super().__init__(gen_specs, History, persis_info, libE_info) + super().__init__(History, persis_info, gen_specs, libE_info) self.sim_id_index = 0 self.all_cancels = [] diff --git a/libensemble/gen_funcs/persistent_gen_wrapper.py b/libensemble/gen_funcs/persistent_gen_wrapper.py index 2ad862864..7fd01ec4d 100644 --- a/libensemble/gen_funcs/persistent_gen_wrapper.py +++ b/libensemble/gen_funcs/persistent_gen_wrapper.py @@ -1,10 +1,8 @@ import inspect -import numpy as np - from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG from libensemble.tools.persistent_support import PersistentSupport -from libensemble.utils.misc import np_to_list_dicts +from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts def persistent_gen_f(H, persis_info, gen_specs, libE_info): @@ -24,11 +22,7 @@ def persistent_gen_f(H, persis_info, gen_specs, libE_info): while tag not in [STOP_TAG, PERSIS_STOP]: H_o = gen.ask(b) if isinstance(H_o, list): - H_o_arr = np.zeros(len(H_o), dtype=gen_specs["out"]) - for i in range(len(H_o)): - for key in H_o[0].keys(): - H_o_arr[i][key] = H_o[i][key] - H_o = H_o_arr + H_o = list_dicts_to_np(H_o) tag, Work, calc_in = ps.send_recv(H_o) gen.tell(np_to_list_dicts(calc_in)) diff --git a/libensemble/generators.py b/libensemble/generators.py index 1ee243954..b13bae31c 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -8,6 +8,7 @@ from libensemble.comms.comms import QComm, QCommThread from libensemble.executors import Executor from libensemble.message_numbers import EVAL_GEN_TAG, PERSIS_STOP +from libensemble.tools.tools import add_unique_random_streams from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts """ @@ -90,6 +91,17 @@ class LibensembleGenerator(Generator): ``ask_numpy/tell_numpy`` methods communicate numpy arrays containing the same data. """ + def __init__( + self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs + ): + self.gen_specs = gen_specs + if len(kwargs) > 0: # so user can specify gen-specific parameters as kwargs to constructor + self.gen_specs["user"] = kwargs + if not persis_info: + self.persis_info = add_unique_random_streams({}, 4, seed=4321)[1] + else: + self.persis_info = persis_info + @abstractmethod def ask_numpy(self, num_points: Optional[int] = 0) -> npt.NDArray: """Request the next set of points to evaluate, as a NumPy array.""" @@ -105,6 +117,8 @@ def ask(self, num_points: Optional[int] = 0) -> List[dict]: def tell(self, results: List[dict]) -> None: """Send the results of evaluations to the generator.""" self.tell_numpy(list_dicts_to_np(results)) + # Note that although we'd prefer to have a complete dtype available, the gen + # doesn't have access to sim_specs["out"] currently. class LibensembleGenThreadInterfacer(LibensembleGenerator): @@ -113,10 +127,10 @@ class LibensembleGenThreadInterfacer(LibensembleGenerator): """ def __init__( - self, gen_specs: dict, History: npt.NDArray = [], persis_info: dict = {}, libE_info: dict = {} + self, History: npt.NDArray = [], persis_info: dict = {}, gen_specs: dict = {}, libE_info: dict = {}, **kwargs ) -> None: + super().__init__(History, persis_info, gen_specs, libE_info, **kwargs) self.gen_f = gen_specs["gen_f"] - self.gen_specs = gen_specs self.History = History self.persis_info = persis_info self.libE_info = libE_info diff --git a/libensemble/tests/regression_tests/test_asktell_surmise.py b/libensemble/tests/regression_tests/test_asktell_surmise.py index a4e5d9ae9..d0aa5310c 100644 --- a/libensemble/tests/regression_tests/test_asktell_surmise.py +++ b/libensemble/tests/regression_tests/test_asktell_surmise.py @@ -80,7 +80,7 @@ } persis_info = add_unique_random_streams({}, 5) - surmise = Surmise(gen_specs, persis_info=persis_info[1]) # we add sim_id as a field to gen_specs["out"] + surmise = Surmise(gen_specs=gen_specs, persis_info=persis_info[1]) # we add sim_id as a field to gen_specs["out"] surmise.setup() initial_sample = surmise.ask() diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py b/libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py index 842573de9..9071e80d4 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_killsims_asktell.py @@ -126,7 +126,7 @@ } persis_info = add_unique_random_streams({}, nworkers + 1) - gen_specs["generator"] = Surmise(gen_specs, persis_info=persis_info) + gen_specs["generator"] = Surmise(gen_specs=gen_specs, persis_info=persis_info) exit_criteria = {"sim_max": max_evals} diff --git a/libensemble/tests/regression_tests/test_proxystore_integration.py b/libensemble/tests/regression_tests/test_proxystore_integration.py index 998a45ab9..a900f876d 100644 --- a/libensemble/tests/regression_tests/test_proxystore_integration.py +++ b/libensemble/tests/regression_tests/test_proxystore_integration.py @@ -11,11 +11,10 @@ """ # Do not change these lines - they are parsed by run-tests.sh -# TESTSUITE_COMMS: local +# TESTSUITE_COMMS: local mpi # TESTSUITE_NPROCS: 4 # TESTSUITE_OS_SKIP: OSX WIN # TESTSUITE_EXTRA: true -# TESTSUITE_EXCLUDE: true from pathlib import Path @@ -39,7 +38,7 @@ def insert_proxy(H0): ) store = get_store("my-store") - picture = Path("libE_logo.png").read_bytes() + picture = Path("libE_logo.png").absolute().read_bytes() proxy = store.proxy(picture) for i in range(len(H0)): H0[i]["proxy"] = proxy diff --git a/libensemble/tests/run-tests.sh b/libensemble/tests/run-tests.sh index 6cdb5b143..a09eebbd4 100755 --- a/libensemble/tests/run-tests.sh +++ b/libensemble/tests/run-tests.sh @@ -154,7 +154,7 @@ cleanup() { filelist=(nodelist_*); [ -e ${filelist[0]} ] && rm nodelist_* filelist=(x_*.txt y_*.txt); [ -e ${filelist[0]} ] && rm x_*.txt y_*.txt filelist=(opt_*.txt_flag); [ -e ${filelist[0]} ] && rm opt_*.txt_flag - filelist=(*.png); [ -e ${filelist[0]} ] && rm *.png + filelist=(logo_id*.png); [ -e ${filelist[0]} ] && rm logo_id*.png done cd $THISDIR } diff --git a/libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py b/libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py index 11cad7c63..9bc097a18 100644 --- a/libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py +++ b/libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py @@ -203,7 +203,7 @@ def test_asktell_with_persistent_aposmm(): }, } - my_APOSMM = APOSMM(gen_specs) + my_APOSMM = APOSMM(gen_specs=gen_specs) my_APOSMM.setup() initial_sample = my_APOSMM.ask(100) @@ -211,7 +211,7 @@ def test_asktell_with_persistent_aposmm(): eval_max = 2000 for point in initial_sample: - point["f"] = six_hump_camel_func(point["x"]) + point["f"] = six_hump_camel_func(np.array([point["x0"], point["x1"]])) total_evals += 1 my_APOSMM.tell(initial_sample) @@ -225,7 +225,7 @@ def test_asktell_with_persistent_aposmm(): for m in detected_minima: potential_minima.append(m) for point in sample: - point["f"] = six_hump_camel_func(point["x"]) + point["f"] = six_hump_camel_func(np.array([point["x0"], point["x1"]])) total_evals += 1 my_APOSMM.tell(sample) H, persis_info, exit_code = my_APOSMM.final_tell(list_dicts_to_np(sample)) # final_tell currently requires numpy diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py new file mode 100644 index 000000000..fd80b8829 --- /dev/null +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -0,0 +1,107 @@ +import numpy as np + +from libensemble.tools.tools import add_unique_random_streams +from libensemble.utils.misc import list_dicts_to_np + + +def _check_conversion(H, npp): + + for field in H.dtype.names: + print(f"Comparing {field}: {H[field]} {npp[field]}") + + if isinstance(H[field], np.ndarray): + assert np.array_equal(H[field], npp[field]), f"Mismatch found in field {field}" + + elif isinstance(H[field], str) and isinstance(npp[field], str): + assert H[field] == npp[field], f"Mismatch found in field {field}" + + elif np.isscalar(H[field]) and np.isscalar(npp[field]): + assert np.isclose(H[field], npp[field]), f"Mismatch found in field {field}" + + else: + raise TypeError(f"Unhandled or mismatched types in field {field}: {type(H[field])} vs {type(npp[field])}") + + +def test_asktell_sampling_and_utils(): + from libensemble.gen_classes.sampling import UniformSample + + persis_info = add_unique_random_streams({}, 5, seed=1234) + gen_specs = { + "out": [("x", float, (2,))], + "user": { + "lb": np.array([-3, -2]), + "ub": np.array([3, 2]), + }, + } + + # Test initialization with libensembley parameters + gen = UniformSample(None, persis_info[1], gen_specs, None) + assert len(gen.ask(10)) == 10 + + # Test initialization gen-specific keyword args + gen = UniformSample(gen_specs=gen_specs, lb=np.array([-3, -2]), ub=np.array([3, 2])) + assert len(gen.ask(10)) == 10 + + out_np = gen.ask_numpy(3) # should get numpy arrays, non-flattened + out = gen.ask(3) # needs to get dicts, 2d+ arrays need to be flattened + assert all([len(x) == 2 for x in out]) # np_to_list_dicts is now tested + + # now we test list_dicts_to_np directly + out_np = list_dicts_to_np(out) + + # check combined values resemble flattened list-of-dicts values + assert out_np.dtype.names == ("x",) + for i, entry in enumerate(out): + for j, value in enumerate(entry.values()): + assert value == out_np["x"][i][j] + + +def test_awkward_list_dict(): + from libensemble.utils.misc import list_dicts_to_np + + # test list_dicts_to_np on a weirdly formatted dictionary + # Unfortunately, we're not really checking against some original + # libE-styled source of truth, like H. + + weird_list_dict = [ + { + "x0": "abcd", + "x1": "efgh", + "y": 56, + "z0": 1, + "z1": 2, + "z2": 3, + "z3": 4, + "z4": 5, + "z5": 6, + "z6": 7, + "z7": 8, + "z8": 9, + "z9": 10, + "z10": 11, + "a0": "B", + } + ] + + out_np = list_dicts_to_np(weird_list_dict) + + assert all([i in ("x", "y", "z", "a0") for i in out_np.dtype.names]) + + +def test_awkward_H(): + from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts + + dtype = [("a", "i4"), ("x", "f4", (3,)), ("y", "f4", (1,)), ("z", "f4", (12,)), ("greeting", "U10"), ("co2", "f8")] + H = np.zeros(2, dtype=dtype) + H[0] = (1, [1.1, 2.2, 3.3], [10.1], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "hello", "1.23") + H[1] = (2, [4.4, 5.5, 6.6], [11.1], [51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "goodbye", "2.23") + + list_dicts = np_to_list_dicts(H) + npp = list_dicts_to_np(list_dicts, dtype=dtype) + _check_conversion(H, npp) + + +if __name__ == "__main__": + test_asktell_sampling_and_utils() + test_awkward_list_dict() + test_awkward_H() diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index db73ccf91..34b7a0931 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -81,33 +81,71 @@ def specs_checker_setattr(obj, key, value): obj.__dict__[key] = value -def _copy_data(array, list_dicts): - for i, entry in enumerate(list_dicts): - for field in entry.keys(): - array[field][i] = entry[field] - return array +def _decide_dtype(name: str, entry, size: int) -> tuple: + if isinstance(entry, str): + output_type = "U" + str(len(entry) + 1) + else: + output_type = type(entry) + if size == 1 or not size: + return (name, output_type) + else: + return (name, output_type, (size,)) -def _decide_dtype(name, entry): - if hasattr(entry, "shape") and len(entry.shape): # numpy type - return (name, entry.dtype, entry.shape) - else: - return (name, type(entry)) +def _combine_names(names: list) -> list: + """combine fields with same name *except* for final digits""" + + out_names = [] + stripped = list(i.rstrip("0123456789") for i in names) # ['x', 'x', y', 'z', 'a'] + for name in names: + stripped_name = name.rstrip("0123456789") + if stripped.count(stripped_name) > 1: # if name appears >= 1, will combine, don't keep int suffix + out_names.append(stripped_name) + else: + out_names.append(name) # name appears once, keep integer suffix, e.g. "co2" + + # intending [x, y, z, a0] from [x0, x1, y, z0, z1, z2, z3, a0] + return list(set(out_names)) -def list_dicts_to_np(list_dicts: list) -> npt.NDArray: +def list_dicts_to_np(list_dicts: list, dtype: list = None) -> npt.NDArray: if list_dicts is None: return None - first = list_dicts[0] - new_dtype_names = [i for i in first.keys()] - new_dtype = [] - for i, entry in enumerate(first.values()): # must inspect values to get presumptive types - name = new_dtype_names[i] - new_dtype.append(_decide_dtype(name, entry)) + if not isinstance(list_dicts, list): # presumably already a numpy array, conversion not necessary + return list_dicts + + first = list_dicts[0] # for determining dtype of output np array + new_dtype_names = _combine_names([i for i in first.keys()]) # -> ['x', 'y'] + combinable_names = [] # [['x0', 'x1'], ['y0', 'y1', 'y2'], ['z']] + for name in new_dtype_names: # is this a necessary search over the keys again? we did it earlier... + combinable_group = [i for i in first.keys() if i.rstrip("0123456789") == name] + if len(combinable_group) > 1: # multiple similar names, e.g. x0, x1 + combinable_names.append(combinable_group) + else: # single name, e.g. local_pt, a0 *AS LONG AS THERE ISNT AN A1* + combinable_names.append([name]) + + if dtype is None: + dtype = [] + + if not len(dtype): + # another loop over names, there's probably a more elegant way, but my brain is fried + for i, entry in enumerate(combinable_names): + name = new_dtype_names[i] + size = len(combinable_names[i]) + dtype.append(_decide_dtype(name, first[entry[0]], size)) + + out = np.zeros(len(list_dicts), dtype=dtype) + + for i, group in enumerate(combinable_names): + new_dtype_name = new_dtype_names[i] + for j, input_dict in enumerate(list_dicts): + if len(group) == 1: # only a single name, e.g. local_pt + out[new_dtype_name][j] = input_dict[new_dtype_name] + else: # combinable names detected, e.g. x0, x1 + out[new_dtype_name][j] = tuple([input_dict[name] for name in group]) - out = np.zeros(len(list_dicts), dtype=new_dtype) - return _copy_data(out, list_dicts) + return out def np_to_list_dicts(array: npt.NDArray) -> List[dict]: @@ -117,6 +155,13 @@ def np_to_list_dicts(array: npt.NDArray) -> List[dict]: for row in array: new_dict = {} for field in row.dtype.names: - new_dict[field] = row[field] + # non-string arrays, lists, etc. + if hasattr(row[field], "__len__") and len(row[field]) > 1 and not isinstance(row[field], str): + for i, x in enumerate(row[field]): + new_dict[field + str(i)] = x + elif hasattr(row[field], "__len__") and len(row[field]) == 1: # single-entry arrays, lists, etc. + new_dict[field] = row[field][0] # will still work on single-char strings + else: + new_dict[field] = row[field] out.append(new_dict) return out diff --git a/libensemble/utils/runners.py b/libensemble/utils/runners.py index d688a427e..08d52a27e 100644 --- a/libensemble/utils/runners.py +++ b/libensemble/utils/runners.py @@ -4,14 +4,13 @@ import time from typing import Optional -import numpy as np import numpy.typing as npt from libensemble.comms.comms import QCommThread from libensemble.generators import LibensembleGenerator, LibensembleGenThreadInterfacer from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG from libensemble.tools.persistent_support import PersistentSupport -from libensemble.utils.misc import np_to_list_dicts +from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts logger = logging.getLogger(__name__) @@ -107,22 +106,9 @@ def __init__(self, specs): super().__init__(specs) self.gen = specs.get("generator") - def _to_array(self, x: list) -> npt.NDArray: - """fast-cast list-of-dicts to NumPy array""" - if isinstance(x, list) and len(x) and isinstance(x[0], dict): - arr = np.zeros(len(x), dtype=self.specs["out"]) - for i in range(len(x)): - for key in x[0].keys(): - arr[i][key] = x[i][key] - return arr - return x - def _get_points_updates(self, batch_size: int) -> (npt.NDArray, npt.NDArray): # no ask_updates on external gens - return ( - self._to_array(self.gen.ask(batch_size)), - None, - ) + return (list_dicts_to_np(self.gen.ask(batch_size), dtype=self.specs.get("out")), None) def _convert_tell(self, x: npt.NDArray) -> list: self.gen.tell(np_to_list_dicts(x)) @@ -155,7 +141,8 @@ def _persistent_result(self, calc_in, persis_info, libE_info): self.gen.libE_info = libE_info if self.gen.thread is None: self.gen.setup() # maybe we're reusing a live gen from a previous run - H_out = self._to_array(self._get_initial_ask(libE_info)) + # libE gens will hit the following line, but list_dicts_to_np will passthrough if the output is a numpy array + H_out = list_dicts_to_np(self._get_initial_ask(libE_info), dtype=self.specs.get("out")) tag, Work, H_in = self.ps.send_recv(H_out) # evaluate the initial sample final_H_in = self._start_generator_loop(tag, Work, H_in) return self.gen.final_tell(final_H_in), FINISHED_PERSISTENT_GEN_TAG