Merge pull request #1298 from Libensemble/release/v_1.3.0

Release/v 1.3.0

.github/workflows/basic.yml

@@ -18,18 +18,18 @@ jobs:
                 os: [ubuntu-latest]
                 mpi-version: [mpich]
                 python-version: [3.9, "3.10", "3.11", "3.12"]
-                pydantic-version: ["2.5.2"]
+                pydantic-version: ["2.6.4"]
                 comms-type: [m, l]
                 include:
                     - os: macos-latest
                       python-version: "3.11"
-                      mpi-version: "mpich=4.0.3"
-                      pydantic-version: "2.5.2"
+                      mpi-version: "mpich=4.2.1"
+                      pydantic-version: "2.6.4"
                       comms-type: m
                     - os: macos-latest
                       python-version: "3.11"
-                      mpi-version: "mpich=4.0.3"
-                      pydantic-version: "2.5.2"
+                      mpi-version: "mpich=4.2.1"
+                      pydantic-version: "2.6.4"
                       comms-type: l
                     - os: ubuntu-latest
                       mpi-version: mpich
@@ -86,11 +86,9 @@ jobs:
           run: conda install gcc_linux-64
 
         # Roundabout solution on macos for proper linking with mpicc
-        - name: Install macOS compilers and older SDK
+        - name: Install macOS compilers
           if: matrix.os == 'macos-latest'
           run: |
-            wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.15.sdk.tar.xz
-            mkdir ../sdk; tar xf MacOSX10.15.sdk.tar.xz -C ../sdk
             conda install clang_osx-64
 
         - name: Install basic testing/feature dependencies
@@ -147,8 +145,6 @@ jobs:
 
         - name: Run simple tests, macOS
           if: matrix.os == 'macos-latest'
-          env:
-              CONDA_BUILD_SYSROOT: /Users/runner/work/libensemble/sdk/MacOSX10.15.sdk
           run: |
             ./libensemble/tests/run-tests.sh -A "-W error" -z -${{ matrix.comms-type }}
 
@@ -167,4 +163,4 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
-        - uses: crate-ci/typos@v1.19.0
+        - uses: crate-ci/typos@v1.20.10

.github/workflows/extra.yml

@@ -12,27 +12,27 @@ jobs:
                 os: [ubuntu-latest]
                 mpi-version: [mpich]
                 python-version: [3.9, "3.10", "3.11", "3.12"]
-                pydantic-version: ["2.5.2"]
+                pydantic-version: ["2.6.4"]
                 comms-type: [m, l]
                 include:
                     - os: macos-latest
                       python-version: 3.11
-                      mpi-version: "mpich=4.0.3"
-                      pydantic-version: "2.5.2"
+                      mpi-version: "mpich=4.2.0"
+                      pydantic-version: "2.6.4"
                       comms-type: m
                     - os: macos-latest
                       python-version: 3.11
-                      mpi-version: "mpich=4.0.3"
-                      pydantic-version: "2.5.2"
+                      mpi-version: "mpich=4.2.0"
+                      pydantic-version: "2.6.4"
                       comms-type: l
                     - os: ubuntu-latest
                       python-version: "3.10"
                       mpi-version: "mpich"
-                      pydantic-version: "2.5.2"
+                      pydantic-version: "2.6.4"
                       comms-type: t
                     - os: ubuntu-latest
                       mpi-version: "openmpi"
-                      pydantic-version: "2.5.2"
+                      pydantic-version: "2.6.4"
                       python-version: "3.12"
                       comms-type: l
                     - os: ubuntu-latest
@@ -91,11 +91,9 @@ jobs:
           run: conda install gcc_linux-64
 
         # Roundabout solution on macos for proper linking with mpicc
-        - name: Install macOS compilers and older SDK
+        - name: Install macOS compilers
           if: matrix.os == 'macos-latest'
           run: |
-            wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.15.sdk.tar.xz
-            mkdir ../sdk; tar xf MacOSX10.15.sdk.tar.xz -C ../sdk
             conda install clang_osx-64
 
         - name: Install mpi4py and MPI from conda
@@ -112,13 +110,16 @@ jobs:
         - name: Install generator dependencies
           run: |
             python -m pip install --upgrade pip
-            pip install mpmath matplotlib
+            pip install mpmath matplotlib ax-platform
             conda env update --file install/gen_deps_environment.yml
 
         - name: Install ax-platform
+          run: |
+            pip install ax-platform
+
+        - name: Install gpcam
           if: matrix.python-version != '3.12'
           run: |
-            pip install ax-platform==0.2.8
             pip install gpcam
 
         - name: Install surmise
@@ -129,8 +130,8 @@ jobs:
         - name: Build ytopt and dependencies
           if: matrix.python-version <= '3.10' && matrix.os != 'macos-latest'
           run: |
-            pip install scikit-learn==1.1.2
-            pip install pandas==1.5.3
+            pip install scikit-learn==1.4.0
+            pip install pandas==2.2.1
             pip install ConfigSpace
             pip install "ray[default]"
             git clone https://github.com/ytopt-team/scikit-optimize.git
@@ -154,11 +155,11 @@ jobs:
             # pip install dragonfly-opt
             # pip install git+https://github.com/dragonfly/dragonfly.git
             pip install git+https://github.com/jlnav/dragonfly.git@fix/remove_npobject
+            pip install scikit-build packaging Tasmanian --user
 
         - name: Install other testing dependencies
           run: |
             conda install octave
-            pip install scikit-build packaging Tasmanian --user
             conda install pyzmq
             pip install -r install/testing_requirements.txt
             pip install -r install/misc_feature_requirements.txt
@@ -215,10 +216,15 @@ jobs:
             rm ./libensemble/tests/regression_tests/test_ytopt_heffte.py
             rm ./libensemble/tests/regression_tests/test_gpCAM.py
             rm ./libensemble/tests/regression_tests/test_persistent_gp.py
-            rm ./libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py
+
+        - name: Remove test for persistent Tasmanian on Python 3.12
+          if: matrix.python-version >= '3.12'
+          run: |
+            rm ./libensemble/tests/regression_tests/test_persistent_tasmanian.py
+            rm ./libensemble/tests/regression_tests/test_persistent_tasmanian_async.py
 
         - name: Remove Balsam/Globus-compute tests on Pydantic 2
-          if: matrix.pydantic-version == '2.5.2'
+          if: matrix.pydantic-version == '2.6.4'
           run: |
             rm ./libensemble/tests/unit_tests/test_ufunc_runners.py
             rm ./libensemble/tests/unit_tests/test_executor_balsam.py
@@ -230,8 +236,6 @@ jobs:
 
         - name: Run extensive tests, macOS
           if: matrix.os == 'macos-latest'
-          env:
-              CONDA_BUILD_SYSROOT: /Users/runner/work/libensemble/sdk/MacOSX10.15.sdk
           run: |
             ./libensemble/tests/run-tests.sh -e -z -${{ matrix.comms-type }}
 
@@ -250,4 +254,4 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v4
-        - uses: crate-ci/typos@v1.19.0
+        - uses: crate-ci/typos@v1.20.10

.typos.toml

@@ -9,6 +9,13 @@ als = "als"
 datas = "datas"
 numer = "numer"
 inout = "inout"
+arange = "arange"
+wrk = "wrk"
+EOF = "EOF"
+HPE = "HPE"
+RO = "RO"
+lst = "lst"
+noy = "noy"
 
 [files]
 extend-exclude = ["*.bib", "*.xml", "docs/nitpicky"]

.wci.yml

@@ -16,8 +16,8 @@ description: |
 language: Python
 
 release:
-  version: 1.2.2
-  date: 2024-03-21
+  version: 1.3.0
+  date: 2024-05-01
 
 documentation:
   general: https://libensemble.readthedocs.io

CHANGELOG.rst

@@ -8,6 +8,28 @@ GitHub issues are referenced, and can be viewed with hyperlinks on the `github r
 
 .. _`github releases page`: https://github.com/Libensemble/libensemble/releases
 
+Release 1.3.0
+--------------
+
+:Date: May 01, 2024
+
+* Support generator running on the manager (on a thread). #1216/#1290
+  * Set `libE_specs["gen_on_manager"] = True`
+  * Then run with `nworkers` equal to the number of simulation workers.
+* Default to local comms when `nworkers` is supplied and no MPI runner is detected. #1169
+* Parse args defaults to **local** comms when `--nworkers` (or `-n`) is set on the command line. #1169
+
+:Note:
+
+* Tests were run on Linux and MacOS with Python versions 3.9, 3.10, 3.11, 3.12
+* Heterogeneous workflows tested on Frontier (OLCF), Polaris (ALCF), and Perlmutter (NERSC).
+* Note that tests have been recently run on Aurora (ALCF), but the system was unavailable at time of release.
+* Tests were also run on Bebop and Improv LCRC systems.
+
+:Known Issues:
+
+* See known issues section in the documentation.
+
 Release 1.2.2
 --------------
 
@@ -203,7 +225,7 @@ Release 0.10.2
   * Ensure relative paths are interpreted from where libEnsemble is run. #1020
   * Create intermediate directories for workflow paths. #1017
 
-* Fixes issue where libEnsemble pre-initialized a shared multiprocssing queue. #1026
+* Fixes issue where libEnsemble pre-initialized a shared multiprocessing queue. #1026
 
 :Note:
 
@@ -786,7 +808,7 @@ Release 0.3.0
 * History is dumped to file on manager or worker exception and MPI aborted (with exit code 1). (#46)
 * Default logging level changed to DEBUG and redirected to file ensemble.log.
 * Added directory of standalone tests (comms, job kills, and nested MPI launches).
-* Improved and speeded up unit tests. (#68)
+* Improved and sped up unit tests. (#68)
 * Considerable documentation enhancements.
 
 :Known issues:

README.rst

@@ -1,39 +1,8 @@
-.. image:: https://raw.githubusercontent.com/Libensemble/libensemble/main/docs/images/libE_logo.png
-   :align: center
-   :alt: libEnsemble
-
-|
-
-.. image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue
-   :target: https://pypi.org/project/libensemble
-
-.. image:: https://img.shields.io/conda/v/conda-forge/libensemble?color=blue
-   :target: https://anaconda.org/conda-forge/libensemble
-
-.. image:: https://img.shields.io/spack/v/py-libensemble?color=blue
-   :target: https://packages.spack.io/package.html?name=py-libensemble
-
-|
-
-.. image:: https://github.com/Libensemble/libensemble/actions/workflows/extra.yml/badge.svg?branch=main
-   :target: https://github.com/Libensemble/libensemble/actions
-
-.. image:: https://codecov.io/github/Libensemble/libensemble/graph/badge.svg
-   :target: https://codecov.io/github/Libensemble/libensemble
-
-.. image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000
-   :target: https://libensemble.readthedocs.org/en/latest/
-   :alt: Documentation Status
-
-.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
-   :target: https://github.com/psf/black
-   :alt: Code style: black
+|libE_logo|
 
-.. image:: https://joss.theoj.org/papers/10.21105/joss.06031/status.svg
-   :target: https://doi.org/10.21105/joss.06031
-   :alt: JOSS Status
+|PyPI| |Conda| |Spack|
 
-|
+|Tests| |Coverage| |Docs| |Style| |JOSS|
 
 .. after_badges_rst_tag
 
@@ -135,6 +104,29 @@ Resources
     doi     = {10.1109/tpds.2021.3082815}
   }
 
+.. |libE_logo| image:: https://raw.githubusercontent.com/Libensemble/libensemble/main/docs/images/libE_logo.png
+   :align: middle
+   :alt: libEnsemble
+.. |PyPI| image:: https://img.shields.io/pypi/v/libensemble.svg?color=blue
+   :target: https://pypi.org/project/libensemble
+.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/libensemble?color=blue
+   :target: https://anaconda.org/conda-forge/libensemble
+.. |Spack| image:: https://img.shields.io/spack/v/py-libensemble?color=blue
+   :target: https://packages.spack.io/package.html?name=py-libensemble
+.. |Tests| image:: https://github.com/Libensemble/libensemble/actions/workflows/extra.yml/badge.svg?branch=main
+   :target: https://github.com/Libensemble/libensemble/actions
+.. |Coverage| image:: https://codecov.io/github/Libensemble/libensemble/graph/badge.svg
+   :target: https://codecov.io/github/Libensemble/libensemble
+.. |Docs| image:: https://readthedocs.org/projects/libensemble/badge/?maxAge=2592000
+   :target: https://libensemble.readthedocs.org/en/latest/
+   :alt: Documentation Status
+.. |Style| image:: https://img.shields.io/badge/code%20style-black-000000.svg
+   :target: https://github.com/psf/black
+   :alt: Code style: black
+.. |JOSS| image:: https://joss.theoj.org/papers/10.21105/joss.06031/status.svg
+   :target: https://doi.org/10.21105/joss.06031
+   :alt: JOSS Status
+
 .. _Community Examples repository: https://github.com/Libensemble/libe-community-examples
 .. _conda-forge: https://conda-forge.org/
 .. _Contributions: https://github.com/Libensemble/libensemble/blob/main/CONTRIBUTING.rst

docs/Makefile

@@ -2,6 +2,7 @@
 #
 
 # You can set these variables from the command line.
+#SPHINXOPTS    = -b spelling
 SPHINXOPTS    =
 SPHINXBUILD   = python3 -msphinx
 SPHINXPROJ    = libensemble

docs/conf.py

@@ -83,6 +83,7 @@ def __getattr__(cls, name):
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    "sphinxcontrib.spelling",
     "sphinxcontrib.bibtex",
     "sphinx.ext.autodoc",
     "sphinx.ext.napoleon",
@@ -94,6 +95,13 @@ def __getattr__(cls, name):
     "sphinx_design",
     "sphinx_copybutton",
 ]
+
+spelling_word_list_filename = "spelling_wordlist.txt"
+spelling_ignore_pypi_package_names = True
+spelling_ignore_acronyms = True
+spelling_ignore_python_builtins = True
+spelling_ignore_importable_modules = True
+
 bibtex_bibfiles = ["references.bib"]
 bibtex_default_style = "unsrt"
 # autosectionlabel_prefix_document = True

docs/data_structures/libE_specs.rst

@@ -26,9 +26,15 @@ libEnsemble is primarily customized by setting options within a ``LibeSpecs`` cl
 
                 **comms** [str] = ``"mpi"``:
                     Manager/Worker communications mode: ``'mpi'``, ``'local'``, or ``'tcp'``.
+                    If ``nworkers`` is specified, then ``local`` comms will be used unless a
+                    parallel MPI environment is detected.
 
                 **nworkers** [int]:
-                    Number of worker processes in ``"local"`` or ``"tcp"``.
+                    Number of worker processes in ``"local"``, ``"threads"``, or ``"tcp"``.
+
+                **gen_on_manager** [bool] = False
+                    Instructs Manager process to run generator functions.
+                    This generator function can access/modify user objects by reference.
 
                 **mpi_comm** [MPI communicator] = ``MPI.COMM_WORLD``:
                     libEnsemble MPI communicator.
@@ -51,6 +57,10 @@ libEnsemble is primarily customized by setting options within a ``LibeSpecs`` cl
                 **disable_log_files** [bool] = ``False``:
                     Disable ``ensemble.log`` and ``libE_stats.txt`` log files.
 
+                **gen_workers** [list of ints]:
+                    List of workers that should run only generators. All other workers will run
+                    only simulator functions.
+
         .. tab-item:: Directories
 
             .. tab-set::