From e9c6f493e339afe5cc88727b0ae7cb162ce26e15 Mon Sep 17 00:00:00 2001 From: paugier Date: Thu, 8 Feb 2024 11:07:40 +0100 Subject: [PATCH] Better pfft --- .gitlab-ci.yml | 8 +-- Makefile | 6 +++ docker/Dockerfile | 5 +- noxfile.py | 23 ++++++--- plugins/fluidfft-pfft/README.md | 16 +++--- plugins/fluidfft-pfft/meson.build | 49 +++++++++++++------ .../src/fluidfft_pfft/meson.build | 2 +- pyproject.toml | 6 --- tests/test_plugins.py | 18 ++++++- 9 files changed, 87 insertions(+), 46 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a21c7d..cb7f852 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,9 +42,9 @@ image:build: stage: image tags: - container-registry-push - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - - if: '$CI_BUILD_IMAGES == "1"' + # rules: + # - if: '$CI_PIPELINE_SOURCE == "schedule"' + # - if: '$CI_BUILD_IMAGES == "1"' image: name: gcr.io/kaniko-project/executor:debug entrypoint: [ "" ] @@ -96,7 +96,7 @@ tests_mpi: - job: "image:build" optional: true script: - - nox -s "tests(with_cov=True, with_mpi=True)" + - nox -s "tests(with_cov=True, with_mpi=True)" -- --with-pfft pages: diff --git a/Makefile b/Makefile index bf4ce4c..ce9ecdb 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,12 @@ develop_mpi_with_fftw: develop_fftwmpi: pdm run pip install -e plugins/fluidfft-fftwmpi --no-build-isolation -v +develop_pfft: + pdm run pip install -e plugins/fluidfft-pfft --no-build-isolation -v + +develop_p3dfft: + pdm run pip install -e plugins/fluidfft-p3dfft --no-build-isolation -v + sync: pdm sync --clean --no-self diff --git a/docker/Dockerfile b/docker/Dockerfile index feec932..8ef3f09 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,19 +43,20 @@ RUN ln -s /usr/include/fftw* $HOME/.local/include RUN ln -s /usr/lib/x86_64-linux-gnu/libfftw3* $HOME/.local/lib ENV LD_LIBRARY_PATH=$HOME/.local/lib +ENV LIBRARY_PATH=$HOME/.local/lib ENV PATH=$HOME/.local/bin:$PATH ENV CPATH=$HOME/.local/include:$CPATH RUN mkdir -p $HOME/.config/matplotlib RUN echo 'backend : agg' > $HOME/.config/matplotlib/matplotlibrc -RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/doc/install/install_p3dfft.sh -O ./install_p3dfft.sh +COPY --chown=appuser:appuser doc/install/install_p3dfft.sh install_p3dfft.sh RUN chmod +x install_p3dfft.sh RUN export FCFLAGS="-w -fallow-argument-mismatch -O2" && \ export FFLAGS="-w -fallow-argument-mismatch -O2" && \ ./install_p3dfft.sh -RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/doc/install/install_pfft.sh -O ./install_pfft.sh +COPY --chown=appuser:appuser doc/install/install_pfft.sh install_pfft.sh RUN chmod +x install_pfft.sh RUN ./install_pfft.sh diff --git a/noxfile.py b/noxfile.py index 5d5672f..f99d0a3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -12,7 +12,7 @@ make -execute ``make list-sessions```` or ``nox -l`` for a list of sessions. +execute `make list-sessions` or `nox -l` for a list of sessions. """ @@ -30,7 +30,7 @@ no_venv_session = partial(nox.session, venv_backend="none") -@nox.session +@nox.session(reuse_venv=True) def validate_code(session): session.run_always( "pdm", "sync", "--clean", "-G", "lint", "--no-self", external=True @@ -40,10 +40,13 @@ def validate_code(session): @nox.parametrize("with_mpi", [True, False]) @nox.parametrize("with_cov", [True, False]) -@nox.session +@nox.session(reuse_venv=True) def tests(session, with_mpi, with_cov): """Execute unit-tests using pytest""" + with_pfft = "--with-pfft" in session.posargs + with_p3dfft = "--with-p3dfft" in session.posargs + command = "pdm sync --clean --no-self -G test -G build -G pyfftw" if with_mpi: command += " -G mpi" @@ -65,6 +68,10 @@ def tests(session, with_mpi, with_cov): "-e", "plugins/fluidfft-mpi_with_fftw", "--no-build-isolation", "-v" ) session.install("-e", "plugins/fluidfft-fftwmpi", "--no-build-isolation", "-v") + if with_pfft: + session.install("-e", "plugins/fluidfft-pfft", "--no-build-isolation", "-v") + if with_p3dfft: + session.install("-e", "plugins/fluidfft-p3dfft", "--no-build-isolation", "-v") if with_cov: path_coverage = Path.cwd() / ".coverage" @@ -78,8 +85,8 @@ def run_command(command, **kwargs): command = "pytest -v -s tests" - run_command(command, *session.posargs) - run_command(command, *session.posargs, env={"TRANSONIC_NO_REPLACE": "1"}) + run_command(command) + run_command(command, env={"TRANSONIC_NO_REPLACE": "1"}) run_command("pytest -v plugins/fluidfft-fftw") if with_mpi: @@ -95,6 +102,10 @@ def test_plugin(package_name): test_plugin("fluidfft-mpi_with_fftw") test_plugin("fluidfft-fftwmpi") + if with_pfft: + test_plugin("fluidfft-pfft") + if with_p3dfft: + test_plugin("fluidfft-p3dfft") if with_cov: if with_mpi: @@ -104,7 +115,7 @@ def test_plugin(package_name): session.run("coverage", "html") -@nox.session +@nox.session(reuse_venv=True) def doc(session): session.run_always( "pdm", "sync", "--clean", "-G", "doc", "--no-self", external=True diff --git a/plugins/fluidfft-pfft/README.md b/plugins/fluidfft-pfft/README.md index 45ec2be..8ba3353 100644 --- a/plugins/fluidfft-pfft/README.md +++ b/plugins/fluidfft-pfft/README.md @@ -5,15 +5,11 @@ This plugin provides a method for parallel FFTs using PFFT: ## Environment variables -- `LIBRARY_PATH` -- `CPATH` -- `LD_LIBRARY_PATH` +The default include path can be expanded with `CPATH` for GCC/Clang and +`INCLUDE` for MSVC. -or +The default library search path can be expanded with `LIBRARY_PATH` for +GCC/Clang and `LIB` for MSVC. -- `PFFT_DIR` - -or - -- `PFFT_LIB_DIR` -- `PFFT_INCLUDE_DIR` +Alternatively, one could define `PFFT_DIR` or `PFFT_LIB_DIR` and +`PFFT_INCLUDE_DIR`. diff --git a/plugins/fluidfft-pfft/meson.build b/plugins/fluidfft-pfft/meson.build index b5bebab..e0983d7 100644 --- a/plugins/fluidfft-pfft/meson.build +++ b/plugins/fluidfft-pfft/meson.build @@ -35,24 +35,41 @@ PFFT_DIR = run_command( ).stdout().strip() message('PFFT_DIR=' + PFFT_DIR) -PFFT_LIB_DIR = run_command( - py, '-c', 'import os; print(os.environ.get("PFFT_LIB_DIR", ""))', - check: true -).stdout().strip() -message('PFFT_LIB_DIR=' + PFFT_LIB_DIR) +if PFFT_DIR != '' + PFFT_LIB_DIR = PFFT_DIR + '/lib' + PFFT_INCLUDE_DIR = PFFT_DIR + '/include' +else + PFFT_LIB_DIR = run_command( + py, '-c', 'import os; print(os.environ.get("PFFT_LIB_DIR", ""))', + check: true + ).stdout().strip() + message('PFFT_LIB_DIR=' + PFFT_LIB_DIR) -PFFT_INCLUDE_DIR = run_command( - py, '-c', 'import os; print(os.environ.get("PFFT_INCLUDE_DIR", ""))', - check: true -).stdout().strip() -message('PFFT_INCLUDE_DIR=' + PFFT_INCLUDE_DIR) + PFFT_INCLUDE_DIR = run_command( + py, '-c', 'import os; print(os.environ.get("PFFT_INCLUDE_DIR", ""))', + check: true + ).stdout().strip() + message('PFFT_INCLUDE_DIR=' + PFFT_INCLUDE_DIR) +endif -# We need to do something with these values -# see https://stackoverflow.com/a/61072768/1779806 +dirs = [] +if PFFT_LIB_DIR != '' + dirs += [PFFT_LIB_DIR] +endif -# set LIBRARY_PATH to include the directory where is libpfft.so -pfft_dep = compiler.find_library('pfft', required: true) -link_args = ['-lpfft', '-lfftw3_mpi'] +include_directories = [] +if PFFT_INCLUDE_DIR != '' + include_directories = include_directories(PFFT_INCLUDE_DIR) +endif + +pfft_dep = compiler.find_library( + 'pfft', + dirs: dirs, + has_headers: ['pfft.h'], + header_include_directories: include_directories, + required: true, +) +link_args = ['-lfftw3_mpi'] dependencies = [fftw_dep, mpi_dep, np_dep, fftwmpi_dep, pfft_dep] @@ -60,6 +77,8 @@ include_path_fluidfft_builder = run_command( 'fluidfft-builder-print-include-dir', check: true ).stdout().strip() +include_directories = [include_directories, include_path_fluidfft_builder] + include_path_cy = run_command( 'fluidfft-builder-print-include-dir-cython', check: true ).stdout().strip() diff --git a/plugins/fluidfft-pfft/src/fluidfft_pfft/meson.build b/plugins/fluidfft-pfft/src/fluidfft_pfft/meson.build index 230c973..66588c3 100644 --- a/plugins/fluidfft-pfft/src/fluidfft_pfft/meson.build +++ b/plugins/fluidfft-pfft/src/fluidfft_pfft/meson.build @@ -25,7 +25,7 @@ py.extension_module( include_path_fluidfft_builder / 'base_fft3dmpi.cpp', dependencies: dependencies, override_options : ['cython_language=cpp'], - include_directories: include_path_fluidfft_builder, + include_directories: include_directories, link_args: link_args, install: true, subdir: 'fluidfft_pfft', diff --git a/pyproject.toml b/pyproject.toml index b399dd3..08faaf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,12 +55,6 @@ fluidfft-bench-analysis = "fluidfft.bench_analysis:run" "fft3d.with_pyfftw" = "fluidfft.fft3d.with_pyfftw" "fft2d.with_dask" = "fluidfft.fft2d.with_dask" -# should be in fluidfft-p3dfft -# "fft3d.mpi_with_p3dfft" = "fluidfft.fft3d.mpi_with_p3dfft" - -# should be in fluidfft-pfft -# "fft3d.mpi_with_pfft" = "fluidfft.fft3d.mpi_with_pfft" - [tool.pdm] distribution = true package-dir = "src" diff --git a/tests/test_plugins.py b/tests/test_plugins.py index bdb4336..5ec301f 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -48,8 +48,22 @@ methodss[3, False].add("fft3d.mpi_with_fftwmpi3d") del fluidfft_fftwmpi -# "fft3d.mpi_with_p3dfft", -# "fft3d.mpi_with_pfft", + +try: + import fluidfft_pfft +except ImportError: + pass +else: + methodss[3, False].add("fft3d.mpi_with_pfft") + del fluidfft_pfft + +try: + import fluidfft_p3dfft +except ImportError: + pass +else: + methodss[3, False].add("fft3d.mpi_with_p3dfft") + del fluidfft_p3dfft def test_plugins():