Better pfft

.gitlab-ci.yml

@@ -42,9 +42,9 @@ image:build:
   stage: image
   tags:
     - container-registry-push
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-    - if: '$CI_BUILD_IMAGES == "1"'
+  # rules:
+  #   - if: '$CI_PIPELINE_SOURCE == "schedule"'
+  #   - if: '$CI_BUILD_IMAGES == "1"'
   image:
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: [ "" ]
@@ -96,7 +96,7 @@ tests_mpi:
     - job: "image:build"
       optional: true
   script:
-    - nox -s "tests(with_cov=True, with_mpi=True)"
+    - nox -s "tests(with_cov=True, with_mpi=True)" -- --with-pfft
 
 
 pages:

Makefile

@@ -11,6 +11,12 @@ develop_mpi_with_fftw:
 develop_fftwmpi:
 	pdm run pip install -e plugins/fluidfft-fftwmpi --no-build-isolation -v
 
+develop_pfft:
+	pdm run pip install -e plugins/fluidfft-pfft --no-build-isolation -v
+
+develop_p3dfft:
+	pdm run pip install -e plugins/fluidfft-p3dfft --no-build-isolation -v
+
 sync:
 	pdm sync --clean --no-self
 

docker/Dockerfile

@@ -43,19 +43,20 @@ RUN ln -s /usr/include/fftw* $HOME/.local/include
 RUN ln -s /usr/lib/x86_64-linux-gnu/libfftw3* $HOME/.local/lib
 
 ENV LD_LIBRARY_PATH=$HOME/.local/lib
+ENV LIBRARY_PATH=$HOME/.local/lib
 ENV PATH=$HOME/.local/bin:$PATH
 ENV CPATH=$HOME/.local/include:$CPATH
 
 RUN mkdir -p $HOME/.config/matplotlib
 RUN echo 'backend      : agg' > $HOME/.config/matplotlib/matplotlibrc
 
-RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/doc/install/install_p3dfft.sh -O ./install_p3dfft.sh
+COPY --chown=appuser:appuser doc/install/install_p3dfft.sh install_p3dfft.sh
 RUN chmod +x install_p3dfft.sh
 RUN export FCFLAGS="-w -fallow-argument-mismatch -O2" && \
     export FFLAGS="-w -fallow-argument-mismatch -O2" && \
     ./install_p3dfft.sh
 
-RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/doc/install/install_pfft.sh -O ./install_pfft.sh
+COPY --chown=appuser:appuser doc/install/install_pfft.sh install_pfft.sh
 RUN chmod +x install_pfft.sh
 RUN ./install_pfft.sh
 

noxfile.py

@@ -12,7 +12,7 @@
 
    make <session>
 
-execute ``make list-sessions```` or ``nox -l`` for a list of sessions.
+execute `make list-sessions` or `nox -l` for a list of sessions.
 
 """
 
@@ -30,7 +30,7 @@
 no_venv_session = partial(nox.session, venv_backend="none")
 
 
-@nox.session
+@nox.session(reuse_venv=True)
 def validate_code(session):
     session.run_always(
         "pdm", "sync", "--clean", "-G", "lint", "--no-self", external=True
@@ -40,10 +40,13 @@ def validate_code(session):
 
 @nox.parametrize("with_mpi", [True, False])
 @nox.parametrize("with_cov", [True, False])
-@nox.session
+@nox.session(reuse_venv=True)
 def tests(session, with_mpi, with_cov):
     """Execute unit-tests using pytest"""
 
+    with_pfft = "--with-pfft" in session.posargs
+    with_p3dfft = "--with-p3dfft" in session.posargs
+
     command = "pdm sync --clean --no-self -G test -G build -G pyfftw"
     if with_mpi:
         command += " -G mpi"
@@ -65,6 +68,10 @@ def tests(session, with_mpi, with_cov):
             "-e", "plugins/fluidfft-mpi_with_fftw", "--no-build-isolation", "-v"
         )
         session.install("-e", "plugins/fluidfft-fftwmpi", "--no-build-isolation", "-v")
+        if with_pfft:
+            session.install("-e", "plugins/fluidfft-pfft", "--no-build-isolation", "-v")
+        if with_p3dfft:
+            session.install("-e", "plugins/fluidfft-p3dfft", "--no-build-isolation", "-v")
 
     if with_cov:
         path_coverage = Path.cwd() / ".coverage"
@@ -78,8 +85,8 @@ def run_command(command, **kwargs):
 
     command = "pytest -v -s tests"
 
-    run_command(command, *session.posargs)
-    run_command(command, *session.posargs, env={"TRANSONIC_NO_REPLACE": "1"})
+    run_command(command)
+    run_command(command, env={"TRANSONIC_NO_REPLACE": "1"})
     run_command("pytest -v plugins/fluidfft-fftw")
 
     if with_mpi:
@@ -95,6 +102,10 @@ def test_plugin(package_name):
 
         test_plugin("fluidfft-mpi_with_fftw")
         test_plugin("fluidfft-fftwmpi")
+        if with_pfft:
+            test_plugin("fluidfft-pfft")
+        if with_p3dfft:
+            test_plugin("fluidfft-p3dfft")
 
     if with_cov:
         if with_mpi:
@@ -104,7 +115,7 @@ def test_plugin(package_name):
         session.run("coverage", "html")
 
 
-@nox.session
+@nox.session(reuse_venv=True)
 def doc(session):
     session.run_always(
         "pdm", "sync", "--clean", "-G", "doc", "--no-self", external=True

plugins/fluidfft-pfft/README.md

@@ -5,15 +5,11 @@ This plugin provides a method for parallel FFTs using PFFT:
 
 ## Environment variables
 
-- `LIBRARY_PATH`
-- `CPATH`
-- `LD_LIBRARY_PATH`
+The default include path can be expanded with `CPATH` for GCC/Clang and
+`INCLUDE` for MSVC.
 
-or
+The default library search path can be expanded with `LIBRARY_PATH` for
+GCC/Clang and `LIB` for MSVC.
 
-- `PFFT_DIR`
-
-or
-
-- `PFFT_LIB_DIR`
-- `PFFT_INCLUDE_DIR`
+Alternatively, one could define `PFFT_DIR` or `PFFT_LIB_DIR` and
+`PFFT_INCLUDE_DIR`.

plugins/fluidfft-pfft/meson.build

@@ -35,31 +35,50 @@ PFFT_DIR = run_command(
 ).stdout().strip()
 message('PFFT_DIR=' + PFFT_DIR)
 
-PFFT_LIB_DIR = run_command(
-  py, '-c', 'import os; print(os.environ.get("PFFT_LIB_DIR", ""))',
-  check: true
-).stdout().strip()
-message('PFFT_LIB_DIR=' + PFFT_LIB_DIR)
+if PFFT_DIR != ''
+  PFFT_LIB_DIR = PFFT_DIR + '/lib'
+  PFFT_INCLUDE_DIR = PFFT_DIR + '/include'
+else
+  PFFT_LIB_DIR = run_command(
+    py, '-c', 'import os; print(os.environ.get("PFFT_LIB_DIR", ""))',
+    check: true
+  ).stdout().strip()
+  message('PFFT_LIB_DIR=' + PFFT_LIB_DIR)
 
-PFFT_INCLUDE_DIR = run_command(
-  py, '-c', 'import os; print(os.environ.get("PFFT_INCLUDE_DIR", ""))',
-  check: true
-).stdout().strip()
-message('PFFT_INCLUDE_DIR=' + PFFT_INCLUDE_DIR)
+  PFFT_INCLUDE_DIR = run_command(
+    py, '-c', 'import os; print(os.environ.get("PFFT_INCLUDE_DIR", ""))',
+    check: true
+  ).stdout().strip()
+  message('PFFT_INCLUDE_DIR=' + PFFT_INCLUDE_DIR)
+endif
 
-# We need to do something with these values
-# see https://stackoverflow.com/a/61072768/1779806
+dirs = []
+if PFFT_LIB_DIR != ''
+   dirs += [PFFT_LIB_DIR]
+endif
 
-# set LIBRARY_PATH to include the directory where is libpfft.so
-pfft_dep = compiler.find_library('pfft', required: true)
-link_args = ['-lpfft', '-lfftw3_mpi']
+include_directories = []
+if PFFT_INCLUDE_DIR != ''
+  include_directories = include_directories(PFFT_INCLUDE_DIR)
+endif
+
+pfft_dep = compiler.find_library(
+  'pfft',
+  dirs: dirs,
+  has_headers: ['pfft.h'],
+  header_include_directories: include_directories,
+  required: true,
+)
+link_args = ['-lfftw3_mpi']
 
 dependencies = [fftw_dep, mpi_dep, np_dep, fftwmpi_dep, pfft_dep]
 
 include_path_fluidfft_builder = run_command(
   'fluidfft-builder-print-include-dir', check: true
 ).stdout().strip()
 
+include_directories = [include_directories, include_path_fluidfft_builder]
+
 include_path_cy = run_command(
   'fluidfft-builder-print-include-dir-cython', check: true
 ).stdout().strip()

plugins/fluidfft-pfft/src/fluidfft_pfft/meson.build

@@ -25,7 +25,7 @@ py.extension_module(
   include_path_fluidfft_builder / 'base_fft3dmpi.cpp',
   dependencies: dependencies,
   override_options : ['cython_language=cpp'],
-  include_directories: include_path_fluidfft_builder,
+  include_directories: include_directories,
   link_args: link_args,
   install: true,
   subdir: 'fluidfft_pfft',

pyproject.toml

@@ -55,12 +55,6 @@ fluidfft-bench-analysis = "fluidfft.bench_analysis:run"
 "fft3d.with_pyfftw" = "fluidfft.fft3d.with_pyfftw"
 "fft2d.with_dask" = "fluidfft.fft2d.with_dask"
 
-# should be in fluidfft-p3dfft
-# "fft3d.mpi_with_p3dfft" = "fluidfft.fft3d.mpi_with_p3dfft"
-
-# should be in fluidfft-pfft
-# "fft3d.mpi_with_pfft" = "fluidfft.fft3d.mpi_with_pfft"
-
 [tool.pdm]
 distribution = true
 package-dir = "src"

tests/test_plugins.py

@@ -48,8 +48,22 @@
     methodss[3, False].add("fft3d.mpi_with_fftwmpi3d")
     del fluidfft_fftwmpi
 
-# "fft3d.mpi_with_p3dfft",
-# "fft3d.mpi_with_pfft",
+
+try:
+    import fluidfft_pfft
+except ImportError:
+    pass
+else:
+    methodss[3, False].add("fft3d.mpi_with_pfft")
+    del fluidfft_pfft
+
+try:
+    import fluidfft_p3dfft
+except ImportError:
+    pass
+else:
+    methodss[3, False].add("fft3d.mpi_with_p3dfft")
+    del fluidfft_p3dfft
 
 
 def test_plugins():