Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add HDF5_DISABLE_TESTS_REGEX option to skip tests #3859

Merged
merged 6 commits into from
Nov 22, 2023
Merged

Add HDF5_DISABLE_TESTS_REGEX option to skip tests #3859

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
edb2a51
Add HDF5_DISABLE_TESTS_REGEX option to skip tests
byrnHDF Nov 16, 2023
1d68e45
Set a default value for regex
byrnHDF Nov 16, 2023
f89ee1c
Add disabled regex to non-tools test folders
byrnHDF Nov 16, 2023
b9db801
Add tool tests to DISABLED check
byrnHDF Nov 17, 2023
537d73d
Add line for CMake
byrnHDF Nov 17, 2023
4ed2f9f
Rework option text
byrnHDF Nov 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
math (EXPR CTEST_LONG_TIMEOUT "${DART_TESTING_TIMEOUT} * 2")
math (EXPR CTEST_VERY_LONG_TIMEOUT "${DART_TESTING_TIMEOUT} * 3")

option (HDF5_DISABLE_TESTS_REGEX "Pattern to set tests to DISABLED" "")
Copy link
Contributor

@brtnfld brtnfld Nov 20, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe "exclude" would be more precise than "disable", "Pattern used to exclude specific tests"

Copy link
Contributor Author

@byrnHDF byrnHDF Nov 20, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not really, the tests are being skipped by disabling the execution of the tests. Besides the CMake property is DISABLED

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think making things clear for the user should be a priority. So far, we have used "ENABLE" and "DISABLE" as boolean values.

at least: "Pattern used to mark specific tests as DISABLED"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed option text

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@brtnfld, is this good now?

mark_as_advanced (HDF5_DISABLE_TESTS_REGEX)

option (HDF5_TEST_API "Execute HDF5 API tests" OFF)
mark_as_advanced (HDF5_TEST_API)
if (HDF5_TEST_API)
Expand Down
3 changes: 3 additions & 0 deletions c++/test/CMakeTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ endif ()
set_tests_properties (CPP_testhdf5 PROPERTIES
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
)
if ("CPP_testhdf5" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (CPP_testhdf5 PROPERTIES DISABLED true)
endif ()

##############################################################################
##############################################################################
Expand Down
3 changes: 3 additions & 0 deletions c++/test/CMakeVFDTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ macro (ADD_VFD_TEST vfdname resultcode)
-P "${HDF_RESOURCES_DIR}/vfdTest.cmake"
)
set_tests_properties (CPP_VFD-${vfdname}-cpp_testhdf5 PROPERTIES TIMEOUT ${CTEST_SHORT_TIMEOUT})
if ("CPP_VFD-${vfdname}-cpp_testhdf5" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (CPP_VFD-${vfdname}-cpp_testhdf5 PROPERTIES DISABLED true)
endif ()
endif ()
endmacro ()

Expand Down
5 changes: 5 additions & 0 deletions config/cmake/scripts/CTestScript.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,11 @@ list (APPEND CTEST_NOTES_FILES
"${CMAKE_CURRENT_LIST_FILE}"
"${CTEST_SOURCE_DIRECTORY}/config/cmake/cacheinit.cmake"
)
if (EXISTS "${CTEST_SCRIPT_DIRECTORY}/SkipTests.log")
list(APPEND CTEST_NOTES_FILES
"${CTEST_SCRIPT_DIRECTORY}/SkipTests.log"
)
endif ()

#-----------------------------------------------------------------------------
# Check for required variables.
Expand Down
18 changes: 18 additions & 0 deletions fortran/test/CMakeTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ endif ()
set_tests_properties (FORTRAN_testhdf5_fortran PROPERTIES
FIXTURES_REQUIRED clear_testhdf5_fortran
)
if ("FORTRAN_testhdf5_fortran" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (FORTRAN_testhdf5_fortran PROPERTIES DISABLED true)
endif ()

#-- Adding test for testhdf5_fortran_1_8
if (HDF5_ENABLE_USING_MEMCHECKER)
Expand All @@ -118,6 +121,9 @@ set_tests_properties (FORTRAN_testhdf5_fortran_1_8 PROPERTIES
DEPENDS FORTRAN_testhdf5_fortran
FIXTURES_REQUIRED clear_testhdf5_fortran
)
if ("FORTRAN_testhdf5_fortran_1_8" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (FORTRAN_testhdf5_fortran_1_8 PROPERTIES DISABLED true)
endif ()

#-- Adding test for fortranlib_test_F03
if (HDF5_ENABLE_USING_MEMCHECKER)
Expand All @@ -142,6 +148,9 @@ set_tests_properties (FORTRAN_fortranlib_test_F03 PROPERTIES
DEPENDS FORTRAN_testhdf5_fortran_1_8
FIXTURES_REQUIRED clear_testhdf5_fortran
)
if ("FORTRAN_fortranlib_test_F03" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (FORTRAN_fortranlib_test_F03 PROPERTIES DISABLED true)
endif ()

#-- Adding test for vol_connector
if (HDF5_ENABLE_USING_MEMCHECKER)
Expand All @@ -164,6 +173,9 @@ endif ()
set_tests_properties (FORTRAN_vol_connector PROPERTIES
FIXTURES_REQUIRED clear_testhdf5_fortran
)
if ("FORTRAN_vol_connector" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (FORTRAN_vol_connector PROPERTIES DISABLED true)
endif ()

#-- Adding test for fflush1
add_test (
Expand All @@ -177,6 +189,9 @@ add_test (
set_tests_properties (FORTRAN_fflush1 PROPERTIES
DEPENDS FORTRAN_flush1-clear-objects
)
if ("FORTRAN_fflush1" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (FORTRAN_fflush1 PROPERTIES DISABLED true)
endif ()

#-- Adding test for fflush2
add_test (
Expand All @@ -186,6 +201,9 @@ add_test (
set_tests_properties (FORTRAN_fflush2 PROPERTIES
DEPENDS FORTRAN_fflush1
)
if ("FORTRAN_fflush2" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (FORTRAN_fflush2 PROPERTIES DISABLED true)
endif ()
add_test (
NAME FORTRAN_flush1-clean-objects
COMMAND ${CMAKE_COMMAND} -E remove flush.h5
Expand Down
11 changes: 11 additions & 0 deletions fortran/testpar/CMakeTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,16 @@
##############################################################################
##############################################################################
add_test (NAME MPI_TEST_FORT_parallel_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $<TARGET_FILE:parallel_test> ${MPIEXEC_POSTFLAGS})
if ("MPI_TEST_FORT_parallel_test" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (MPI_TEST_FORT_parallel_test PROPERTIES DISABLED true)
endif ()

add_test (NAME MPI_TEST_FORT_subfiling_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $<TARGET_FILE:subfiling_test> ${MPIEXEC_POSTFLAGS})
if ("MPI_TEST_FORT_subfiling_test" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (MPI_TEST_FORT_subfiling_test PROPERTIES DISABLED true)
endif ()

add_test (NAME MPI_TEST_FORT_async_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $<TARGET_FILE:async_test> ${MPIEXEC_POSTFLAGS})
if ("MPI_TEST_FORT_async_test" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (MPI_TEST_FORT_async_test PROPERTIES DISABLED true)
endif ()
4 changes: 4 additions & 0 deletions hl/c++/test/CMakeTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ else ()
)
endif ()
set_tests_properties (HL_CPP_ptableTest PROPERTIES DEPENDS HL_CPP_ptableTest-clear-objects)
if ("HL_CPP_ptableTest" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (HL_CPP_ptableTest PROPERTIES DISABLED true)
endif ()
add_test (
NAME HL_CPP_ptableTest-clean-objects
COMMAND ${CMAKE_COMMAND}
Expand All @@ -53,3 +56,4 @@ set_tests_properties (HL_CPP_ptableTest-clean-objects PROPERTIES
DEPENDS HL_CPP_ptableTest
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
)

3 changes: 3 additions & 0 deletions hl/fortran/test/CMakeTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ macro (ADD_H5_FORTRAN_TEST file)
FIXTURES_REQUIRED clear_HL_FORTRAN_test
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
)
if ("HL_FORTRAN_f90_${file}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (HL_FORTRAN_f90_${file} PROPERTIES DISABLED true)
endif ()
endmacro ()

foreach (h5_test ${H5_TESTS})
Expand Down
3 changes: 3 additions & 0 deletions hl/test/CMakeTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ macro (HL_ADD_TEST hl_name)
ENVIRONMENT "srcdir=${HDF5_HL_TEST_BINARY_DIR}"
WORKING_DIRECTORY ${HDF5_HL_TEST_BINARY_DIR}
)
if ("HL_${hl_name}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (HL_${hl_name} PROPERTIES DISABLED true)
endif ()
endmacro ()

HL_add_test (test_lite )
Expand Down
9 changes: 9 additions & 0 deletions hl/tools/h5watch/CMakeTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ add_custom_target(H5WATCH_files ALL COMMENT "Copying files needed by H5WATCH tes
DEPENDS ${last_test}
FIXTURES_REQUIRED gen_test_watch
)
if ("H5WATCH_ARGS-h5watch-${resultfile}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (H5WATCH_ARGS-h5watch-${resultfile} PROPERTIES DISABLED true)
endif ()
set (last_test "H5WATCH_ARGS-h5watch-${resultfile}")
endif ()
endmacro ()
Expand All @@ -109,6 +112,9 @@ add_custom_target(H5WATCH_files ALL COMMENT "Copying files needed by H5WATCH tes
DEPENDS ${last_test}
FIXTURES_REQUIRED gen_test_watch
)
if ("H5WATCH_ARGS-h5watch-${resultfile}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (H5WATCH_ARGS-h5watch-${resultfile} PROPERTIES DISABLED true)
endif ()
set (last_test "H5WATCH_ARGS-h5watch-${resultfile}")
endif ()
endmacro ()
Expand All @@ -131,6 +137,9 @@ add_custom_target(H5WATCH_files ALL COMMENT "Copying files needed by H5WATCH tes
DEPENDS ${last_test}
FIXTURES_REQUIRED gen_test_watch
)
if ("H5WATCH-${resultfile}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (H5WATCH-${resultfile} PROPERTIES DISABLED true)
endif ()
set (last_test "H5WATCH-${resultfile}")
endif ()
endmacro ()
Expand Down
6 changes: 6 additions & 0 deletions java/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,9 @@ if (HDF5_TEST_JAVA AND HDF5_TEST_SERIAL)
FIXTURES_REQUIRED clear_JUnit
WORKING_DIRECTORY ${HDF5_BINARY_DIR}/java/test
)
if ("JUnit-${test_file}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (JUnit-${test_file} PROPERTIES DISABLED true)
endif ()
endforeach ()


Expand Down Expand Up @@ -208,6 +211,9 @@ if (HDF5_TEST_JAVA AND HDF5_TEST_SERIAL)
ENVIRONMENT "HDF5_PLUGIN_PATH=${CMAKE_BINARY_DIR}/testdir2"
WORKING_DIRECTORY ${HDF5_BINARY_DIR}/java/test/${volname}
)
if ("JUnit-VOL-${volname}-${voltest}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (JUnit-VOL-${volname}-${voltest} PROPERTIES DISABLED true)
endif ()
add_test (
NAME JUnit-VOL-${volname}-${voltest}-clean-objects
COMMAND ${CMAKE_COMMAND} -E remove
Expand Down
1 change: 1 addition & 0 deletions release_docs/INSTALL_CMake.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -879,6 +879,7 @@ if (BUILD_TESTING)
if (HDF5_TEST_VFD)
HDF5_TEST_FHEAP_VFD "Execute fheap test with different VFDs" ON
TEST_SHELL_SCRIPTS "Enable shell script tests" ON
HDF5_DISABLE_TESTS_REGEX "Pattern to set tests to DISABLED" ""

---------------- External Library Options ---------------------
HDF5_ALLOW_EXTERNAL_SUPPORT "Allow External Library Building (NO GIT TGZ)" "NO"
Expand Down
8 changes: 8 additions & 0 deletions release_docs/RELEASE.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ New Features

Configuration:
-------------
- Added new option for CMake to mark tests as SKIPPED.

HDF5_DISABLE_TESTS_REGEX is a REGEX string that will be checked with
test names and if there is a match then that test's property will be
set to DISABLED. HDF5_DISABLE_TESTS_REGEX can be initialized on the
command line: "-DHDF5_DISABLE_TESTS_REGEX:STRING=<regex>"
See CMake documentation for regex-specification.

- Added defaults to CMake for long double conversion checks

HDF5 performs a couple of checks at build time to see if long double
Expand Down
Loading