Merge branch 'master' into rootcling

.github/workflows/key4hep.yml

@@ -30,7 +30,8 @@ jobs:
             -DCMAKE_INSTALL_PREFIX=../install \
             -DCMAKE_CXX_STANDARD=17 \
             -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \
-            -DUSE_EXTERNAL_CATCH2=AUTO \
+            -DUSE_EXTERNAL_CATCH2=ON \
+            -DENABLE_RNTUPLE=ON \
             -G Ninja ..
           echo "::endgroup::"
           echo "::group::Build"

.github/workflows/pre-commit.yml

@@ -15,19 +15,24 @@ jobs:
     - uses: cvmfs-contrib/github-action-cvmfs@v3
     - uses: aidasoft/run-lcg-view@v4
       with:
-        release-platform: LCG_102/x86_64-centos7-clang12-opt
+        container: centos7
+        view-path: /cvmfs/sw-nightlies.hsf.org/key4hep
         run: |
           echo "::group::Setup pre-commit"
-          export PYTHONPATH=$(python -m site --user-site):$PYTHONPATH
           export PATH=/root/.local/bin:$PATH
+          # Newer versions of git are more cautious around the github runner
+          # environment and without this git rev-parse --show-cdup in pre-commit
+          # fails
+          git config --global --add safe.directory $(pwd)
           pip install pre-commit
-          # Use virtualenv from the LCG release
-          pip uninstall --yes virtualenv
+          pip install pylint==2.12.2
+          pip install flake8
           echo "::endgroup::"
           echo "::group::Run CMake"
           mkdir build
           cd build
           cmake .. -DENABLE_SIO=ON \
+            -DENABLE_RNTUPLE=ON \
             -DCMAKE_CXX_STANDARD=17 \
             -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror "\
             -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \

.github/workflows/test.yml

@@ -13,12 +13,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        sio: [ON]
-        LCG: ["LCG_102/x86_64-centos7-clang12-opt",
-              "LCG_102/x86_64-centos8-gcc11-opt",
-              "dev3/x86_64-centos7-clang12-opt",
+        RNTUPLE: [ON]
+        LCG: ["dev3/x86_64-centos7-clang12-opt",
               "dev4/x86_64-centos7-gcc11-opt",
               "dev4/x86_64-centos7-clang12-opt"]
+        include:
+          - LCG: "LCG_102/x86_64-centos7-clang12-opt"
+            RNTUPLE: OFF
+          - LCG: "LCG_102/x86_64-centos8-gcc11-opt"
+            RNTUPLE: OFF
     steps:
     - uses: actions/checkout@v3
     - uses: cvmfs-contrib/github-action-cvmfs@v3
@@ -29,7 +32,8 @@ jobs:
           echo "::group::Run CMake"
           mkdir build install
           cd build
-          cmake -DENABLE_SIO=${{ matrix.sio }} \
+          cmake -DENABLE_SIO=ON \
+            -DENABLE_RNTUPLE=${{ matrix.RNTUPLE }} \
             -DCMAKE_INSTALL_PREFIX=../install \
             -DCMAKE_CXX_STANDARD=17 \
             -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \

.github/workflows/ubuntu.yml

@@ -30,9 +30,10 @@ jobs:
             -DCMAKE_INSTALL_PREFIX=../install \
             -DCMAKE_CXX_STANDARD=17 \
             -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \
-            -DUSE_EXTERNAL_CATCH2=OFF \
-            -DPODIO_SET_RPATH=ON \
-            -G Ninja ..
+                 -DUSE_EXTERNAL_CATCH2=OFF \
+                 -DPODIO_SET_RPATH=ON \
+                 -DENABLE_RNTUPLE=ON \
+                 -G Ninja ..
           echo "::endgroup::"
           echo "::group::Build"
           ninja -k0

CMakeLists.txt

@@ -68,12 +68,20 @@ ADD_CLANG_TIDY()
 option(CREATE_DOC        "Whether or not to create doxygen doc target." OFF)
 option(ENABLE_SIO        "Build SIO I/O support" OFF)
 option(PODIO_RELAX_PYVER "Do not require exact python version match with ROOT" OFF)
+option(ENABLE_RNTUPLE    "Build with support for the new ROOT NTtuple format" OFF)
 
 
 #--- Declare ROOT dependency ---------------------------------------------------
 list(APPEND CMAKE_PREFIX_PATH $ENV{ROOTSYS})
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
-find_package(ROOT REQUIRED COMPONENTS RIO Tree)
+if(NOT ENABLE_RNTUPLE)
+  find_package(ROOT REQUIRED COMPONENTS RIO Tree)
+else()
+  find_package(ROOT REQUIRED COMPONENTS RIO Tree ROOTNTuple)
+  if(${ROOT_VERSION} VERSION_LESS 6.28.02)
+    message(FATAL_ERROR "You are trying to build podio with support for the new ROOT NTuple format, but your ROOT version is too old. Please update ROOT to at least version 6.28.02")
+  endif()
+endif()
 
 # Check that root is compiled with a modern enough c++ standard
 get_target_property(ROOT_COMPILE_FEATURES ROOT::Core INTERFACE_COMPILE_FEATURES)

cmake/podioCPack.cmake

@@ -15,9 +15,9 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "[email protected]")
 set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${CPACK_PACKAGE_HOMEPAGE_URL})
 set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "hep-root, python-yaml")
 
-set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_SOURCE_DIR}/README.md")
-set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE")
-set(CPACK_RESOURCE_FILE_README "${CMAKE_SOURCE_DIR}/README.md")
+set(CPACK_PACKAGE_DESCRIPTION_FILE "${PROJECT_SOURCE_DIR}/README.md")
+set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE")
+set(CPACK_RESOURCE_FILE_README "${PROJECT_SOURCE_DIR}/README.md")
 
 #--- source package settings ---------------------------------------------------
 set(CPACK_SOURCE_IGNORE_FILES

cmake/podioCreateConfig.cmake

@@ -19,7 +19,7 @@ configure_package_config_file(${PROJECT_SOURCE_DIR}/cmake/podioConfig.cmake.in
 
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/podioConfig.cmake
               ${CMAKE_CURRENT_BINARY_DIR}/podioConfigVersion.cmake
-              ${CMAKE_SOURCE_DIR}/cmake/podioMacros.cmake
+              ${PROJECT_SOURCE_DIR}/cmake/podioMacros.cmake
               DESTINATION ${CMAKE_INSTALL_CMAKEDIR}/${PROJECT_NAME} )
 install(EXPORT podioTargets
   DESTINATION ${CMAKE_INSTALL_CMAKEDIR}/${PROJECT_NAME}

cmake/podioDoxygen.cmake

@@ -8,7 +8,7 @@ if(DOXYGEN_FOUND)
   configure_file(${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile.in
                  ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
   configure_file(${CMAKE_CURRENT_SOURCE_DIR}/doc/doxy-boot.js.in
-                 ${CMAKE_BINARY_DIR}/doxygen/html/doxy-boot.js)
+                 ${PROJECT_BINARY_DIR}/doxygen/html/doxy-boot.js)
   add_custom_target(doc
                     ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
                     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}

cmake/podioMacros.cmake

@@ -356,7 +356,7 @@ function(PODIO_CHECK_CPP_FS FS_LIBS)
   # After that it should be built-in
   FOREACH(FS_LIB_NAME "" stdc++fs c++fs)
     # MESSAGE(STATUS "Linking against ${FS_LIB_NAME}")
-    try_compile(have_filesystem ${CMAKE_BINARY_DIR}/try ${PROJECT_SOURCE_DIR}/cmake/try_filesystem.cpp
+    try_compile(have_filesystem ${PROJECT_BINARY_DIR}/try ${PROJECT_SOURCE_DIR}/cmake/try_filesystem.cpp
       CXX_STANDARD ${CMAKE_CXX_STANDARD}
       CXX_EXTENSIONS False
       OUTPUT_VARIABLE HAVE_FS_OUTPUT

cmake/podioTest.cmake

@@ -5,14 +5,14 @@ function(PODIO_SET_TEST_ENV test)
   list(JOIN PODIO_IO_HANDLERS " " IO_HANDLERS)
   set_property(TEST ${test}
     PROPERTY ENVIRONMENT
-      LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/tests:${CMAKE_BINARY_DIR}/src:$<TARGET_FILE_DIR:ROOT::Tree>:$<$<TARGET_EXISTS:SIO::sio>:$<TARGET_FILE_DIR:SIO::sio>>:$ENV{LD_LIBRARY_PATH}
-      PYTHONPATH=${CMAKE_SOURCE_DIR}/python:$ENV{PYTHONPATH}
-      PODIO_SIOBLOCK_PATH=${CMAKE_BINARY_DIR}/tests
-      ROOT_INCLUDE_PATH=${CMAKE_BINARY_DIR}/tests/datamodel:${CMAKE_SOURCE_DIR}/include
+      LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/tests:${PROJECT_BINARY_DIR}/src:$<TARGET_FILE_DIR:ROOT::Tree>:$<$<TARGET_EXISTS:SIO::sio>:$<TARGET_FILE_DIR:SIO::sio>>:$ENV{LD_LIBRARY_PATH}
+      PYTHONPATH=${PROJECT_SOURCE_DIR}/python:$ENV{PYTHONPATH}
+      PODIO_SIOBLOCK_PATH=${PROJECT_BINARY_DIR}/tests
+      ROOT_INCLUDE_PATH=${PROJECT_BINARY_DIR}/tests/datamodel:${PROJECT_SOURCE_DIR}/include
       SKIP_SIO_TESTS=$<NOT:$<BOOL:${ENABLE_SIO}>>
       IO_HANDLERS=${IO_HANDLERS}
       PODIO_USE_CLANG_FORMAT=${PODIO_USE_CLANG_FORMAT}
-      PODIO_BASE=${CMAKE_SOURCE_DIR}
+      PODIO_BASE=${PROJECT_SOURCE_DIR}
       ENABLE_SIO=${ENABLE_SIO}
   )
 endfunction()
@@ -35,7 +35,7 @@ macro(PODIO_DOWNLOAD_LEGACY_INPUTS legacy_versions)
   if (NOT DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR} OR NOT EXISTS ${PODIO_TEST_INPUT_DATA_DIR}/v00-16-05/example_frame.root)
     message(STATUS "Getting test input files")
     execute_process(
-      COMMAND bash ${CMAKE_SOURCE_DIR}/tests/scripts/get_test_inputs.sh ${legacy_versions}
+      COMMAND bash ${PROJECT_SOURCE_DIR}/tests/scripts/get_test_inputs.sh ${legacy_versions}
       OUTPUT_VARIABLE podio_test_input_data_dir
       RESULT_VARIABLE test_inputs_available
     )

doc/Doxyfile.in

@@ -58,7 +58,7 @@ PROJECT_LOGO           =
 # entered, it will be relative to the location where doxygen was started. If
 # left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       = @CMAKE_BINARY_DIR@/doxygen
+OUTPUT_DIRECTORY       = @PROJECT_BINARY_DIR@/doxygen
 
 # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub-
 # directories (in 2 levels) under the output directory of each output format and
@@ -144,7 +144,7 @@ FULL_PATH_NAMES        = YES
 # will be relative from the directory where doxygen is started.
 # This tag requires that the tag FULL_PATH_NAMES is set to YES.
 
-STRIP_FROM_PATH        = @CMAKE_SOURCE_DIR@ @CMAKE_BINARY_DIR@
+STRIP_FROM_PATH        = @PROJECT_SOURCE_DIR@ @PROJECT_BINARY_DIR@
 
 # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
 # path mentioned in the documentation of a class, which tells the reader which
@@ -153,7 +153,7 @@ STRIP_FROM_PATH        = @CMAKE_SOURCE_DIR@ @CMAKE_BINARY_DIR@
 # specify the list of include paths that are normally passed to the compiler
 # using the -I flag.
 
-STRIP_FROM_INC_PATH    = @DOXYGEN_INCLUDE_DIRS@ @CMAKE_BINARY_DIR@/include
+STRIP_FROM_INC_PATH    = @DOXYGEN_INCLUDE_DIRS@ @PROJECT_BINARY_DIR@/include
 
 # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
 # less readable) file names. This can be useful is your file systems doesn't
@@ -731,7 +731,7 @@ WARN_FORMAT            = "$file:$line: $text"
 # messages should be written. If left blank the output is written to standard
 # error (stderr).
 
-WARN_LOGFILE           = @CMAKE_BINARY_DIR@/doxygen-warnings.log
+WARN_LOGFILE           = @PROJECT_BINARY_DIR@/doxygen-warnings.log
 
 #---------------------------------------------------------------------------
 # Configuration options related to the input files
@@ -743,8 +743,8 @@ WARN_LOGFILE           = @CMAKE_BINARY_DIR@/doxygen-warnings.log
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = @CMAKE_SOURCE_DIR@
-INPUT                 += @CMAKE_BINARY_DIR@/include
+INPUT                  = @PROJECT_SOURCE_DIR@
+INPUT                 += @PROJECT_BINARY_DIR@/include
 INPUT                 += @CMAKE_CURRENT_BINARY_DIR@
 
 # This tag can be used to specify the character encoding of the source files
@@ -801,7 +801,7 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       =  */tests/* */dict/* */cmake/* @CMAKE_BINARY_DIR@
+EXCLUDE_PATTERNS       =  */tests/* */dict/* */cmake/* @PROJECT_BINARY_DIR@
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the
@@ -818,7 +818,7 @@ EXCLUDE_SYMBOLS        =
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           = @CMAKE_SOURCE_DIR@
+EXAMPLE_PATH           = @PROJECT_SOURCE_DIR@
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and

include/podio/CollectionBuffers.h

@@ -27,6 +27,7 @@ using VectorMembersInfo = std::vector<std::pair<std::string, void*>>;
  */
 struct CollectionWriteBuffers {
   void* data{nullptr};
+  void* vecPtr{nullptr};
   CollRefCollection* references{nullptr};
   VectorMembersInfo* vectorMembers{nullptr};
 

include/podio/Frame.h

@@ -4,6 +4,7 @@
 #include "podio/CollectionBase.h"
 #include "podio/CollectionBufferFactory.h"
 #include "podio/CollectionIDTable.h"
+#include "podio/FrameCategories.h" // mainly for convenience
 #include "podio/GenericParameters.h"
 #include "podio/ICollectionProvider.h"
 #include "podio/SchemaEvolution.h"

include/podio/FrameCategories.h

@@ -0,0 +1,43 @@
+#ifndef PODIO_FRAMECATEGORIES_H
+#define PODIO_FRAMECATEGORIES_H
+
+#include <string>
+
+namespace podio {
+
+/**
+ * Create a parameterName that encodes the collection name and the parameter
+ * Name into one string.
+ *
+ * This codifies a convention that was decided on to store collection level
+ * parameters. These are parameters / metadata that are valid for all
+ * collections of a given name in a file, e.g. CellID encoding strings. These
+ * parameters are usually stored in a dedicated metadata Frame inside a file,
+ * see the predefined category names in the Cateogry namespace.
+ *
+ * @param collName the name of the collection
+ * @param paramName the name of the parameter
+ *
+ * @returns A single key string that combines the collection and parameter name
+ */
+inline std::string collMetadataParamName(const std::string& collName, const std::string& paramName) {
+  return collName + "__" + paramName;
+}
+
+/**
+ * This namespace mimics an enum (at least in its usage) and simply defines
+ * either commonly used category names, or category names that form a
+ * convention.
+ */
+namespace Category {
+  /// The event category
+  constexpr const auto Event = "events";
+  /// The run category
+  constexpr const auto Run = "runs";
+  /// The metadata cateogry that is used to store a single Frame that holds data
+  /// that is valid for a whole file, e.g. collection level parameters
+  constexpr const auto Metadata = "metadata";
+} // namespace Category
+} // namespace podio
+
+#endif

include/podio/GenericParameters.h

@@ -18,6 +18,11 @@ class write_device;
 using version_type = uint32_t; // from sio/definitions
 } // namespace sio
 
+namespace podio {
+class ROOTNTupleReader;
+class ROOTNTupleWriter;
+} // namespace podio
+
 #define DEPR_NON_TEMPLATE                                                                                              \
   [[deprecated("Non-templated access will be removed. Switch to templated access functionality")]]
 
@@ -145,6 +150,8 @@ class GenericParameters {
 
   friend void writeGenericParameters(sio::write_device& device, const GenericParameters& parameters);
   friend void readGenericParameters(sio::read_device& device, GenericParameters& parameters, sio::version_type version);
+  friend ROOTNTupleReader;
+  friend ROOTNTupleWriter;
 
   /// Get a reference to the internal map for a given type
   template <typename T>
@@ -187,6 +194,7 @@ class GenericParameters {
     }
   }
 
+private:
   /// Get the mutex that guards the map for the given type
   template <typename T>
   std::mutex& getMutex() const {

include/podio/ROOTFrameWriter.h

@@ -24,7 +24,7 @@ class GenericParameters;
 class ROOTFrameWriter {
 public:
   ROOTFrameWriter(const std::string& filename);
-  ~ROOTFrameWriter() = default;
+  ~ROOTFrameWriter();
 
   ROOTFrameWriter(const ROOTFrameWriter&) = delete;
   ROOTFrameWriter& operator=(const ROOTFrameWriter&) = delete;
@@ -83,6 +83,8 @@ class ROOTFrameWriter {
   std::unordered_map<std::string, CategoryInfo> m_categories{}; ///< All categories
 
   DatamodelDefinitionCollector m_datamodelCollector{};
+
+  bool m_finished{false}; ///< Whether writing has been actually done
 };
 
 } // namespace podio