Incorporating Julia code generation in the existing Python interface (#…

…473) * Parsing Julia types from c++ types in yaml description (#310) * Julia preprocessing (#311) * adding pre-processing/"include" logic for julia * Add templates for Julia code * Add Julia unittests to test suite (#322) * grouping code into modules * add documentation for Julia code generation (#329) * Added abstract types in default parameters and '-l' language argument * Added --upstream-edm code generation support * Created _sort_components_and_datatypes function to perform topological sort * Added abstract support for builtin Vector Members * Added conditional check for StaticArrays * Suggested Changes * Added ENABLE_JULIA toggle --------- Co-authored-by: soumilbaldota <[email protected]> Co-authored-by: Thomas Madlener <[email protected]>
AIDASoft · Dec 4, 2023 · 360dad2 · 360dad2
1 parent e0bffc8
commit 360dad2
Show file tree

Hide file tree

Showing 21 changed files with 706 additions and 83 deletions.
diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
@@ -13,7 +13,7 @@ jobs:
     - uses: cvmfs-contrib/github-action-cvmfs@v3
     - uses: aidasoft/run-lcg-view@v4
       with:
-        coverity-cmake-command: 'cmake -DCMAKE_CXX_STANDARD=17  -DENABLE_SIO=ON -DUSE_EXTERNAL_CATCH2=OFF ..'
+        coverity-cmake-command: 'cmake -DCMAKE_CXX_STANDARD=17  -DENABLE_SIO=ON -DENABLE_JULIA=ON -DUSE_EXTERNAL_CATCH2=OFF ..'
         coverity-project: 'AIDASoft%2Fpodio'
         coverity-project-token: ${{ secrets.PODIO_COVERITY_TOKEN }}
         github-pat: ${{ secrets.READ_COVERITY_IMAGE }}

diff --git a/.github/workflows/edm4hep.yaml b/.github/workflows/edm4hep.yaml
@@ -44,6 +44,7 @@ jobs:
             cd $STARTDIR/podio
             mkdir build && cd build
             cmake -DENABLE_SIO=ON \
+              -DENABLE_JULIA=ON \
               -DCMAKE_INSTALL_PREFIX=../install \
               -DCMAKE_CXX_STANDARD=17 \
               -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \

diff --git a/.github/workflows/key4hep.yml b/.github/workflows/key4hep.yml
@@ -28,6 +28,7 @@ jobs:
           mkdir build install
           cd build
           cmake -DENABLE_SIO=ON \
+            -DENABLE_JULIA=OFF \
             -DCMAKE_INSTALL_PREFIX=../install \
             -DCMAKE_CXX_STANDARD=17 \
             -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \

diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -32,6 +32,7 @@ jobs:
           mkdir build
           cd build
           cmake .. -DENABLE_SIO=ON \
+            -DENABLE_JULIA=ON \
             -DENABLE_RNTUPLE=ON \
             -DCMAKE_CXX_STANDARD=17 \
             -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror "\

diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml
@@ -43,11 +43,15 @@ jobs:
               -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always " \
               -DUSE_EXTERNAL_CATCH2=OFF \
               -DENABLE_SIO=ON \
+              -DENABLE_JULIA=ON \
               -G Ninja ..
             echo "::endgroup::"
             echo "::group::Build"
             ninja -k0
             echo "::endgroup::"
+            echo "::group::Julia StaticArrays Package Install"
+            julia -e 'import Pkg; Pkg.add("StaticArrays")'
+            echo "::endgroup"
             echo "::group::Run tests"
             ctest --output-on-failure
-            echo "::endgroup::"
+            echo "::endgroup::"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -37,6 +37,7 @@ jobs:
           mkdir build install
           cd build
           cmake -DENABLE_SIO=ON \
+            -DENABLE_JULIA=ON \
             -DENABLE_RNTUPLE=${{ matrix.RNTUPLE }} \
             -DCMAKE_INSTALL_PREFIX=../install \
             -DCMAKE_CXX_STANDARD=${{ matrix.CXX_STANDARD }} \
@@ -48,9 +49,12 @@ jobs:
           echo "::group::Build"
           ninja -k0
           echo "::endgroup"
+          echo "::group::Julia StaticArrays Package Install"
+          julia -e 'import Pkg; Pkg.add("StaticArrays")'
+          echo "::endgroup"
           echo "::group::Run tests"
           ctest --output-on-failure
           echo "::endgroup::"
           echo "::group::Install"
           ninja install
-          echo "::endgroup::"
+          echo "::endgroup::"
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -27,6 +27,7 @@ jobs:
           mkdir build install
           cd build
           cmake -DENABLE_SIO=${{ matrix.sio }} \
+            -DENABLE_JULIA=ON \
             -DCMAKE_INSTALL_PREFIX=../install \
             -DCMAKE_CXX_STANDARD=17 \
             -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \
@@ -38,9 +39,12 @@ jobs:
           echo "::group::Build"
           ninja -k0
           echo "::endgroup"
+          echo "::group::Julia StaticArrays Package Install"
+          julia -e 'import Pkg; Pkg.add("StaticArrays")'
+          echo "::endgroup"
           echo "::group::Run tests"
           ctest --output-on-failure
           echo "::endgroup::"
           echo "::group::Install"
           ninja install
-          echo "::endgroup::"
+          echo "::endgroup::"
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,9 @@ install
 tests/src
 tests/datamodel
 tests/extension_model
+tests/datamodeljulia
+tests/unittests/Project.toml
+tests/unittests/Manifest.toml
 
 # Python
 *pyc

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -70,6 +70,7 @@ option(ENABLE_SIO        "Build SIO I/O support" OFF)
 option(PODIO_RELAX_PYVER "Do not require exact python version match with ROOT" OFF)
 option(ENABLE_RNTUPLE    "Build with support for the new ROOT NTtuple format" OFF)
 option(PODIO_USE_CLANG_FORMAT "Use clang-format to format the code" OFF)
+option(ENABLE_JULIA      "Enable Julia support. When enabled, Julia datamodels will be generated, and Julia tests will run." OFF)
 
 
 #--- Declare ROOT dependency ---------------------------------------------------

diff --git a/README.md b/README.md
@@ -162,6 +162,7 @@ The generation script has the following additional options:
 - `--clangformat` (`-c`): Apply clang-format after file creation (uses [option `-style=file`](https://clang.llvm.org/docs/ClangFormatStyleOptions.html) with llvm as backup style), needs clang-format in `$PATH`.
 - `--quiet` (`-q`): Suppress all print out to STDOUT
 - `--dryrun` (`-d`): Only run the generation logic and validate yaml, do not write files to disk
+- `--lang` (`-l`): Specify the programming language (default: cpp), choices: cpp, julia
 
 ## Running tests
 After compilation you can run rudimentary tests with

diff --git a/cmake/podioMacros.cmake b/cmake/podioMacros.cmake
@@ -128,13 +128,15 @@ set_property(CACHE PODIO_USE_CLANG_FORMAT PROPERTY STRINGS AUTO ON OFF)
 #                           passed directly to podio_class_generator.py and validated there
 #                           Default is ROOT
 #      SCHEMA_EVOLUTION     OPTIONAL: The path to the yaml file declaring the necessary schema evolution
-#  )
+#      LANG                 OPTIONAL: The programming language choice
+#                           Default is cpp
+# )
 #
 # Note that the create_${datamodel} target will always be called, but if the YAML_FILE has not changed
 # this is essentially a no-op, and should not cause re-compilation.
 #---------------------------------------------------------------------------------------------------
 function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOURCES)
-  CMAKE_PARSE_ARGUMENTS(ARG "" "OLD_DESCRIPTION;OUTPUT_FOLDER;UPSTREAM_EDM;SCHEMA_EVOLUTION" "IO_BACKEND_HANDLERS" ${ARGN})
+  CMAKE_PARSE_ARGUMENTS(ARG "" "OLD_DESCRIPTION;OUTPUT_FOLDER;UPSTREAM_EDM;SCHEMA_EVOLUTION" "IO_BACKEND_HANDLERS;LANG" ${ARGN})
   IF(NOT ARG_OUTPUT_FOLDER)
     SET(ARG_OUTPUT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR})
   ENDIF()
@@ -153,6 +155,13 @@ function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOUR
     SET(ARG_IO_BACKEND_HANDLERS "ROOT")
   ENDIF()
 
+  # Check if the LANG argument is specified and set the language accordingly.
+  IF(ARG_LANG)
+    SET(LANGUAGE_ARG "--lang=${ARG_LANG}")
+  ELSE()
+    SET(LANGUAGE_ARG "--lang=cpp")  # Default to C++
+  ENDIF()
+
   SET(SCHEMA_EVOLUTION_ARG "")
   IF (ARG_SCHEMA_EVOLUTION)
     SET(SCHEMA_EVOLUTION_ARG "--evolution_file=${ARG_SCHEMA_EVOLUTION}")
@@ -201,7 +210,7 @@ function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOUR
   message(STATUS "Creating '${datamodel}' datamodel")
   # we need to boostrap the data model, so this has to be executed in the cmake run
   execute_process(
-    COMMAND ${Python_EXECUTABLE} ${podio_PYTHON_DIR}/podio_class_generator.py ${CLANG_FORMAT_ARG} ${OLD_DESCRIPTION_ARG} ${SCHEMA_EVOLUTION_ARG} ${UPSTREAM_EDM_ARG} ${YAML_FILE} ${ARG_OUTPUT_FOLDER} ${datamodel} ${ARG_IO_BACKEND_HANDLERS}
+    COMMAND ${Python_EXECUTABLE} ${podio_PYTHON_DIR}/podio_class_generator.py ${CLANG_FORMAT_ARG} ${OLD_DESCRIPTION_ARG} ${SCHEMA_EVOLUTION_ARG} ${UPSTREAM_EDM_ARG} ${YAML_FILE} ${ARG_OUTPUT_FOLDER} ${datamodel} ${ARG_IO_BACKEND_HANDLERS} ${LANGUAGE_ARG}
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     RESULT_VARIABLE podio_generate_command_retval
     )

diff --git a/doc/templates.md b/doc/templates.md
@@ -32,6 +32,9 @@ They are broadly split along the classes that are generated for each datatype or
 | `CollectionData.{h,cc}.jinja2`  | The classes managing the collection storage (not user facing!)                                          | `[<package>/]<datatype-name>CollectionData.h`, `src/<datatype-name>CollectionData.cc` |
 | `selection.xml.jinja2`          | The `selection.xml` file that is necessary for generating a root dictionary for the generated datamodel | `src/selection.xml`                                                                   |
 | `SIOBlock.{h,cc}.jinja2`        | The SIO blocks that are necessary for the SIO backend                                                   | `[<package>/]<datatype-name>SIOBlock.h`, `src/<datatype-name>SIOBlock.cc`             |
+| `MutableStruct.jl.jinja2`		  | The mutable struct definitions of components and datatypes for julia 													|`[<package>/]<datatype-name>Struct.jl`, `[<package>/]<component-name>Struct.jl`	   |
+| `ParentModule.jl.jinja2`	      | The constructor and collection definitions of components and datatypes in the data model are contained within a single module named after the package-name										    				|`[<package>/]<package-name>.jl`											   |
+
 
 The presence of a `[<package>]` subdirectory for the header files is controlled by the `includeSubfolder` option in the yaml definition file.
 
@@ -55,7 +58,7 @@ The main entry point to the generation is the `process` method which essentially
 The following gives an overview of the information that is available from the dictionary that is passed to the templates from the `ClassGenerator`.
 Each (top level) key in this dict is directly available as a variable in the Jinja2 templates, e.g.
 ```python
-component['include'] = # list of includes
+component['includes'] = # list of includes
 ```
 will become available as
 ```jinja2
@@ -123,6 +126,8 @@ In principle all members are accessible in the templates, however, the most impo
 | `getter_name` | Method for generating the correct name for getter functions, depending on the `getSyntax` option in the yaml definition file.                                               |
 | `setter_name` | Method for generating the correct name for setter functions, depending on the `getSyntax` option in the yaml definition file and on whether the member is a relation or not |
 | `signature`   | The signature of a data member that can be used in function signatures, corresponds to `{{ full_type }} {{ name }}`                                                         |
+| `jl_imports`  | Import required for `StaticArrays: MVector`                                                                                                                                 |
+| `julia_type`  | Equivalent julia type for the c++ type                                                                                                                                      |
 
 ### `DataType`
 Defined in [`python/generator_utils.py`](/python/generator_utils.py).
@@ -134,3 +139,20 @@ The available fields are
 | `bare_type` | The type without the namespace                                                 |
 | `namespace` | The (potentially empty) namespace                                              |
 | `full_type` | The fully qualified type, corresponding to `{{ namespace }}::{{ bare_type }}`. |
+
+### Julia code generation
+It is an experimental feature.
+Builtin types mapping in Julia
+| cpp       | julia                                                                    |
+|-------------|--------------------------------------------------------------------------------|
+| `bool` | `Bool` |
+| `char` | `Char` |
+| `short` | `Int16` |
+| `int` | `Int32` |
+| `unsigned int` | `UInt32` |
+| `float` | `Float32` |
+| `double` | `Float64` |
+| `long` | `Int64` |
+| `unsigned long` | `UInt64` |
+| `long long` | `Int64` |
+| `unsigned long long` | `UInt64` |