Restructure python parts, splitting code generation and python bindin…

…gs into separate modules (AIDASoft#511)

* Decouple generation tools and files from the rest of podio python files

* Move some tests to podio_gen

* Simplify __init__.py

* Make sure to always have the Frame python wrapper when imported

---------

Co-authored-by: jmcarcell <[email protected]>
Co-authored-by: tmadlener <[email protected]>

python/CMakeLists.txt

@@ -23,17 +23,25 @@ else()
     )
 endif()
 
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/podio_gen
+  DESTINATION ${podio_PYTHON_INSTALLDIR}
+  REGEX test_.*\\.py$ EXCLUDE  # Do not install test files
+  PATTERN __pycache__ EXCLUDE  # Or pythons caches
+  )
+
 #--- install templates ---------------------------------------------------------
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/templates
   DESTINATION ${podio_PYTHON_INSTALLDIR})
 
 IF (BUILD_TESTING)
   add_test( NAME pyunittest COMMAND python3 -m unittest discover -s ${PROJECT_SOURCE_DIR}/python/podio)
   PODIO_SET_TEST_ENV(pyunittest)
+  add_test( NAME pyunittest_gen COMMAND python3 -m unittest discover -s ${PROJECT_SOURCE_DIR}/python/podio_gen)
+  PODIO_SET_TEST_ENV(pyunittest_gen)
 
   set_property(TEST pyunittest PROPERTY DEPENDS write write_frame_root)
   if (TARGET write_sio)
     set_property(TEST pyunittest PROPERTY DEPENDS write_sio write_frame_sio)
   endif()
-  set_property(TEST pyunittest PROPERTY WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/tests)
+  set_tests_properties(pyunittest pyunittest_gen PROPERTIES WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/tests)
 ENDIF()

python/podio/__init__.py

@@ -1,56 +1,33 @@
 """Python module for the podio EDM toolkit and its utilities"""
-import sys
-
 from .__version__ import __version__
 
-from .podio_config_reader import *  # noqa: F403, F401
-
-import ROOT  # pylint: disable=wrong-import-order
-
-# Track whether we were able to dynamially load the library that is built by
-# podio and enable certain features of the bindings only if they are actually
-# available.
-_DYNAMIC_LIBS_LOADED = False
-
-# Check if we can locate the dictionary wthout loading it as this allows us to
-# silence any ouptput. If we can find it, we can also safely load it
-if ROOT.gSystem.DynamicPathName("libpodioDict.so", True):
-  ROOT.gSystem.Load("libpodioDict.so")
-  from ROOT import podio
-
-  _DYNAMIC_LIBS_LOADED = True
-
-if _DYNAMIC_LIBS_LOADED:
-  from .frame import Frame
-  from . import root_io, reading
-
-  try:
-    # We try to import the sio bindings which may fail if ROOT is not able to
-    # load the dictionary in this case they have most likely not been built and
-    # we just move on
-    from . import sio_io
-  except ImportError:
-    pass
-
-  from . import EventStore
-
-  try:
-    # For some reason the test_utils only work at (test) runtime if they are
-    # imported with the rest of podio. Otherwise they produce a weird c++ error.
-    # This happens even if we import the *exact* same file.
-    from . import test_utils  # noqa: F401
-  except ImportError:
-    pass
-
-  # Make sure that this module is actually usable as podio even though most of
-  # it is dynamically populated by cppyy
-  sys.modules["podio"] = podio
-
-  __all__ = [
-      "__version__",
-      "Frame",
-      "root_io",
-      "sio_io",
-      "reading",
-      "EventStore"
-      ]
+# Try to load podio, this is equivalent to trying to load libpodio.so and will
+# error if libpodio.so is not found but work if it's found
+try:
+  from ROOT import podio  # noqa: F401
+except ImportError:
+  print('Unable to load podio, make sure that libpodio.so is in LD_LIBRARY_PATH')
+  raise
+
+from .frame import Frame
+from . import root_io, reading
+
+try:
+  # We try to import the sio bindings which may fail if ROOT is not able to
+  # load the dictionary. In this case they have most likely not been built and
+  # we just move on
+  from . import sio_io
+except ImportError:
+  pass
+
+from . import EventStore
+
+
+__all__ = [
+    "__version__",
+    "Frame",
+    "root_io",
+    "sio_io",
+    "reading",
+    "EventStore"
+    ]

python/podio/sio_io.py

@@ -2,9 +2,9 @@
 """Python module for reading sio files containing podio Frames"""
 
 from ROOT import gSystem
-ret = gSystem.Load('libpodioSioIO')  # noqa: 402
-# Return values: -1 when it doesn't exist and -2 when there is a version mismatch
-if ret < 0:
+if gSystem.DynamicPathName("libpodioSioIO.so", True):
+  gSystem.Load('libpodioSioIO')  # noqa: 402
+else:
   raise ImportError('Error when importing libpodioSioIO')
 from ROOT import podio  # noqa: 402 # pylint: disable=wrong-import-position
 

python/podio/test_Frame.py

@@ -4,7 +4,7 @@
 import unittest
 
 # pylint: disable=import-error
-from test_utils import ExampleHitCollection  # noqa: E402
+from ROOT import ExampleHitCollection
 
 from podio.frame import Frame
 # using root_io as that should always be present regardless of which backends are built

python/podio/test_utils.py

@@ -2,59 +2,5 @@
 """Utilities for python unittests"""
 
 import os
-import ROOT
-
-if ROOT.gSystem.DynamicPathName("libTestDataModelDict.so", True):  # noqa: E402
-  ROOT.gSystem.Load("libTestDataModelDict.so")  # noqa: E402
-from ROOT import ExampleHitCollection, ExampleClusterCollection  # noqa: E402 # pylint: disable=wrong-import-position
-
-from podio import Frame  # pylint: disable=wrong-import-position
-
 
 SKIP_SIO_TESTS = os.environ.get("SKIP_SIO_TESTS", "1") == "1"
-
-
-def create_hit_collection():
-  """Create a simple hit collection with two hits for testing"""
-  hits = ExampleHitCollection()
-  hits.create(0xBAD, 0.0, 0.0, 0.0, 23.0)
-  hits.create(0xCAFFEE, 1.0, 0.0, 0.0, 12.0)
-
-  return hits
-
-
-def create_cluster_collection():
-  """Create a simple cluster collection with two clusters"""
-  clusters = ExampleClusterCollection()
-  clu0 = clusters.create()
-  clu0.energy(3.14)
-  clu1 = clusters.create()
-  clu1.energy(1.23)
-
-  return clusters
-
-
-def create_frame():
-  """Create a frame with an ExampleHit and an ExampleCluster collection"""
-  frame = Frame()
-  hits = create_hit_collection()
-  frame.put(hits, "hits_from_python")
-  clusters = create_cluster_collection()
-  frame.put(clusters, "clusters_from_python")
-
-  frame.put_parameter("an_int", 42)
-  frame.put_parameter("some_floats", [1.23, 7.89, 3.14])
-  frame.put_parameter("greetings", ["from", "python"])
-  frame.put_parameter("real_float", 3.14, as_type="float")
-  frame.put_parameter("more_real_floats", [1.23, 4.56, 7.89], as_type="float")
-
-  return frame
-
-
-def write_file(writer_type, filename):
-  """Write a file using the given Writer type and put one Frame into it under
-  the events category
-  """
-  writer = writer_type(filename)
-  event = create_frame()
-  writer.write_frame(event, "events")

python/podio_class_generator.py

@@ -16,8 +16,8 @@
 
 from podio_schema_evolution import DataModelComparator  # dealing with cyclic imports
 from podio_schema_evolution import RenamedMember, root_filter, RootIoRule
-from podio.podio_config_reader import PodioConfigReader
-from podio.generator_utils import DataType, DefinitionError, DataModelJSONEncoder
+from podio_gen.podio_config_reader import PodioConfigReader
+from podio_gen.generator_utils import DataType, DefinitionError, DataModelJSONEncoder
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 TEMPLATE_DIR = os.path.join(THIS_DIR, 'templates')

python/podio/podio_config_reader.py → python/podio_gen/podio_config_reader.py

@@ -6,7 +6,7 @@
 import warnings
 import yaml
 
-from podio.generator_utils import MemberVariable, DefinitionError, BUILTIN_TYPES, DataModel
+from podio_gen.generator_utils import MemberVariable, DefinitionError, BUILTIN_TYPES, DataModel
 
 
 class MemberParser:

python/podio/test_ClassDefinitionValidator.py → python/podio_gen/test_ClassDefinitionValidator.py

@@ -8,8 +8,8 @@
 import unittest
 from copy import deepcopy
 
-from podio.podio_config_reader import ClassDefinitionValidator, MemberVariable, DefinitionError
-from podio.generator_utils import DataModel
+from podio_gen.podio_config_reader import ClassDefinitionValidator, MemberVariable, DefinitionError
+from podio_gen.generator_utils import DataModel
 
 
 def make_dm(components, datatypes, options=None):

python/podio/test_DataModelJSONEncoder.py → python/podio_gen/test_DataModelJSONEncoder.py

@@ -3,8 +3,8 @@
 
 import unittest
 
-from podio.generator_utils import DataModelJSONEncoder
-from podio.podio_config_reader import MemberParser
+from podio_gen.generator_utils import DataModelJSONEncoder
+from podio_gen.podio_config_reader import MemberParser
 
 
 def get_member_var_json(string):

python/podio/test_MemberParser.py → python/podio_gen/test_MemberParser.py

@@ -6,7 +6,7 @@
 
 import unittest
 
-from podio.podio_config_reader import MemberParser, DefinitionError
+from podio_gen.podio_config_reader import MemberParser, DefinitionError
 
 
 class MemberParserTest(unittest.TestCase):

python/podio_schema_evolution.py

@@ -5,7 +5,7 @@
 
 import yaml
 
-from podio.podio_config_reader import PodioConfigReader
+from podio_gen.podio_config_reader import PodioConfigReader
 
 
 # @TODO: not really a good class model here

tests/root_io/CMakeLists.txt

@@ -80,6 +80,6 @@ endforeach()
 
 #--- Write via python and the ROOT backend and see if we can read it back in in
 #--- c++
-add_test(NAME write_python_frame_root COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/write_frame_root.py)
+add_test(NAME write_python_frame_root COMMAND python3 ${PROJECT_SOURCE_DIR}/tests/write_frame.py example_frame_with_py.root)
 PODIO_SET_TEST_ENV(write_python_frame_root)
 set_property(TEST read_python_frame_root PROPERTY DEPENDS write_python_frame_root)

tests/sio_io/CMakeLists.txt

@@ -8,6 +8,7 @@ set(sio_dependent_tests
   write_frame_sio.cpp
   read_frame_legacy_sio.cpp
   read_and_write_frame_sio.cpp
+  read_python_frame_sio.cpp
 )
 set(sio_libs podio::podioSioIO)
 foreach( sourcefile ${sio_dependent_tests} )
@@ -36,3 +37,9 @@ set_tests_properties(
 
 add_test(NAME check_benchmark_outputs_sio COMMAND check_benchmark_outputs write_benchmark_sio.root read_benchmark_sio.root)
 set_property(TEST check_benchmark_outputs_sio PROPERTY DEPENDS read_timed_sio write_timed_sio)
+
+#--- Write via python and the SIO backend and see if we can read it back in in
+#--- c++
+add_test(NAME write_python_frame_sio COMMAND python3 ${PROJECT_SOURCE_DIR}/tests/write_frame.py example_frame_with_py.sio)
+PODIO_SET_TEST_ENV(write_python_frame_sio)
+set_property(TEST read_python_frame_sio PROPERTY DEPENDS write_python_frame_sio)

tests/write_frame.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Utilities for python unittests"""
+
+import importlib
+
+
+import ROOT
+
+if ROOT.gSystem.Load("libTestDataModelDict.so") < 0:  # noqa: E402
+  raise RuntimeError("Could not load TestDataModel dictionary")
+
+from ROOT import (  # pylint: disable=wrong-import-position
+    ExampleHitCollection,
+    ExampleClusterCollection,
+    )  # noqa: E402
+
+from podio import Frame  # pylint: disable=wrong-import-position
+
+
+def create_hit_collection():
+  """Create a simple hit collection with two hits for testing"""
+  hits = ExampleHitCollection()
+  hits.create(0xBAD, 0.0, 0.0, 0.0, 23.0)
+  hits.create(0xCAFFEE, 1.0, 0.0, 0.0, 12.0)
+
+  return hits
+
+
+def create_cluster_collection():
+  """Create a simple cluster collection with two clusters"""
+  clusters = ExampleClusterCollection()
+  clu0 = clusters.create()
+  clu0.energy(3.14)
+  clu1 = clusters.create()
+  clu1.energy(1.23)
+
+  return clusters
+
+
+def create_frame():
+  """Create a frame with an ExampleHit and an ExampleCluster collection"""
+  frame = Frame()
+  hits = create_hit_collection()
+  frame.put(hits, "hits_from_python")
+  clusters = create_cluster_collection()
+  frame.put(clusters, "clusters_from_python")
+
+  frame.put_parameter("an_int", 42)
+  frame.put_parameter("some_floats", [1.23, 7.89, 3.14])
+  frame.put_parameter("greetings", ["from", "python"])
+  frame.put_parameter("real_float", 3.14, as_type="float")
+  frame.put_parameter("more_real_floats", [1.23, 4.56, 7.89], as_type="float")
+
+  return frame
+
+
+def write_file(io_backend, filename):
+  """Write a file using the given Writer type and put one Frame into it under
+  the events category
+  """
+  io_module = importlib.import_module(f"podio.{io_backend}")
+
+  writer = io_module.Writer(filename)
+  event = create_frame()
+  writer.write_frame(event, "events")
+
+
+if __name__ == "__main__":
+  import argparse
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument("outputfile", help="Output file name")
+
+  args = parser.parse_args()
+
+  io_format = args.outputfile.split(".")[-1]
+
+  write_file(f"{io_format}_io", args.outputfile)

tools/podio-vis

@@ -4,7 +4,7 @@
 import sys
 import argparse
 import yaml
-from podio.podio_config_reader import PodioConfigReader
+from podio_gen.podio_config_reader import PodioConfigReader
 try:
   from graphviz import Digraph
 except ImportError: