Skip to content

Commit

Permalink
scripts: dts: extract pickled EDT generation
Browse files Browse the repository at this point in the history
Separate the pickled EDT generation from the C-Macro header
generation in gen_defines.py to have a more clear responsibility
of the scripts in the DTS parsing process.

Signed-off-by: Benedikt Schmidt <[email protected]>
  • Loading branch information
benediktibk committed Sep 10, 2024
1 parent a04f0b4 commit 20f0185
Show file tree
Hide file tree
Showing 6 changed files with 205 additions and 77 deletions.
57 changes: 49 additions & 8 deletions cmake/modules/dts.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ find_package(Dtc 1.4.6)
# The directory containing devicetree related scripts.
set(DT_SCRIPTS ${ZEPHYR_BASE}/scripts/dts)

# This parses and collects the DT information
set(GEN_EDT_SCRIPT ${DT_SCRIPTS}/gen_edt.py)
# This generates DT information needed by the C macro APIs,
# along with a few other things.
set(GEN_DEFINES_SCRIPT ${DT_SCRIPTS}/gen_defines.py)
Expand Down Expand Up @@ -216,7 +218,7 @@ foreach(dts_root ${DTS_ROOT})

set(vendor_prefixes ${dts_root}/${VENDOR_PREFIXES})
if(EXISTS ${vendor_prefixes})
list(APPEND EXTRA_GEN_DEFINES_ARGS --vendor-prefixes ${vendor_prefixes})
list(APPEND EXTRA_GEN_EDT_ARGS --vendor-prefixes ${vendor_prefixes})
endif()
endforeach()

Expand Down Expand Up @@ -266,23 +268,64 @@ toolchain_parse_make_rule(${DTS_DEPS}
set_property(DIRECTORY APPEND PROPERTY
CMAKE_CONFIGURE_DEPENDS
${DTS_INCLUDE_FILES}
${GEN_EDT_SCRIPT}
${GEN_DEFINES_SCRIPT}
${GEN_DRIVER_KCONFIG_SCRIPT}
${GEN_DTS_CMAKE_SCRIPT}
)

#
# Run GEN_DEFINES_SCRIPT.
# Run GEN_EDT_SCRIPT.
#

string(REPLACE ";" " " EXTRA_DTC_FLAGS_RAW "${EXTRA_DTC_FLAGS}")
set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT}
set(CMD_GEN_EDT ${PYTHON_EXECUTABLE} ${GEN_EDT_SCRIPT}
--dts ${DTS_POST_CPP}
--dtc-flags '${EXTRA_DTC_FLAGS_RAW}'
--bindings-dirs ${DTS_ROOT_BINDINGS}
--header-out ${DEVICETREE_GENERATED_H}.new
--dts-out ${ZEPHYR_DTS}.new # for debugging and dtc
--edt-pickle-out ${EDT_PICKLE}
--edt-pickle-out ${EDT_PICKLE}.new
${EXTRA_GEN_EDT_ARGS}
)

execute_process(
COMMAND ${CMD_GEN_EDT}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
RESULT_VARIABLE ret
ERROR_VARIABLE stderr
)
if(NOT "${ret}" STREQUAL "0")
if (stderr)
# gen_edt.py failed after printing message(s) to stderr.
# Append stream content to the FATAL_ERROR message on a new line.
set(stderr "\n${stderr}")
else()
# gen_edt.py did not print anything on stderr. To inform users
# of this condition, set ${stderr} to "<empty>" to have this printed
# in the error message. Note that we do NOT want a newline, such that
# the error message is printed as a single line, e.g.:
#
# gen_edt.py failed with result code: 1 - stderr contents: <empty>
#
set(stderr "<empty>")
endif()
message(STATUS "In: ${PROJECT_BINARY_DIR}, command: ${CMD_GEN_EDT}")
message(FATAL_ERROR "gen_edt.py failed with result code: ${ret} - stderr contents: ${stderr}")
else()
zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT)
zephyr_file_copy(${EDT_PICKLE}.new ${EDT_PICKLE} ONLY_IF_DIFFERENT)
file(REMOVE ${ZEPHYR_DTS}.new ${EDT_PICKLE}.new)
message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}")
message(STATUS "Generated pickled edt: ${EDT_PICKLE}")
endif()

#
# Run GEN_DEFINES_SCRIPT.
#

set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT}
--header-out ${DEVICETREE_GENERATED_H}.new
--edt-pickle ${EDT_PICKLE}
${EXTRA_GEN_DEFINES_ARGS}
)

Expand Down Expand Up @@ -310,10 +353,8 @@ if(NOT "${ret}" STREQUAL "0")
message(STATUS "In: ${PROJECT_BINARY_DIR}, command: ${CMD_GEN_DEFINES}")
message(FATAL_ERROR "gen_defines.py failed with result code: ${ret} - stderr contents: ${stderr}")
else()
zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT)
zephyr_file_copy(${DEVICETREE_GENERATED_H}.new ${DEVICETREE_GENERATED_H} ONLY_IF_DIFFERENT)
file(REMOVE ${ZEPHYR_DTS}.new ${DEVICETREE_GENERATED_H}.new)
message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}")
file(REMOVE ${DEVICETREE_GENERATED_H}.new)
message(STATUS "Generated devicetree_generated.h: ${DEVICETREE_GENERATED_H}")
endif()

Expand Down
11 changes: 11 additions & 0 deletions doc/releases/release-notes-4.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,17 @@ Build system and Infrastructure

* Added support for .elf files to the west flash command for jlink, pyocd and linkserver runners.

* Extracted pickled EDT generation from gen_defines.py into gen_edt.py. This moved the following
parameters from the cmake variable ``EXTRA_GEN_DEFINES_ARGS`` to ``EXTRA_GEN_EDT_ARGS``:

* ``--dts``
* ``--dtc-flags``
* ``--bindings-dirs``
* ``--dts-out``
* ``--edt-pickle-out``
* ``--vendor-prefixes``
* ``--edtlib-Werror``

Documentation
*************

Expand Down
71 changes: 7 additions & 64 deletions scripts/dts/gen_defines.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,11 @@

# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA
# Copyright (c) 2019 Linaro Limited
# Copyright (c) 2024 SILA Embedded Solutions GmbH
# SPDX-License-Identifier: BSD-3-Clause

# This script uses edtlib to generate a header file from a devicetree
# (.dts) file. Information from binding files in YAML format is used
# as well.
#
# Bindings are files that describe devicetree nodes. Devicetree nodes are
# usually mapped to bindings via their 'compatible = "..."' property.
#
# See Zephyr's Devicetree user guide for details.
# This script uses edtlib to generate a header file from a pickled
# edt file.
#
# Note: Do not access private (_-prefixed) identifiers from edtlib here (and
# also note that edtlib is not meant to expose the dtlib API directly).
Expand Down Expand Up @@ -52,28 +47,11 @@ def main():

setup_edtlib_logging()

vendor_prefixes = {}
for prefixes_file in args.vendor_prefixes:
vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file))

try:
edt = edtlib.EDT(args.dts, args.bindings_dirs,
# Suppress this warning if it's suppressed in dtc
warn_reg_unit_address_mismatch=
"-Wno-simple_bus_reg" not in args.dtc_flags,
default_prop_types=True,
infer_binding_for_paths=["/zephyr,user"],
werror=args.edtlib_Werror,
vendor_prefixes=vendor_prefixes)
except edtlib.EDTError as e:
sys.exit(f"devicetree error: {e}")
with open(args.edt_pickle, 'rb') as f:
edt = pickle.load(f)

flash_area_num = 0

# Save merged DTS source, as a debugging aid
with open(args.dts_out, "w", encoding="utf-8") as f:
print(edt.dts_source, file=f)

# Create the generated header.
with open(args.header_out, "w", encoding="utf-8") as header_file:
write_top_comment(edt)
Expand Down Expand Up @@ -133,9 +111,6 @@ def main():
write_chosen(edt)
write_global_macros(edt)

if args.edt_pickle_out:
write_pickled_edt(edt, args.edt_pickle_out)


def setup_edtlib_logging() -> None:
# The edtlib module emits logs using the standard 'logging' module.
Expand Down Expand Up @@ -173,27 +148,10 @@ def parse_args() -> argparse.Namespace:
# Returns parsed command-line arguments

parser = argparse.ArgumentParser(allow_abbrev=False)
parser.add_argument("--dts", required=True, help="DTS file")
parser.add_argument("--dtc-flags",
help="'dtc' devicetree compiler flags, some of which "
"might be respected here")
parser.add_argument("--bindings-dirs", nargs='+', required=True,
help="directory with bindings in YAML format, "
"we allow multiple")
parser.add_argument("--header-out", required=True,
help="path to write header to")
parser.add_argument("--dts-out", required=True,
help="path to write merged DTS source code to (e.g. "
"as a debugging aid)")
parser.add_argument("--edt-pickle-out",
help="path to write pickled edtlib.EDT object to")
parser.add_argument("--vendor-prefixes", action='append', default=[],
help="vendor-prefixes.txt path; used for validation; "
"may be given multiple times")
parser.add_argument("--edtlib-Werror", action="store_true",
help="if set, edtlib-specific warnings become errors. "
"(this does not apply to warnings shared "
"with dtc.)")
parser.add_argument("--edt-pickle",
help="path to read pickled edtlib.EDT object from")

return parser.parse_args()

Expand Down Expand Up @@ -1099,21 +1057,6 @@ def quote_str(s: str) -> str:
return f'"{escape(s)}"'


def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None:
# Writes the edt object in pickle format to out_file.

with open(out_file, 'wb') as f:
# Pickle protocol version 4 is the default as of Python 3.8
# and was introduced in 3.4, so it is both available and
# recommended on all versions of Python that Zephyr supports
# (at time of writing, Python 3.6 was Zephyr's minimum
# version, and 3.8 the most recent CPython release).
#
# Using a common protocol version here will hopefully avoid
# reproducibility issues in different Python installations.
pickle.dump(edt, f, protocol=4)


def err(s: str) -> NoReturn:
raise Exception(s)

Expand Down
133 changes: 133 additions & 0 deletions scripts/dts/gen_edt.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
#!/usr/bin/env python3

# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA
# Copyright (c) 2019 Linaro Limited
# Copyright (c) 2024 SILA Embedded Solutions GmbH
# SPDX-License-Identifier: Apache-2.0

# This script uses edtlib to generate a pickled edt from a devicetree
# (.dts) file. Information from binding files in YAML format is used
# as well.
#
# Bindings are files that describe devicetree nodes. Devicetree nodes are
# usually mapped to bindings via their 'compatible = "..."' property.
#
# See Zephyr's Devicetree user guide for details.
#
# Note: Do not access private (_-prefixed) identifiers from edtlib here (and
# also note that edtlib is not meant to expose the dtlib API directly).
# Instead, think of what API you need, and add it as a public documented API in
# edtlib. This will keep this script simple.

import argparse
import logging
import os
import pickle
import sys
from typing import NoReturn

sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'python-devicetree',
'src'))

from devicetree import edtlib

class LogFormatter(logging.Formatter):
'''A log formatter that prints the level name in lower case,
for compatibility with earlier versions of edtlib.'''

def __init__(self):
super().__init__(fmt='%(levelnamelower)s: %(message)s')

def format(self, record):
record.levelnamelower = record.levelname.lower()
return super().format(record)

def main():
args = parse_args()

setup_edtlib_logging()

vendor_prefixes = {}
for prefixes_file in args.vendor_prefixes:
vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file))

try:
edt = edtlib.EDT(args.dts, args.bindings_dirs,
# Suppress this warning if it's suppressed in dtc
warn_reg_unit_address_mismatch=
"-Wno-simple_bus_reg" not in args.dtc_flags,
default_prop_types=True,
infer_binding_for_paths=["/zephyr,user"],
werror=args.edtlib_Werror,
vendor_prefixes=vendor_prefixes)
except edtlib.EDTError as e:
sys.exit(f"devicetree error: {e}")

# Save merged DTS source, as a debugging aid
with open(args.dts_out, "w", encoding="utf-8") as f:
print(edt.dts_source, file=f)

write_pickled_edt(edt, args.edt_pickle_out)


def setup_edtlib_logging() -> None:
# The edtlib module emits logs using the standard 'logging' module.
# Configure it so that warnings and above are printed to stderr,
# using the LogFormatter class defined above to format each message.

handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(LogFormatter())

logger = logging.getLogger('edtlib')
logger.setLevel(logging.WARNING)
logger.addHandler(handler)


def parse_args() -> argparse.Namespace:
# Returns parsed command-line arguments

parser = argparse.ArgumentParser(allow_abbrev=False)
parser.add_argument("--dts", required=True, help="DTS file")
parser.add_argument("--dtc-flags",
help="'dtc' devicetree compiler flags, some of which "
"might be respected here")
parser.add_argument("--bindings-dirs", nargs='+', required=True,
help="directory with bindings in YAML format, "
"we allow multiple")
parser.add_argument("--dts-out", required=True,
help="path to write merged DTS source code to (e.g. "
"as a debugging aid)")
parser.add_argument("--edt-pickle-out",
help="path to write pickled edtlib.EDT object to", required=True)
parser.add_argument("--vendor-prefixes", action='append', default=[],
help="vendor-prefixes.txt path; used for validation; "
"may be given multiple times")
parser.add_argument("--edtlib-Werror", action="store_true",
help="if set, edtlib-specific warnings become errors. "
"(this does not apply to warnings shared "
"with dtc.)")

return parser.parse_args()


def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None:
# Writes the edt object in pickle format to out_file.

with open(out_file, 'wb') as f:
# Pickle protocol version 4 is the default as of Python 3.8
# and was introduced in 3.4, so it is both available and
# recommended on all versions of Python that Zephyr supports
# (at time of writing, Python 3.6 was Zephyr's minimum
# version, and 3.8 the most recent CPython release).
#
# Using a common protocol version here will hopefully avoid
# reproducibility issues in different Python installations.
pickle.dump(edt, f, protocol=4)


def err(s: str) -> NoReturn:
raise Exception(s)


if __name__ == "__main__":
main()
6 changes: 3 additions & 3 deletions scripts/pylib/twister/twisterlib/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,10 +330,10 @@ def run_cmake(self, args="", filter_stages=[]):

if not self.options.disable_warnings_as_errors:
warnings_as_errors = 'y'
gen_defines_args = "--edtlib-Werror"
gen_edt_args = "--edtlib-Werror"
else:
warnings_as_errors = 'n'
gen_defines_args = ""
gen_edt_args = ""

warning_command = 'CONFIG_COMPILER_WARNINGS_AS_ERRORS'
if self.instance.sysbuild:
Expand All @@ -344,7 +344,7 @@ def run_cmake(self, args="", filter_stages=[]):
f'-B{self.build_dir}',
f'-DTC_RUNID={self.instance.run_id}',
f'-D{warning_command}={warnings_as_errors}',
f'-DEXTRA_GEN_DEFINES_ARGS={gen_defines_args}',
f'-DEXTRA_GEN_EDT_ARGS={gen_edt_args}',
f'-G{self.env.generator}'
]

Expand Down
Loading

0 comments on commit 20f0185

Please sign in to comment.