From 20f01856576166ffe7128ad0d3dbdc30f74fad1a Mon Sep 17 00:00:00 2001 From: Benedikt Schmidt Date: Mon, 9 Sep 2024 11:18:56 +0200 Subject: [PATCH] scripts: dts: extract pickled EDT generation Separate the pickled EDT generation from the C-Macro header generation in gen_defines.py to have a more clear responsibility of the scripts in the DTS parsing process. Signed-off-by: Benedikt Schmidt --- cmake/modules/dts.cmake | 57 +++++++-- doc/releases/release-notes-4.0.rst | 11 ++ scripts/dts/gen_defines.py | 71 ++--------- scripts/dts/gen_edt.py | 133 +++++++++++++++++++++ scripts/pylib/twister/twisterlib/runner.py | 6 +- scripts/tests/twister/test_runner.py | 4 +- 6 files changed, 205 insertions(+), 77 deletions(-) create mode 100755 scripts/dts/gen_edt.py diff --git a/cmake/modules/dts.cmake b/cmake/modules/dts.cmake index 88abeb3ed43fa51..d0dff6dbc16bd81 100644 --- a/cmake/modules/dts.cmake +++ b/cmake/modules/dts.cmake @@ -94,6 +94,8 @@ find_package(Dtc 1.4.6) # The directory containing devicetree related scripts. set(DT_SCRIPTS ${ZEPHYR_BASE}/scripts/dts) +# This parses and collects the DT information +set(GEN_EDT_SCRIPT ${DT_SCRIPTS}/gen_edt.py) # This generates DT information needed by the C macro APIs, # along with a few other things. set(GEN_DEFINES_SCRIPT ${DT_SCRIPTS}/gen_defines.py) @@ -216,7 +218,7 @@ foreach(dts_root ${DTS_ROOT}) set(vendor_prefixes ${dts_root}/${VENDOR_PREFIXES}) if(EXISTS ${vendor_prefixes}) - list(APPEND EXTRA_GEN_DEFINES_ARGS --vendor-prefixes ${vendor_prefixes}) + list(APPEND EXTRA_GEN_EDT_ARGS --vendor-prefixes ${vendor_prefixes}) endif() endforeach() @@ -266,23 +268,64 @@ toolchain_parse_make_rule(${DTS_DEPS} set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${DTS_INCLUDE_FILES} + ${GEN_EDT_SCRIPT} ${GEN_DEFINES_SCRIPT} ${GEN_DRIVER_KCONFIG_SCRIPT} ${GEN_DTS_CMAKE_SCRIPT} ) # -# Run GEN_DEFINES_SCRIPT. +# Run GEN_EDT_SCRIPT. # string(REPLACE ";" " " EXTRA_DTC_FLAGS_RAW "${EXTRA_DTC_FLAGS}") -set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT} +set(CMD_GEN_EDT ${PYTHON_EXECUTABLE} ${GEN_EDT_SCRIPT} --dts ${DTS_POST_CPP} --dtc-flags '${EXTRA_DTC_FLAGS_RAW}' --bindings-dirs ${DTS_ROOT_BINDINGS} ---header-out ${DEVICETREE_GENERATED_H}.new --dts-out ${ZEPHYR_DTS}.new # for debugging and dtc ---edt-pickle-out ${EDT_PICKLE} +--edt-pickle-out ${EDT_PICKLE}.new +${EXTRA_GEN_EDT_ARGS} +) + +execute_process( + COMMAND ${CMD_GEN_EDT} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + RESULT_VARIABLE ret + ERROR_VARIABLE stderr + ) +if(NOT "${ret}" STREQUAL "0") + if (stderr) + # gen_edt.py failed after printing message(s) to stderr. + # Append stream content to the FATAL_ERROR message on a new line. + set(stderr "\n${stderr}") + else() + # gen_edt.py did not print anything on stderr. To inform users + # of this condition, set ${stderr} to "" to have this printed + # in the error message. Note that we do NOT want a newline, such that + # the error message is printed as a single line, e.g.: + # + # gen_edt.py failed with result code: 1 - stderr contents: + # + set(stderr "") + endif() + message(STATUS "In: ${PROJECT_BINARY_DIR}, command: ${CMD_GEN_EDT}") + message(FATAL_ERROR "gen_edt.py failed with result code: ${ret} - stderr contents: ${stderr}") +else() + zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT) + zephyr_file_copy(${EDT_PICKLE}.new ${EDT_PICKLE} ONLY_IF_DIFFERENT) + file(REMOVE ${ZEPHYR_DTS}.new ${EDT_PICKLE}.new) + message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}") + message(STATUS "Generated pickled edt: ${EDT_PICKLE}") +endif() + +# +# Run GEN_DEFINES_SCRIPT. +# + +set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT} +--header-out ${DEVICETREE_GENERATED_H}.new +--edt-pickle ${EDT_PICKLE} ${EXTRA_GEN_DEFINES_ARGS} ) @@ -310,10 +353,8 @@ if(NOT "${ret}" STREQUAL "0") message(STATUS "In: ${PROJECT_BINARY_DIR}, command: ${CMD_GEN_DEFINES}") message(FATAL_ERROR "gen_defines.py failed with result code: ${ret} - stderr contents: ${stderr}") else() - zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT) zephyr_file_copy(${DEVICETREE_GENERATED_H}.new ${DEVICETREE_GENERATED_H} ONLY_IF_DIFFERENT) - file(REMOVE ${ZEPHYR_DTS}.new ${DEVICETREE_GENERATED_H}.new) - message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}") + file(REMOVE ${DEVICETREE_GENERATED_H}.new) message(STATUS "Generated devicetree_generated.h: ${DEVICETREE_GENERATED_H}") endif() diff --git a/doc/releases/release-notes-4.0.rst b/doc/releases/release-notes-4.0.rst index 05225a2dc58e031..a083123ab100925 100644 --- a/doc/releases/release-notes-4.0.rst +++ b/doc/releases/release-notes-4.0.rst @@ -112,6 +112,17 @@ Build system and Infrastructure * Added support for .elf files to the west flash command for jlink, pyocd and linkserver runners. +* Extracted pickled EDT generation from gen_defines.py into gen_edt.py. This moved the following + parameters from the cmake variable ``EXTRA_GEN_DEFINES_ARGS`` to ``EXTRA_GEN_EDT_ARGS``: + + * ``--dts`` + * ``--dtc-flags`` + * ``--bindings-dirs`` + * ``--dts-out`` + * ``--edt-pickle-out`` + * ``--vendor-prefixes`` + * ``--edtlib-Werror`` + Documentation ************* diff --git a/scripts/dts/gen_defines.py b/scripts/dts/gen_defines.py index f049f7bc2e41da2..acb655156497fdd 100755 --- a/scripts/dts/gen_defines.py +++ b/scripts/dts/gen_defines.py @@ -2,16 +2,11 @@ # Copyright (c) 2019 - 2020 Nordic Semiconductor ASA # Copyright (c) 2019 Linaro Limited +# Copyright (c) 2024 SILA Embedded Solutions GmbH # SPDX-License-Identifier: BSD-3-Clause -# This script uses edtlib to generate a header file from a devicetree -# (.dts) file. Information from binding files in YAML format is used -# as well. -# -# Bindings are files that describe devicetree nodes. Devicetree nodes are -# usually mapped to bindings via their 'compatible = "..."' property. -# -# See Zephyr's Devicetree user guide for details. +# This script uses edtlib to generate a header file from a pickled +# edt file. # # Note: Do not access private (_-prefixed) identifiers from edtlib here (and # also note that edtlib is not meant to expose the dtlib API directly). @@ -52,28 +47,11 @@ def main(): setup_edtlib_logging() - vendor_prefixes = {} - for prefixes_file in args.vendor_prefixes: - vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file)) - - try: - edt = edtlib.EDT(args.dts, args.bindings_dirs, - # Suppress this warning if it's suppressed in dtc - warn_reg_unit_address_mismatch= - "-Wno-simple_bus_reg" not in args.dtc_flags, - default_prop_types=True, - infer_binding_for_paths=["/zephyr,user"], - werror=args.edtlib_Werror, - vendor_prefixes=vendor_prefixes) - except edtlib.EDTError as e: - sys.exit(f"devicetree error: {e}") + with open(args.edt_pickle, 'rb') as f: + edt = pickle.load(f) flash_area_num = 0 - # Save merged DTS source, as a debugging aid - with open(args.dts_out, "w", encoding="utf-8") as f: - print(edt.dts_source, file=f) - # Create the generated header. with open(args.header_out, "w", encoding="utf-8") as header_file: write_top_comment(edt) @@ -133,9 +111,6 @@ def main(): write_chosen(edt) write_global_macros(edt) - if args.edt_pickle_out: - write_pickled_edt(edt, args.edt_pickle_out) - def setup_edtlib_logging() -> None: # The edtlib module emits logs using the standard 'logging' module. @@ -173,27 +148,10 @@ def parse_args() -> argparse.Namespace: # Returns parsed command-line arguments parser = argparse.ArgumentParser(allow_abbrev=False) - parser.add_argument("--dts", required=True, help="DTS file") - parser.add_argument("--dtc-flags", - help="'dtc' devicetree compiler flags, some of which " - "might be respected here") - parser.add_argument("--bindings-dirs", nargs='+', required=True, - help="directory with bindings in YAML format, " - "we allow multiple") parser.add_argument("--header-out", required=True, help="path to write header to") - parser.add_argument("--dts-out", required=True, - help="path to write merged DTS source code to (e.g. " - "as a debugging aid)") - parser.add_argument("--edt-pickle-out", - help="path to write pickled edtlib.EDT object to") - parser.add_argument("--vendor-prefixes", action='append', default=[], - help="vendor-prefixes.txt path; used for validation; " - "may be given multiple times") - parser.add_argument("--edtlib-Werror", action="store_true", - help="if set, edtlib-specific warnings become errors. " - "(this does not apply to warnings shared " - "with dtc.)") + parser.add_argument("--edt-pickle", + help="path to read pickled edtlib.EDT object from") return parser.parse_args() @@ -1099,21 +1057,6 @@ def quote_str(s: str) -> str: return f'"{escape(s)}"' -def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None: - # Writes the edt object in pickle format to out_file. - - with open(out_file, 'wb') as f: - # Pickle protocol version 4 is the default as of Python 3.8 - # and was introduced in 3.4, so it is both available and - # recommended on all versions of Python that Zephyr supports - # (at time of writing, Python 3.6 was Zephyr's minimum - # version, and 3.8 the most recent CPython release). - # - # Using a common protocol version here will hopefully avoid - # reproducibility issues in different Python installations. - pickle.dump(edt, f, protocol=4) - - def err(s: str) -> NoReturn: raise Exception(s) diff --git a/scripts/dts/gen_edt.py b/scripts/dts/gen_edt.py new file mode 100755 index 000000000000000..75a4e81c8df1b1b --- /dev/null +++ b/scripts/dts/gen_edt.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA +# Copyright (c) 2019 Linaro Limited +# Copyright (c) 2024 SILA Embedded Solutions GmbH +# SPDX-License-Identifier: Apache-2.0 + +# This script uses edtlib to generate a pickled edt from a devicetree +# (.dts) file. Information from binding files in YAML format is used +# as well. +# +# Bindings are files that describe devicetree nodes. Devicetree nodes are +# usually mapped to bindings via their 'compatible = "..."' property. +# +# See Zephyr's Devicetree user guide for details. +# +# Note: Do not access private (_-prefixed) identifiers from edtlib here (and +# also note that edtlib is not meant to expose the dtlib API directly). +# Instead, think of what API you need, and add it as a public documented API in +# edtlib. This will keep this script simple. + +import argparse +import logging +import os +import pickle +import sys +from typing import NoReturn + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'python-devicetree', + 'src')) + +from devicetree import edtlib + +class LogFormatter(logging.Formatter): + '''A log formatter that prints the level name in lower case, + for compatibility with earlier versions of edtlib.''' + + def __init__(self): + super().__init__(fmt='%(levelnamelower)s: %(message)s') + + def format(self, record): + record.levelnamelower = record.levelname.lower() + return super().format(record) + +def main(): + args = parse_args() + + setup_edtlib_logging() + + vendor_prefixes = {} + for prefixes_file in args.vendor_prefixes: + vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file)) + + try: + edt = edtlib.EDT(args.dts, args.bindings_dirs, + # Suppress this warning if it's suppressed in dtc + warn_reg_unit_address_mismatch= + "-Wno-simple_bus_reg" not in args.dtc_flags, + default_prop_types=True, + infer_binding_for_paths=["/zephyr,user"], + werror=args.edtlib_Werror, + vendor_prefixes=vendor_prefixes) + except edtlib.EDTError as e: + sys.exit(f"devicetree error: {e}") + + # Save merged DTS source, as a debugging aid + with open(args.dts_out, "w", encoding="utf-8") as f: + print(edt.dts_source, file=f) + + write_pickled_edt(edt, args.edt_pickle_out) + + +def setup_edtlib_logging() -> None: + # The edtlib module emits logs using the standard 'logging' module. + # Configure it so that warnings and above are printed to stderr, + # using the LogFormatter class defined above to format each message. + + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(LogFormatter()) + + logger = logging.getLogger('edtlib') + logger.setLevel(logging.WARNING) + logger.addHandler(handler) + + +def parse_args() -> argparse.Namespace: + # Returns parsed command-line arguments + + parser = argparse.ArgumentParser(allow_abbrev=False) + parser.add_argument("--dts", required=True, help="DTS file") + parser.add_argument("--dtc-flags", + help="'dtc' devicetree compiler flags, some of which " + "might be respected here") + parser.add_argument("--bindings-dirs", nargs='+', required=True, + help="directory with bindings in YAML format, " + "we allow multiple") + parser.add_argument("--dts-out", required=True, + help="path to write merged DTS source code to (e.g. " + "as a debugging aid)") + parser.add_argument("--edt-pickle-out", + help="path to write pickled edtlib.EDT object to", required=True) + parser.add_argument("--vendor-prefixes", action='append', default=[], + help="vendor-prefixes.txt path; used for validation; " + "may be given multiple times") + parser.add_argument("--edtlib-Werror", action="store_true", + help="if set, edtlib-specific warnings become errors. " + "(this does not apply to warnings shared " + "with dtc.)") + + return parser.parse_args() + + +def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None: + # Writes the edt object in pickle format to out_file. + + with open(out_file, 'wb') as f: + # Pickle protocol version 4 is the default as of Python 3.8 + # and was introduced in 3.4, so it is both available and + # recommended on all versions of Python that Zephyr supports + # (at time of writing, Python 3.6 was Zephyr's minimum + # version, and 3.8 the most recent CPython release). + # + # Using a common protocol version here will hopefully avoid + # reproducibility issues in different Python installations. + pickle.dump(edt, f, protocol=4) + + +def err(s: str) -> NoReturn: + raise Exception(s) + + +if __name__ == "__main__": + main() diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 12f77df4c37dfc8..058169003f3e08f 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -330,10 +330,10 @@ def run_cmake(self, args="", filter_stages=[]): if not self.options.disable_warnings_as_errors: warnings_as_errors = 'y' - gen_defines_args = "--edtlib-Werror" + gen_edt_args = "--edtlib-Werror" else: warnings_as_errors = 'n' - gen_defines_args = "" + gen_edt_args = "" warning_command = 'CONFIG_COMPILER_WARNINGS_AS_ERRORS' if self.instance.sysbuild: @@ -344,7 +344,7 @@ def run_cmake(self, args="", filter_stages=[]): f'-B{self.build_dir}', f'-DTC_RUNID={self.instance.run_id}', f'-D{warning_command}={warnings_as_errors}', - f'-DEXTRA_GEN_DEFINES_ARGS={gen_defines_args}', + f'-DEXTRA_GEN_EDT_ARGS={gen_edt_args}', f'-G{self.env.generator}' ] diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index cde4a26456c3281..807591b193ec3c0 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -371,7 +371,7 @@ def mock_popen(*args, **kwargs): [os.path.join('dummy', 'cmake'), '-B' + os.path.join('build', 'dir'), '-DTC_RUNID=1', '-DSB_CONFIG_COMPILER_WARNINGS_AS_ERRORS=y', - '-DEXTRA_GEN_DEFINES_ARGS=--edtlib-Werror', '-Gdummy_generator', + '-DEXTRA_GEN_EDT_ARGS=--edtlib-Werror', '-Gdummy_generator', '-S' + os.path.join('source', 'dir'), 'arg1', 'arg2', '-DBOARD=', @@ -385,7 +385,7 @@ def mock_popen(*args, **kwargs): [os.path.join('dummy', 'cmake'), '-B' + os.path.join('build', 'dir'), '-DTC_RUNID=1', '-DSB_CONFIG_COMPILER_WARNINGS_AS_ERRORS=n', - '-DEXTRA_GEN_DEFINES_ARGS=', '-Gdummy_generator', + '-DEXTRA_GEN_EDT_ARGS=', '-Gdummy_generator', '-Szephyr_base/share/sysbuild', '-DAPP_DIR=' + os.path.join('source', 'dir'), 'arg1', 'arg2',