From 00345a2df7eee18807dc398b42d70824e55b10b8 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Fri, 5 Jul 2024 07:54:34 +0200 Subject: [PATCH] [libc++] Adds a new feature-test macro generator class. (#90889) The new feature-test macro generator uses a JSON file as input. Separating the code from the data allows for testing the code. The generator has several tests. This JSON format is based on the new format proposed in #88630 At the moment the FTM script has the existing code and an unused new generator. Followup patches will complete the generator and convert the existing Python `dict` to the new JSON format. Since that conversion is a manual job and quite a bit of work the transition path has some unused code for some time. --- .../feature_test_macro/implemented_ftms.sh.py | 54 +++++ .../libcxx/feature_test_macro/invalid.sh.py | 108 ++++++++++ .../feature_test_macro/standard_ftms.sh.py | 54 +++++ .../feature_test_macro/std_dialects.sh.py | 30 +++ .../libcxx/feature_test_macro/test_data.json | 151 ++++++++++++++ .../generate_feature_test_macro_components.py | 194 ++++++++++++++++++ 6 files changed, 591 insertions(+) create mode 100644 libcxx/test/libcxx/feature_test_macro/implemented_ftms.sh.py create mode 100644 libcxx/test/libcxx/feature_test_macro/invalid.sh.py create mode 100644 libcxx/test/libcxx/feature_test_macro/standard_ftms.sh.py create mode 100644 libcxx/test/libcxx/feature_test_macro/std_dialects.sh.py create mode 100644 libcxx/test/libcxx/feature_test_macro/test_data.json diff --git a/libcxx/test/libcxx/feature_test_macro/implemented_ftms.sh.py b/libcxx/test/libcxx/feature_test_macro/implemented_ftms.sh.py new file mode 100644 index 00000000000000..67353fc41e5098 --- /dev/null +++ b/libcxx/test/libcxx/feature_test_macro/implemented_ftms.sh.py @@ -0,0 +1,54 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +# RUN: %{python} %s %{libcxx-dir}/utils %{libcxx-dir}/test/libcxx/feature_test_macro/test_data.json + +import sys + +sys.path.append(sys.argv[1]) +from generate_feature_test_macro_components import FeatureTestMacros + + +def test(output, expected): + assert output == expected, f"expected\n{expected}\n\noutput\n{output}" + + +ftm = FeatureTestMacros(sys.argv[2]) +test( + ftm.implemented_ftms, + { + "__cpp_lib_any": { + "c++17": "201606L", + "c++20": "201606L", + "c++23": "201606L", + "c++26": "201606L", + }, + "__cpp_lib_barrier": { + "c++20": "201907L", + "c++23": "201907L", + "c++26": "201907L", + }, + "__cpp_lib_format": { + "c++20": None, + "c++23": None, + "c++26": None, + }, + "__cpp_lib_parallel_algorithm": { + "c++17": "201603L", + "c++20": "201603L", + "c++23": "201603L", + "c++26": "201603L", + }, + "__cpp_lib_variant": { + "c++17": "202102L", + "c++20": "202102L", + "c++23": "202102L", + "c++26": "202102L", + }, + }, +) diff --git a/libcxx/test/libcxx/feature_test_macro/invalid.sh.py b/libcxx/test/libcxx/feature_test_macro/invalid.sh.py new file mode 100644 index 00000000000000..ae457f6e1a545a --- /dev/null +++ b/libcxx/test/libcxx/feature_test_macro/invalid.sh.py @@ -0,0 +1,108 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +# RUN: %{python} %s %{libcxx-dir}/utils %t + +import sys +import json + +sys.path.append(sys.argv[1]) +from generate_feature_test_macro_components import FeatureTestMacros + + +def test(output, expected): + assert output == expected, f"expected\n{expected}\n\noutput\n{output}" + + +def test_error(data, type, message): + tmp = sys.argv[2] + with open(tmp, "w") as file: + file.write(json.dumps(data)) + ftm = FeatureTestMacros(tmp) + try: + ftm.implemented_ftms + except type as error: + test(str(error), message) + else: + assert False, "no exception was thrown" + + +test_error( + [ + { + "values": { + "c++17": { + "197001": [ + { + "implemented": False, + }, + ], + }, + }, + "headers": [], + }, + ], + KeyError, + "'name'", +) + +test_error( + [ + { + "name": "a", + "headers": [], + }, + ], + KeyError, + "'values'", +) + +test_error( + [ + { + "name": "a", + "values": {}, + "headers": [], + }, + ], + AssertionError, + "'values' is empty", +) + + +test_error( + [ + { + "name": "a", + "values": { + "c++17": {}, + }, + "headers": [], + }, + ], + AssertionError, + "a[c++17] has no entries", +) + +test_error( + [ + { + "name": "a", + "values": { + "c++17": { + "197001": [ + {}, + ], + }, + }, + "headers": [], + }, + ], + KeyError, + "'implemented'", +) diff --git a/libcxx/test/libcxx/feature_test_macro/standard_ftms.sh.py b/libcxx/test/libcxx/feature_test_macro/standard_ftms.sh.py new file mode 100644 index 00000000000000..43c90b131bff16 --- /dev/null +++ b/libcxx/test/libcxx/feature_test_macro/standard_ftms.sh.py @@ -0,0 +1,54 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +# RUN: %{python} %s %{libcxx-dir}/utils %{libcxx-dir}/test/libcxx/feature_test_macro/test_data.json + +import sys + +sys.path.append(sys.argv[1]) +from generate_feature_test_macro_components import FeatureTestMacros + + +def test(output, expected): + assert output == expected, f"expected\n{expected}\n\noutput\n{output}" + + +ftm = FeatureTestMacros(sys.argv[2]) +test( + ftm.standard_ftms, + { + "__cpp_lib_any": { + "c++17": "201606L", + "c++20": "201606L", + "c++23": "201606L", + "c++26": "201606L", + }, + "__cpp_lib_barrier": { + "c++20": "201907L", + "c++23": "201907L", + "c++26": "201907L", + }, + "__cpp_lib_format": { + "c++20": "202110L", + "c++23": "202207L", + "c++26": "202311L", + }, + "__cpp_lib_parallel_algorithm": { + "c++17": "201603L", + "c++20": "201603L", + "c++23": "201603L", + "c++26": "201603L", + }, + "__cpp_lib_variant": { + "c++17": "202102L", + "c++20": "202106L", + "c++23": "202106L", + "c++26": "202306L", + }, + }, +) diff --git a/libcxx/test/libcxx/feature_test_macro/std_dialects.sh.py b/libcxx/test/libcxx/feature_test_macro/std_dialects.sh.py new file mode 100644 index 00000000000000..368020c91e1d2b --- /dev/null +++ b/libcxx/test/libcxx/feature_test_macro/std_dialects.sh.py @@ -0,0 +1,30 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +# RUN: %{python} %s %{libcxx-dir}/utils %{libcxx-dir}/test/libcxx/feature_test_macro/test_data.json + +import sys + +sys.path.append(sys.argv[1]) +from generate_feature_test_macro_components import FeatureTestMacros + + +def test(output, expected): + assert output == expected, f"expected\n{expected}\n\noutput\n{output}" + + +ftm = FeatureTestMacros(sys.argv[2]) +test( + ftm.std_dialects, + [ + "c++17", + "c++20", + "c++23", + "c++26", + ], +) diff --git a/libcxx/test/libcxx/feature_test_macro/test_data.json b/libcxx/test/libcxx/feature_test_macro/test_data.json new file mode 100644 index 00000000000000..1f8bbe5d769b5b --- /dev/null +++ b/libcxx/test/libcxx/feature_test_macro/test_data.json @@ -0,0 +1,151 @@ +[ + { + "name": "__cpp_lib_any", + "values": { + "c++17": { + "201606": [ + { + "implemented": true + } + ] + } + }, + "headers": [ + "any" + ] + }, + { + "name": "__cpp_lib_barrier", + "values": { + "c++20": { + "201907": [ + { + "implemented": true + } + ] + } + }, + "headers": [ + "barrier" + ], + "test_suite_guard": + "!defined(_LIBCPP_HAS_NO_THREADS) && (!defined(_LIBCPP_VERSION) || _LIBCPP_AVAILABILITY_HAS_SYNC)", + "libcxx_guard": "!defined(_LIBCPP_HAS_NO_THREADS) && _LIBCPP_AVAILABILITY_HAS_SYNC" + }, + { + "name": "__cpp_lib_format", + "values": { + "c++20": { + "201907": [ + { + "number": "P0645R10", + "title": "Text Formatting", + "implemented": true + }, + { + "number": "P1361R2", + "title": "Integration of chrono with text formatting", + "implemented": false + } + ], + "202106": [ + { + "number": "P2216R3", + "title": "std::format improvements", + "implemented": true + } + ], + "202110": [ + { + "number": "P2372R3", + "title": "Fixing locale handling in chrono formatters", + "implemented": false + }, + { + "number": "P2418R2", + "title": "FAdd support for std::generator-like types to std::format", + "implemented": true + } + ] + }, + "c++23": { + "202207": [ + { + "number": "P2419R2", + "title": "Clarify handling of encodings in localized formatting of chrono types", + "implemented": false + } + ] + }, + "c++26": { + "202306": [ + { + "number": "P2637R3", + "title": "Member Visit", + "implemented": true + } + ], + "202311": [ + { + "number": "P2918R2", + "title": "Runtime format strings II", + "implemented": true + } + ] + } + }, + "headers": [ + "format" + ] + }, + { + "name": "__cpp_lib_parallel_algorithm", + "values": { + "c++17": { + "201603": [ + { + "implemented": true + } + ] + } + }, + "headers": [ + "algorithm", + "numeric" + ] + }, + { + "name": "__cpp_lib_variant", + "values": { + "c++17": { + "202102": [ + { + "title": "``std::visit`` for classes derived from ``std::variant``", + "implemented": true + } + ] + }, + "c++20": { + "202106": [ + { + "number": "", + "title": "Fully constexpr ``std::variant``", + "implemented": false + } + ] + }, + "c++26": { + "202306": [ + { + "number": "", + "title": "Member visit", + "implemented": true + } + ] + } + }, + "headers": [ + "variant" + ] + } +] diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py index fe5bab05195a38..3f8ecc26321ee1 100755 --- a/libcxx/utils/generate_feature_test_macro_components.py +++ b/libcxx/utils/generate_feature_test_macro_components.py @@ -3,6 +3,9 @@ import os from builtins import range from functools import reduce +from typing import Any, Dict, List # Needed for python 3.8 compatibility. +import functools +import json def get_libcxx_paths(): @@ -1867,6 +1870,197 @@ def produce_docs(): f.write(doc_str) +def get_ftms( + data, std_dialects: List[str], use_implemented_status: bool +) -> Dict[str, Dict[str, Any]]: + """Impementation for FeatureTestMacros.(standard|implemented)_ftms().""" + result = dict() + for feature in data: + last = None + entry = dict() + implemented = True + for std in std_dialects: + if std not in feature["values"].keys(): + if last == None: + continue + else: + entry[std] = last + else: + if implemented: + values = feature["values"][std] + assert len(values) > 0, f"{feature['name']}[{std}] has no entries" + for value in values: + papers = list(values[value]) + assert ( + len(papers) > 0 + ), f"{feature['name']}[{std}][{value}] has no entries" + for paper in papers: + if use_implemented_status and not paper["implemented"]: + implemented = False + break + if implemented: + last = f"{value}L" + else: + break + + entry[std] = last + result[feature["name"]] = entry + + return result + + +class FeatureTestMacros: + """Provides all feature-test macro (FTM) output components. + + The class has several generators to use the feature-test macros in libc++: + - FTM status page + - The version header and its tests + + This class is not intended to duplicate + https://isocpp.org/std/standing-documents/sd-6-sg10-feature-test-recommendations#library-feature-test-macros + SD-FeatureTest: Feature-Test Macros and Policies + + Historically libc++ did not list all papers affecting a FTM, the new data + structure is able to do that. However there is no intention to add the + historical data. After papers have been implemented this information can be + removed. For example, __cpp_lib_format's value 201907 requires 3 papers, + once implemented it can be reduced to 1 paper and remove the paper number + and title. This would reduce the size of the data. + + The input data is stored in the following JSON format: + [ # A list with multiple feature-test macro entries. + { + # required + # The name of the feature test macro. These names should be unique and + # sorted in the list. + "name": "__cpp_lib_any", + + # required + # A map with the value of the FTM based on the language standard. Only + # the versions in which the value of the FTM changes are listed. For + # example, this macro's value does not change in C++20 so it does not + # list C++20. If it changes in C++26, it will have entries for C++17 and + # C++26. + "values": { + + # required + # The language standard, also named dialect in this class. + "c++17": { + + # required + # The value of the feature test macro. This contains an array with + # one or more papers that need to be implemented before this value + # is considered implemented. + "201606": [ + { + # optional + # Contains the paper number that is part of the FTM version. + "number": "P0220R1", + + # optional + # Contains the title of the paper that is part of the FTM + # version. + "title": "Adopt Library Fundamentals V1 TS Components for C++17" + + # required + # The implementation status of the paper. + "implemented": true + } + ] + } + }, + + # required + # A sorted list of headers that should provide the FTM. The header + # is automatically added to this list. This list could be + # empty. For example, __cpp_lib_modules is only present in version. + # Requiring the field makes it easier to detect accidental omission. + "headers": [ + "any" + ], + + # optional, required when libcxx_guard is present + # This field is used only to generate the unit tests for the + # feature-test macros. It can't depend on macros defined in <__config> + # because the `test/std/` parts of the test suite are intended to be + # portable to any C++ standard library implementation, not just libc++. + # It may depend on + # * macros defined by the compiler itself, or + # * macros generated by CMake. + # In some cases we add also depend on macros defined in + # <__availability>. + "test_suite_guard": "!defined(_LIBCPP_VERSION) || _LIBCPP_AVAILABILITY_HAS_PMR" + + # optional, required when test_suite_guard is present + # This field is used only to guard the feature-test macro in + # . It may be the same as `test_suite_guard`, or it may + # depend on macros defined in <__config>. + "libcxx_guard": "_LIBCPP_AVAILABILITY_HAS_PMR" + }, + ] + """ + + # The JSON data structure. + __data = None + + def __init__(self, filename: str): + """Initializes the class with the JSON data in the file 'filename'.""" + self.__data = json.load(open(filename)) + + @functools.cached_property + def std_dialects(self) -> List[str]: + """Returns the C++ dialects avaiable. + + The available dialects are based on the 'c++xy' keys found the 'values' + entries in '__data'. So when WG21 starts to feature-test macros for a + future C++ Standard this dialect will automatically be available. + + The return value is a sorted list with the C++ dialects used. Since FTM + were added in C++14 the list will not contain C++03 or C++11. + """ + dialects = set() + for feature in self.__data: + keys = feature["values"].keys() + assert len(keys) > 0, "'values' is empty" + dialects |= keys + + return sorted(list(dialects)) + + @functools.cached_property + def standard_ftms(self) -> Dict[str, Dict[str, Any]]: + """Returns the FTM versions per dialect in the Standard. + + This function does not use the 'implemented' flag. The output contains + the versions used in the Standard. When a FTM in libc++ is not + implemented according to the Standard to output may opt to show the + expected value. + + The result is a dict with the following content + - key: Name of the feature test macro. + - value: A dict with the following content: + * key: The version of the C++ dialect. + * value: The value of the feature-test macro. + """ + return get_ftms(self.__data, self.std_dialects, False) + + @functools.cached_property + def implemented_ftms(self) -> Dict[str, Dict[str, Any]]: + """Returns the FTM versions per dialect implemented in libc++. + + Unlike `get_std_dialect_versions` this function uses the 'implemented' + flag. This returns the actual implementation status in libc++. + + The result is a dict with the following content + - key: Name of the feature test macro. + - value: A dict with the following content: + * key: The version of the C++ dialect. + * value: The value of the feature-test macro. When a feature-test + macro is not implemented its value is None. + """ + + return get_ftms(self.__data, self.std_dialects, True) + + def main(): produce_version_header() produce_tests()