diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index ae563bc..33c3749 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -112,6 +112,10 @@ jobs: - name: Install test dependencies run: python -m pip --disable-pip-version-check install -r tests/requirements.txt + - name: Setup MSVC compiler + uses: ilammy/msvc-dev-cmd@v1 + if: matrix.os == 'windows-latest' + - name: Test wheel shell: bash run: | diff --git a/cxxheaderparser/options.py b/cxxheaderparser/options.py index 6e0930c..82a59f6 100644 --- a/cxxheaderparser/options.py +++ b/cxxheaderparser/options.py @@ -2,7 +2,7 @@ from typing import Callable, Optional #: arguments are (filename, content) -PreprocessorFunction = Callable[[str, str], str] +PreprocessorFunction = Callable[[str, Optional[str]], str] @dataclass diff --git a/cxxheaderparser/parser.py b/cxxheaderparser/parser.py index a2c031a..18614c5 100644 --- a/cxxheaderparser/parser.py +++ b/cxxheaderparser/parser.py @@ -74,9 +74,10 @@ class CxxParser: def __init__( self, filename: str, - content: str, + content: typing.Optional[str], visitor: CxxVisitor, options: typing.Optional[ParserOptions] = None, + encoding: typing.Optional[str] = None, ) -> None: self.visitor = visitor self.filename = filename @@ -85,6 +86,13 @@ def __init__( if options and options.preprocessor is not None: content = options.preprocessor(filename, content) + if content is None: + if encoding is None: + encoding = "utf-8-sig" + + with open(filename, "r", encoding=encoding) as fp: + content = fp.read() + self.lex: lexer.TokenStream = lexer.LexerTokenStream(filename, content) global_ns = NamespaceDecl([], False) diff --git a/cxxheaderparser/preprocessor.py b/cxxheaderparser/preprocessor.py index 1e5719b..f3f001f 100644 --- a/cxxheaderparser/preprocessor.py +++ b/cxxheaderparser/preprocessor.py @@ -7,6 +7,7 @@ import os import subprocess import sys +import tempfile import typing from .options import PreprocessorFunction @@ -74,7 +75,7 @@ def make_gcc_preprocessor( if not encoding: encoding = "utf-8" - def _preprocess_file(filename: str, content: str) -> str: + def _preprocess_file(filename: str, content: typing.Optional[str]) -> str: cmd = gcc_args + ["-w", "-E", "-C"] for p in include_paths: @@ -86,6 +87,8 @@ def _preprocess_file(filename: str, content: str) -> str: if filename == "": cmd.append("-") filename = "" + if content is None: + raise PreprocessorError("no content specified for stdin") kwargs["input"] = content else: cmd.append(filename) @@ -102,6 +105,110 @@ def _preprocess_file(filename: str, content: str) -> str: return _preprocess_file +# +# Microsoft Visual Studio preprocessor support +# + + +def _msvc_filter(fp: typing.TextIO) -> str: + # MSVC outputs the original file as the very first #line directive + # so we just use that + new_output = io.StringIO() + keep = True + + first = fp.readline() + assert first.startswith("#line") + fname = first[first.find('"') :] + + for line in fp: + if line.startswith("#line"): + keep = line.endswith(fname) + + if keep: + new_output.write(line) + + new_output.seek(0) + return new_output.read() + + +def make_msvc_preprocessor( + *, + defines: typing.List[str] = [], + include_paths: typing.List[str] = [], + retain_all_content: bool = False, + encoding: typing.Optional[str] = None, + msvc_args: typing.List[str] = ["cl.exe"], + print_cmd: bool = True, +) -> PreprocessorFunction: + """ + Creates a preprocessor function that uses cl.exe from Microsoft Visual Studio + to preprocess the input text. cl.exe is not typically on the path, so you + may need to open the correct developer tools shell or pass in the correct path + to cl.exe in the `msvc_args` parameter. + + cl.exe will throw an error if a file referenced by an #include directive is not found. + + :param defines: list of #define macros specified as "key value" + :param include_paths: list of directories to search for included files + :param retain_all_content: If False, only the parsed file content will be retained + :param encoding: If specified any include files are opened with this encoding + :param msvc_args: This is the path to cl.exe and any extra args you might want + :param print_cmd: Prints the command as its executed + + .. code-block:: python + + pp = make_msvc_preprocessor() + options = ParserOptions(preprocessor=pp) + + parse_file(content, options=options) + + """ + + if not encoding: + encoding = "utf-8" + + def _preprocess_file(filename: str, content: typing.Optional[str]) -> str: + cmd = msvc_args + ["/nologo", "/E", "/C"] + + for p in include_paths: + cmd.append(f"/I{p}") + for d in defines: + cmd.append(f"/D{d.replace(' ', '=')}") + + tfpname = None + + try: + kwargs = {"encoding": encoding} + if filename == "": + if content is None: + raise PreprocessorError("no content specified for stdin") + + tfp = tempfile.NamedTemporaryFile( + mode="w", encoding=encoding, suffix=".h", delete=False + ) + tfpname = tfp.name + tfp.write(content) + tfp.close() + + cmd.append(tfpname) + else: + cmd.append(filename) + + if print_cmd: + print("+", " ".join(cmd), file=sys.stderr) + + result: str = subprocess.check_output(cmd, **kwargs) # type: ignore + if not retain_all_content: + result = _msvc_filter(io.StringIO(result)) + finally: + if tfpname: + os.unlink(tfpname) + + return result + + return _preprocess_file + + # # PCPP preprocessor support (not installed by default) # @@ -191,7 +298,7 @@ def make_pcpp_preprocessor( if pcpp is None: raise PreprocessorError("pcpp is not installed") - def _preprocess_file(filename: str, content: str) -> str: + def _preprocess_file(filename: str, content: typing.Optional[str]) -> str: pp = _CustomPreprocessor(encoding, passthru_includes) if include_paths: for p in include_paths: @@ -203,6 +310,10 @@ def _preprocess_file(filename: str, content: str) -> str: if not retain_all_content: pp.line_directive = "#line" + if content is None: + with open(filename, "r", encoding=encoding) as fp: + content = fp.read() + pp.parse(content, filename) if pp.errors: diff --git a/cxxheaderparser/simple.py b/cxxheaderparser/simple.py index 421666b..3efb048 100644 --- a/cxxheaderparser/simple.py +++ b/cxxheaderparser/simple.py @@ -348,7 +348,10 @@ def parse_file( if filename == "-": content = sys.stdin.read() else: - with open(filename, encoding=encoding) as fp: - content = fp.read() + content = None - return parse_string(content, filename=filename, options=options) + visitor = SimpleCxxVisitor() + parser = CxxParser(filename, content, visitor, options) + parser.parse() + + return visitor.data diff --git a/tests/test_preprocessor.py b/tests/test_preprocessor.py index e54f86e..bff60c1 100644 --- a/tests/test_preprocessor.py +++ b/tests/test_preprocessor.py @@ -26,7 +26,7 @@ ) -@pytest.fixture(params=["gcc", "pcpp"]) +@pytest.fixture(params=["gcc", "msvc", "pcpp"]) def make_pp(request) -> typing.Callable[..., PreprocessorFunction]: param = request.param if param == "gcc": @@ -36,6 +36,12 @@ def make_pp(request) -> typing.Callable[..., PreprocessorFunction]: subprocess.run([gcc_path, "--version"]) return preprocessor.make_gcc_preprocessor + elif param == "msvc": + gcc_path = shutil.which("cl.exe") + if not gcc_path: + pytest.skip("cl.exe not found") + + return preprocessor.make_msvc_preprocessor elif param == "pcpp": if preprocessor.pcpp is None: pytest.skip("pcpp not installed")