refactor: common parser infrastructure for different input types

Enables: #38 and #41

Makefile

@@ -50,6 +50,7 @@ export LATEST_DOXYGEN_VERSION := 1.9.2
 DOCKER_IMAGE_NAME ?= silvester747/asciidoxy
 DOCKER_IMAGE_VERSION ?= testing
 DOCKER_IMAGE_PLATFORM ?= linux/amd64
+LOG_LEVEL ?= WARNING
 
 help:
 	@python3 -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
@@ -161,6 +162,7 @@ visual-test-$(notdir $(basename $(1))): $(patsubst %.toml,%.adoc,$(1))
 		--debug \
 		--require asciidoctor-diagram \
 		--failure-level ERROR \
+		--log $(LOG_LEVEL) \
 		--multipage
 	mv $(VISUAL_TEST_CASE_BUILD_DIR)/debug.json $(VISUAL_TEST_CASE_BUILD_DIR)/$(notdir $(basename $(1))).debug.json
 

asciidoxy/api_reference.py

@@ -13,12 +13,17 @@
 # limitations under the License.
 """API reference storage and search."""
 
+import logging
 import re
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from typing import Dict, List, Optional, Tuple, TypeVar
+from typing import Dict, List, Optional, Set, Tuple, TypeVar
 
-from .model import Compound, ReferableElement
+from tqdm import tqdm
+
+from .model import Compound, ReferableElement, TypeRef
+
+logger = logging.getLogger(__name__)
 
 
 class AmbiguousLookupError(Exception):
@@ -359,11 +364,25 @@ class ApiReference:
     elements: List[ReferableElement]
     _id_index: Dict[str, ReferableElement]
     _name_index: Dict[str, List[ReferableElement]]
+    _unresolved_refs: List[TypeRef]
+    _unchecked_refs: List[TypeRef]
+    _inner_type_refs: List[Tuple[Compound, TypeRef]]
 
     def __init__(self):
         self.elements = []
         self._id_index = {}
         self._name_index = defaultdict(list)
+        self._unresolved_refs = []
+        self._unchecked_refs = []
+        self._inner_type_refs = []
+
+    @property
+    def unresolved_ref_count(self):
+        return len(self._unresolved_refs) + len(self._inner_type_refs)
+
+    @property
+    def unchecked_ref_count(self):
+        return len(self._unchecked_refs)
 
     def append(self, element: ReferableElement) -> None:
         self.elements.append(element)
@@ -448,6 +467,79 @@ def find(self,
 
         raise AmbiguousLookupError(matches)
 
+    def add_unresolved_reference(self, ref: TypeRef) -> None:
+        self._unresolved_refs.append(ref)
+
+    def add_inner_type_reference(self, parent: Compound, ref: TypeRef) -> None:
+        self._inner_type_refs.append((parent, ref))
+
+    def add_unchecked_reference(self, ref: TypeRef) -> None:
+        self._unchecked_refs.append(ref)
+
+    def resolve_references(self, progress: Optional[tqdm] = None) -> None:
+        """Resolve all references between objects from different XML files."""
+
+        unresolved_names: Set[str] = set()
+        if progress is not None:
+            progress.total = len(self._unresolved_refs) + len(self._inner_type_refs)
+
+        still_unresolved_refs = []
+        for ref in self._unresolved_refs:
+            if progress is not None:
+                progress.update()
+            assert ref.name
+            element = self.resolve_reference(ref)
+            if element is not None:
+                ref.resolve(element)
+            else:
+                still_unresolved_refs.append(ref)
+                unresolved_names.add(ref.name)
+
+        still_unresolved_inner_type_refs: List[Tuple[Compound, TypeRef]] = []
+        for parent, ref in self._inner_type_refs:
+            if progress is not None:
+                progress.update()
+            assert ref.name
+            element = self.resolve_reference(ref)
+            if element is not None:
+                assert isinstance(element, Compound)
+                if ref.prot:
+                    element.prot = ref.prot
+                parent.members.append(element)
+            else:
+                still_unresolved_inner_type_refs.append((parent, ref))
+                unresolved_names.add(ref.name)
+
+        resolved_ref_count = len(self._unresolved_refs) - len(still_unresolved_refs)
+        resolved_inner_type_ref_count = (len(self._inner_type_refs) -
+                                         len(still_unresolved_inner_type_refs))
+        unresolved_ref_count = len(still_unresolved_refs) + len(still_unresolved_inner_type_refs)
+        logger.debug(f"Resolved refs: {resolved_ref_count + resolved_inner_type_ref_count}")
+        logger.debug(f"Still unresolved: {unresolved_ref_count}: {', '.join(unresolved_names)}")
+
+        self._unresolved_refs = still_unresolved_refs
+        self._inner_type_refs = still_unresolved_inner_type_refs
+
+    def check_references(self, progress: Optional[tqdm] = None) -> None:
+        """Verify all references point to an existing element."""
+        if progress is not None:
+            progress.total = len(self._unchecked_refs)
+
+        for ref in self._unchecked_refs:
+            if progress is not None:
+                progress.update()
+            if self.resolve_reference(ref) is None:
+                logger.warning(f"Unknown reference id `{ref.id}`. Some XML files may be missing or"
+                               " cannot be parsed.")
+                ref.id = None
+        self._unchecked_refs = []
+
+    def resolve_reference(self, ref: TypeRef) -> Optional[ReferableElement]:
+        try:
+            return self.find(ref.name, target_id=ref.id, lang=ref.language, namespace=ref.namespace)
+        except AmbiguousLookupError:
+            return None
+
 
 MaybeOptionalStr = TypeVar("MaybeOptionalStr", str, Optional[str])
 

asciidoxy/cli.py

@@ -28,7 +28,6 @@
 from .generator import process_adoc
 from .model import json_repr
 from .packaging import CollectError, PackageManager, SpecificationError
-from .parser.doxygen import Driver as DoxygenDriver
 
 
 def error(*args, **kwargs) -> None:
@@ -54,6 +53,7 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
     logger = logging.getLogger(__name__)
 
     pkg_mgr = PackageManager(config.build_dir, config.warnings_are_errors)
+    api_reference = ApiReference()
     if config.spec_file is not None:
         try:
             with tqdm(desc="Collecting packages     ", unit="pkg") as progress:
@@ -65,24 +65,19 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
             logger.exception("Failed to collect packages.")
             sys.exit(1)
 
-        xml_parser = DoxygenDriver(force_language=config.force_language)
         with tqdm(desc="Loading API reference   ", unit="pkg") as progress:
-            pkg_mgr.load_reference(xml_parser, progress)
+            pkg_mgr.load_reference(api_reference, config, progress)
 
         with tqdm(desc="Resolving references    ", unit="ref") as progress:
-            xml_parser.resolve_references(progress)
+            api_reference.resolve_references(progress)
 
         with tqdm(desc="Checking references     ", unit="ref") as progress:
-            xml_parser.check_references(progress)
+            api_reference.check_references(progress)
 
         if config.debug:
             logger.info("Writing debug data, sorry for the delay!")
             with (config.build_dir / "debug.json").open("w", encoding="utf-8") as f:
-                json.dump(xml_parser.api_reference.elements, f, default=json_repr, indent=2)
-
-        api_reference = xml_parser.api_reference
-    else:
-        api_reference = ApiReference()
+                json.dump(api_reference.elements, f, default=json_repr, indent=2)
 
     if config.backend == "adoc":
         pkg_mgr.work_dir = config.destination_dir

asciidoxy/packaging/manager.py

@@ -21,8 +21,10 @@
 
 from tqdm import tqdm
 
+from ..api_reference import ApiReference
+from ..config import Configuration
 from ..document import Document, Package
-from ..parser.doxygen import Driver
+from ..parser import parser_factory
 from .collect import CollectError, collect, specs_from_file
 
 logger = logging.getLogger(__name__)
@@ -141,21 +143,24 @@ def collect(self,
         packages = loop.run_until_complete(collect(specs, download_dir, progress))
         self.packages.update({pkg.name: pkg for pkg in packages})
 
-    def load_reference(self, parser: Driver, progress: Optional[tqdm] = None) -> None:
+    def load_reference(self,
+                       api_reference: ApiReference,
+                       config: Configuration,
+                       progress: Optional[tqdm] = None) -> None:
         """Load API reference from available packages.
 
         Args:
-            parser:   Parser to feed the API reference.
-            progress: Optional progress reporting.
+            api_reference: API reference collection to store loaded reference in.
+            config:        Application configuration.
+            progress:      Optional progress reporting.
         """
         if progress is not None:
             progress.total = len(self.packages)
             progress.update(0)
 
         for pkg in self.packages.values():
-            if pkg.reference_dir is not None:
-                for xml_file in pkg.reference_dir.glob("**/*.xml"):
-                    parser.parse(xml_file)
+            if pkg.reference_type and pkg.reference_dir is not None:
+                parser_factory(pkg.reference_type, api_reference, config).parse(pkg.reference_dir)
             if progress is not None:
                 progress.update()
 

asciidoxy/parser/__init__.py

@@ -12,3 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Parsers for API reference information."""
+
+from .base import ReferenceParserBase
+from .factory import UnsupportedReferenceTypeError, parser_factory
+
+__all__ = "UnsupportedReferenceTypeError", "ReferenceParserBase", "parser_factory"

asciidoxy/parser/base.py

@@ -0,0 +1,45 @@
+# Copyright (C) 2019, TomTom (http://tomtom.com).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Factory methods for API reference information parsers."""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Union
+
+from ..api_reference import ApiReference
+
+
+class ReferenceParserBase(ABC):
+    """Abstract base class for reference parser.
+
+    Attributes:
+        api_reference: API reference holder where parsed reference documentation is added.
+    """
+
+    api_reference: ApiReference
+
+    def __init__(self, api_reference: ApiReference):
+        self.api_reference = api_reference
+
+    @abstractmethod
+    def parse(self, reference_path: Union[Path, str]) -> bool:
+        """Parse reference documentation from the given path.
+
+        Args:
+            reference_path File or directory containing the reference documentation.
+
+        Returns:
+            True if the reference has been parsed. False if the reference path does not contain
+            valid content for this parser.
+        """

asciidoxy/parser/doxygen/__init__.py

@@ -13,6 +13,6 @@
 # limitations under the License.
 """Parser for Doxygen XML output."""
 
-from .driver import Driver, safe_language_tag
+from .parser import Parser, safe_language_tag
 
-__all__ = ["Driver", "safe_language_tag"]
+__all__ = ["Parser", "safe_language_tag"]