Add scan option to build packages in subdirectories

.github/workflows/test.yml

@@ -7,42 +7,45 @@ on:
     - main
 
 jobs:
-  install_tests:
-    name: Install check using pipx
-    runs-on: ubuntu-latest
+  venv_tests:
+    name: Install test using venv
     strategy:
       matrix:
-        container: ['ros:iron', 'ros:jazzy', 'ros:rolling']
-    container: ${{ matrix.container }}
+        os: ['ubuntu-22.04', 'ubuntu-24.04']
+    runs-on: ${{ matrix.os }}
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Install pip
-        run: apt update && apt install -y python3-pip python3-pytest
-
-      - name: Install apt dependencies
-        run: apt update && apt install -y doxygen graphviz pipx python3-sphinx
-
-      - name: Smoke test of pipx install
-        run: |
-          PATH="$HOME/.local/bin:$PATH"
-          pipx install .
-          rosdoc2 build -d tmp/docs_build -c /tmp/cross_references -o /tmp/docs_output -p test/packages/full_package
-          if [ ! -f /tmp/docs_output/full_package/index.html ]; then
-            echo "Failed to find any output from rosdoc2"
-            exit 2
-          else
-          echo "rosdoc2 ran successfully under pipx"
-          fi
+    - name: Checkout
+      uses: actions/checkout@v4
 
+    - name: Install apt dependencies
+      run: |
+        sudo apt update && sudo apt install -y \
+        python3-pip \
+        python3-venv \
+        doxygen \
+        graphviz
+
+    - name: Activate python virtual environment
+      run: |
+        python3 -m venv venv
+        source venv/bin/activate
+        echo PATH=$PATH >> $GITHUB_ENV
+
+    - name: Install pip dependencies
+      run: |
+        python3 -m pip install .[test]
+        python3 -m pip freeze
+
+    - name: Run tests
+      run: python3 -m pytest --verbose test
+
   unit_tests:
     name: Unit Tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        container: ['ros:iron', 'ros:rolling']
+        container: ['ros:iron', 'ros:jazzy', 'ros:rolling']
     container: ${{ matrix.container }}
 
     env:

rosdoc2/verbs/build/builders/doxygen_builder.py

@@ -17,6 +17,7 @@
 import os
 import shutil
 import subprocess
+import sys
 
 from ..builder import Builder
 from ..collect_tag_files import collect_tag_files
@@ -243,7 +244,8 @@ def build(self, *, doc_build_folder, output_staging_directory):
         logger.info(
             f"Running Doxygen: '{' '.join(cmd)}' in '{working_directory}'"
         )
-        completed_process = subprocess.run(cmd, cwd=working_directory)
+        completed_process = subprocess.run(
+            cmd, cwd=working_directory, stdout=sys.stdout, stderr=sys.stderr)
         logger.info(
             f"Doxygen exited with return code '{completed_process.returncode}'")
 

rosdoc2/verbs/build/builders/sphinx_builder.py

@@ -19,10 +19,10 @@
 from pathlib import Path
 import shutil
 import subprocess
+import sys
 
 from jinja2 import Template
 import setuptools
-from sphinx.cmd.build import main as sphinx_main
 
 from ..builder import Builder
 from ..collect_inventory_files import collect_inventory_files
@@ -523,7 +523,8 @@ def build(self, *, doc_build_folder, output_staging_directory):
                 logger.info(
                     f"Running sphinx-apidoc: '{' '.join(cmd)}' in '{wrapped_sphinx_directory}'"
                 )
-                completed_process = subprocess.run(cmd, cwd=wrapped_sphinx_directory)
+                completed_process = subprocess.run(cmd, cwd=wrapped_sphinx_directory,
+                                                   stdout=sys.stdout, stderr=sys.stderr)
                 msg = f"sphinx-apidoc exited with return code '{completed_process.returncode}'"
                 if completed_process.returncode == 0:
                     logger.debug(msg)
@@ -533,12 +534,18 @@ def build(self, *, doc_build_folder, output_staging_directory):
         # Invoke Sphinx-build.
         sphinx_output_dir = os.path.abspath(
             os.path.join(wrapped_sphinx_directory, 'sphinx_output'))
+        cmd = [
+            'sphinx-build',
+            wrapped_sphinx_directory,
+            sphinx_output_dir,
+        ]
         logger.info(
-            f"Running sphinx_build with: [{wrapped_sphinx_directory}, '{sphinx_output_dir}]'"
+            f"Running Sphinx-build: '{' '.join(cmd)}' in '{wrapped_sphinx_directory}'"
         )
-        returncode = sphinx_main([wrapped_sphinx_directory, sphinx_output_dir])
-        msg = f"sphinx_build exited with return code '{returncode}'"
-        if returncode == 0:
+        completed_process = subprocess.run(cmd, cwd=wrapped_sphinx_directory,
+                                           stdout=sys.stdout, stderr=sys.stderr)
+        msg = f"Sphinx-build exited with return code '{completed_process.returncode}'"
+        if completed_process.returncode == 0:
             logger.info(msg)
         else:
             raise RuntimeError(msg)

rosdoc2/verbs/scan/__init__.py

@@ -0,0 +1,29 @@
+# Copyright 2024 R. Kent James <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .impl import main
+from .impl import prepare_arguments
+
+__all__ = [
+    'entry_point_data',
+]
+
+entry_point_data = {
+    'verb': 'scan',
+    'description': 'Scan subdirectories looking for packages, then build those packages.',
+    # Called for execution, given parsed arguments object
+    'main': main,
+    # Called first to setup argparse, given argparse parser
+    'prepare_arguments': prepare_arguments,
+}

rosdoc2/verbs/scan/impl.py

@@ -0,0 +1,217 @@
+# Copyright 2024 R. Kent James <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import multiprocessing as mp
+import os
+import signal
+import sys
+import threading
+import time
+
+from catkin_pkg.packages import find_packages_allowing_duplicates
+from rosdoc2.verbs.build.impl import main_impl as build_main_impl
+from rosdoc2.verbs.build.impl import prepare_arguments as build_prepare_arguments
+
+logging.basicConfig(
+    format='[%(name)s] [%(levelname)s] %(message)s', level=logging.INFO)
+logger_scan = logging.getLogger('rosdoc2.scan')
+
+goptions = None
+# Setting the BATCH_SIZE or MAX_PACKAGES to a smaller value may be useful in debugging
+# this module, to reduce run time or isolate sections that cause hangs.
+BATCH_SIZE = 10000
+MAX_PACKAGES = 10000
+WATCHDOG_TIMEOUT = 15 * 60  # Seconds
+
+
+def main(options):
+    """Execute the program, catching errors."""
+    try:
+        return main_impl(options)
+    except Exception as e:  # noqa: B902
+        if options.debug:
+            raise
+        else:
+            sys.exit(str(e))
+
+
+class Struct:
+    """Wrap argparse options to allow copies."""
+
+    def __init__(self, **entries):
+        """Create a dictionary from option entries."""
+        self.__dict__.update(entries)
+
+
+def prepare_arguments(parser):
+    """Add command-line arguments to the argparse object."""
+    # Wrap the builder arguments to include their choices.
+    build_prepare_arguments(parser)
+
+    # Additional options for scan
+    parser.add_argument(
+        '--timeout',
+        '-t',
+        default=WATCHDOG_TIMEOUT,
+        help='maximum time in seconds allowed per package',
+    )
+    parser.add_argument(
+        '--max-packages',
+        '-m',
+        default=MAX_PACKAGES,
+        help='maximum number of packages to process'
+    )
+    return parser
+
+
+def main_impl(options):
+    """Execute the program."""
+    global goptions
+    goptions = options
+
+    if options.install_directory is not None:
+        # Check that the install directory exists.
+        if not os.path.exists(options.install_directory):
+            sys.exit(
+                f"Error: given install directory '{options.install_directory}' does not exist")
+
+    # Locate the packages to document.
+    packages = find_packages_allowing_duplicates(options.package_path)
+    if len(packages) == 0:
+        logger_scan.error(f'No packages found in subdirectories of {options.package_path}')
+        exit(1)
+    max_packages = int(options.max_packages)
+    if len(packages) > max_packages:
+        packages = packages[0:max_packages]
+
+    packages_total = len(packages)
+    packages_done = 0
+    logger_scan.info(f'Processing {packages_total} packages')
+    batch_packages = []
+    batch_no = 0
+    failed_packages = []
+    while len(packages) > 0:
+        batch_no += 1
+        batch_packages.clear()
+        packages = list(packages.values())
+        for i in range(len(packages)):
+            batch_packages.append(packages.pop())
+            if len(batch_packages) >= BATCH_SIZE:
+                break
+        logger_scan.info(f'Begin batch # {batch_no}')
+        for package in batch_packages:
+            logger_scan.info(f'Adding {package.name} for processing')
+        pool = mp.Pool(maxtasksperchild=1)
+        pool_results = pool.imap_unordered(package_impl, batch_packages)
+        while True:
+            try:
+                (package, returns, message) = pool_results.next()
+                packages_done += 1
+                if returns != 0:
+                    logger_scan.warning(f'{package.name} ({packages_done}/{packages_total})'
+                                        f' returned {returns}: {message}')
+                    failed_packages.append((package, returns, message))
+                else:
+                    logger_scan.info(
+                        f'{package.name} successful ({packages_done}/{packages_total})')
+            except StopIteration:
+                break
+            except BaseException as e:  # noqa: B902
+                logger_scan.error(f'Unexpected error in scan: {type(e).__name__ + " " + str(e)}')
+                break
+        logger_scan.info(f'Finished batch {batch_no}')
+        # I'd prefer close() then join() but that seems to sometimes hang.
+        pool.terminate()
+
+    logger_scan.info('scan complete')
+    if len(failed_packages) > 0:
+        print(f'{len(failed_packages)} packages failed:')
+        for failed in failed_packages:
+            (package, returns, message) = failed
+            print(f'{package.name}: retval={returns}: {message}')
+    else:
+        print('All packages succeeded')
+
+
+def _clocktime():
+    return time.strftime('%H:%M:%S')
+
+
+def package_impl(package):
+    """Execute for a single function."""
+    global goptions
+    options = Struct(**goptions.__dict__)
+    package_path = os.path.dirname(package.filename)
+    options.package_path = package_path
+    return_value = 100
+    message = 'Unknown error'
+    start = time.time()
+    had_timeout = threading.Event()
+
+    def watchdog():
+        """Kill the process after a timeout."""
+        time.sleep(float(options.timeout))
+        had_timeout.set()
+        os.kill(os.getpid(), signal.SIGINT)
+    threading.Thread(target=watchdog, daemon=True).start()
+
+    # Generate the doc build directory.
+    os.makedirs(options.doc_build_directory, exist_ok=True)
+
+    print(f'{_clocktime()} Begin processing {package.name}', flush=True)
+    # remap output
+    outfile = open(os.path.join(options.doc_build_directory, f'{package.name}.txt'), 'w')
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = outfile
+    sys.stderr = outfile
+    logging.basicConfig(
+        format='%(asctime)s [%(name)s] [%(levelname)s] %(message)s',
+        level=logging.INFO, stream=outfile, force=True)
+    logger = logging.getLogger('rosdoc2')
+    logger.info(f'Processing package build at {package_path}')
+
+    try:
+        # run rosdoc2 for the package
+        build_main_impl(options)
+        return_value = 0
+        message = 'OK'
+    except RuntimeError as e:
+        return_value = 1
+        message = type(e).__name__ + ' ' + str(e)
+    except KeyboardInterrupt as e:
+        return_value = 2
+        if had_timeout.is_set():
+            e = TimeoutError(f'runtime {"{:.3f}".format(time.time() - start)} seconds')
+        message = type(e).__name__ + ' ' + str(e)
+    except BaseException as e:  # noqa: B902
+        return_value = 3
+        message = type(e).__name__ + ' ' + str(e)
+    finally:
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+        elapsed_time = '{:.3f}'.format(time.time() - start)
+        if return_value != 0:
+            print(f'{_clocktime()} Package at {package_path} failed {return_value}: {message}',
+                  flush=True)
+            logger.error(f'Package at {package_path} failed {return_value}: {message}')
+        else:
+            print(f'{_clocktime()} Package {package.name} succeeded '
+                  f'in {elapsed_time} seconds', flush=True)
+            logger.info(f'Completed rosdoc2 build for {package_path} '
+                        f'in {elapsed_time} seconds')
+        if not outfile.closed:
+            outfile.close()
+        return (package, return_value, message)

setup.cfg

@@ -76,6 +76,7 @@ rosdoc2.verbs =
     build = rosdoc2.verbs.build:entry_point_data
     open = rosdoc2.verbs.open:entry_point_data
     default_config = rosdoc2.verbs.default_config:entry_point_data
+    scan = rosdoc2.verbs.scan:entry_point_data
 console_scripts =
     rosdoc2 = rosdoc2.main:main