Document variables by default (#575)

* document variables by default, even if they have no docstring or type annotation. refs #574 * [autofix.ci] apply automated fixes * fixup * tests++ --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
mitmproxy · Jun 19, 2023 · 2f767b9 · 2f767b9
1 parent 82bd69e
commit 2f767b9
Show file tree

Hide file tree

Showing 24 changed files with 679 additions and 76 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,10 @@
 
 <!-- ✨ You do not need to add a pull request reference or an author, this will be added automatically by CI. ✨ -->
 
+ - pdoc now documents variables by default, even if they have no docstring or type annotation.
+   Please make yourself heard in [#574](https://github.com/mitmproxy/pdoc/issues/574) if that
+   introduces significant issues for your use case.
+   ([#575](https://github.com/mitmproxy/pdoc/pull/575), @mhils)
  - Add support for Python 3.12.
    ([#570](https://github.com/mitmproxy/pdoc/pull/570), @mhils)
  - Remove support for Python 3.7, which has reached end-of-life on 2023-06-27.

diff --git a/pdoc/doc.py b/pdoc/doc.py
@@ -489,7 +489,7 @@ def include(name: str) -> bool:
         return submodules
 
     @cached_property
-    def _documented_members(self) -> set[str]:
+    def _ast_keys(self) -> set[str]:
         return (
             self._var_docstrings.keys()
             | self._func_docstrings.keys()
@@ -525,24 +525,21 @@ def _member_objects(self) -> dict[str, Any]:
         else:
             # Starting with Python 3.10, __annotations__ is created on demand,
             # so we make a copy here as obj.__dict__ is changed while we iterate over it.
-            # Additionally, accessing self._documented_members may lead to the execution of TYPE_CHECKING blocks,
+            # Additionally, accessing self._ast_keys may lead to the execution of TYPE_CHECKING blocks,
             # which may also modify obj.__dict__. (https://github.com/mitmproxy/pdoc/issues/351)
             for name, obj in list(self.obj.__dict__.items()):
-                # We already exclude everything here that is imported, only a TypeVar,
-                # or a variable without annotation and docstring.
-                # If one needs to document one of these things, __all__ is the correct way.
+                # We already exclude everything here that is imported.
                 obj_module = inspect.getmodule(obj)
                 declared_in_this_module = self.obj.__name__ == _safe_getattr(
                     obj_module, "__name__", None
                 )
-                include_in_docs = name in self._documented_members or (
-                    declared_in_this_module and not isinstance(obj, TypeVar)
-                )
+                include_in_docs = declared_in_this_module or name in self._ast_keys
                 if include_in_docs:
                     members[name] = obj
+
             for name in self._var_docstrings:
                 members.setdefault(name, empty)
-            for name in self._func_docstrings:
+            for name in self._var_annotations:
                 members.setdefault(name, empty)
 
             members, notfound = doc_ast.sort_by_source(self.obj, {}, members)
@@ -1087,6 +1084,14 @@ def is_classvar(self) -> bool:
         else:
             return False
 
+    @cached_property
+    def is_typevar(self) -> bool:
+        """`True` if the variable is a `typing.TypeVar`, `False` otherwise."""
+        if isinstance(self.default_value, TypeVar):
+            return True
+        else:
+            return False
+
     @cached_property
     def is_enum_member(self) -> bool:
         """`True` if the variable is an enum member, `False` otherwise."""

diff --git a/pdoc/doc_ast.py b/pdoc/doc_ast.py
@@ -13,6 +13,7 @@
 from itertools import tee
 from itertools import zip_longest
 import types
+from typing import TYPE_CHECKING
 from typing import Any
 from typing import TypeVar
 from typing import overload
@@ -23,6 +24,9 @@
 from ._compat import ast_unparse
 from ._compat import cache
 
+if TYPE_CHECKING:
+    import pdoc.doc_types
+
 
 def get_source(obj: Any) -> str:
     """
@@ -90,7 +94,7 @@ class AstInfo:
     """A qualname -> docstring mapping."""
     func_docstrings: dict[str, str]
     """A qualname -> docstring mapping for functions."""
-    annotations: dict[str, str]
+    annotations: dict[str, str | type[pdoc.doc_types.empty]]
     """A qualname -> annotation mapping.
     
     Annotations are not evaluated by this module and only returned as strings."""

diff --git a/pdoc/doc_types.py b/pdoc/doc_types.py
@@ -33,7 +33,7 @@ class empty:
         pass
 
 
-empty = inspect.Signature.empty  # type: ignore  # noqa
+empty: type = inspect.Signature.empty  # type: ignore  # noqa
 """
 A "special" object signaling the absence of a type annotation. 
 This is useful to distinguish it from an actual annotation with `None`.

diff --git a/pdoc/templates/default/module.html.jinja2 b/pdoc/templates/default/module.html.jinja2
@@ -243,12 +243,14 @@ See https://pdoc.dev/docs/pdoc/render_helpers.html#DefaultMacroExtension for an
         true
     {% elif doc.name == "__doc__" %}
         {# We don't want to document __doc__ itself, https://github.com/mitmproxy/pdoc/issues/235 #}
+    {% elif doc.kind == "variable" and doc.is_typevar and not doc.docstring %}
+        {# do not document TypeVars, that only clutters the docs. #}
     {% elif doc.kind == "module" and doc.fullname not in all_modules %}
         {# Skip modules that were manually excluded, https://github.com/mitmproxy/pdoc/issues/334 #}
     {% elif (doc.qualname or doc.name) is in(module.obj.__all__ or []) %}
         {# members starting with an underscore are still public if mentioned in __all__ #}
         true
-    {% elif not doc.name.startswith("_") and (doc.kind != "variable" or doc.is_enum_member or doc.docstring) %}
+    {% elif not doc.name.startswith("_") %}
         {# members not starting with an underscore are considered public by default #}
         true
     {% endif %}

diff --git a/test/test_doc.py b/test/test_doc.py
@@ -46,6 +46,16 @@ def test_attrs():
     assert c.staticmethods
     assert c.methods
 
+    e = m.members["EnumDemo"]
+    assert isinstance(e, Class)
+    v = e.members["RED"]
+    assert isinstance(v, Variable)
+    assert v.is_enum_member
+
+    c = m.members["FOO_CONSTANT"]
+    assert isinstance(c, Variable)
+    assert not c.is_enum_member
+
 
 def test_all_with_import_err():
     mod = extract.load_module(extract.parse_spec(here / "import_err"))

diff --git a/test/test_snapshot.py b/test/test_snapshot.py
@@ -95,6 +95,7 @@ def outfile(self, format: str) -> Path:
 
 
 snapshots = [
+    Snapshot("ast_parsing"),
     Snapshot("demo", min_version=(3, 9)),
     Snapshot("flavors_google"),
     Snapshot("flavors_numpy"),

diff --git a/test/testdata/ast_parsing.html b/test/testdata/ast_parsing.html
diff --git a/test/testdata/ast_parsing.py b/test/testdata/ast_parsing.py
@@ -0,0 +1,5 @@
+class Foo:
+    def __init__(self):
+        self.no_docstring = 42
+        self.with_docstring = 43
+        """This is an attribute docstring."""
diff --git a/test/testdata/ast_parsing.txt b/test/testdata/ast_parsing.txt
@@ -0,0 +1,7 @@
+<module ast_parsing
+    <class ast_parsing.Foo
+        <method def __init__(self): ...>
+        <var no_docstring>
+        <var with_docstring  # This is an attribute…>
+    >
+>
diff --git a/test/testdata/demo_long.html b/test/testdata/demo_long.html
@@ -35,6 +35,9 @@ <h2>API Documentation</h2>
             <li>
                     <a class="variable" href="#FOO_SINGLETON">FOO_SINGLETON</a>
             </li>
+            <li>
+                    <a class="variable" href="#NO_DOCSTRING">NO_DOCSTRING</a>
+            </li>
             <li>
                     <a class="function" href="#a_simple_function">a_simple_function</a>
             </li>
@@ -56,6 +59,9 @@ <h2>API Documentation</h2>
                         <li>
                                 <a class="variable" href="#Foo.a_constructor_only_attribute">a_constructor_only_attribute</a>
                         </li>
+                        <li>
+                                <a class="variable" href="#Foo.undocumented_constructor_attribute">undocumented_constructor_attribute</a>
+                        </li>
                         <li>
                                 <a class="function" href="#Foo.a_regular_function">a_regular_function</a>
                         </li>
@@ -116,6 +122,9 @@ <h2>API Documentation</h2>
             <li>
                     <a class="variable" href="#CONST_B">CONST_B</a>
             </li>
+            <li>
+                    <a class="variable" href="#CONST_NO_DOC">CONST_NO_DOC</a>
+            </li>
             <li>
                     <a class="class" href="#DataDemo">DataDemo</a>
                             <ul class="memberlist">
@@ -125,6 +134,15 @@ <h2>API Documentation</h2>
                         <li>
                                 <a class="variable" href="#DataDemo.a">a</a>
                         </li>
+                        <li>
+                                <a class="variable" href="#DataDemo.a2">a2</a>
+                        </li>
+                        <li>
+                                <a class="variable" href="#DataDemo.a3">a3</a>
+                        </li>
+                        <li>
+                                <a class="variable" href="#DataDemo.a4">a4</a>
+                        </li>
                         <li>
                                 <a class="variable" href="#DataDemo.b">b</a>
                         </li>
@@ -272,7 +290,7 @@ <h1 id="a-second-section">A Second Section</h1>
 </span><span id="L-59"><a href="#L-59"><span class="linenos"> 59</span></a><span class="sd">&quot;&quot;&quot;</span>
 </span><span id="L-60"><a href="#L-60"><span class="linenos"> 60</span></a>
 </span><span id="L-61"><a href="#L-61"><span class="linenos"> 61</span></a><span class="n">NO_DOCSTRING</span><span class="p">:</span> <span class="nb">int</span>
-</span><span id="L-62"><a href="#L-62"><span class="linenos"> 62</span></a><span class="c1"># this variable has a type annotation but not docstring, so it does not show up.</span>
+</span><span id="L-62"><a href="#L-62"><span class="linenos"> 62</span></a><span class="c1"># this variable has a type annotation but not docstring.</span>
 </span><span id="L-63"><a href="#L-63"><span class="linenos"> 63</span></a>
 </span><span id="L-64"><a href="#L-64"><span class="linenos"> 64</span></a>
 </span><span id="L-65"><a href="#L-65"><span class="linenos"> 65</span></a><span class="k">def</span> <span class="nf">a_simple_function</span><span class="p">(</span><span class="n">a</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
@@ -532,6 +550,17 @@ <h1 id="a-second-section">A Second Section</h1>
 </div>
 
 
+                </section>
+                <section id="NO_DOCSTRING">
+                    <div class="attr variable">
+            <span class="name">NO_DOCSTRING</span><span class="annotation">: int</span>
+
+
+    </div>
+    <a class="headerlink" href="#NO_DOCSTRING"></a>
+
+
+
                 </section>
                 <section id="a_simple_function">
                             <input id="a_simple_function-view-source" class="view-source-toggle-state" type="checkbox" aria-hidden="true" tabindex="-1">
@@ -736,6 +765,17 @@ <h1 id="a-second-section">A Second Section</h1>
 </div>
 
 
+                            </div>
+                            <div id="Foo.undocumented_constructor_attribute" class="classattr">
+                                <div class="attr variable">
+            <span class="name">undocumented_constructor_attribute</span>
+
+
+    </div>
+    <a class="headerlink" href="#Foo.undocumented_constructor_attribute"></a>
+
+
+
                             </div>
                             <div id="Foo.a_regular_function" class="classattr">
                                         <input id="Foo.a_regular_function-view-source" class="view-source-toggle-state" type="checkbox" aria-hidden="true" tabindex="-1">
@@ -933,6 +973,7 @@ <h5>Inherited Members</h5>
                 <dd id="Bar.an_attribute" class="variable"><a href="#Foo.an_attribute">an_attribute</a></dd>
                 <dd id="Bar.a_class_attribute" class="variable"><a href="#Foo.a_class_attribute">a_class_attribute</a></dd>
                 <dd id="Bar.a_constructor_only_attribute" class="variable"><a href="#Foo.a_constructor_only_attribute">a_constructor_only_attribute</a></dd>
+                <dd id="Bar.undocumented_constructor_attribute" class="variable"><a href="#Foo.undocumented_constructor_attribute">undocumented_constructor_attribute</a></dd>
                 <dd id="Bar.a_regular_function" class="function"><a href="#Foo.a_regular_function">a_regular_function</a></dd>
                 <dd id="Bar.a_property" class="variable"><a href="#Foo.a_property">a_property</a></dd>
                 <dd id="Bar.a_cached_property" class="variable"><a href="#Foo.a_cached_property">a_cached_property</a></dd>
@@ -1127,6 +1168,7 @@ <h5>Inherited Members</h5>
                 <dd id="DoubleInherit.an_attribute" class="variable"><a href="#Foo.an_attribute">an_attribute</a></dd>
                 <dd id="DoubleInherit.a_class_attribute" class="variable"><a href="#Foo.a_class_attribute">a_class_attribute</a></dd>
                 <dd id="DoubleInherit.a_constructor_only_attribute" class="variable"><a href="#Foo.a_constructor_only_attribute">a_constructor_only_attribute</a></dd>
+                <dd id="DoubleInherit.undocumented_constructor_attribute" class="variable"><a href="#Foo.undocumented_constructor_attribute">undocumented_constructor_attribute</a></dd>
                 <dd id="DoubleInherit.a_regular_function" class="function"><a href="#Foo.a_regular_function">a_regular_function</a></dd>
                 <dd id="DoubleInherit.a_property" class="variable"><a href="#Foo.a_property">a_property</a></dd>
                 <dd id="DoubleInherit.a_cached_property" class="variable"><a href="#Foo.a_cached_property">a_cached_property</a></dd>
@@ -1156,6 +1198,18 @@ <h5>Inherited Members</h5>
 </div>
 
 
+                </section>
+                <section id="CONST_NO_DOC">
+                    <div class="attr variable">
+            <span class="name">CONST_NO_DOC</span>        =
+<span class="default_value">&#39;SHOULD NOT APPEAR&#39;</span>
+
+
+    </div>
+    <a class="headerlink" href="#CONST_NO_DOC"></a>
+
+
+
                 </section>
                 <section id="DataDemo">
                             <input id="DataDemo-view-source" class="view-source-toggle-state" type="checkbox" aria-hidden="true" tabindex="-1">
@@ -1220,6 +1274,41 @@ <h5>Inherited Members</h5>
 </div>
 
 
+                            </div>
+                            <div id="DataDemo.a2" class="classattr">
+                                <div class="attr variable">
+            <span class="name">a2</span><span class="annotation">: Sequence[str]</span>
+
+
+    </div>
+    <a class="headerlink" href="#DataDemo.a2"></a>
+
+
+
+                            </div>
+                            <div id="DataDemo.a3" class="classattr">
+                                <div class="attr variable">
+            <span class="name">a3</span>        =
+<span class="default_value">&#39;a3&#39;</span>
+
+
+    </div>
+    <a class="headerlink" href="#DataDemo.a3"></a>
+
+
+
+                            </div>
+                            <div id="DataDemo.a4" class="classattr">
+                                <div class="attr variable">
+            <span class="name">a4</span><span class="annotation">: str</span>        =
+<span class="default_value">&#39;a4&#39;</span>
+
+
+    </div>
+    <a class="headerlink" href="#DataDemo.a4"></a>
+
+
+
                             </div>
                             <div id="DataDemo.b" class="classattr">
                                 <div class="attr variable">
@@ -1288,6 +1377,9 @@ <h5>Inherited Members</h5>
                                 <dl>
                                     <div><dt><a href="#DataDemo">DataDemo</a></dt>
                                 <dd id="DataDemoExtended.a" class="variable"><a href="#DataDemo.a">a</a></dd>
+                <dd id="DataDemoExtended.a2" class="variable"><a href="#DataDemo.a2">a2</a></dd>
+                <dd id="DataDemoExtended.a3" class="variable"><a href="#DataDemo.a3">a3</a></dd>
+                <dd id="DataDemoExtended.a4" class="variable"><a href="#DataDemo.a4">a4</a></dd>
                 <dd id="DataDemoExtended.b" class="variable"><a href="#DataDemo.b">b</a></dd>
 
             </div>

diff --git a/test/testdata/demo_long.py b/test/testdata/demo_long.py
@@ -59,7 +59,7 @@
 """
 
 NO_DOCSTRING: int
-# this variable has a type annotation but not docstring, so it does not show up.
+# this variable has a type annotation but not docstring.
 
 
 def a_simple_function(a: str) -> str:

diff --git a/test/testdata/demo_long.txt b/test/testdata/demo_long.txt
@@ -3,6 +3,7 @@
 This …
     <var FOO_CONSTANT: int = 42  # A happy constant. ✨ …>
     <var FOO_SINGLETON: demo_long.Foo  # This variable is ann…>
+    <var NO_DOCSTRING: int>
     <function def a_simple_function(a: str) -> str: ...  # This is a basic modu…>
     <var T = ~T>
     <function def a_complex_function(

diff --git a/test/testdata/demopackage_dir.html b/test/testdata/demopackage_dir.html