From cf7fd7d7c23f0ca7dfacee57157c7f565660c722 Mon Sep 17 00:00:00 2001 From: Eli Date: Fri, 4 Sep 2026 21:08:05 -0400 Subject: [PATCH] Generate API docs with `sphinx-autodoc2` `sphinx.ext.autodoc` imports the package and introspects live objects; `sphinx-autodoc2` reads the source with `astroid` instead. Two things motivate the switch. The docstrings in `handlers.llm` are Markdown, and RST rendered them as literal `##` and broken fence anchors. They are model-facing -- the legibility framework and every `PromptInjectingInterpretation` inject them into the system prompt verbatim, and `serialization` parses their ATX headings to rebase heading levels -- so the renderer moves to the docstrings rather than the other way round. `autodoc2_docstring_parser_regexes` names the six that are Markdown; the rest of the package stays reStructuredText. The hand-written `.. autofunction::` overrides also go away. They existed because `automodule :members:` skips names whose `__module__` differs: the torch wrappers `functools.wraps` relabels as `torch.func`, and the jax operations re-exported from `._handlers`. Static analysis sees the `def` regardless, so all eleven torch functions render on their own; jax needs an `__all__` to resolve its re-exports, since analysis never follows imports. Two configuration choices are less obvious than they look. `autodoc2_render_plugin` stays `rst`: MyST pages would make MyST the default parser for every docstring, and CommonMark reads a `>>>` prompt as a blockquote, so `ops.syntax` alone turned 31 doctest blocks into 186 nested quotes. And `autodoc2_docstrings` is `direct`, not `all`, despite the latter looking like the analogue of `autodoc_inherit_docstrings`: the old `effectful.rst` never passed `:inherited-members:`, and `all` repeats a base class's docstring on every subclass -- 40 copies of `_DistributionTerm` across the numpyro wrappers, `Operation`'s doctests onto two subclasses in `ops.syntax`, and stdlib boilerplate onto classes whose base is a `dict` or an `ABC`. The API reference is now a page per module under `apidocs/`, so `effectful.rst` becomes a redirect and per-object anchors no longer resolve. `sphinx_autodoc_typehints` is dropped because it only hooks autodoc events, and `sphinxcontrib-bibtex` because nothing used it. The docs jobs keep their unpinned interpreter. `sphinx-autodoc2` pins `astroid<4`, so 3.14 looked like a risk, but building under 3.14.7 -- what `uv python install` resolves to -- gives an API tree byte-identical to 3.13, the same 30 warnings and the same 36 `_modules` pages. Static analysis does not see everything an import does. PEP 695 `type` aliases (including `Expr`), definitions inside module-level `try:` or `if TYPE_CHECKING:` blocks (`Encodable`), `@overload`-only members, and six of the seven torch functorch wrappers' signatures are absent, and per-module `:private-members:` is no longer expressible. --- .gitignore | 3 + docs/source/conf.py | 48 ++++- docs/source/effectful.rst | 294 +---------------------------- docs/source/index.rst | 11 +- effectful/handlers/jax/__init__.py | 2 + pyproject.toml | 5 +- 6 files changed, 64 insertions(+), 299 deletions(-) diff --git a/.gitignore b/.gitignore index 25b344a5e..6a169acdc 100644 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,9 @@ venv/ docs/source/lightning_logs docs/preconvert docs/build +docs/_build +# Written into the source tree by sphinx-autodoc2 at build time. +docs/source/apidocs/ site/ out *.matrix.gz diff --git a/docs/source/conf.py b/docs/source/conf.py index be54e2e36..b49996ce6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,27 +28,61 @@ # ones. extensions = [ "nbsphinx", - "sphinx.ext.autodoc", "sphinx.ext.imgconverter", "sphinx.ext.intersphinx", "sphinx.ext.mathjax", "sphinx.ext.viewcode", "myst_parser", "sphinxcontrib.jquery", - "sphinx_autodoc_typehints", + "autodoc2", ] -typehints_use_signature = True -typehints_use_signature_return = True +autodoc2_packages = [{"path": "../../effectful", "module": "effectful"}] +autodoc2_render_plugin = "rst" -# Enable documentation inheritance -autodoc_inherit_docstrings = True +# The ``automodule`` blocks this replaced all set ``:undoc-members:``, so leave +# "undoc" out: adding it drops roughly 40% of the published objects. +autodoc2_hidden_objects = ["inherited", "private", "dunder"] + +# Document only an object's own docstring. Falling back to a base class's +# repeats it once per subclass -- the 40 distribution wrappers in +# ``handlers.numpyro`` would all carry ``_DistributionTerm``'s text -- and +# reaches stdlib boilerplate for classes whose base is a ``dict`` or an ``ABC``. +autodoc2_docstrings = "direct" # Render the ``__init__`` docstring alongside the class docstring. The LLM # harness handlers rely on this split: a handler's class docstring is injected # into the model's system prompt, so constructor documentation -- which only # the caller of ``Handler(...)`` can act on -- lives on ``__init__`` instead. -autoclass_content = "both" +autodoc2_class_docstring = "merge" + +# ``effectful.handlers.jax`` re-exports its operations from ``._handlers``, which +# static analysis cannot follow; resolving through ``__all__`` recovers them. +# Scoped to that one module: a regex matching a module with no ``__all__`` raises +# an uncaught NoAllError, and a matched module loses its submodule toctree. +autodoc2_module_all_regexes = [r"effectful\.handlers\.jax"] + +# These docstrings are Markdown because they double as model-facing prompts (see +# ``PromptInjectingInterpretation``). Named individually rather than by subtree: +# most of ``handlers.llm`` is reStructuredText, and the ``:parser:`` path this +# option takes bloats doctrees, so its blast radius is kept small. +autodoc2_docstring_parser_regexes = [ + (r"effectful\.handlers\.llm\.types", "myst"), + (r"effectful\.handlers\.llm\.types\.(Tool|Skill|Agent)", "myst"), + (r"effectful\.handlers\.llm\.harness\.hooks\.AgentLoop", "myst"), + ( + r"effectful\.handlers\.llm\.harness\.synthesis\.snippet" + r"\.StatefulReplSynthesizer", + "myst", + ), + ( + r"effectful\.handlers\.llm\.harness\.durability\.retrying\.TenacityRetryer", + "myst", + ), +] + +# One per ``singledispatch`` registration written as ``def _(...)``. +suppress_warnings = ["autodoc2.dup_item"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: diff --git a/docs/source/effectful.rst b/docs/source/effectful.rst index 17c97d4ea..cc45760e8 100644 --- a/docs/source/effectful.rst +++ b/docs/source/effectful.rst @@ -1,294 +1,10 @@ +:orphan: + Effectful ========= -Operations ----------- - -.. automodule:: effectful.ops - :members: - :undoc-members: - -Syntax -^^^^^^ - -.. automodule:: effectful.ops.syntax - :members: - :undoc-members: - - .. autofunction:: effectful.ops.syntax.defdata(value: Term[T]) -> Expr[T] - -Semantics -^^^^^^^^^ - -.. automodule:: effectful.ops.semantics - :members: - :undoc-members: - -Types -^^^^^ - -.. automodule:: effectful.ops.types - :members: - :undoc-members: - - -Handlers --------- - -.. automodule:: effectful.handlers - :members: - :undoc-members: - - -LLM -^^^ - -.. automodule:: effectful.handlers.llm - :members: - :undoc-members: - -Types -""""" - -.. automodule:: effectful.handlers.llm.types - :members: - :undoc-members: - -Harness -""""""" - -.. automodule:: effectful.handlers.llm.harness - :members: - :undoc-members: - -Command-line launcher -~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.__main__ - :members: - :undoc-members: - -Hooks -~~~~~ - -.. automodule:: effectful.handlers.llm.harness.hooks - :members: - :undoc-members: - -Serialization -~~~~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.serialization - :members: - :undoc-members: - -Provision -~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.provision - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.provision.litellm - :members: - :undoc-members: - -Legibility -~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.legibility - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.legibility.framework - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.legibility.lexical - :members: - :undoc-members: - -Execution -~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.execution - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.execution.hooks - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.execution.builtin - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.execution.restricted - :members: - :undoc-members: - :private-members: - -Validation -~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.validation - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.validation.hooks - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.validation.pydantic - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.validation.mypy - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.validation.ty - :members: - :undoc-members: - -Synthesis -~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.synthesis - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.synthesis.snippet - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.synthesis.function - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.synthesis.body - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.synthesis.toolcall - :members: - :undoc-members: - -Durability -~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.durability - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.durability.transaction - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.durability.retrying - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.durability.persistence - :members: - :undoc-members: - -Observability -~~~~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.observability - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.observability.rich - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.observability.dump - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.observability.langfuse - :members: - :undoc-members: - - -Jax -^^^ - -.. automodule:: effectful.handlers.jax - :members: - :undoc-members: - - .. autofunction:: effectful.handlers.jax.bind_dims - .. autofunction:: effectful.handlers.jax.jax_getitem - .. autofunction:: effectful.handlers.jax.jit - .. autofunction:: effectful.handlers.jax.sizesof - .. autofunction:: effectful.handlers.jax.unbind_dims - -.. automodule:: effectful.handlers.jax.numpy - :members: - :undoc-members: - -.. automodule:: effectful.handlers.jax.scipy - :members: - :undoc-members: - - -Numpyro -^^^^^^^ - -.. automodule:: effectful.handlers.numpyro - :members: - :undoc-members: - -Pyro -^^^^ - -.. automodule:: effectful.handlers.pyro - :members: - :undoc-members: - -Torch -^^^^^ - -.. automodule:: effectful.handlers.torch - :members: - :undoc-members: - - .. autofunction:: effectful.handlers.torch.grad - .. autofunction:: effectful.handlers.torch.jacfwd - .. autofunction:: effectful.handlers.torch.jacrev - .. autofunction:: effectful.handlers.torch.hessian - .. autofunction:: effectful.handlers.torch.jvp - .. autofunction:: effectful.handlers.torch.vjp - .. autofunction:: effectful.handlers.torch.vmap - -Indexed -^^^^^^^ - -.. automodule:: effectful.handlers.indexed - :members: - :undoc-members: - - -Internals ---------- - -.. automodule:: effectful.internals - :members: - :undoc-members: - -Runtime -^^^^^^^ - -.. automodule:: effectful.internals.runtime - :members: - :undoc-members: +.. raw:: html -Unification -^^^^^^^^^^^ + -.. automodule:: effectful.internals.unification - :members: - :undoc-members: +The API reference has moved to :doc:`apidocs/index`. diff --git a/docs/source/index.rst b/docs/source/index.rst index 92aa02071..d0e037af6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,7 +22,16 @@ Table of Contents :maxdepth: 2 :caption: Documentation - effectful + apidocs/index + +.. Resolving ``effectful.handlers.jax`` through its ``__all__`` is what surfaces + its re-exported operations, but it also empties that package's generated + submodule toctree, so its one subpackage is linked here instead. + +.. toctree:: + :hidden: + + apidocs/effectful/effectful.handlers.jax.numpy Indices and Tables ================== diff --git a/effectful/handlers/jax/__init__.py b/effectful/handlers/jax/__init__.py index 44ca93cc6..b0fcade20 100644 --- a/effectful/handlers/jax/__init__.py +++ b/effectful/handlers/jax/__init__.py @@ -12,3 +12,5 @@ from ._handlers import jit as jit from ._handlers import sizesof as sizesof from ._handlers import unbind_dims as unbind_dims + +__all__ = ["bind_dims", "jax_getitem", "jit", "sizesof", "unbind_dims"] diff --git a/pyproject.toml b/pyproject.toml index 6020fd5c7..a571a0eb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,11 +69,12 @@ docs = [ # to an editor over the Agent Client Protocol. Imported as `acp`. "agent-client-protocol>=0.12,<0.13", "sphinx", - "sphinxcontrib-bibtex", "sphinx_rtd_theme", "myst-parser", "nbsphinx", - "sphinx_autodoc_typehints>=3.6,<3.9", + # Generates the API reference by static analysis instead of by importing the + # package. + "sphinx-autodoc2>=0.5,<0.6", "pypandoc_binary<1.16", ] test = [