diff --git a/.gitignore b/.gitignore index 25b344a5e..6a169acdc 100644 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,9 @@ venv/ docs/source/lightning_logs docs/preconvert docs/build +docs/_build +# Written into the source tree by sphinx-autodoc2 at build time. +docs/source/apidocs/ site/ out *.matrix.gz diff --git a/docs/source/conf.py b/docs/source/conf.py index be54e2e36..b49996ce6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,27 +28,61 @@ # ones. extensions = [ "nbsphinx", - "sphinx.ext.autodoc", "sphinx.ext.imgconverter", "sphinx.ext.intersphinx", "sphinx.ext.mathjax", "sphinx.ext.viewcode", "myst_parser", "sphinxcontrib.jquery", - "sphinx_autodoc_typehints", + "autodoc2", ] -typehints_use_signature = True -typehints_use_signature_return = True +autodoc2_packages = [{"path": "../../effectful", "module": "effectful"}] +autodoc2_render_plugin = "rst" -# Enable documentation inheritance -autodoc_inherit_docstrings = True +# The ``automodule`` blocks this replaced all set ``:undoc-members:``, so leave +# "undoc" out: adding it drops roughly 40% of the published objects. +autodoc2_hidden_objects = ["inherited", "private", "dunder"] + +# Document only an object's own docstring. Falling back to a base class's +# repeats it once per subclass -- the 40 distribution wrappers in +# ``handlers.numpyro`` would all carry ``_DistributionTerm``'s text -- and +# reaches stdlib boilerplate for classes whose base is a ``dict`` or an ``ABC``. +autodoc2_docstrings = "direct" # Render the ``__init__`` docstring alongside the class docstring. The LLM # harness handlers rely on this split: a handler's class docstring is injected # into the model's system prompt, so constructor documentation -- which only # the caller of ``Handler(...)`` can act on -- lives on ``__init__`` instead. -autoclass_content = "both" +autodoc2_class_docstring = "merge" + +# ``effectful.handlers.jax`` re-exports its operations from ``._handlers``, which +# static analysis cannot follow; resolving through ``__all__`` recovers them. +# Scoped to that one module: a regex matching a module with no ``__all__`` raises +# an uncaught NoAllError, and a matched module loses its submodule toctree. +autodoc2_module_all_regexes = [r"effectful\.handlers\.jax"] + +# These docstrings are Markdown because they double as model-facing prompts (see +# ``PromptInjectingInterpretation``). Named individually rather than by subtree: +# most of ``handlers.llm`` is reStructuredText, and the ``:parser:`` path this +# option takes bloats doctrees, so its blast radius is kept small. +autodoc2_docstring_parser_regexes = [ + (r"effectful\.handlers\.llm\.types", "myst"), + (r"effectful\.handlers\.llm\.types\.(Tool|Skill|Agent)", "myst"), + (r"effectful\.handlers\.llm\.harness\.hooks\.AgentLoop", "myst"), + ( + r"effectful\.handlers\.llm\.harness\.synthesis\.snippet" + r"\.StatefulReplSynthesizer", + "myst", + ), + ( + r"effectful\.handlers\.llm\.harness\.durability\.retrying\.TenacityRetryer", + "myst", + ), +] + +# One per ``singledispatch`` registration written as ``def _(...)``. +suppress_warnings = ["autodoc2.dup_item"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: diff --git a/docs/source/effectful.rst b/docs/source/effectful.rst index 17c97d4ea..cc45760e8 100644 --- a/docs/source/effectful.rst +++ b/docs/source/effectful.rst @@ -1,294 +1,10 @@ +:orphan: + Effectful ========= -Operations ----------- - -.. automodule:: effectful.ops - :members: - :undoc-members: - -Syntax -^^^^^^ - -.. automodule:: effectful.ops.syntax - :members: - :undoc-members: - - .. autofunction:: effectful.ops.syntax.defdata(value: Term[T]) -> Expr[T] - -Semantics -^^^^^^^^^ - -.. automodule:: effectful.ops.semantics - :members: - :undoc-members: - -Types -^^^^^ - -.. automodule:: effectful.ops.types - :members: - :undoc-members: - - -Handlers --------- - -.. automodule:: effectful.handlers - :members: - :undoc-members: - - -LLM -^^^ - -.. automodule:: effectful.handlers.llm - :members: - :undoc-members: - -Types -""""" - -.. automodule:: effectful.handlers.llm.types - :members: - :undoc-members: - -Harness -""""""" - -.. automodule:: effectful.handlers.llm.harness - :members: - :undoc-members: - -Command-line launcher -~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.__main__ - :members: - :undoc-members: - -Hooks -~~~~~ - -.. automodule:: effectful.handlers.llm.harness.hooks - :members: - :undoc-members: - -Serialization -~~~~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.serialization - :members: - :undoc-members: - -Provision -~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.provision - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.provision.litellm - :members: - :undoc-members: - -Legibility -~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.legibility - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.legibility.framework - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.legibility.lexical - :members: - :undoc-members: - -Execution -~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.execution - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.execution.hooks - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.execution.builtin - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.execution.restricted - :members: - :undoc-members: - :private-members: - -Validation -~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.validation - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.validation.hooks - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.validation.pydantic - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.validation.mypy - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.validation.ty - :members: - :undoc-members: - -Synthesis -~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.synthesis - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.synthesis.snippet - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.synthesis.function - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.synthesis.body - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.synthesis.toolcall - :members: - :undoc-members: - -Durability -~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.durability - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.durability.transaction - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.durability.retrying - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.durability.persistence - :members: - :undoc-members: - -Observability -~~~~~~~~~~~~~ - -.. automodule:: effectful.handlers.llm.harness.observability - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.observability.rich - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.observability.dump - :members: - :undoc-members: - -.. automodule:: effectful.handlers.llm.harness.observability.langfuse - :members: - :undoc-members: - - -Jax -^^^ - -.. automodule:: effectful.handlers.jax - :members: - :undoc-members: - - .. autofunction:: effectful.handlers.jax.bind_dims - .. autofunction:: effectful.handlers.jax.jax_getitem - .. autofunction:: effectful.handlers.jax.jit - .. autofunction:: effectful.handlers.jax.sizesof - .. autofunction:: effectful.handlers.jax.unbind_dims - -.. automodule:: effectful.handlers.jax.numpy - :members: - :undoc-members: - -.. automodule:: effectful.handlers.jax.scipy - :members: - :undoc-members: - - -Numpyro -^^^^^^^ - -.. automodule:: effectful.handlers.numpyro - :members: - :undoc-members: - -Pyro -^^^^ - -.. automodule:: effectful.handlers.pyro - :members: - :undoc-members: - -Torch -^^^^^ - -.. automodule:: effectful.handlers.torch - :members: - :undoc-members: - - .. autofunction:: effectful.handlers.torch.grad - .. autofunction:: effectful.handlers.torch.jacfwd - .. autofunction:: effectful.handlers.torch.jacrev - .. autofunction:: effectful.handlers.torch.hessian - .. autofunction:: effectful.handlers.torch.jvp - .. autofunction:: effectful.handlers.torch.vjp - .. autofunction:: effectful.handlers.torch.vmap - -Indexed -^^^^^^^ - -.. automodule:: effectful.handlers.indexed - :members: - :undoc-members: - - -Internals ---------- - -.. automodule:: effectful.internals - :members: - :undoc-members: - -Runtime -^^^^^^^ - -.. automodule:: effectful.internals.runtime - :members: - :undoc-members: +.. raw:: html -Unification -^^^^^^^^^^^ + -.. automodule:: effectful.internals.unification - :members: - :undoc-members: +The API reference has moved to :doc:`apidocs/index`. diff --git a/docs/source/index.rst b/docs/source/index.rst index 92aa02071..d0e037af6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,7 +22,16 @@ Table of Contents :maxdepth: 2 :caption: Documentation - effectful + apidocs/index + +.. Resolving ``effectful.handlers.jax`` through its ``__all__`` is what surfaces + its re-exported operations, but it also empties that package's generated + submodule toctree, so its one subpackage is linked here instead. + +.. toctree:: + :hidden: + + apidocs/effectful/effectful.handlers.jax.numpy Indices and Tables ================== diff --git a/effectful/handlers/jax/__init__.py b/effectful/handlers/jax/__init__.py index 44ca93cc6..b0fcade20 100644 --- a/effectful/handlers/jax/__init__.py +++ b/effectful/handlers/jax/__init__.py @@ -12,3 +12,5 @@ from ._handlers import jit as jit from ._handlers import sizesof as sizesof from ._handlers import unbind_dims as unbind_dims + +__all__ = ["bind_dims", "jax_getitem", "jit", "sizesof", "unbind_dims"] diff --git a/pyproject.toml b/pyproject.toml index 6020fd5c7..a571a0eb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,11 +69,12 @@ docs = [ # to an editor over the Agent Client Protocol. Imported as `acp`. "agent-client-protocol>=0.12,<0.13", "sphinx", - "sphinxcontrib-bibtex", "sphinx_rtd_theme", "myst-parser", "nbsphinx", - "sphinx_autodoc_typehints>=3.6,<3.9", + # Generates the API reference by static analysis instead of by importing the + # package. + "sphinx-autodoc2>=0.5,<0.6", "pypandoc_binary<1.16", ] test = [