Introduce ecoli.library.xarray_emitter

.gitignore

@@ -69,4 +69,4 @@ trace-*
 test_sherlock/
 
 # SMS API #
-.hpc_env
+.hpc_env

configs/test_configs/moving_avg_analysis.json

@@ -0,0 +1,63 @@
+{
+  "experiment_id": "moving_avg",
+  "sim_data_path": "out/kb/simData.cPickle",
+  "suffix_time": false,
+  "fail_at_max_duration": true,
+  "generations": 6,
+  "n_init_sims": 10,
+  "skip_baseline": false,
+  "variants": {
+    "condition": {"condition": {"value": ["with_aa", "acetate"]}}
+  },
+  "emitter": "xarray",
+  "emitter_arg": {
+    "out_dir": "out/experiments",
+    "transducer": {
+      "predicate": [
+        [{"subsample": {"interval": 3}}]
+      ],
+      "buffer": {"size": 200}
+    },
+    "writer": {
+      "threaded": true,
+      "buffers_per_chunk": 1,
+      "backend": "zarr",
+      "backend_config": {
+        "format": 3,
+        "async.concurrency": 3,
+        "threading.max_workers": 3
+      }
+    },
+    "view": [
+      {
+        "root": ["listeners"],
+        "variables": {
+          "fba_results": {
+            "reaction_fluxes": [{
+              "path": "metabolic_fluxes",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }]
+          }
+        }
+      }
+    ]
+  },
+  "analysis_options": {
+    "zarr_mapreduce": {
+      "moving_avg": {
+        "paths": {
+          "result": "moving_avg.zip",
+          "figure": "moving_avg.svg"
+        },
+        "variables": {
+          "metabolic_fluxes": "FUM"
+        },
+        "parameters": {
+          "window": 21,
+          "min_window": 9
+        }
+      }
+    }
+  }
+}

configs/test_configs/test_xarray_emitter.json

@@ -0,0 +1,143 @@
+{
+  "experiment_id": "test_xarray_emitter",
+  "fixed_media": "minimal",
+  "suffix_time": false,
+  "max_duration": 10.0,
+  "fail_at_max_duration": false,
+  "generations": 1,
+  "n_init_sims": 1,
+  "log_updates": true,
+  "variants": {},
+  "emitter": "xarray",
+  "emitter_arg": {
+    "out_dir": "out/tests",
+    "debug": false,
+    "transducer": {
+      "predicate": [
+        [{"subsample": {"interval": 1}}]
+      ],
+      "buffer": {
+        "size": 3
+      }
+    },
+    "writer": {
+      "threaded": true,
+      "buffers_per_chunk": 2,
+      "backend": "zarr",
+      "backend_config": {
+        "format": 3,
+        "async.concurrency": 3,
+        "threading.max_workers": 3
+      }
+    },
+    "view": [
+      {
+        "root": [],
+        "variables": {
+          "bulk": [{
+            "path": "bulk/bulk_molecule",
+            "dtype": "<i8",
+            "codecs": {
+              "filters_v2": [],
+              "filters_v3": [],
+              "compressors_v2": [{
+                "id": "lzma", "check": -1, "preset": null,
+                "format": 3, "filters": [{"id": 33, "preset": 6}]
+              }],
+              "compressors_v3": [{
+                "name": "numcodecs.lzma",
+                "configuration": {
+                  "format": 3, "filters": [{"id": 33, "preset": 6}]
+                }
+              }]
+            }
+          }]
+        }
+      },
+      {
+        "root": ["listeners", "mass"],
+        "metadata": false,
+        "variables": {
+          "cell_mass": [{
+            "path": "gauges/cell_mass",
+            "unit": "[fg]",
+            "dtype": "<f4"
+          }],
+          "dry_mass": [{
+            "path": "gauges/dry_mass",
+            "unit": "[fg]",
+            "dtype": "<f4"
+          }]
+        }
+      },
+      {
+        "root": ["listeners"],
+        "variables": {
+          "fba_results": {
+            "reaction_fluxes": [{
+              "path": "metabolism/fluxes/internal/rxn_global_listener",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }]
+          }
+        }
+      },
+      {
+        "root": ["log_update", "ecoli-metabolism", "listeners"],
+        "variables": {
+          "fba_results": {
+            "coefficient": [{
+              "path": "gauges/mass_per_vol",
+              "unit": "[g/L]",
+              "dtype": "<f4"
+            }],
+            "reaction_fluxes": [{
+              "path": "metabolism/fluxes/internal/rxn",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }],
+            "external_exchange_fluxes": [{
+              "path": "metabolism/fluxes/exchange/molecule",
+              "unit": "[mmol/g.h]",
+              "dtype": "<f4"
+            }]
+          },
+          "enzyme_kinetics": {
+            "counts_to_molar": [{
+              "path": "gauges/conc_per_count",
+              "unit": "[mmol/L]",
+              "dtype": "<f4"
+            }],
+            "actual_fluxes": [{
+              "path": "metabolism/fluxes/internal/rxn_constrained",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }],
+            "target_fluxes": [{
+              "path": "metabolism/fluxes/internal/rxn_target",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }]
+          }
+        }
+      }
+    ]
+  },
+  "analysis_options": {
+    "zarr_mapreduce": {
+      "moving_avg": {
+        "paths": {
+          "result": "moving_avg.zip",
+          "figure": "moving_avg.svg"
+        },
+        "variables": {
+          "bulk/bulk_molecule": "FUM",
+          "metabolism/fluxes/internal/rxn": "FUM"
+        },
+        "parameters": {
+          "window": 3
+        }
+      }
+    }
+  }
+}

doc/apidoc_templates/module.rst_t

@@ -1,7 +1,5 @@
 {{- "``{}``".format(basename) | heading }}
 .. automodule:: {{ qualname }}
-   :private-members:
-   :exclude-members: _abc_impl 
 {%- for option in automodule_options %}
    :{{ option }}:
 {%- endfor %}

doc/composites.rst

@@ -72,6 +72,9 @@ their corresponding topologies to the final composite model.
 - ``generations``: Adds :py:class:`~ecoli.processes.cell_division.StopAfterDivision`
   when ``True`` but only if ``divide`` is ``True``
 
+
+.. _log_updates:
+
 -----------
 Log Updates
 -----------
@@ -90,12 +93,14 @@ The analysis plots located in :py:mod:`~ecoli.analysis.single.blame` can be used
 to visualize these updates.
 
 .. warning::
-    This feature should only be turned on for debugging purposes and
-    only when using the in-memory emitter (see :ref:`ram_emitter`).
+    This feature should only be turned on for debugging purposes, and only when
+    using the :ref:`in-memory emitter <ram_emitter>` or the
+    :py:mod:`ecoli.library.xarray_emitter`.
     The Parquet emitter is unable to serialize many of the objects
     contained in process updates (e.g., nested lists of inconsistent
     depth like ``[[1], 2]``).
 
+
 -------------
 Initial State
 -------------

doc/conf.py

@@ -75,17 +75,43 @@
     ("py:class", "numpy.int64"),
     ("py:class", "numpy.int32"),
     ("py:class", "numpy.bool_"),
+    ("py:class", "numpy._typing._array_like._ScalarType_co"),
     ("py:class", "duckdb.duckdb.DuckDBPyConnection"),
+    ("py:class", "pandas.core.series.Series"),
     ("py:class", "pandas.core.frame.DataFrame"),
     ("py:class", "polars.series.series.Series"),
     ("py:class", "polars.dataframe.frame.DataFrame"),
     ("py:class", "polars.datatypes.classes.DataTypeClass"),
+    ("py:class", "altair.vegalite.v5.api.LayerChart"),
+    ("py:class", "altair.vegalite.v5.api.VConcatChart"),
     # No docs for fsspec
     ("py:class", "fsspec.spec.AbstractFileSystem"),
     ("py:class", "fsspec.core.OpenFile"),
     # Silence warning in ecoli.processes.environment.field_timeline.FieldTimeline
     ("py:class", "vivarium.processes.timeline.TimelineProcess"),
     ("py:class", "concurrent.futures._base.Future"),
+    # Type annotations using library internals
+    ("py:class", "_asyncio.Future"),
+    ("py:class", "asyncio.taskgroups.TaskGroup"),
+    ("py:class", "concurrent.futures._base.Executor"),
+    ("py:class", "unittest.mock._patch"),
+    ("py:class", "xarray.backends.common.ArrayWriter"),
+    ("py:class", "xarray.core.treenode.NodePath"),
+    ("py:class", "zarr.core.group.ConsolidatedMetadata"),
+    ("py:class", "zarr.core._tree.TreeRepr"),
+    # Sphinx does not recognize type parameters in generic classes
+    ("py:class", "ArrT"), ("py:class", "NodeT"), ("py:class", "StoreT"),
+    ("py:class", "ElemT"),
+    ("py:class", "KeyT"), ("py:class", "KeyT1"), ("py:class", "KeyT2"),
+    ("py:class", "InputT"), ("py:class", "ResultT"),
+    ("py:class", "AnalysisConfigT"), ("py:class", "PlotConfigT"),
+    # Sphinx does not recognize type aliases
+    ("py:type", "VariableEncoding"), ("py:class", "VariableEncoding"),
+    ("py:type", "VariablePath"), ("py:class", "VariablePath"),
+    ("py:type", "AnyAsyncArray"), ("py:class", "AnyAsyncArray"),
+    ("py:type", "AnyGenerator"), ("py:class", "AnyGenerator"),
+    ("py:type", "Selector"), ("py:class", "Selector"),
+    ("py:type", "NDArrayLikeOrScalar"), ("py:class", "NDArrayLikeOrScalar"),
 ]
 
 
@@ -115,15 +141,17 @@
 # -- sphinx.ext.intersphinx options --
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
-    "vivarium": (
-        "https://vivarium-core.readthedocs.io/en/latest/",
-        None,
-    ),
+    "pytest": ("https://docs.pytest.org/en/latest", None),
+    "vivarium": ("https://vivarium-core.readthedocs.io/en/latest/", None),
     "numpy": ("https://numpy.org/doc/stable", None),
+    "xarray": ("https://docs.xarray.dev/en/latest", None),
+    "zarr": ("https://zarr.readthedocs.io/en/latest", None),
     "matplotlib": ("https://matplotlib.org/stable/", None),
     "pandas": ("http://pandas.pydata.org/pandas-docs/dev", None),
     "polars": ("https://docs.pola.rs/api/python/stable", None),
     "sympy": ("https://docs.sympy.org/latest", None),
+    "pint": ("https://pint.readthedocs.io/en/stable", None),
+    "unum": ("https://unum.readthedocs.io/en/stable", None),
 }
 
 
@@ -143,8 +171,24 @@
 ]
 # Move typehints from signature into description
 autodoc_typehints = "description"
-# Concatenate class and __init__ docstrings
-autoclass_content = "both"
+# Only use the class’s docstring. __init__ docstrings are now listed separately.
+autoclass_content = "class"
+# Default options for all autodoc directives.
+autodoc_default_options = {
+    "member-order": "bysource",
+    "private-members": True,
+    "special-members": (
+        # object
+        "__init__,  __del__, __call__"
+    ),
+    "exclude-members": (
+        # abc.ABC
+        "_abc_impl, "
+        # enum.Flag
+        "_flag_mask_, _singles_mask_, _all_bits_, _boundary_, _inverted_, "
+        "_generate_next_value_"
+    )
+}
 # Remove domain objects (e.g. functions, classes, attributes) from
 # table of contents
 toc_object_entries = False

doc/experiments.rst

@@ -396,6 +396,9 @@ Here are some general rules to remember when writing your own JSON config files:
     without dividing, this results in a more informative error message instead
     of a Nextflow error about missing daughter cell states.
 
+
+.. _experiment_output:
+
 ------
 Output
 ------

doc/stores.rst

@@ -368,6 +368,8 @@ any number of attributes for all active (``_entryState`` is 1) unique molecules
 of a given type (e.g. RNA, active RNAP, etc.).
 
 
+.. _listeners:
+
 ---------
 Listeners
 ---------

doc/workflows.rst

@@ -330,6 +330,7 @@ it has access to.
   folder, you can just create stub files in the appropriate folders
   that simply import the ``plot`` function from a primary analysis script.
 
+
 .. _analysis_config:
 
 Configuration
@@ -502,6 +503,9 @@ Refer to :ref:`/output.rst` for more information about how
 to use DuckDB to read and analyze simulation output inside
 analysis scripts.
 
+
+.. _workflows:
+
 ---------
 Workflows
 ---------
@@ -706,7 +710,7 @@ is a list workflow behaviors enabled in our model to handle unexpected errors.
   The only exceptions are changes to resource allocation options (e.g., ``SIM_MEM``),
   allowing users to retry failed jobs with higher resource limits without triggering
   re-execution of already completed jobs. If you want to change other options, you
-  must launch a new workflow. 
+  must launch a new workflow.
 
 .. _output: