Lazy handling and saving of timestamps (and remove load_from/save_to folder_metadata)

.github/scripts/serialization/objects.py

@@ -29,10 +29,13 @@
     "json": ".json",
     "pickle": ".pkl",
     "binary": "_binary",
+    "binary_parallel": "_binary_parallel",
     "numpy_folder": "_numpy_folder",
     "zarr": ".zarr",
+    "zarr_parallel": "_parallel.zarr",
 }
 
+DEFAULT_DURATION = 2.0  # seconds, for recordings and sortings
 
 # --- json (recipe) entries: moved class + a second class -------------------------
 
@@ -46,7 +49,12 @@ def _build_noise_generator_recording():
     except ImportError:
         from spikeinterface.core.generate import NoiseGeneratorRecording
 
-    return NoiseGeneratorRecording(num_channels=4, sampling_frequency=30000.0, durations=[1.0, 1.5], seed=0)
+    return NoiseGeneratorRecording(
+        num_channels=4,
+        sampling_frequency=30000.0,
+        durations=[DEFAULT_DURATION, DEFAULT_DURATION + 0.5],
+        seed=0,
+    )
 
 
 def _check_noise_generator_recording(rec):
@@ -59,7 +67,7 @@ def _check_noise_generator_recording(rec):
 def _build_mock_recording():
     from spikeinterface.core import generate_recording
 
-    return generate_recording(num_channels=4, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    return generate_recording(num_channels=4, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
 
 
 def _check_mock_recording(rec):
@@ -77,7 +85,7 @@ def _build_recording_with_properties():
     import numpy as np
     from spikeinterface.core import generate_recording
 
-    rec = generate_recording(num_channels=4, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    rec = generate_recording(num_channels=4, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
     rec.set_property("quality", np.array(["good", "good", "bad", "good"]))
     rec.annotate(experimenter="test")
     return rec
@@ -94,7 +102,7 @@ def _build_recording_with_probe():
     from probeinterface import generate_linear_probe
     from spikeinterface.core import generate_recording
 
-    rec = generate_recording(num_channels=8, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    rec = generate_recording(num_channels=8, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
     probe = generate_linear_probe(num_elec=8)
     probe.set_device_channel_indices(np.arange(8))
     if parse(si_version) <= parse("0.105.0"):
@@ -104,6 +112,20 @@ def _build_recording_with_probe():
     return rec_with_probe
 
 
+def _build_recording_with_timestamps():
+    rec = _build_mock_recording()
+    times = rec.get_times(segment_index=0) + 100
+    rec.set_times(times, segment_index=0)
+    return rec
+
+
+def _check_recording_with_timestamps(rec):
+    import numpy as np
+    expected_times = np.arange(int(DEFAULT_DURATION * 30000)) / 30000.0 + 100
+    times = rec.get_times(segment_index=0)
+    assert np.allclose(times, expected_times)
+
+
 def _check_recording_with_probe(rec):
     import numpy as np
 
@@ -121,7 +143,7 @@ def _build_recording_with_interleaved_probes():
     from probeinterface import ProbeGroup, generate_linear_probe
     from spikeinterface.core import generate_recording
 
-    rec = generate_recording(num_channels=8, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    rec = generate_recording(num_channels=8, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
     probe0 = generate_linear_probe(num_elec=4)
     probe1 = generate_linear_probe(num_elec=4)
     probe1.move([100.0, 0.0])
@@ -164,7 +186,7 @@ def _build_preprocessed_chain():
     from spikeinterface.core import generate_recording
     from spikeinterface.preprocessing import common_reference, scale
 
-    rec = generate_recording(num_channels=4, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    rec = generate_recording(num_channels=4, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
     # Two nested scipy-free preprocessing wrappers (scale then common_reference): this
     # exercises recursive parent reload without pulling scipy into the environments.
     return common_reference(scale(rec, gain=2.0))
@@ -181,7 +203,7 @@ def _check_preprocessed_chain(rec):
 def _build_sorting():
     from spikeinterface.core import generate_sorting
 
-    return generate_sorting(num_units=5, sampling_frequency=30000.0, durations=[1.0])
+    return generate_sorting(num_units=5, sampling_frequency=30000.0, durations=[DEFAULT_DURATION])
 
 
 def _check_sorting(sorting):
@@ -195,7 +217,7 @@ def _build_sorting_with_properties():
     import numpy as np
     from spikeinterface.core import generate_sorting
 
-    sorting = generate_sorting(num_units=4, sampling_frequency=30000.0, durations=[1.0])
+    sorting = generate_sorting(num_units=4, sampling_frequency=30000.0, durations=[DEFAULT_DURATION])
     sorting.set_property("quality", np.array(["good", "good", "bad", "good"]))
     sorting.annotate(experimenter="test")
     return sorting
@@ -224,19 +246,25 @@ def _check_sorting_with_properties(sorting):
         "id": "recording_with_properties",
         "build": _build_recording_with_properties,
         "check": _check_recording_with_properties,
-        "formats": ["binary", "zarr"],
+        "formats": ["binary", "zarr", "binary_parallel", "zarr_parallel"],
+    },
+    {
+        "id": "recording_with_timestamps",
+        "build": _build_recording_with_timestamps,
+        "check": _check_recording_with_timestamps,
+        "formats": ["binary", "zarr", "binary_parallel", "zarr_parallel"],
     },
     {
         "id": "recording_with_probe",
         "build": _build_recording_with_probe,
         "check": _check_recording_with_probe,
-        "formats": ["binary", "zarr"],
+        "formats": ["binary", "zarr", "binary_parallel", "zarr_parallel"],
     },
     {
         "id": "recording_with_interleaved_probes",
         "build": _build_recording_with_interleaved_probes,
         "check": _check_recording_with_interleaved_probes,
-        "formats": ["binary", "zarr"],
+        "formats": ["binary", "zarr", "binary_parallel", "zarr_parallel"],
     },
     {
         "id": "preprocessed_chain",

.github/scripts/serialization/serialize_objects.py

@@ -33,10 +33,14 @@
             obj.dump_to_pickle(dest)
         elif fmt == "binary":
             obj.save(folder=dest, format="binary", overwrite=True)
+        elif fmt == "binary_parallel":
+            obj.save(folder=dest, format="binary", overwrite=True, n_jobs=2)
         elif fmt == "numpy_folder":
             obj.save(folder=dest, format="numpy_folder", overwrite=True)
         elif fmt == "zarr":
             obj.save(folder=dest, format="zarr", overwrite=True)
+        elif fmt == "zarr_parallel":
+            obj.save(folder=dest, format="zarr", overwrite=True, n_jobs=2)
         print(f"  wrote {dest.name} ({fmt})")
 
 print(f"Fixtures written to: {out_dir.resolve()}")

.github/workflows/cross_version_serialization.yml

@@ -6,9 +6,8 @@ name: Cross-version serialization
 #
 # Delivery model: Live generation. Fixtures are regenerated from a real old install each
 # run in an isolated uv environment (nothing committed). The version list is computed
-# from PyPI (latest patch of each minor at or above MIN_VERSION_TO_TEST): on a pull
-# request only the latest released minor is tested; the full matrix runs weekly and on
-# manual dispatch.
+# from PyPI (latest patch of each minor at or above MIN_VERSION_TO_TEST), and the full
+# matrix is tested on every pull request and on manual dispatch.
 
 on:
   pull_request:
@@ -20,11 +19,9 @@ on:
     # watches core only; a break introduced in generation/ or preprocessing/ would not
     # trigger it.
     paths:
-      - "src/spikeinterface/core/**"
-      - ".github/scripts/serialization/**"
-      - ".github/workflows/cross_version_serialization.yml"
-  schedule:
-    - cron: "0 4 * * 0" # weekly, Sunday 04:00 UTC
+      - 'src/spikeinterface/core/**'
+      - '.github/scripts/serialization/**'
+      - '.github/workflows/cross_version_serialization.yml'
   workflow_dispatch:
 
 concurrency:
@@ -33,8 +30,8 @@ concurrency:
 
 jobs:
   # Compute which released versions the matrix below tests loading from, exposed as the
-  # job output `list`. Latest patch of each minor at or above the floor on schedule /
-  # dispatch; only the latest minor on a pull request (keeps per-PR runs cheap).
+  # job output `list`: the latest patch of each minor at or above the floor, tested on
+  # every trigger.
   versions:
     name: Select versions
     runs-on: ubuntu-latest
@@ -46,7 +43,6 @@ jobs:
           python-version: "3.10"
       - id: set
         env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
           # Support floor: oldest minor to test. Below this, installs fail on the CI
           # Python (numcodecs dependency rot in 0.100/0.101). Raise when 0.102 rots.
           MIN_VERSION_TO_TEST: "0.102" # We will change this when making breaking changes (hopefully not too often)
@@ -67,10 +63,6 @@ jobs:
           candidates = [v for v in available if not v.is_prerelease and (v.major, v.minor) >= (floor.major, floor.minor)]
           minor_releases = [max(group) for _, group in groupby(candidates, key=lambda v: (v.major, v.minor))]
 
-          # on a pull request, test only the latest minor
-          if os.environ.get('GITHUB_EVENT_NAME') == 'pull_request':
-              minor_releases = minor_releases[-1:]
-
           print(json.dumps([str(v) for v in minor_releases]))
           ")