Add Cowley2026 V4 neural benchmark

brainscore_vision/benchmarks/cowley2026/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import Cowley2026_190923_V4PLS
+
+benchmark_registry['Cowley2026.190923.V4-pls'] = Cowley2026_190923_V4PLS

brainscore_vision/benchmarks/cowley2026/benchmark.py

@@ -0,0 +1,55 @@
+from brainscore_vision import load_metric, load_ceiling, load_dataset
+from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition
+
+VISUAL_DEGREES = 11.2
+NUMBER_OF_TRIALS = 14  # mode of the per-image repeat counts
+BIBTEX = """@article{cowley2026compact,
+  title={Compact deep neural network models of the visual cortex},
+  author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A},
+  journal={Nature},
+  volume={652},
+  number={8111},
+  pages={947--954},
+  year={2026},
+  publisher={Nature Publishing Group}}"""
+
+# no object categories -> plain random CV splits, not object_name stratification
+pls_metric = lambda: load_metric('pls', crossvalidation_kwargs=dict(stratification_coord=None))
+
+
+def _Cowley2026V4PLS(session: str):
+    identifier = f'Cowley2026.{session}'
+    assembly_repetition = alternate_repetition_halves(load_assembly(identifier, average_repetitions=False))
+    assembly = load_assembly(identifier, average_repetitions=True)
+    return NeuralBenchmark(
+        identifier=f'{identifier}.V4-pls', version=1,
+        assembly=assembly, similarity_metric=pls_metric(),
+        visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS,
+        ceiling_func=lambda: load_ceiling('internal_consistency')(assembly_repetition),
+        parent='V4', bibtex=BIBTEX)
+
+
+def alternate_repetition_halves(assembly):
+    """Relabel repetitions to even/odd halves so the split-half ceiling balances per image."""
+    names = list(assembly.indexes['presentation'].names)
+    half = (assembly['repetition'].values % 2).astype(int)
+    assembly = assembly.reset_index('presentation')
+    assembly['repetition'] = 'presentation', half
+    return assembly.set_index(presentation=names)
+
+
+def load_assembly(identifier: str, average_repetitions: bool):
+    assembly = load_dataset(identifier)
+    assembly = assembly.sel(region='V4')
+    assembly = assembly.stack(neuroid=['neuroid_id'])  # work around xarray multiindex issues
+    assembly['region'] = 'neuroid', ['V4'] * len(assembly['neuroid'])
+    assembly.load()
+    if 'time_bin' in assembly.dims:  # single static window (50, 150) ms
+        assembly = assembly.squeeze('time_bin')
+    if average_repetitions:
+        assembly = average_repetition(assembly)
+    return assembly
+
+
+def Cowley2026_190923_V4PLS():
+    return _Cowley2026V4PLS('190923')

brainscore_vision/benchmarks/cowley2026/test.py

@@ -0,0 +1,25 @@
+import pytest
+from pytest import approx
+
+from brainscore_vision import load_benchmark, load_model
+
+
+@pytest.mark.private_access
+class TestExist:
+    @pytest.mark.parametrize('identifier', ['Cowley2026.190923.V4-pls'])
+    def test_benchmark_loads(self, identifier):
+        benchmark = load_benchmark(identifier)
+        assert benchmark is not None
+        assert benchmark.identifier == identifier
+
+
+@pytest.mark.private_access
+@pytest.mark.slow
+class TestAlexNet:
+    @pytest.mark.parametrize('benchmark, expected_score', [
+        ('Cowley2026.190923.V4-pls', approx(0.34609011, abs=0.005)),
+    ])
+    def test_model_score(self, benchmark, expected_score):
+        benchmark = load_benchmark(benchmark)
+        score = benchmark(load_model('alexnet'))
+        assert score.values == expected_score

brainscore_vision/data/cowley2026/__init__.py

@@ -0,0 +1,42 @@
+from brainscore_vision import data_registry, stimulus_set_registry, load_stimulus_set
+from brainscore_core.supported_data_standards.brainio.s3 import load_stimulus_set_from_s3, load_assembly_from_s3
+from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly
+
+BIBTEX = """@article{cowley2026compact,
+  title={Compact deep neural network models of the visual cortex},
+  author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A},
+  journal={Nature},
+  volume={652},
+  number={8111},
+  pages={947--954},
+  year={2026},
+  publisher={Nature Publishing Group}}"""
+
+BUCKET = "brainscore-storage/brainscore-vision/data/user_718/"
+
+
+# keep literal `*_registry['<id>'] =` lines below: plugin discovery greps for that substring
+def stimulus_set(identifier, csv_sha1, zip_sha1, csv_version_id, zip_version_id):
+    return lambda: load_stimulus_set_from_s3(
+        identifier=identifier, bucket=BUCKET,
+        csv_sha1=csv_sha1, zip_sha1=zip_sha1,
+        csv_version_id=csv_version_id, zip_version_id=zip_version_id)
+
+
+def assembly(identifier, sha1, version_id):
+    return lambda: load_assembly_from_s3(
+        identifier=identifier, bucket=BUCKET, sha1=sha1, version_id=version_id,
+        cls=NeuroidAssembly, stimulus_set_loader=lambda: load_stimulus_set(identifier))
+
+
+# session 190923
+stimulus_set_registry['Cowley2026.190923'] = stimulus_set(
+    'Cowley2026.190923',
+    csv_sha1="7752f43fc809c193334dd97171867e733291b8fd",
+    zip_sha1="a14f9d4cfc98cb253f23d4eaa159c60666903668",
+    csv_version_id="4ZuvTJxZptY8V04ayk2CRLb209BihWis",
+    zip_version_id="tWyQAXN_fM4Y2fLnQtITbi0QMLawr4Nd")
+data_registry['Cowley2026.190923'] = assembly(
+    'Cowley2026.190923',
+    sha1="2ac7f60f21ccc5137074633c0614f52566acff6a",
+    version_id="kpX10KvUti_Vg4WAMHsiayllYlStMcgI")

brainscore_vision/data/cowley2026/data_packaging/data_packaging.py

@@ -0,0 +1,115 @@
+
+import numpy as np
+import brainscore_vision
+
+from brainscore_core.supported_data_standards.brainio.stimuli import StimulusSet
+from brainscore_core.supported_data_standards.brainio.packaging import package_stimulus_set_locally
+from brainscore_core.supported_data_standards.brainio.assemblies import DataAssembly
+from brainscore_core.supported_data_standards.brainio import packaging
+from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly
+
+import statistics
+
+
+# Note:
+# Running local on mac
+
+session_id = 190923 # 190923 201025 210225 211022
+date_experiment = '2019-09-23' # '2019-09-23' '2020-10-25' '2021-02-25' '2021-10-22'
+
+responses = np.load('/Users/cowley/Desktop/brainscore_upload/data_raw/responses_{:d}.npy'.format(session_id))
+	# (num_neurons, num_images, num_possible_repeats)
+
+num_neurons = responses.shape[0]
+num_images = responses.shape[1]
+
+
+## STIMULUS SET
+
+# Create a dataframe tracking image paths and attributes
+print('---STIMULUS SET---')
+stimuli_data = [{'stimulus_id': 'image{:04d}'.format(iimage), 'object_name': '{:04d}'.format(iimage)} for iimage in range(1,num_images+1)]
+stimulus_set = StimulusSet(stimuli_data)
+
+stimulus_set.stimulus_paths = {
+	'image{:04d}'.format(iimage): '/Users/cowley/Desktop/brainscore_upload/data_raw/images_{:d}/image{:04d}.jpg'.format(session_id, iimage)
+	for iimage in range(1,num_images+1)
+	}
+
+stimulus_set.name = 'Cowley2026.{:d}'.format(session_id)
+
+package_output = package_stimulus_set_locally(
+    proto_stimulus_set=stimulus_set,
+    stimulus_set_identifier=stimulus_set.name,
+)
+
+print(package_output)
+
+
+
+
+
+
+## NEURAL DATA
+
+print()
+print('---NEURAL DATA---')
+
+
+## flatten responses to be (num_neurons, num_repeats*num_images)
+
+responses_data_matrix = []
+stimulus_ids = []
+object_names = []
+repeat_ids = []
+
+
+for iimage in range(num_images):
+	num_repeats = np.sum(~np.isnan(responses[0,iimage,:]))
+
+	for irepeat in range(num_repeats):
+
+		responses_data_matrix.append(responses[:,iimage,irepeat])
+		stimulus_ids.append('image{:04d}'.format(iimage+1))
+		object_names.append('{:04d}'.format(iimage+1))
+		repeat_ids.append(irepeat)
+
+
+responses_data_matrix = np.stack(responses_data_matrix)
+	# (num_presentations, num_neurons)
+responses_data_matrix = np.expand_dims(responses_data_matrix, axis=2)  # include time_bin dimension (dummy)
+
+
+assembly = NeuroidAssembly(
+    responses_data_matrix,
+    coords={
+        # Coordinates tracking the 'presentation' dimension (axis 1)
+        'stimulus_id': ('presentation', stimulus_ids),
+        'object_name': ('presentation', object_names),
+        'repetition': ('presentation', repeat_ids),
+
+        # Coordinates tracking the 'neuroid' dimension (axis 0)
+        'neuroid_id': ('neuroid', [f'neuron_{i}' for i in range(num_neurons)]),
+        'region': ('neuroid', ['V4'] * num_neurons),  # e.g., 'V4', 'IT', or 'AL'
+
+        'time_bin_start': ('time_bin', [50]),
+        'time_bin_end': ('time_bin', [150])
+    },
+    dims=['presentation', 'neuroid', 'time_bin']
+)
+
+
+assembly.attrs['experiment_date'] = date_experiment
+assembly.name = 'Cowley2026.{:d}'.format(session_id)
+
+
+package_output = packaging.package_data_assembly_locally(
+	proto_data_assembly=assembly,
+	assembly_identifier=stimulus_set.name, # We use the same stimulusSet name for the assembly
+	stimulus_set_identifier=stimulus_set.name,
+	assembly_class_name="NeuroidAssembly", # For most neural data, use NeuroidAssembly. For behavioral data, use BehavioralAssembly
+)
+
+print(package_output)
+
+

brainscore_vision/data/cowley2026/data_packaging/output_hash.txt

@@ -0,0 +1,5 @@
+---STIMULUS SET---
+{'identifier': 'Cowley2026.190923', 'csv_path': '/Users/cowley/Downloads/brainscore_packages/stimulus_Cowley2026_190923.csv', 'zip_path': '/Users/cowley/Downloads/brainscore_packages/stimulus_Cowley2026_190923.zip', 'csv_sha1': '7752f43fc809c193334dd97171867e733291b8fd', 'zip_sha1': 'a14f9d4cfc98cb253f23d4eaa159c60666903668'}
+
+---NEURAL DATA---
+{'identifier': 'Cowley2026.190923', 'path': '/Users/cowley/Downloads/brainscore_packages/assy_Cowley2026_190923.nc', 'sha1': '2ac7f60f21ccc5137074633c0614f52566acff6a', 'cls': 'NeuroidAssembly'}

brainscore_vision/data/cowley2026/test.py

@@ -0,0 +1,35 @@
+import numpy as np
+import pytest
+from brainscore_vision import load_dataset, load_stimulus_set
+
+
+@pytest.mark.private_access
+class TestStimulusSet:
+    def test_stimulus_set_exists(self):
+        stimulus_set = load_stimulus_set('Cowley2026.190923')
+        assert stimulus_set is not None
+        assert stimulus_set.identifier == 'Cowley2026.190923'
+
+    def test_stimulus_set_counts(self):
+        stimulus_set = load_stimulus_set('Cowley2026.190923')
+        assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1200
+
+
+@pytest.mark.private_access
+class TestAssembly:
+    def test_assembly_exists(self):
+        assembly = load_dataset('Cowley2026.190923')
+        assert assembly is not None
+        assert assembly.identifier == 'Cowley2026.190923'
+
+    def test_assembly_structure(self):
+        assembly = load_dataset('Cowley2026.190923')
+        assert 'presentation' in assembly.dims
+        assert 'stimulus_id' in assembly.indexes['presentation'].names
+        assert set(np.unique(assembly['region'].values)) == {'V4'}
+
+    def test_assembly_alignment(self):
+        assembly = load_dataset('Cowley2026.190923')
+        assembly_stimuli = set(assembly['stimulus_id'].values)
+        stimulus_set_stimuli = set(assembly.stimulus_set['stimulus_id'].values)
+        assert assembly_stimuli.issubset(stimulus_set_stimuli)