From 4cbc1658549ef2f1fb35108e797a7130f0c5ea26 Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Tue, 30 Jun 2026 11:52:49 -0400 Subject: [PATCH 1/2] feat(benchmarks): add Cowley2026 V4 benchmark (contributor submission) Files as submitted by Benjamin Cowley (CSHL), unmodified. --- .../benchmarks/cowley2026/__init__.py | 19 +++ .../benchmarks/benchmark_Cowley2026_190923.py | 81 ++++++++++++ .../benchmarks/cowley2026/test.py | 31 +++++ brainscore_vision/data/cowley2026/__init__.py | 52 ++++++++ .../data_packaging/data_packaging.py | 115 ++++++++++++++++++ .../cowley2026/data_packaging/output_hash.txt | 5 + brainscore_vision/data/cowley2026/test.py | 115 ++++++++++++++++++ 7 files changed, 418 insertions(+) create mode 100644 brainscore_vision/benchmarks/cowley2026/__init__.py create mode 100644 brainscore_vision/benchmarks/cowley2026/benchmarks/benchmark_Cowley2026_190923.py create mode 100644 brainscore_vision/benchmarks/cowley2026/test.py create mode 100644 brainscore_vision/data/cowley2026/__init__.py create mode 100644 brainscore_vision/data/cowley2026/data_packaging/data_packaging.py create mode 100644 brainscore_vision/data/cowley2026/data_packaging/output_hash.txt create mode 100644 brainscore_vision/data/cowley2026/test.py diff --git a/brainscore_vision/benchmarks/cowley2026/__init__.py b/brainscore_vision/benchmarks/cowley2026/__init__.py new file mode 100644 index 000000000..379e31723 --- /dev/null +++ b/brainscore_vision/benchmarks/cowley2026/__init__.py @@ -0,0 +1,19 @@ +from brainscore_vision import benchmark_registry + + +BIBTEX = """@article{cowley2026compact, + title={Compact deep neural network models of the visual cortex}, + author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, + journal={Nature}, + volume={652}, + number={8111}, + pages={947--954}, + year={2026}, + publisher={Nature Publishing Group}}""" + + +from .benchmarks.benchmark_Cowley2026_190923 import benchmark_Cowley2026_190923 +benchmark_registry['Cowley2026_190923'] = benchmark_Cowley2026_190923 + + + diff --git a/brainscore_vision/benchmarks/cowley2026/benchmarks/benchmark_Cowley2026_190923.py b/brainscore_vision/benchmarks/cowley2026/benchmarks/benchmark_Cowley2026_190923.py new file mode 100644 index 000000000..bb96131ec --- /dev/null +++ b/brainscore_vision/benchmarks/cowley2026/benchmarks/benchmark_Cowley2026_190923.py @@ -0,0 +1,81 @@ +from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition +from brainscore_vision import load_metric, load_ceiling, load_dataset, load_stimulus_set + +import numpy as np +import pandas as pd + +import xarray as xr +from brainscore_vision import load_stimulus_set +from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly + + +BIBTEX = """@article{cowley2026compact, + title={Compact deep neural network models of the visual cortex}, + author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, + journal={Nature}, + volume={652}, + number={8111}, + pages={947--954}, + year={2026}, + publisher={Nature Publishing Group}}""" + + + +def benchmark_Cowley2026_190923(): + + assembly = load_assembly(average_repetitions=True) + assembly_repetition = load_assembly(average_repetitions=False) + + return NeuralBenchmark( + identifier='Cowley2026_190923.V4-pls', + version=1, + assembly=assembly, + similarity_metric=load_metric('pls'), + visual_degrees=11.2, + number_of_trials=14, # mode of num repeats --- different images had different repeats + ceiling_func=lambda: load_ceiling('internal_consistency')(assembly_repetition), + parent='V4', + bibtex=BIBTEX + ) + + +def load_assembly(average_repetitions, access='private'): + # 1. Load the pristine dataset from your disk + assembly = load_dataset('Cowley2026.190923') + assembly = assembly.sel(region='V4') + assembly = assembly.stack(neuroid=['neuroid_id']) # work around xarray multiindex issues + assembly['region'] = 'neuroid', ['V4'] * len(assembly['neuroid']) + assembly.load() + + # 2. Squeeze out the physical time_bin axis to align shapes with static models + if 'time_bin' in assembly.dims: + assembly = assembly.squeeze('time_bin') + + if average_repetitions: + # 3. Collapse repetitions down to unique stimulus_ids + # Your clean packaging ensures object_name and category survive natively! + assembly = average_repetition(assembly) + + return assembly + + +# # copying FreemanZiemba2013??? +# assembly = load_dataset('Cowley2026.190923') +# assembly = assembly.sel(region='V4') +# assembly = assembly.stack(neuroid=['neuroid_id']) # work around xarray multiindex issues +# assembly['region'] = 'neuroid', ['V4'] * len(assembly['neuroid']) +# assembly.load() +# time_window = (50, 150) +# assembly = assembly.sel(time_bin=[(t, t + 1) for t in range(*time_window)]) +# assembly = assembly.mean(dim='time_bin', keep_attrs=True) +# assembly = assembly.expand_dims('time_bin_start').expand_dims('time_bin_end') +# assembly['time_bin_start'], assembly['time_bin_end'] = [time_window[0]], [time_window[1]] +# assembly = assembly.stack(time_bin=['time_bin_start', 'time_bin_end']) +# assembly = assembly.squeeze('time_bin') +# assembly = assembly.transpose('presentation', 'neuroid') +# if average_repetitions: +# assembly = average_repetition(assembly) +# return assembly + + + diff --git a/brainscore_vision/benchmarks/cowley2026/test.py b/brainscore_vision/benchmarks/cowley2026/test.py new file mode 100644 index 000000000..ece9e2b48 --- /dev/null +++ b/brainscore_vision/benchmarks/cowley2026/test.py @@ -0,0 +1,31 @@ +import pytest +from pytest import approx + +from brainscore_vision import benchmark_registry, load_benchmark, load_model + +@pytest.mark.private_access +class TestExist: + + @pytest.mark.parametrize("identifier", [ + 'Cowley2026_190923' + ]) + def test_benchmark_loads(self, identifier): + """Verify benchmark can be loaded.""" + benchmark = load_benchmark(identifier) + assert benchmark is not None + assert benchmark.identifier == identifier + '.V4-pls' + + +@pytest.mark.private_access +class TestAlexNet: + + @pytest.mark.slow + @pytest.mark.parametrize('benchmark, expected_score', [ + ('Cowley2026_190923', approx(0.61209661, abs=0.001)), + ]) + def test_model_score(self, benchmark, expected_score): + benchmark = load_benchmark(benchmark) + model = load_model('alexnet') + score = benchmark(model) + print(score) + assert score == expected_score \ No newline at end of file diff --git a/brainscore_vision/data/cowley2026/__init__.py b/brainscore_vision/data/cowley2026/__init__.py new file mode 100644 index 000000000..371b1418f --- /dev/null +++ b/brainscore_vision/data/cowley2026/__init__.py @@ -0,0 +1,52 @@ + +from brainscore_vision import data_registry, stimulus_set_registry, load_stimulus_set +from brainscore_core.supported_data_standards.brainio.s3 import load_stimulus_set_from_s3, load_assembly_from_s3 +from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly + +# Provide bibtex for proper citation of the original dataset +BIBTEX = """@article{cowley2026compact, + title={Compact deep neural network models of the visual cortex}, + author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, + journal={Nature}, + volume={652}, + number={8111}, + pages={947--954}, + year={2026}, + publisher={Nature Publishing Group}}""" + + +# stimulus set: 190923 +stimulus_set_registry['Cowley2026.190923'] = lambda: load_stimulus_set_from_s3( + identifier="Cowley2026.190923", + bucket="brainscore-storage/brainscore-vision/data/user_718/", + csv_sha1="7752f43fc809c193334dd97171867e733291b8fd", + zip_sha1="a14f9d4cfc98cb253f23d4eaa159c60666903668", + csv_version_id="4ZuvTJxZptY8V04ayk2CRLb209BihWis", + zip_version_id="kpX10KvUti_Vg4WAMHsiayllYlStMcgI") + + +# neural assembly 190923: +data_registry['Cowley2026.190923'] = lambda: load_assembly_from_s3( + identifier="Cowley2026.190923", + sha1="2ac7f60f21ccc5137074633c0614f52566acff6a", # From local packaging output + version_id="kpX10KvUti_Vg4WAMHsiayllYlStMcgI", # S3 version ID + bucket="brainscore-storage/brainscore-vision/data/user_718/", # S3 bucket path (provided after upload) + cls=NeuroidAssembly, # Assembly class (NeuroidAssembly or BehavioralAssembly) + stimulus_set_loader=lambda: load_stimulus_set('Cowley2026.190923') # Links to the stimulus set +) + + + + + + + + + + + + + + + + diff --git a/brainscore_vision/data/cowley2026/data_packaging/data_packaging.py b/brainscore_vision/data/cowley2026/data_packaging/data_packaging.py new file mode 100644 index 000000000..3b8679207 --- /dev/null +++ b/brainscore_vision/data/cowley2026/data_packaging/data_packaging.py @@ -0,0 +1,115 @@ + +import numpy as np +import brainscore_vision + +from brainscore_core.supported_data_standards.brainio.stimuli import StimulusSet +from brainscore_core.supported_data_standards.brainio.packaging import package_stimulus_set_locally +from brainscore_core.supported_data_standards.brainio.assemblies import DataAssembly +from brainscore_core.supported_data_standards.brainio import packaging +from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly + +import statistics + + +# Note: +# Running local on mac + +session_id = 190923 # 190923 201025 210225 211022 +date_experiment = '2019-09-23' # '2019-09-23' '2020-10-25' '2021-02-25' '2021-10-22' + +responses = np.load('/Users/cowley/Desktop/brainscore_upload/data_raw/responses_{:d}.npy'.format(session_id)) + # (num_neurons, num_images, num_possible_repeats) + +num_neurons = responses.shape[0] +num_images = responses.shape[1] + + +## STIMULUS SET + +# Create a dataframe tracking image paths and attributes +print('---STIMULUS SET---') +stimuli_data = [{'stimulus_id': 'image{:04d}'.format(iimage), 'object_name': '{:04d}'.format(iimage)} for iimage in range(1,num_images+1)] +stimulus_set = StimulusSet(stimuli_data) + +stimulus_set.stimulus_paths = { + 'image{:04d}'.format(iimage): '/Users/cowley/Desktop/brainscore_upload/data_raw/images_{:d}/image{:04d}.jpg'.format(session_id, iimage) + for iimage in range(1,num_images+1) + } + +stimulus_set.name = 'Cowley2026.{:d}'.format(session_id) + +package_output = package_stimulus_set_locally( + proto_stimulus_set=stimulus_set, + stimulus_set_identifier=stimulus_set.name, +) + +print(package_output) + + + + + + +## NEURAL DATA + +print() +print('---NEURAL DATA---') + + +## flatten responses to be (num_neurons, num_repeats*num_images) + +responses_data_matrix = [] +stimulus_ids = [] +object_names = [] +repeat_ids = [] + + +for iimage in range(num_images): + num_repeats = np.sum(~np.isnan(responses[0,iimage,:])) + + for irepeat in range(num_repeats): + + responses_data_matrix.append(responses[:,iimage,irepeat]) + stimulus_ids.append('image{:04d}'.format(iimage+1)) + object_names.append('{:04d}'.format(iimage+1)) + repeat_ids.append(irepeat) + + +responses_data_matrix = np.stack(responses_data_matrix) + # (num_presentations, num_neurons) +responses_data_matrix = np.expand_dims(responses_data_matrix, axis=2) # include time_bin dimension (dummy) + + +assembly = NeuroidAssembly( + responses_data_matrix, + coords={ + # Coordinates tracking the 'presentation' dimension (axis 1) + 'stimulus_id': ('presentation', stimulus_ids), + 'object_name': ('presentation', object_names), + 'repetition': ('presentation', repeat_ids), + + # Coordinates tracking the 'neuroid' dimension (axis 0) + 'neuroid_id': ('neuroid', [f'neuron_{i}' for i in range(num_neurons)]), + 'region': ('neuroid', ['V4'] * num_neurons), # e.g., 'V4', 'IT', or 'AL' + + 'time_bin_start': ('time_bin', [50]), + 'time_bin_end': ('time_bin', [150]) + }, + dims=['presentation', 'neuroid', 'time_bin'] +) + + +assembly.attrs['experiment_date'] = date_experiment +assembly.name = 'Cowley2026.{:d}'.format(session_id) + + +package_output = packaging.package_data_assembly_locally( + proto_data_assembly=assembly, + assembly_identifier=stimulus_set.name, # We use the same stimulusSet name for the assembly + stimulus_set_identifier=stimulus_set.name, + assembly_class_name="NeuroidAssembly", # For most neural data, use NeuroidAssembly. For behavioral data, use BehavioralAssembly +) + +print(package_output) + + diff --git a/brainscore_vision/data/cowley2026/data_packaging/output_hash.txt b/brainscore_vision/data/cowley2026/data_packaging/output_hash.txt new file mode 100644 index 000000000..4b3e6d053 --- /dev/null +++ b/brainscore_vision/data/cowley2026/data_packaging/output_hash.txt @@ -0,0 +1,5 @@ +---STIMULUS SET--- +{'identifier': 'Cowley2026.190923', 'csv_path': '/Users/cowley/Downloads/brainscore_packages/stimulus_Cowley2026_190923.csv', 'zip_path': '/Users/cowley/Downloads/brainscore_packages/stimulus_Cowley2026_190923.zip', 'csv_sha1': '7752f43fc809c193334dd97171867e733291b8fd', 'zip_sha1': 'a14f9d4cfc98cb253f23d4eaa159c60666903668'} + +---NEURAL DATA--- +{'identifier': 'Cowley2026.190923', 'path': '/Users/cowley/Downloads/brainscore_packages/assy_Cowley2026_190923.nc', 'sha1': '2ac7f60f21ccc5137074633c0614f52566acff6a', 'cls': 'NeuroidAssembly'} diff --git a/brainscore_vision/data/cowley2026/test.py b/brainscore_vision/data/cowley2026/test.py new file mode 100644 index 000000000..853453d79 --- /dev/null +++ b/brainscore_vision/data/cowley2026/test.py @@ -0,0 +1,115 @@ +import numpy as np +import pytest +from brainscore_vision import load_dataset, load_stimulus_set + +@pytest.mark.private_access +class TestStimulusSet: + def test_stimulus_set_exists(self): + """Test that stimulus set loads correctly""" + stimulus_set = load_stimulus_set('Cowley2026.190923') + assert stimulus_set is not None + assert stimulus_set.identifier == 'Cowley2026.190923' + + # stimulus_set = load_stimulus_set('Cowley2026.201025') + # assert stimulus_set is not None + # assert stimulus_set.identifier == 'Cowley2026.201025' + + # stimulus_set = load_stimulus_set('Cowley2026.210225') + # assert stimulus_set is not None + # assert stimulus_set.identifier == 'Cowley2026.210225' + + # stimulus_set = load_stimulus_set('Cowley2026.211022') + # assert stimulus_set is not None + # assert stimulus_set.identifier == 'Cowley2026.211022' + + + + def test_stimulus_set_counts(self): + """Test expected number of stimuli""" + stimulus_set = load_stimulus_set('Cowley2026.190923') + assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1200 + + # stimulus_set = load_stimulus_set('Cowley2026.201025') + # assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1200 + + # stimulus_set = load_stimulus_set('Cowley2026.210225') + # assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1200 + + # stimulus_set = load_stimulus_set('Cowley2026.211022') + # assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1600 + + + + +@pytest.mark.private_access +class TestAssembly: + def test_assembly_exists(self): + """Test that assembly loads correctly""" + assembly = load_dataset('Cowley2026.190923') + assert assembly is not None + assert assembly.identifier == 'Cowley2026.190923' + + # assembly = load_dataset('Cowley2026.201025') + # assert assembly is not None + # assert assembly.identifier == 'Cowley2026.201025' + + # assembly = load_dataset('Cowley2026.210225') + # assert assembly is not None + # assert assembly.identifier == 'Cowley2026.210225' + + # assembly = load_dataset('Cowley2026.211022') + # assert assembly is not None + # assert assembly.identifier == 'Cowley2026.211022' + + + + def test_assembly_structure(self): + """Test assembly has required dimensions and coordinates""" + assembly = load_dataset('Cowley2026.190923') + assert 'presentation' in assembly.dims + assert 'stimulus_id' in assembly.indexes['presentation'].names + + # assembly = load_dataset('Cowley2026.201025') + # assert 'presentation' in assembly.dims + # assert 'stimulus_id' in assembly.indexes['presentation'].names + + # assembly = load_dataset('Cowley2026.210225') + # assert 'presentation' in assembly.dims + # assert 'stimulus_id' in assembly.indexes['presentation'].names + + # assembly = load_dataset('Cowley2026.211022') + # assert 'presentation' in assembly.dims + # assert 'stimulus_id' in assembly.indexes['presentation'].names + + + def test_assembly_alignment(self): + """Test stimulus set and assembly are properly linked""" + assembly = load_dataset('Cowley2026.190923') + stimulus_set = assembly.stimulus_set + + # All assembly stimulus IDs should exist in stimulus set + assembly_stimuli = set(assembly['stimulus_id'].values) + stimulus_set_stimuli = set(stimulus_set['stimulus_id'].values) + assert assembly_stimuli.issubset(stimulus_set_stimuli) + + # assembly = load_dataset('Cowley2026.201025') + # stimulus_set = assembly.stimulus_set + # assembly_stimuli = set(assembly['stimulus_id'].values) + # stimulus_set_stimuli = set(stimulus_set['stimulus_id'].values) + # assert assembly_stimuli.issubset(stimulus_set_stimuli) + + # assembly = load_dataset('Cowley2026.210225') + # stimulus_set = assembly.stimulus_set + # assembly_stimuli = set(assembly['stimulus_id'].values) + # stimulus_set_stimuli = set(stimulus_set['stimulus_id'].values) + # assert assembly_stimuli.issubset(stimulus_set_stimuli) + + # assembly = load_dataset('Cowley2026.211022') + # stimulus_set = assembly.stimulus_set + # assembly_stimuli = set(assembly['stimulus_id'].values) + # stimulus_set_stimuli = set(stimulus_set['stimulus_id'].values) + # assert assembly_stimuli.issubset(stimulus_set_stimuli) + + + + From 86868f872ae26bf36eaff7c9284947bedd7de63d Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Tue, 30 Jun 2026 11:53:15 -0400 Subject: [PATCH 2/2] fix(benchmarks): make Cowley2026 V4 benchmark run - disable CV stratification (no object categories) - even/odd repetition relabel for ragged-repeat ceiling - correct zip S3 version_id - literal registry lines for plugin discovery; flatten to benchmark.py alexnet 0.346 (raw r 0.528, ceiling 0.804); data and benchmark tests pass. --- .../benchmarks/cowley2026/__init__.py | 19 +--- .../benchmarks/cowley2026/benchmark.py | 55 ++++++++++++ .../benchmarks/benchmark_Cowley2026_190923.py | 81 ----------------- .../benchmarks/cowley2026/test.py | 22 ++--- brainscore_vision/data/cowley2026/__init__.py | 54 +++++------- brainscore_vision/data/cowley2026/test.py | 86 +------------------ 6 files changed, 90 insertions(+), 227 deletions(-) create mode 100644 brainscore_vision/benchmarks/cowley2026/benchmark.py delete mode 100644 brainscore_vision/benchmarks/cowley2026/benchmarks/benchmark_Cowley2026_190923.py diff --git a/brainscore_vision/benchmarks/cowley2026/__init__.py b/brainscore_vision/benchmarks/cowley2026/__init__.py index 379e31723..06875eac9 100644 --- a/brainscore_vision/benchmarks/cowley2026/__init__.py +++ b/brainscore_vision/benchmarks/cowley2026/__init__.py @@ -1,19 +1,4 @@ from brainscore_vision import benchmark_registry +from .benchmark import Cowley2026_190923_V4PLS - -BIBTEX = """@article{cowley2026compact, - title={Compact deep neural network models of the visual cortex}, - author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, - journal={Nature}, - volume={652}, - number={8111}, - pages={947--954}, - year={2026}, - publisher={Nature Publishing Group}}""" - - -from .benchmarks.benchmark_Cowley2026_190923 import benchmark_Cowley2026_190923 -benchmark_registry['Cowley2026_190923'] = benchmark_Cowley2026_190923 - - - +benchmark_registry['Cowley2026.190923.V4-pls'] = Cowley2026_190923_V4PLS diff --git a/brainscore_vision/benchmarks/cowley2026/benchmark.py b/brainscore_vision/benchmarks/cowley2026/benchmark.py new file mode 100644 index 000000000..14aa693b7 --- /dev/null +++ b/brainscore_vision/benchmarks/cowley2026/benchmark.py @@ -0,0 +1,55 @@ +from brainscore_vision import load_metric, load_ceiling, load_dataset +from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition + +VISUAL_DEGREES = 11.2 +NUMBER_OF_TRIALS = 14 # mode of the per-image repeat counts +BIBTEX = """@article{cowley2026compact, + title={Compact deep neural network models of the visual cortex}, + author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, + journal={Nature}, + volume={652}, + number={8111}, + pages={947--954}, + year={2026}, + publisher={Nature Publishing Group}}""" + +# no object categories -> plain random CV splits, not object_name stratification +pls_metric = lambda: load_metric('pls', crossvalidation_kwargs=dict(stratification_coord=None)) + + +def _Cowley2026V4PLS(session: str): + identifier = f'Cowley2026.{session}' + assembly_repetition = alternate_repetition_halves(load_assembly(identifier, average_repetitions=False)) + assembly = load_assembly(identifier, average_repetitions=True) + return NeuralBenchmark( + identifier=f'{identifier}.V4-pls', version=1, + assembly=assembly, similarity_metric=pls_metric(), + visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS, + ceiling_func=lambda: load_ceiling('internal_consistency')(assembly_repetition), + parent='V4', bibtex=BIBTEX) + + +def alternate_repetition_halves(assembly): + """Relabel repetitions to even/odd halves so the split-half ceiling balances per image.""" + names = list(assembly.indexes['presentation'].names) + half = (assembly['repetition'].values % 2).astype(int) + assembly = assembly.reset_index('presentation') + assembly['repetition'] = 'presentation', half + return assembly.set_index(presentation=names) + + +def load_assembly(identifier: str, average_repetitions: bool): + assembly = load_dataset(identifier) + assembly = assembly.sel(region='V4') + assembly = assembly.stack(neuroid=['neuroid_id']) # work around xarray multiindex issues + assembly['region'] = 'neuroid', ['V4'] * len(assembly['neuroid']) + assembly.load() + if 'time_bin' in assembly.dims: # single static window (50, 150) ms + assembly = assembly.squeeze('time_bin') + if average_repetitions: + assembly = average_repetition(assembly) + return assembly + + +def Cowley2026_190923_V4PLS(): + return _Cowley2026V4PLS('190923') diff --git a/brainscore_vision/benchmarks/cowley2026/benchmarks/benchmark_Cowley2026_190923.py b/brainscore_vision/benchmarks/cowley2026/benchmarks/benchmark_Cowley2026_190923.py deleted file mode 100644 index bb96131ec..000000000 --- a/brainscore_vision/benchmarks/cowley2026/benchmarks/benchmark_Cowley2026_190923.py +++ /dev/null @@ -1,81 +0,0 @@ -from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition -from brainscore_vision import load_metric, load_ceiling, load_dataset, load_stimulus_set - -import numpy as np -import pandas as pd - -import xarray as xr -from brainscore_vision import load_stimulus_set -from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly - - -BIBTEX = """@article{cowley2026compact, - title={Compact deep neural network models of the visual cortex}, - author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, - journal={Nature}, - volume={652}, - number={8111}, - pages={947--954}, - year={2026}, - publisher={Nature Publishing Group}}""" - - - -def benchmark_Cowley2026_190923(): - - assembly = load_assembly(average_repetitions=True) - assembly_repetition = load_assembly(average_repetitions=False) - - return NeuralBenchmark( - identifier='Cowley2026_190923.V4-pls', - version=1, - assembly=assembly, - similarity_metric=load_metric('pls'), - visual_degrees=11.2, - number_of_trials=14, # mode of num repeats --- different images had different repeats - ceiling_func=lambda: load_ceiling('internal_consistency')(assembly_repetition), - parent='V4', - bibtex=BIBTEX - ) - - -def load_assembly(average_repetitions, access='private'): - # 1. Load the pristine dataset from your disk - assembly = load_dataset('Cowley2026.190923') - assembly = assembly.sel(region='V4') - assembly = assembly.stack(neuroid=['neuroid_id']) # work around xarray multiindex issues - assembly['region'] = 'neuroid', ['V4'] * len(assembly['neuroid']) - assembly.load() - - # 2. Squeeze out the physical time_bin axis to align shapes with static models - if 'time_bin' in assembly.dims: - assembly = assembly.squeeze('time_bin') - - if average_repetitions: - # 3. Collapse repetitions down to unique stimulus_ids - # Your clean packaging ensures object_name and category survive natively! - assembly = average_repetition(assembly) - - return assembly - - -# # copying FreemanZiemba2013??? -# assembly = load_dataset('Cowley2026.190923') -# assembly = assembly.sel(region='V4') -# assembly = assembly.stack(neuroid=['neuroid_id']) # work around xarray multiindex issues -# assembly['region'] = 'neuroid', ['V4'] * len(assembly['neuroid']) -# assembly.load() -# time_window = (50, 150) -# assembly = assembly.sel(time_bin=[(t, t + 1) for t in range(*time_window)]) -# assembly = assembly.mean(dim='time_bin', keep_attrs=True) -# assembly = assembly.expand_dims('time_bin_start').expand_dims('time_bin_end') -# assembly['time_bin_start'], assembly['time_bin_end'] = [time_window[0]], [time_window[1]] -# assembly = assembly.stack(time_bin=['time_bin_start', 'time_bin_end']) -# assembly = assembly.squeeze('time_bin') -# assembly = assembly.transpose('presentation', 'neuroid') -# if average_repetitions: -# assembly = average_repetition(assembly) -# return assembly - - - diff --git a/brainscore_vision/benchmarks/cowley2026/test.py b/brainscore_vision/benchmarks/cowley2026/test.py index ece9e2b48..334c8715e 100644 --- a/brainscore_vision/benchmarks/cowley2026/test.py +++ b/brainscore_vision/benchmarks/cowley2026/test.py @@ -1,31 +1,25 @@ import pytest from pytest import approx -from brainscore_vision import benchmark_registry, load_benchmark, load_model +from brainscore_vision import load_benchmark, load_model + @pytest.mark.private_access class TestExist: - - @pytest.mark.parametrize("identifier", [ - 'Cowley2026_190923' - ]) + @pytest.mark.parametrize('identifier', ['Cowley2026.190923.V4-pls']) def test_benchmark_loads(self, identifier): - """Verify benchmark can be loaded.""" benchmark = load_benchmark(identifier) assert benchmark is not None - assert benchmark.identifier == identifier + '.V4-pls' + assert benchmark.identifier == identifier @pytest.mark.private_access +@pytest.mark.slow class TestAlexNet: - - @pytest.mark.slow @pytest.mark.parametrize('benchmark, expected_score', [ - ('Cowley2026_190923', approx(0.61209661, abs=0.001)), + ('Cowley2026.190923.V4-pls', approx(0.34609011, abs=0.005)), ]) def test_model_score(self, benchmark, expected_score): benchmark = load_benchmark(benchmark) - model = load_model('alexnet') - score = benchmark(model) - print(score) - assert score == expected_score \ No newline at end of file + score = benchmark(load_model('alexnet')) + assert score.values == expected_score diff --git a/brainscore_vision/data/cowley2026/__init__.py b/brainscore_vision/data/cowley2026/__init__.py index 371b1418f..6ae101d49 100644 --- a/brainscore_vision/data/cowley2026/__init__.py +++ b/brainscore_vision/data/cowley2026/__init__.py @@ -1,9 +1,7 @@ - from brainscore_vision import data_registry, stimulus_set_registry, load_stimulus_set from brainscore_core.supported_data_standards.brainio.s3 import load_stimulus_set_from_s3, load_assembly_from_s3 from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly -# Provide bibtex for proper citation of the original dataset BIBTEX = """@article{cowley2026compact, title={Compact deep neural network models of the visual cortex}, author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, @@ -14,39 +12,31 @@ year={2026}, publisher={Nature Publishing Group}}""" - -# stimulus set: 190923 -stimulus_set_registry['Cowley2026.190923'] = lambda: load_stimulus_set_from_s3( - identifier="Cowley2026.190923", - bucket="brainscore-storage/brainscore-vision/data/user_718/", - csv_sha1="7752f43fc809c193334dd97171867e733291b8fd", - zip_sha1="a14f9d4cfc98cb253f23d4eaa159c60666903668", - csv_version_id="4ZuvTJxZptY8V04ayk2CRLb209BihWis", - zip_version_id="kpX10KvUti_Vg4WAMHsiayllYlStMcgI") - - -# neural assembly 190923: -data_registry['Cowley2026.190923'] = lambda: load_assembly_from_s3( - identifier="Cowley2026.190923", - sha1="2ac7f60f21ccc5137074633c0614f52566acff6a", # From local packaging output - version_id="kpX10KvUti_Vg4WAMHsiayllYlStMcgI", # S3 version ID - bucket="brainscore-storage/brainscore-vision/data/user_718/", # S3 bucket path (provided after upload) - cls=NeuroidAssembly, # Assembly class (NeuroidAssembly or BehavioralAssembly) - stimulus_set_loader=lambda: load_stimulus_set('Cowley2026.190923') # Links to the stimulus set -) - - - - - - - - - - +BUCKET = "brainscore-storage/brainscore-vision/data/user_718/" +# keep literal `*_registry[''] =` lines below: plugin discovery greps for that substring +def stimulus_set(identifier, csv_sha1, zip_sha1, csv_version_id, zip_version_id): + return lambda: load_stimulus_set_from_s3( + identifier=identifier, bucket=BUCKET, + csv_sha1=csv_sha1, zip_sha1=zip_sha1, + csv_version_id=csv_version_id, zip_version_id=zip_version_id) +def assembly(identifier, sha1, version_id): + return lambda: load_assembly_from_s3( + identifier=identifier, bucket=BUCKET, sha1=sha1, version_id=version_id, + cls=NeuroidAssembly, stimulus_set_loader=lambda: load_stimulus_set(identifier)) +# session 190923 +stimulus_set_registry['Cowley2026.190923'] = stimulus_set( + 'Cowley2026.190923', + csv_sha1="7752f43fc809c193334dd97171867e733291b8fd", + zip_sha1="a14f9d4cfc98cb253f23d4eaa159c60666903668", + csv_version_id="4ZuvTJxZptY8V04ayk2CRLb209BihWis", + zip_version_id="tWyQAXN_fM4Y2fLnQtITbi0QMLawr4Nd") +data_registry['Cowley2026.190923'] = assembly( + 'Cowley2026.190923', + sha1="2ac7f60f21ccc5137074633c0614f52566acff6a", + version_id="kpX10KvUti_Vg4WAMHsiayllYlStMcgI") diff --git a/brainscore_vision/data/cowley2026/test.py b/brainscore_vision/data/cowley2026/test.py index 853453d79..a977e50db 100644 --- a/brainscore_vision/data/cowley2026/test.py +++ b/brainscore_vision/data/cowley2026/test.py @@ -2,114 +2,34 @@ import pytest from brainscore_vision import load_dataset, load_stimulus_set + @pytest.mark.private_access class TestStimulusSet: def test_stimulus_set_exists(self): - """Test that stimulus set loads correctly""" stimulus_set = load_stimulus_set('Cowley2026.190923') assert stimulus_set is not None assert stimulus_set.identifier == 'Cowley2026.190923' - # stimulus_set = load_stimulus_set('Cowley2026.201025') - # assert stimulus_set is not None - # assert stimulus_set.identifier == 'Cowley2026.201025' - - # stimulus_set = load_stimulus_set('Cowley2026.210225') - # assert stimulus_set is not None - # assert stimulus_set.identifier == 'Cowley2026.210225' - - # stimulus_set = load_stimulus_set('Cowley2026.211022') - # assert stimulus_set is not None - # assert stimulus_set.identifier == 'Cowley2026.211022' - - - def test_stimulus_set_counts(self): - """Test expected number of stimuli""" stimulus_set = load_stimulus_set('Cowley2026.190923') assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1200 - # stimulus_set = load_stimulus_set('Cowley2026.201025') - # assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1200 - - # stimulus_set = load_stimulus_set('Cowley2026.210225') - # assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1200 - - # stimulus_set = load_stimulus_set('Cowley2026.211022') - # assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1600 - - - @pytest.mark.private_access class TestAssembly: def test_assembly_exists(self): - """Test that assembly loads correctly""" assembly = load_dataset('Cowley2026.190923') assert assembly is not None assert assembly.identifier == 'Cowley2026.190923' - # assembly = load_dataset('Cowley2026.201025') - # assert assembly is not None - # assert assembly.identifier == 'Cowley2026.201025' - - # assembly = load_dataset('Cowley2026.210225') - # assert assembly is not None - # assert assembly.identifier == 'Cowley2026.210225' - - # assembly = load_dataset('Cowley2026.211022') - # assert assembly is not None - # assert assembly.identifier == 'Cowley2026.211022' - - - def test_assembly_structure(self): - """Test assembly has required dimensions and coordinates""" assembly = load_dataset('Cowley2026.190923') assert 'presentation' in assembly.dims assert 'stimulus_id' in assembly.indexes['presentation'].names - - # assembly = load_dataset('Cowley2026.201025') - # assert 'presentation' in assembly.dims - # assert 'stimulus_id' in assembly.indexes['presentation'].names - - # assembly = load_dataset('Cowley2026.210225') - # assert 'presentation' in assembly.dims - # assert 'stimulus_id' in assembly.indexes['presentation'].names - - # assembly = load_dataset('Cowley2026.211022') - # assert 'presentation' in assembly.dims - # assert 'stimulus_id' in assembly.indexes['presentation'].names - + assert set(np.unique(assembly['region'].values)) == {'V4'} def test_assembly_alignment(self): - """Test stimulus set and assembly are properly linked""" assembly = load_dataset('Cowley2026.190923') - stimulus_set = assembly.stimulus_set - - # All assembly stimulus IDs should exist in stimulus set assembly_stimuli = set(assembly['stimulus_id'].values) - stimulus_set_stimuli = set(stimulus_set['stimulus_id'].values) + stimulus_set_stimuli = set(assembly.stimulus_set['stimulus_id'].values) assert assembly_stimuli.issubset(stimulus_set_stimuli) - - # assembly = load_dataset('Cowley2026.201025') - # stimulus_set = assembly.stimulus_set - # assembly_stimuli = set(assembly['stimulus_id'].values) - # stimulus_set_stimuli = set(stimulus_set['stimulus_id'].values) - # assert assembly_stimuli.issubset(stimulus_set_stimuli) - - # assembly = load_dataset('Cowley2026.210225') - # stimulus_set = assembly.stimulus_set - # assembly_stimuli = set(assembly['stimulus_id'].values) - # stimulus_set_stimuli = set(stimulus_set['stimulus_id'].values) - # assert assembly_stimuli.issubset(stimulus_set_stimuli) - - # assembly = load_dataset('Cowley2026.211022') - # stimulus_set = assembly.stimulus_set - # assembly_stimuli = set(assembly['stimulus_id'].values) - # stimulus_set_stimuli = set(stimulus_set['stimulus_id'].values) - # assert assembly_stimuli.issubset(stimulus_set_stimuli) - - - -