diff --git a/brainscore_vision/benchmarks/cowley2026/__init__.py b/brainscore_vision/benchmarks/cowley2026/__init__.py new file mode 100644 index 000000000..06875eac9 --- /dev/null +++ b/brainscore_vision/benchmarks/cowley2026/__init__.py @@ -0,0 +1,4 @@ +from brainscore_vision import benchmark_registry +from .benchmark import Cowley2026_190923_V4PLS + +benchmark_registry['Cowley2026.190923.V4-pls'] = Cowley2026_190923_V4PLS diff --git a/brainscore_vision/benchmarks/cowley2026/benchmark.py b/brainscore_vision/benchmarks/cowley2026/benchmark.py new file mode 100644 index 000000000..14aa693b7 --- /dev/null +++ b/brainscore_vision/benchmarks/cowley2026/benchmark.py @@ -0,0 +1,55 @@ +from brainscore_vision import load_metric, load_ceiling, load_dataset +from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition + +VISUAL_DEGREES = 11.2 +NUMBER_OF_TRIALS = 14 # mode of the per-image repeat counts +BIBTEX = """@article{cowley2026compact, + title={Compact deep neural network models of the visual cortex}, + author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, + journal={Nature}, + volume={652}, + number={8111}, + pages={947--954}, + year={2026}, + publisher={Nature Publishing Group}}""" + +# no object categories -> plain random CV splits, not object_name stratification +pls_metric = lambda: load_metric('pls', crossvalidation_kwargs=dict(stratification_coord=None)) + + +def _Cowley2026V4PLS(session: str): + identifier = f'Cowley2026.{session}' + assembly_repetition = alternate_repetition_halves(load_assembly(identifier, average_repetitions=False)) + assembly = load_assembly(identifier, average_repetitions=True) + return NeuralBenchmark( + identifier=f'{identifier}.V4-pls', version=1, + assembly=assembly, similarity_metric=pls_metric(), + visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS, + ceiling_func=lambda: load_ceiling('internal_consistency')(assembly_repetition), + parent='V4', bibtex=BIBTEX) + + +def alternate_repetition_halves(assembly): + """Relabel repetitions to even/odd halves so the split-half ceiling balances per image.""" + names = list(assembly.indexes['presentation'].names) + half = (assembly['repetition'].values % 2).astype(int) + assembly = assembly.reset_index('presentation') + assembly['repetition'] = 'presentation', half + return assembly.set_index(presentation=names) + + +def load_assembly(identifier: str, average_repetitions: bool): + assembly = load_dataset(identifier) + assembly = assembly.sel(region='V4') + assembly = assembly.stack(neuroid=['neuroid_id']) # work around xarray multiindex issues + assembly['region'] = 'neuroid', ['V4'] * len(assembly['neuroid']) + assembly.load() + if 'time_bin' in assembly.dims: # single static window (50, 150) ms + assembly = assembly.squeeze('time_bin') + if average_repetitions: + assembly = average_repetition(assembly) + return assembly + + +def Cowley2026_190923_V4PLS(): + return _Cowley2026V4PLS('190923') diff --git a/brainscore_vision/benchmarks/cowley2026/test.py b/brainscore_vision/benchmarks/cowley2026/test.py new file mode 100644 index 000000000..334c8715e --- /dev/null +++ b/brainscore_vision/benchmarks/cowley2026/test.py @@ -0,0 +1,25 @@ +import pytest +from pytest import approx + +from brainscore_vision import load_benchmark, load_model + + +@pytest.mark.private_access +class TestExist: + @pytest.mark.parametrize('identifier', ['Cowley2026.190923.V4-pls']) + def test_benchmark_loads(self, identifier): + benchmark = load_benchmark(identifier) + assert benchmark is not None + assert benchmark.identifier == identifier + + +@pytest.mark.private_access +@pytest.mark.slow +class TestAlexNet: + @pytest.mark.parametrize('benchmark, expected_score', [ + ('Cowley2026.190923.V4-pls', approx(0.34609011, abs=0.005)), + ]) + def test_model_score(self, benchmark, expected_score): + benchmark = load_benchmark(benchmark) + score = benchmark(load_model('alexnet')) + assert score.values == expected_score diff --git a/brainscore_vision/data/cowley2026/__init__.py b/brainscore_vision/data/cowley2026/__init__.py new file mode 100644 index 000000000..6ae101d49 --- /dev/null +++ b/brainscore_vision/data/cowley2026/__init__.py @@ -0,0 +1,42 @@ +from brainscore_vision import data_registry, stimulus_set_registry, load_stimulus_set +from brainscore_core.supported_data_standards.brainio.s3 import load_stimulus_set_from_s3, load_assembly_from_s3 +from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly + +BIBTEX = """@article{cowley2026compact, + title={Compact deep neural network models of the visual cortex}, + author={Cowley, Benjamin R and Stan, Patricia L and Pillow, Jonathan W and Smith, Matthew A}, + journal={Nature}, + volume={652}, + number={8111}, + pages={947--954}, + year={2026}, + publisher={Nature Publishing Group}}""" + +BUCKET = "brainscore-storage/brainscore-vision/data/user_718/" + + +# keep literal `*_registry[''] =` lines below: plugin discovery greps for that substring +def stimulus_set(identifier, csv_sha1, zip_sha1, csv_version_id, zip_version_id): + return lambda: load_stimulus_set_from_s3( + identifier=identifier, bucket=BUCKET, + csv_sha1=csv_sha1, zip_sha1=zip_sha1, + csv_version_id=csv_version_id, zip_version_id=zip_version_id) + + +def assembly(identifier, sha1, version_id): + return lambda: load_assembly_from_s3( + identifier=identifier, bucket=BUCKET, sha1=sha1, version_id=version_id, + cls=NeuroidAssembly, stimulus_set_loader=lambda: load_stimulus_set(identifier)) + + +# session 190923 +stimulus_set_registry['Cowley2026.190923'] = stimulus_set( + 'Cowley2026.190923', + csv_sha1="7752f43fc809c193334dd97171867e733291b8fd", + zip_sha1="a14f9d4cfc98cb253f23d4eaa159c60666903668", + csv_version_id="4ZuvTJxZptY8V04ayk2CRLb209BihWis", + zip_version_id="tWyQAXN_fM4Y2fLnQtITbi0QMLawr4Nd") +data_registry['Cowley2026.190923'] = assembly( + 'Cowley2026.190923', + sha1="2ac7f60f21ccc5137074633c0614f52566acff6a", + version_id="kpX10KvUti_Vg4WAMHsiayllYlStMcgI") diff --git a/brainscore_vision/data/cowley2026/data_packaging/data_packaging.py b/brainscore_vision/data/cowley2026/data_packaging/data_packaging.py new file mode 100644 index 000000000..3b8679207 --- /dev/null +++ b/brainscore_vision/data/cowley2026/data_packaging/data_packaging.py @@ -0,0 +1,115 @@ + +import numpy as np +import brainscore_vision + +from brainscore_core.supported_data_standards.brainio.stimuli import StimulusSet +from brainscore_core.supported_data_standards.brainio.packaging import package_stimulus_set_locally +from brainscore_core.supported_data_standards.brainio.assemblies import DataAssembly +from brainscore_core.supported_data_standards.brainio import packaging +from brainscore_core.supported_data_standards.brainio.assemblies import NeuroidAssembly + +import statistics + + +# Note: +# Running local on mac + +session_id = 190923 # 190923 201025 210225 211022 +date_experiment = '2019-09-23' # '2019-09-23' '2020-10-25' '2021-02-25' '2021-10-22' + +responses = np.load('/Users/cowley/Desktop/brainscore_upload/data_raw/responses_{:d}.npy'.format(session_id)) + # (num_neurons, num_images, num_possible_repeats) + +num_neurons = responses.shape[0] +num_images = responses.shape[1] + + +## STIMULUS SET + +# Create a dataframe tracking image paths and attributes +print('---STIMULUS SET---') +stimuli_data = [{'stimulus_id': 'image{:04d}'.format(iimage), 'object_name': '{:04d}'.format(iimage)} for iimage in range(1,num_images+1)] +stimulus_set = StimulusSet(stimuli_data) + +stimulus_set.stimulus_paths = { + 'image{:04d}'.format(iimage): '/Users/cowley/Desktop/brainscore_upload/data_raw/images_{:d}/image{:04d}.jpg'.format(session_id, iimage) + for iimage in range(1,num_images+1) + } + +stimulus_set.name = 'Cowley2026.{:d}'.format(session_id) + +package_output = package_stimulus_set_locally( + proto_stimulus_set=stimulus_set, + stimulus_set_identifier=stimulus_set.name, +) + +print(package_output) + + + + + + +## NEURAL DATA + +print() +print('---NEURAL DATA---') + + +## flatten responses to be (num_neurons, num_repeats*num_images) + +responses_data_matrix = [] +stimulus_ids = [] +object_names = [] +repeat_ids = [] + + +for iimage in range(num_images): + num_repeats = np.sum(~np.isnan(responses[0,iimage,:])) + + for irepeat in range(num_repeats): + + responses_data_matrix.append(responses[:,iimage,irepeat]) + stimulus_ids.append('image{:04d}'.format(iimage+1)) + object_names.append('{:04d}'.format(iimage+1)) + repeat_ids.append(irepeat) + + +responses_data_matrix = np.stack(responses_data_matrix) + # (num_presentations, num_neurons) +responses_data_matrix = np.expand_dims(responses_data_matrix, axis=2) # include time_bin dimension (dummy) + + +assembly = NeuroidAssembly( + responses_data_matrix, + coords={ + # Coordinates tracking the 'presentation' dimension (axis 1) + 'stimulus_id': ('presentation', stimulus_ids), + 'object_name': ('presentation', object_names), + 'repetition': ('presentation', repeat_ids), + + # Coordinates tracking the 'neuroid' dimension (axis 0) + 'neuroid_id': ('neuroid', [f'neuron_{i}' for i in range(num_neurons)]), + 'region': ('neuroid', ['V4'] * num_neurons), # e.g., 'V4', 'IT', or 'AL' + + 'time_bin_start': ('time_bin', [50]), + 'time_bin_end': ('time_bin', [150]) + }, + dims=['presentation', 'neuroid', 'time_bin'] +) + + +assembly.attrs['experiment_date'] = date_experiment +assembly.name = 'Cowley2026.{:d}'.format(session_id) + + +package_output = packaging.package_data_assembly_locally( + proto_data_assembly=assembly, + assembly_identifier=stimulus_set.name, # We use the same stimulusSet name for the assembly + stimulus_set_identifier=stimulus_set.name, + assembly_class_name="NeuroidAssembly", # For most neural data, use NeuroidAssembly. For behavioral data, use BehavioralAssembly +) + +print(package_output) + + diff --git a/brainscore_vision/data/cowley2026/data_packaging/output_hash.txt b/brainscore_vision/data/cowley2026/data_packaging/output_hash.txt new file mode 100644 index 000000000..4b3e6d053 --- /dev/null +++ b/brainscore_vision/data/cowley2026/data_packaging/output_hash.txt @@ -0,0 +1,5 @@ +---STIMULUS SET--- +{'identifier': 'Cowley2026.190923', 'csv_path': '/Users/cowley/Downloads/brainscore_packages/stimulus_Cowley2026_190923.csv', 'zip_path': '/Users/cowley/Downloads/brainscore_packages/stimulus_Cowley2026_190923.zip', 'csv_sha1': '7752f43fc809c193334dd97171867e733291b8fd', 'zip_sha1': 'a14f9d4cfc98cb253f23d4eaa159c60666903668'} + +---NEURAL DATA--- +{'identifier': 'Cowley2026.190923', 'path': '/Users/cowley/Downloads/brainscore_packages/assy_Cowley2026_190923.nc', 'sha1': '2ac7f60f21ccc5137074633c0614f52566acff6a', 'cls': 'NeuroidAssembly'} diff --git a/brainscore_vision/data/cowley2026/test.py b/brainscore_vision/data/cowley2026/test.py new file mode 100644 index 000000000..a977e50db --- /dev/null +++ b/brainscore_vision/data/cowley2026/test.py @@ -0,0 +1,35 @@ +import numpy as np +import pytest +from brainscore_vision import load_dataset, load_stimulus_set + + +@pytest.mark.private_access +class TestStimulusSet: + def test_stimulus_set_exists(self): + stimulus_set = load_stimulus_set('Cowley2026.190923') + assert stimulus_set is not None + assert stimulus_set.identifier == 'Cowley2026.190923' + + def test_stimulus_set_counts(self): + stimulus_set = load_stimulus_set('Cowley2026.190923') + assert len(np.unique(stimulus_set['stimulus_id'].values)) == 1200 + + +@pytest.mark.private_access +class TestAssembly: + def test_assembly_exists(self): + assembly = load_dataset('Cowley2026.190923') + assert assembly is not None + assert assembly.identifier == 'Cowley2026.190923' + + def test_assembly_structure(self): + assembly = load_dataset('Cowley2026.190923') + assert 'presentation' in assembly.dims + assert 'stimulus_id' in assembly.indexes['presentation'].names + assert set(np.unique(assembly['region'].values)) == {'V4'} + + def test_assembly_alignment(self): + assembly = load_dataset('Cowley2026.190923') + assembly_stimuli = set(assembly['stimulus_id'].values) + stimulus_set_stimuli = set(assembly.stimulus_set['stimulus_id'].values) + assert assembly_stimuli.issubset(stimulus_set_stimuli)