From 55bbcbb3fd7cbc54c6187fb5698c9035c0240614 Mon Sep 17 00:00:00 2001 From: Jackson Loper Date: Wed, 29 Jul 2026 11:52:08 -0400 Subject: [PATCH 1/7] Stream ballots into a compact per-race projection for tabulation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tabulating a large election loaded every ballot as a full verbose row before handing it to the tabulators, which is one of the main drivers of star-server OOMKills. The algorithms are batch, so they genuinely need the whole election — but "the whole election" doesn't have to mean 30k verbose JSON ballot objects. getElectionResults now streams ballots from a db cursor (selecting only the votes column) and projects each row into a CompactVoteStore: flat typed arrays holding one race's marks positionally against its candidate order. The verbose row is dropped immediately, and the tabulator's own input is expanded from the store one race at a time. The positional layout is the same one bulk uploads already use, so it's now defined once in shared/domain_model/OrderedVoteCodec: the frontend uploader encodes with it, castVoteController's mapOrderedNewBallot decodes with it, and the projection stores it unrolled. Measured on a synthetic 31k-ballot election with 10 candidates/race, peak retained memory (heap + external): 1 race: 38.1 MB -> 10.0 MB 3 races: 72.1 MB -> 16.3 MB The compact store is 3.2 MB/race and lives off the JS heap, so it also stops adding GC-tracing pressure. What remains is the expanded rawVote cvr the tabulator API requires. A bug in a projection layer miscounts elections silently rather than failing loudly, so BallotProjection.test.ts pins the new path against a verbatim copy of the old inline projection: for every voting method, the tabulator inputs must match and tabulating both must give identical results. Fixtures cover null vs. absent marks, write-ins (approved, unapproved, unrecognized, aliased, shared aliases), duplicate scores, overvote_rank, has_duplicate_rank, skipped ranks and multi-race ballots. The distinction that most needed pinning: a candidate the ballot never mentions is not the same as one left explicitly blank. STAR treats an all-equal ballot as an abstention, so filling an approved write-in's missing slot with 0 would silently reclassify those ballots. Closes #1425 Co-Authored-By: Claude Opus 5 --- .../Controllers/Ballot/castVoteController.ts | 34 +- .../Election/getElectionResultsController.ts | 100 +--- packages/backend/src/Models/Ballots.ts | 22 +- packages/backend/src/Models/IBallotStore.ts | 8 + .../backend/src/Models/__mocks__/Ballots.ts | 13 +- .../src/Tabulators/BallotProjection.test.ts | 472 ++++++++++++++++++ .../src/Tabulators/BallotProjection.ts | 197 ++++++++ .../src/Tabulators/CompactVoteStore.ts | 181 +++++++ .../backend/src/test/orderedVoteCodec.test.ts | 97 ++++ .../src/components/UploadElections.tsx | 3 +- .../src/domain_model/OrderedVoteCodec.ts | 76 +++ packages/shared/src/domain_model/Vote.ts | 12 +- 12 files changed, 1110 insertions(+), 105 deletions(-) create mode 100644 packages/backend/src/Tabulators/BallotProjection.test.ts create mode 100644 packages/backend/src/Tabulators/BallotProjection.ts create mode 100644 packages/backend/src/Tabulators/CompactVoteStore.ts create mode 100644 packages/backend/src/test/orderedVoteCodec.test.ts create mode 100644 packages/shared/src/domain_model/OrderedVoteCodec.ts diff --git a/packages/backend/src/Controllers/Ballot/castVoteController.ts b/packages/backend/src/Controllers/Ballot/castVoteController.ts index 7ccca89bf..273cf4f42 100644 --- a/packages/backend/src/Controllers/Ballot/castVoteController.ts +++ b/packages/backend/src/Controllers/Ballot/castVoteController.ts @@ -16,8 +16,7 @@ import { io } from "../../socketHandler"; import { Server } from "socket.io"; import { expectPermission } from "../controllerUtils"; import { permissions } from "@equal-vote/star-vote-shared/domain_model/permissions"; -import { OrderedVote } from "@equal-vote/star-vote-shared/domain_model/Vote"; -import { Score } from "@equal-vote/star-vote-shared/domain_model/Score"; +import { OrderedVoteFormatError, orderedVotesToVotes } from "@equal-vote/star-vote-shared/domain_model/OrderedVoteCodec"; import { makeUniqueID, ID_LENGTHS, ID_PREFIXES } from "@equal-vote/star-vote-shared/utils/makeID"; const ElectionsModel = ServiceLocator.electionsDb(); @@ -124,28 +123,15 @@ async function makeBallotEvent(req: IElectionRequest, targetElection: Election, } const mapOrderedNewBallot = (ballot: OrderedNewBallot, raceOrder: RaceCandidateOrder[]): NewBallot => { - let subBallot: any = {...ballot}; - delete subBallot.orderedVotes; - if(ballot.orderedVotes.length != raceOrder.length){ - throw new BadRequest(`Ballot contains different number of races than race_order: ${ballot.orderedVotes.length} != ${raceOrder.length}`) - } - return { - ...subBallot, - votes: ballot.orderedVotes.map((vote: OrderedVote, i) => { - // +2 accounts for overvote_rank and has_duplicate_rank - if(vote.length != raceOrder[i].candidate_id_order.length+2){ - throw new BadRequest(`Race ${i} contains different number of candidates than race_order: ${vote.length} != ${raceOrder[i].candidate_id_order.length+2}`) - } - return { - race_id: raceOrder[i].race_id, - scores: vote.slice(0, -2).map((s, j) => ({ - candidate_id: raceOrder[i].candidate_id_order[j], - score: s - } as Score)), - overvote_rank: vote.at(-2), - has_duplicate_rank: vote.at(-1) == 1, - } - }) + const {orderedVotes, ...subBallot} = ballot; + try { + return { + ...subBallot, + votes: orderedVotesToVotes(orderedVotes, raceOrder) + } + } catch (err: any) { + if (err instanceof OrderedVoteFormatError) throw new BadRequest(err.message); + throw err; } } async function uploadBallotsController(req: IElectionRequest, res: Response, next: NextFunction) { diff --git a/packages/backend/src/Controllers/Election/getElectionResultsController.ts b/packages/backend/src/Controllers/Election/getElectionResultsController.ts index 512eaf09d..b3a58681f 100644 --- a/packages/backend/src/Controllers/Election/getElectionResultsController.ts +++ b/packages/backend/src/Controllers/Election/getElectionResultsController.ts @@ -1,16 +1,13 @@ import ServiceLocator from "../../ServiceLocator"; import Logger from "../../Services/Logging/Logger"; -import { BadRequest, Forbidden } from "@curveball/http-errors"; -import { Ballot } from '@equal-vote/star-vote-shared/domain_model/Ballot'; +import { Forbidden } from "@curveball/http-errors"; import { expectPermission } from "../controllerUtils"; import { permissions } from '@equal-vote/star-vote-shared/domain_model/permissions'; import { VotingMethods } from '../../Tabulators/VotingMethodSelecter'; import { IElectionRequest } from "../../IRequest"; import { Response, NextFunction } from 'express'; -import { ElectionResults, candidate, rawVote } from "@equal-vote/star-vote-shared/domain_model/ITabulators"; -import { makeWriteInCandidateId } from "@equal-vote/star-vote-shared/utils/makeID"; -import { Candidate } from "@equal-vote/star-vote-shared/domain_model/Candidate"; -import { trimLower } from "@equal-vote/star-vote-shared/domain_model/Util"; +import { ElectionResults, rawVote } from "@equal-vote/star-vote-shared/domain_model/ITabulators"; +import { projectBallots } from "../../Tabulators/BallotProjection"; import shuffleCandidatesForRandomTiebreak from "../../Tabulators/shuffleCandidatesForRandomTiebreak"; const BallotModel = ServiceLocator.ballotsDb(); @@ -30,87 +27,34 @@ const getElectionResults = async (req: IElectionRequest, res: Response, next: Ne expectPermission(req.user_auth.roles, permissions.canViewPreliminaryResults) } - const ballots = await BallotModel.getBallotsByElectionID(String(electionId), req); + // Stream ballots from a db cursor and project each one into the compact + // per-race representation, so the verbose rows are never all in memory at + // once (see Tabulators/BallotProjection). + const projections = await projectBallots( + election.races, + BallotModel.streamVotesByElectionID(String(electionId), req), + { + debug: (msg) => Logger.debug(req, msg), + warn: (msg) => Logger.warn(req, msg), + } + ) let results: ElectionResults[] = [] for (let race_index = 0; race_index < election.races.length; race_index++) { const race = election.races[race_index] - const useWriteIns = race.enable_write_in && race.write_in_candidates && race.write_in_candidates.length > 0 - const writeInCandidates = useWriteIns && race.write_in_candidates ? race.write_in_candidates : [] - - // Build candidate list including approved write-in candidates - const candidates: candidate[] = race.candidates.map((c: Candidate, i) => ({ - id: c.candidate_id, - name: c.candidate_name, - tieBreakOrder: -1, - votesPreferredOver: {}, - winsAgainst: {} - })) - - Logger.debug(req, `[WriteIn Debug] race=${race.race_id} useWriteIns=${useWriteIns} writeInCandidates=${JSON.stringify(writeInCandidates.map(wc => ({name: wc.candidate_name, approved: wc.approved, aliases: wc.aliases})))}`); + const projection = projections[race_index] + const candidates = projection.candidates + const numUnprocessedWriteIns = projection.numUnprocessedWriteIns + const numExcludedWriteIns = projection.numExcludedWriteIns - if (useWriteIns) { - writeInCandidates.forEach((wc, i) => { - if (wc.approved) { - candidates.push({ - id: makeWriteInCandidateId(wc.candidate_name), - name: wc.candidate_name, - tieBreakOrder: -1, - votesPreferredOver: {}, - winsAgainst: {} - }) - } - }) - } + Logger.debug(req, `[WriteIn Debug] race=${race.race_id} useWriteIns=${projection.useWriteIns} writeInCandidates=${JSON.stringify(projection.writeInCandidates.map(wc => ({name: wc.candidate_name, approved: wc.approved, aliases: wc.aliases})))}`); Logger.debug(req, `[WriteIn Debug] candidates for tabulation: ${JSON.stringify(candidates.map(c => ({id: c.id, name: c.name})))}`); - const race_id = race.race_id - const cvr: rawVote[] = [] const num_winners = race.num_winners const voting_method = race.voting_method - let numUnprocessedWriteIns = 0 - let numExcludedWriteIns = 0 - - ballots.forEach((ballot: Ballot) => { - const vote = ballot.votes.find((vote) => vote.race_id === race_id) - if (vote) { - const marks: {[key: string]: number | null} = {} - vote.scores.forEach(score => { - const isRegularCandidate = race.candidates.some((c: Candidate) => c.candidate_id === score.candidate_id) - if (isRegularCandidate) { - if (score.candidate_id in marks) { - Logger.warn(req, `[Tabulation] Duplicate score for candidate "${score.candidate_id}" on same ballot, keeping first score`); - } else { - marks[score.candidate_id] = score.score - } - } else if (race.enable_write_in && score.write_in_name) { - const write_in_name = score.write_in_name - const writeInCandidate = writeInCandidates.find(wc => wc.aliases.includes(trimLower(write_in_name))) - Logger.debug(req, `[WriteIn Debug] ballot write_in_name="${write_in_name}" matched=${!!writeInCandidate} approved=${writeInCandidate?.approved} matchedAliases=${JSON.stringify(writeInCandidate?.aliases)}`); - if (!writeInCandidate) { - numUnprocessedWriteIns += 1 - numExcludedWriteIns += 1 - } else if (writeInCandidate.approved) { - const wcId = makeWriteInCandidateId(writeInCandidate.candidate_name) - if (!(wcId in marks)) { - marks[wcId] = score.score - } else { - Logger.warn(req, `[WriteIn] Duplicate write-in score for "${writeInCandidate.candidate_name}" on same ballot, keeping first score`); - } - } else { - numExcludedWriteIns += 1 - } - } - }) - cvr.push({ - marks, - overvote_rank: vote?.overvote_rank, - has_duplicate_rank: vote?.has_duplicate_rank, - }) - } - }) if (candidates.length < 1) { + projection.release() results[race_index] = { votingMethod: voting_method, elected: [], @@ -138,6 +82,10 @@ const getElectionResults = async (req: IElectionRequest, res: Response, next: Ne throw new Error(`Invalid Voting Method: ${voting_method}`) } + // Expand the compact store into the tabulator's input only now, and only + // for this race, so at most one race's verbose cvr is alive at a time. + const cvr: rawVote[] = projection.takeRawVotes() + shuffleCandidatesForRandomTiebreak(election.create_date, candidates, cvr.length, race.race_id); const perm = candidates.map(candidate => candidate.id); diff --git a/packages/backend/src/Models/Ballots.ts b/packages/backend/src/Models/Ballots.ts index 2983d0063..d3c08cae0 100644 --- a/packages/backend/src/Models/Ballots.ts +++ b/packages/backend/src/Models/Ballots.ts @@ -3,7 +3,7 @@ import { Uid } from '@equal-vote/star-vote-shared/domain_model/Uid'; import { ILoggingContext } from '../Services/Logging/ILogger'; import Logger from '../Services/Logging/Logger'; import { logSafeHash } from '../Services/Logging/logSafeHash'; -import { IBallotStore } from './IBallotStore'; +import { BallotVotes, IBallotStore } from './IBallotStore'; import { Kysely, sql, Transaction } from 'kysely'; import { Database } from './Database'; import { InternalServerError } from '@curveball/http-errors'; @@ -135,6 +135,26 @@ export default class BallotsDB implements IBallotStore { .stream(500) as AsyncIterableIterator; } + // Tabulation only reads the marks, so select just the `votes` column and + // stream it from a cursor. Skipping ballot_id/history/timestamps and never + // holding the full result set is what keeps a large election's tabulation + // from inflating the heap (see issue #1425). + // + // The filter deliberately matches getBallotsByElectionID (head only, any + // status) rather than streamSubmittedBallotsByElectionID — narrowing it to + // submitted ballots here would silently change election results. + streamVotesByElectionID(election_id: string, ctx: ILoggingContext, db?: Kysely | Transaction): AsyncIterableIterator { + Logger.debug(ctx, `${tableName}.streamVotesByElectionID ${election_id}`); + const client = db || this._postgresClient; + + return client + .selectFrom(tableName) + .select('votes') + .where('election_id', '=', election_id) + .where('head', '=', true) + .stream(500) as AsyncIterableIterator; + } + getBallotByVoterID(voter_id: string, election_id: string, ctx: ILoggingContext, db?: Kysely | Transaction): Promise { Logger.debug(ctx, `${tableName}.getBallotByVoterID ${logSafeHash(voter_id)} ${election_id}`); const client = db || this._postgresClient; diff --git a/packages/backend/src/Models/IBallotStore.ts b/packages/backend/src/Models/IBallotStore.ts index 9bb228fe0..ec73ec245 100644 --- a/packages/backend/src/Models/IBallotStore.ts +++ b/packages/backend/src/Models/IBallotStore.ts @@ -1,9 +1,15 @@ import { Ballot } from "@equal-vote/star-vote-shared/domain_model/Ballot"; import { Uid } from "@equal-vote/star-vote-shared/domain_model/Uid"; +import { Vote } from "@equal-vote/star-vote-shared/domain_model/Vote"; import { ILoggingContext } from "../Services/Logging/ILogger"; import { Kysely, Transaction } from 'kysely'; import { Database } from './Database'; +/** The only column tabulation reads off a ballot row. */ +export interface BallotVotes { + votes: Vote[]; +} + export interface IBallotStore { submitBallot: (ballot: Ballot, ctx: ILoggingContext, reason: string, db?: Kysely | Transaction) => Promise; updateBallot: (ballot: Ballot, ctx: ILoggingContext, reason: string, db?: Kysely | Transaction) => Promise; @@ -12,6 +18,8 @@ export interface IBallotStore { getBallotsByElectionID: (election_id: string, ctx: ILoggingContext, db?: Kysely | Transaction) => Promise; // Streams head, submitted ballots in random order (see Ballots.ts for the anonymity rationale) streamSubmittedBallotsByElectionID: (election_id: string, ctx: ILoggingContext, db?: Kysely | Transaction) => AsyncIterableIterator; + // Streams just the votes column of every head ballot, for tabulation + streamVotesByElectionID: (election_id: string, ctx: ILoggingContext, db?: Kysely | Transaction) => AsyncIterableIterator; getBallotByVoterID: (voter_id: string, election_id: string, ctx: ILoggingContext, db?: Kysely | Transaction) => Promise; delete(ballot_id: Uid, ctx: ILoggingContext, reason: string, db?: Kysely | Transaction): Promise; deleteAllBallotsForElectionID: (election_id: string, ctx: ILoggingContext, db?: Kysely | Transaction) => Promise; diff --git a/packages/backend/src/Models/__mocks__/Ballots.ts b/packages/backend/src/Models/__mocks__/Ballots.ts index 9a634e3a0..a8a412894 100644 --- a/packages/backend/src/Models/__mocks__/Ballots.ts +++ b/packages/backend/src/Models/__mocks__/Ballots.ts @@ -1,7 +1,7 @@ import { Ballot } from "@equal-vote/star-vote-shared/domain_model/Ballot"; import { Uid } from "@equal-vote/star-vote-shared/domain_model/Uid"; import { ILoggingContext } from "../../Services/Logging/ILogger"; -import { IBallotStore } from "../IBallotStore"; +import { BallotVotes, IBallotStore } from "../IBallotStore"; export default class BallotsDB implements IBallotStore { ballots: Ballot[] = []; @@ -47,6 +47,17 @@ export default class BallotsDB implements IBallotStore { } } + // mirrors getBallotsByElectionID's filter (see Ballots.ts) so tabulation + // sees exactly the same ballots as the old non-streaming path + async *streamVotesByElectionID(election_id: string, ctx:ILoggingContext): AsyncIterableIterator { + const ballots = this.ballots.filter( + (ballot) => ballot.election_id === election_id + ); + for (const ballot of ballots) { + yield { votes: JSON.parse(JSON.stringify(ballot.votes)) }; + } + } + getBallotByVoterID(voter_id: string, election_id: string, ctx:ILoggingContext): Promise { const ballots = this.ballots.filter( (ballot) => ballot.user_id === voter_id diff --git a/packages/backend/src/Tabulators/BallotProjection.test.ts b/packages/backend/src/Tabulators/BallotProjection.test.ts new file mode 100644 index 000000000..e43d546d6 --- /dev/null +++ b/packages/backend/src/Tabulators/BallotProjection.test.ts @@ -0,0 +1,472 @@ +import { Race, VotingMethod } from "@equal-vote/star-vote-shared/domain_model/Race"; +import { Candidate } from "@equal-vote/star-vote-shared/domain_model/Candidate"; +import { Vote } from "@equal-vote/star-vote-shared/domain_model/Vote"; +import { Score } from "@equal-vote/star-vote-shared/domain_model/Score"; +import { ElectionSettings } from "@equal-vote/star-vote-shared/domain_model/ElectionSettings"; +import { candidate, rawVote } from "@equal-vote/star-vote-shared/domain_model/ITabulators"; +import { makeWriteInCandidateId } from "@equal-vote/star-vote-shared/utils/makeID"; +import { trimLower } from "@equal-vote/star-vote-shared/domain_model/Util"; +import { orderedVotesToVotes } from "@equal-vote/star-vote-shared/domain_model/OrderedVoteCodec"; +import { BallotVotes } from "../Models/IBallotStore"; +import { projectBallots } from "./BallotProjection"; +import { VotingMethods } from "./VotingMethodSelecter"; + +// The compact projection re-encodes the semantic content of every ballot, so a +// bug in it miscounts elections silently rather than failing loudly. These tests +// pin it against `referenceProjection` below — a verbatim copy of the inline +// projection getElectionResultsController used before the streaming rewrite — +// and assert both that the tabulator inputs match and that tabulating each +// voting method over the two produces identical results. + +// --------------------------------------------------------------------------- +// Reference implementation (the pre-streaming code path, unchanged) +// --------------------------------------------------------------------------- + +const referenceProjection = (race: Race, ballots: BallotVotes[]) => { + const useWriteIns = race.enable_write_in && race.write_in_candidates && race.write_in_candidates.length > 0 + const writeInCandidates = useWriteIns && race.write_in_candidates ? race.write_in_candidates : [] + + const candidates: candidate[] = race.candidates.map((c: Candidate) => ({ + id: c.candidate_id, + name: c.candidate_name, + tieBreakOrder: -1, + votesPreferredOver: {}, + winsAgainst: {} + })) + + if (useWriteIns) { + writeInCandidates.forEach((wc) => { + if (wc.approved) { + candidates.push({ + id: makeWriteInCandidateId(wc.candidate_name), + name: wc.candidate_name, + tieBreakOrder: -1, + votesPreferredOver: {}, + winsAgainst: {} + }) + } + }) + } + + const race_id = race.race_id + const cvr: rawVote[] = [] + let numUnprocessedWriteIns = 0 + let numExcludedWriteIns = 0 + + ballots.forEach((ballot) => { + const vote = ballot.votes.find((vote) => vote.race_id === race_id) + if (vote) { + const marks: {[key: string]: number | null} = {} + vote.scores.forEach(score => { + const isRegularCandidate = race.candidates.some((c: Candidate) => c.candidate_id === score.candidate_id) + if (isRegularCandidate) { + if (!(score.candidate_id in marks)) { + marks[score.candidate_id] = score.score + } + } else if (race.enable_write_in && score.write_in_name) { + const write_in_name = score.write_in_name + const writeInCandidate = writeInCandidates.find(wc => wc.aliases.includes(trimLower(write_in_name))) + if (!writeInCandidate) { + numUnprocessedWriteIns += 1 + numExcludedWriteIns += 1 + } else if (writeInCandidate.approved) { + const wcId = makeWriteInCandidateId(writeInCandidate.candidate_name) + if (!(wcId in marks)) { + marks[wcId] = score.score + } + } else { + numExcludedWriteIns += 1 + } + } + }) + cvr.push({ + marks, + overvote_rank: vote?.overvote_rank, + has_duplicate_rank: vote?.has_duplicate_rank, + }) + } + }) + + return { candidates, cvr, numUnprocessedWriteIns, numExcludedWriteIns } +} + +// --------------------------------------------------------------------------- +// Fixture helpers +// --------------------------------------------------------------------------- + +async function* asStream(ballots: BallotVotes[]): AsyncIterableIterator { + for (const ballot of ballots) yield ballot; +} + +const makeRace = (overrides: Partial & {race_id: string, voting_method: VotingMethod, candidateNames: string[]}): Race => ({ + race_id: overrides.race_id, + title: overrides.race_id, + voting_method: overrides.voting_method, + num_winners: overrides.num_winners ?? 1, + candidates: overrides.candidateNames.map(name => ({candidate_id: `c-${name}`, candidate_name: name} as Candidate)), + enable_write_in: overrides.enable_write_in, + write_in_candidates: overrides.write_in_candidates, +} as Race); + +// deterministic RNG so a failure is always reproducible +const mulberry32 = (seed: number) => () => { + seed = (seed + 0x6D2B79F5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; +}; + +const shuffled = (items: T[], rand: () => number): T[] => { + const copy = [...items]; + for (let i = copy.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)); + [copy[i], copy[j]] = [copy[j], copy[i]]; + } + return copy; +}; + +/** + * Generates ballots that exercise every edge case the projection has to + * preserve: null marks, omitted candidates, scores out of candidate order, + * duplicate scores, write-ins (approved / unapproved / unrecognized / aliased), + * overvote_rank, has_duplicate_rank, skipped ranks, and ballots that skip the + * race entirely. + */ +const generateBallots = (race: Race, count: number, seed: number): BallotVotes[] => { + const rand = mulberry32(seed); + const maxMark = ['IRV', 'STV', 'RankedRobin'].includes(race.voting_method) ? race.candidates.length + : ['Approval', 'Plurality'].includes(race.voting_method) ? 1 : 5; + const ballots: BallotVotes[] = []; + + for (let b = 0; b < count; b++) { + // ~8% of ballots never voted in this race + if (rand() < 0.08) { + ballots.push({votes: [{race_id: 'some-other-race', scores: []}]}); + continue; + } + + const scores: Score[] = []; + race.candidates.forEach(c => { + const roll = rand(); + if (roll < 0.1) return; // candidate omitted entirely + if (roll < 0.25) { scores.push({candidate_id: c.candidate_id, score: null}); return; } + scores.push({candidate_id: c.candidate_id, score: Math.floor(rand() * (maxMark + 1))}); + }); + + // out-of-bounds mark + if (rand() < 0.05 && scores.length > 0) scores[0].score = maxMark + 3; + // duplicate score for a candidate already marked + if (rand() < 0.1 && scores.length > 0) scores.push({...scores[0], score: 2}); + + if (race.enable_write_in) { + const writeInNames = ['Charlie', ' charlie ', 'chuck', 'Dana', 'Nobody At All']; + const n = Math.floor(rand() * 3); + for (let i = 0; i < n; i++) { + const name = writeInNames[Math.floor(rand() * writeInNames.length)]; + scores.push({ + candidate_id: makeWriteInCandidateId(name.trim()), + score: Math.floor(rand() * (maxMark + 1)), + write_in_name: name, + }); + } + } + + const vote: Vote = {race_id: race.race_id, scores: shuffled(scores, rand)}; + const overvoteRoll = rand(); + if (overvoteRoll < 0.1) vote.overvote_rank = 1 + Math.floor(rand() * maxMark); + const duplicateRoll = rand(); + if (duplicateRoll < 0.1) vote.has_duplicate_rank = true; + else if (duplicateRoll < 0.2) vote.has_duplicate_rank = false; + // otherwise left undefined + + ballots.push({votes: [vote]}); + } + return ballots; +}; + +const WRITE_INS = [ + {candidate_name: 'Charlie', aliases: ['charlie', 'chuck'], approved: true}, + {candidate_name: 'Dana', aliases: ['dana'], approved: false}, +]; + +const SETTINGS = {max_rankings: 5} as ElectionSettings; + +const compareProjection = async (race: Race, ballots: BallotVotes[]) => { + const reference = referenceProjection(race, ballots); + const [projection] = await projectBallots([race], asStream(ballots)); + + expect(projection.candidates).toEqual(reference.candidates); + expect(projection.numUnprocessedWriteIns).toBe(reference.numUnprocessedWriteIns); + expect(projection.numExcludedWriteIns).toBe(reference.numExcludedWriteIns); + expect(projection.count).toBe(reference.cvr.length); + + const cvr = projection.takeRawVotes(); + // toEqual ignores key insertion order, which the tabulators are also + // insensitive to (they only ever read marks by candidate id, or reduce over + // all of them with order-independent operations) + expect(cvr).toEqual(reference.cvr); + return {reference, cvr, candidates: projection.candidates}; +}; + +const compareTabulation = async (race: Race, ballots: BallotVotes[]) => { + const {reference, cvr, candidates} = await compareProjection(race, ballots); + const tabulate = VotingMethods[race.voting_method]; + const expected = tabulate(reference.candidates, reference.cvr, race.num_winners, SETTINGS); + const actual = tabulate(candidates, cvr, race.num_winners, SETTINGS); + expect(JSON.stringify(actual)).toEqual(JSON.stringify(expected)); +}; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("BallotProjection matches the verbose projection", () => { + const methods: VotingMethod[] = ['STAR', 'STAR_PR', 'Approval', 'Plurality', 'RankedRobin', 'IRV', 'STV']; + + methods.forEach(voting_method => { + test(`${voting_method}: plain race`, async () => { + const race = makeRace({race_id: 'r0', voting_method, candidateNames: ['Allison', 'Bill', 'Carmen', 'Doug'], num_winners: voting_method === 'STV' || voting_method === 'STAR_PR' ? 2 : 1}); + await compareTabulation(race, generateBallots(race, 200, 11)); + }); + + test(`${voting_method}: with write-ins`, async () => { + const race = makeRace({ + race_id: 'r0', + voting_method, + candidateNames: ['Allison', 'Bill', 'Carmen'], + num_winners: voting_method === 'STV' || voting_method === 'STAR_PR' ? 2 : 1, + enable_write_in: true, + write_in_candidates: WRITE_INS, + }); + await compareTabulation(race, generateBallots(race, 200, 23)); + }); + }); + + test("multi-race ballots project independently", async () => { + const races = [ + makeRace({race_id: 'r0', voting_method: 'STAR', candidateNames: ['Allison', 'Bill']}), + makeRace({race_id: 'r1', voting_method: 'IRV', candidateNames: ['Carmen', 'Doug', 'Elle']}), + makeRace({race_id: 'r2', voting_method: 'Approval', candidateNames: ['Frank', 'Gina'], enable_write_in: true, write_in_candidates: WRITE_INS}), + ]; + // each ballot votes in a different subset of the races + const perRace = races.map((race, i) => generateBallots(race, 120, 31 + i)); + const ballots: BallotVotes[] = perRace[0].map((_, i) => ({ + votes: races.flatMap((race, r) => perRace[r][i].votes.filter(v => v.race_id === race.race_id)) + })); + + const projections = await projectBallots(races, asStream(ballots)); + races.forEach((race, r) => { + const reference = referenceProjection(race, ballots); + expect(projections[r].candidates).toEqual(reference.candidates); + expect(projections[r].numUnprocessedWriteIns).toBe(reference.numUnprocessedWriteIns); + expect(projections[r].numExcludedWriteIns).toBe(reference.numExcludedWriteIns); + expect(projections[r].takeRawVotes()).toEqual(reference.cvr); + }); + }); + + test("a candidate the ballot never mentions stays absent, not zero", async () => { + // This is the case that makes null and undefined marks non-interchangeable: + // STAR treats an all-equal ballot as an abstention, and a ballot that + // omits the approved write-in must not gain an implicit 0 for them. + const race = makeRace({ + race_id: 'r0', + voting_method: 'STAR', + candidateNames: ['Allison', 'Bill'], + enable_write_in: true, + write_in_candidates: WRITE_INS, + }); + const ballots: BallotVotes[] = [ + {votes: [{race_id: 'r0', scores: [ + {candidate_id: 'c-Allison', score: 5}, + {candidate_id: 'c-Bill', score: 5}, + ]}]}, + {votes: [{race_id: 'r0', scores: [ + {candidate_id: 'c-Allison', score: 5}, + {candidate_id: 'c-Bill', score: 1}, + {candidate_id: makeWriteInCandidateId('Charlie'), score: 3, write_in_name: 'Charlie'}, + ]}]}, + ]; + const {cvr} = await compareProjection(race, ballots); + expect(cvr[0].marks).toEqual({'c-Allison': 5, 'c-Bill': 5}); + expect(makeWriteInCandidateId('Charlie') in cvr[0].marks).toBe(false); + await compareTabulation(race, ballots); + }); + + test("duplicate scores keep the first, for both regular and write-in candidates", async () => { + const race = makeRace({ + race_id: 'r0', + voting_method: 'STAR', + candidateNames: ['Allison', 'Bill'], + enable_write_in: true, + write_in_candidates: WRITE_INS, + }); + const warnings: string[] = []; + const ballots: BallotVotes[] = [ + {votes: [{race_id: 'r0', scores: [ + {candidate_id: 'c-Allison', score: 4}, + {candidate_id: 'c-Allison', score: 1}, + {candidate_id: 'c-Bill', score: null}, + {candidate_id: 'c-Bill', score: 5}, + {candidate_id: 'cwi-x', score: 2, write_in_name: 'chuck'}, + {candidate_id: 'cwi-y', score: 5, write_in_name: 'Charlie'}, + ]}]}, + ]; + const [projection] = await projectBallots([race], asStream(ballots), {warn: (m) => warnings.push(m)}); + expect(projection.takeRawVotes()[0].marks).toEqual({ + 'c-Allison': 4, + 'c-Bill': null, + [makeWriteInCandidateId('Charlie')]: 2, + }); + expect(warnings).toEqual([ + '[Tabulation] Duplicate score for candidate "c-Allison" on same ballot, keeping first score', + '[Tabulation] Duplicate score for candidate "c-Bill" on same ballot, keeping first score', + '[WriteIn] Duplicate write-in score for "Charlie" on same ballot, keeping first score', + ]); + }); + + test("write-in aliases, unapproved write-ins and unrecognized names are counted the old way", async () => { + const race = makeRace({ + race_id: 'r0', + voting_method: 'Approval', + candidateNames: ['Allison'], + enable_write_in: true, + write_in_candidates: WRITE_INS, + }); + const ballots: BallotVotes[] = [ + // alias match (trimmed + lowercased) onto the approved write-in + {votes: [{race_id: 'r0', scores: [{candidate_id: 'x', score: 1, write_in_name: ' CHUCK '}]}]}, + // matched but unapproved -> excluded only + {votes: [{race_id: 'r0', scores: [{candidate_id: 'x', score: 1, write_in_name: 'Dana'}]}]}, + // unmatched -> unprocessed and excluded + {votes: [{race_id: 'r0', scores: [{candidate_id: 'x', score: 1, write_in_name: 'Nobody At All'}]}]}, + // write_in_name absent -> silently dropped, no diagnostics + {votes: [{race_id: 'r0', scores: [{candidate_id: 'cwi-Charlie', score: 1}]}]}, + ]; + const {reference, cvr} = await compareProjection(race, ballots); + expect(reference.numUnprocessedWriteIns).toBe(1); + expect(reference.numExcludedWriteIns).toBe(2); + expect(cvr[0].marks).toEqual({[makeWriteInCandidateId('Charlie')]: 1}); + expect(cvr[3].marks).toEqual({}); + }); + + test("when two write-in candidates share an alias, the first one still wins", async () => { + // the old code resolved aliases with writeInCandidates.find(...), so the + // earlier entry won; an alias index has to preserve that + const race = makeRace({ + race_id: 'r0', + voting_method: 'Approval', + candidateNames: ['Allison'], + enable_write_in: true, + write_in_candidates: [ + {candidate_name: 'Charlie', aliases: ['chuck'], approved: true}, + {candidate_name: 'Chuck Jones', aliases: ['chuck'], approved: true}, + ], + }); + const ballots: BallotVotes[] = [ + {votes: [{race_id: 'r0', scores: [{candidate_id: 'x', score: 1, write_in_name: 'Chuck'}]}]}, + ]; + const {cvr} = await compareProjection(race, ballots); + expect(cvr[0].marks).toEqual({[makeWriteInCandidateId('Charlie')]: 1}); + }); + + test("an unapproved write-in listed before an approved one still shadows it", async () => { + const race = makeRace({ + race_id: 'r0', + voting_method: 'Approval', + candidateNames: ['Allison'], + enable_write_in: true, + write_in_candidates: [ + {candidate_name: 'Dana', aliases: ['chuck'], approved: false}, + {candidate_name: 'Charlie', aliases: ['chuck'], approved: true}, + ], + }); + const ballots: BallotVotes[] = [ + {votes: [{race_id: 'r0', scores: [{candidate_id: 'x', score: 1, write_in_name: 'chuck'}]}]}, + ]; + const {reference, cvr} = await compareProjection(race, ballots); + expect(cvr[0].marks).toEqual({}); + expect(reference.numExcludedWriteIns).toBe(1); + expect(reference.numUnprocessedWriteIns).toBe(0); + }); + + test("write-ins are ignored when the race has no write-in list", async () => { + // enable_write_in without write_in_candidates: every write-in is unprocessed + const race = makeRace({race_id: 'r0', voting_method: 'STAR', candidateNames: ['Allison'], enable_write_in: true}); + const ballots: BallotVotes[] = [ + {votes: [{race_id: 'r0', scores: [ + {candidate_id: 'c-Allison', score: 3}, + {candidate_id: 'x', score: 5, write_in_name: 'Charlie'}, + ]}]}, + ]; + const {reference} = await compareProjection(race, ballots); + expect(reference.numUnprocessedWriteIns).toBe(1); + }); + + test("overvote_rank and has_duplicate_rank round-trip, including when unset", async () => { + const race = makeRace({race_id: 'r0', voting_method: 'IRV', candidateNames: ['Allison', 'Bill', 'Carmen']}); + const ballots: BallotVotes[] = [ + {votes: [{race_id: 'r0', scores: [{candidate_id: 'c-Allison', score: 1}], overvote_rank: 2, has_duplicate_rank: true}]}, + {votes: [{race_id: 'r0', scores: [{candidate_id: 'c-Allison', score: 1}], has_duplicate_rank: false}]}, + // skipped rankings, nothing set + {votes: [{race_id: 'r0', scores: [{candidate_id: 'c-Allison', score: 1}, {candidate_id: 'c-Bill', score: null}, {candidate_id: 'c-Carmen', score: 3}]}]}, + ]; + const {cvr} = await compareProjection(race, ballots); + expect(cvr[0].overvote_rank).toBe(2); + expect(cvr[0].has_duplicate_rank).toBe(true); + expect(cvr[1].overvote_rank).toBeUndefined(); + expect(cvr[1].has_duplicate_rank).toBe(false); + expect(cvr[2].overvote_rank).toBeUndefined(); + expect(cvr[2].has_duplicate_rank).toBeUndefined(); + await compareTabulation(race, ballots); + }); + + test("a race with no candidates projects empty ballots", async () => { + const race = makeRace({race_id: 'r0', voting_method: 'STAR', candidateNames: []}); + const ballots: BallotVotes[] = [ + {votes: [{race_id: 'r0', scores: [{candidate_id: 'c-Ghost', score: 5}]}]}, + ]; + const {cvr} = await compareProjection(race, ballots); + expect(cvr).toEqual([{marks: {}, overvote_rank: undefined, has_duplicate_rank: undefined}]); + }); + + test("takeRawVotes releases the compact store", async () => { + const race = makeRace({race_id: 'r0', voting_method: 'STAR', candidateNames: ['Allison']}); + const ballots = generateBallots(race, 5, 7); + const [projection] = await projectBallots([race], asStream(ballots)); + const count = projection.count; + expect(projection.takeRawVotes().length).toBe(count); + // the buffers are gone; expanding again would read zeroed memory, so it + // fails loudly rather than quietly producing a second, wrong cvr + expect(() => projection.takeRawVotes()).toThrow(/released/); + expect(projection.count).toBe(count); + }); + + test("the store holds exactly the shared OrderedVote layout", async () => { + // the compact store is the OrderedVote layout unrolled into typed arrays, + // so decoding it with the shared codec must reproduce the ballots + const race = makeRace({ + race_id: 'r0', + voting_method: 'STAR', + candidateNames: ['Allison', 'Bill', 'Carmen'], + enable_write_in: true, + write_in_candidates: WRITE_INS, + }); + const ballots = generateBallots(race, 50, 91); + const [projection] = await projectBallots([race], asStream(ballots)); + const order = projection.candidateOrder; + const decoded = Array.from({length: projection.count}, (_, i) => + orderedVotesToVotes([projection.store.toOrderedVote(i)], [order])[0]); + + const cvr = projection.takeRawVotes(); + decoded.forEach((vote, i) => { + // a score of undefined is the codec's way of saying "no entry", which is + // exactly the key rawVote leaves out + const marks = Object.fromEntries( + vote.scores.filter(s => s.score !== undefined).map(s => [s.candidate_id, s.score]) + ); + expect(marks).toEqual(cvr[i].marks); + expect(vote.overvote_rank).toEqual(cvr[i].overvote_rank); + expect(vote.has_duplicate_rank).toEqual(cvr[i].has_duplicate_rank); + }); + }); +}); diff --git a/packages/backend/src/Tabulators/BallotProjection.ts b/packages/backend/src/Tabulators/BallotProjection.ts new file mode 100644 index 000000000..8a37bd05c --- /dev/null +++ b/packages/backend/src/Tabulators/BallotProjection.ts @@ -0,0 +1,197 @@ +import { Race } from "@equal-vote/star-vote-shared/domain_model/Race"; +import { Candidate } from "@equal-vote/star-vote-shared/domain_model/Candidate"; +import { WriteInCandidate } from "@equal-vote/star-vote-shared/domain_model/WriteIn"; +import { Uid } from "@equal-vote/star-vote-shared/domain_model/Uid"; +import { Vote } from "@equal-vote/star-vote-shared/domain_model/Vote"; +import { RaceCandidateOrder } from "@equal-vote/star-vote-shared/domain_model/Ballot"; +import { candidate, rawVote } from "@equal-vote/star-vote-shared/domain_model/ITabulators"; +import { trimLower } from "@equal-vote/star-vote-shared/domain_model/Util"; +import { makeWriteInCandidateId } from "@equal-vote/star-vote-shared/utils/makeID"; +import { BallotVotes } from "../Models/IBallotStore"; +import { CompactVoteStore, MARK_ABSENT, MARK_NULL } from "./CompactVoteStore"; + +// Tabulation needs the whole election at once, but "the whole election" doesn't +// have to mean a heap full of verbose ballot rows. Each streamed ballot is +// projected straight into a CompactVoteStore — the same positional layout an +// OrderedVote uses, backed by flat typed arrays — and the verbose row is dropped +// immediately. The tabulator's own input is expanded from the store one race at +// a time, so a race's worth of it is the largest thing alive. +// +// The projection re-encodes the semantic content of every ballot, so a bug here +// silently miscounts an election rather than failing loudly. Everything below +// mirrors what getElectionResultsController used to do inline; see +// BallotProjection.test.ts, which pins the two against each other. + +export interface ProjectionHooks { + debug?: (msg: string) => void; + warn?: (msg: string) => void; +} + +export class RaceProjection { + readonly race: Race; + /** write-ins only participate when the race enables them AND has a write-in list */ + readonly useWriteIns: boolean; + readonly writeInCandidates: WriteInCandidate[]; + /** the candidate list this race is tabulated over: race candidates, then approved write-ins */ + readonly candidates: candidate[]; + readonly store: CompactVoteStore; + + numUnprocessedWriteIns = 0; + numExcludedWriteIns = 0; + + // race.candidates only — an approved write-in id reached via score.candidate_id + // (rather than via write_in_name) is not a "regular candidate" + private readonly regularIndexById: Map; + private readonly writeInIndexByName: Map; + private readonly writeInByAlias: Map; + private readonly hooks: ProjectionHooks; + + constructor(race: Race, hooks: ProjectionHooks = {}) { + this.race = race; + this.hooks = hooks; + this.useWriteIns = !!(race.enable_write_in && race.write_in_candidates && race.write_in_candidates.length > 0); + this.writeInCandidates = this.useWriteIns && race.write_in_candidates ? race.write_in_candidates : []; + + this.candidates = race.candidates.map((c: Candidate) => ({ + id: c.candidate_id, + name: c.candidate_name, + tieBreakOrder: -1, + votesPreferredOver: {}, + winsAgainst: {}, + })); + + this.regularIndexById = new Map(); + race.candidates.forEach((c: Candidate, i) => { + // first occurrence wins, matching the `race.candidates.some(...)` lookup this replaced + if (!this.regularIndexById.has(c.candidate_id)) this.regularIndexById.set(c.candidate_id, i); + }); + + this.writeInIndexByName = new Map(); + this.writeInCandidates.forEach(wc => { + if (!wc.approved) return; + this.writeInIndexByName.set(wc.candidate_name, this.candidates.length); + this.candidates.push({ + id: makeWriteInCandidateId(wc.candidate_name), + name: wc.candidate_name, + tieBreakOrder: -1, + votesPreferredOver: {}, + winsAgainst: {}, + }); + }); + + this.writeInByAlias = new Map(); + this.writeInCandidates.forEach(wc => { + // first match wins, matching the `writeInCandidates.find(...)` lookup this replaced + wc.aliases.forEach(alias => { + if (!this.writeInByAlias.has(alias)) this.writeInByAlias.set(alias, wc); + }); + }); + + this.store = new CompactVoteStore(this.candidates.length); + } + + /** number of ballots that contained a vote for this race */ + get count() { + return this.store.count; + } + + /** the candidate order the store's marks are positional against */ + get candidateOrder(): RaceCandidateOrder { + return {race_id: this.race.race_id, candidate_id_order: this.candidates.map(c => c.id)}; + } + + addVote(vote: Vote) { + const store = this.store; + store.startRow(); + + vote.scores.forEach(score => { + // a score of undefined (a row missing the field) is recorded as null so + // it stays distinguishable from "no entry for this candidate"; the + // tabulators treat null and undefined marks identically + const mark = score.score === undefined ? null : score.score; + + const index = this.regularIndexById.get(score.candidate_id); + if (index !== undefined) { + if (store.markTagOf(index) !== MARK_ABSENT) { + this.hooks.warn?.(`[Tabulation] Duplicate score for candidate "${score.candidate_id}" on same ballot, keeping first score`); + return; + } + store.setMark(index, mark); + return; + } + if (!this.race.enable_write_in || !score.write_in_name) return; + + const writeInCandidate = this.writeInByAlias.get(trimLower(score.write_in_name)); + this.hooks.debug?.(`[WriteIn Debug] ballot write_in_name="${score.write_in_name}" matched=${!!writeInCandidate} approved=${writeInCandidate?.approved} matchedAliases=${JSON.stringify(writeInCandidate?.aliases)}`); + if (!writeInCandidate) { + this.numUnprocessedWriteIns += 1; + this.numExcludedWriteIns += 1; + } else if (writeInCandidate.approved) { + const writeInIndex = this.writeInIndexByName.get(writeInCandidate.candidate_name)!; + if (store.markTagOf(writeInIndex) === MARK_ABSENT) { + store.setMark(writeInIndex, mark); + } else { + this.hooks.warn?.(`[WriteIn] Duplicate write-in score for "${writeInCandidate.candidate_name}" on same ballot, keeping first score`); + } + } else { + this.numExcludedWriteIns += 1; + } + }); + + store.commitRow(vote.overvote_rank, vote.has_duplicate_rank); + } + + /** + * Expand this race's compact store into the tabulator's input and release + * the store. Called one race at a time so only a single race's verbose + * tabulator input is ever alive. + */ + takeRawVotes(): rawVote[] { + const store = this.store; + if (store.isReleased) throw new Error('RaceProjection: takeRawVotes after the store was released'); + const candidateIds = this.candidates.map(c => c.id); + const candidateCount = store.candidateCount; + const cvr: rawVote[] = new Array(store.count); + for (let b = 0; b < store.count; b++) { + const marks: {[key: string]: number | null} = {}; + for (let i = 0; i < candidateCount; i++) { + const tag = store.markTag(b, i); + // MARK_ABSENT means the ballot had no entry for this candidate, which + // is not the same as an explicit blank — leave the key out entirely + if (tag === MARK_ABSENT) continue; + marks[candidateIds[i]] = tag === MARK_NULL ? null : store.markValue(b, i); + } + cvr[b] = { + marks, + overvote_rank: store.overvoteRank(b) as number | undefined, + has_duplicate_rank: store.hasDuplicateRank(b) as boolean | undefined, + }; + } + store.release(); + return cvr; + } + + /** Drop the compact store without expanding it (for races that skip tabulation). */ + release() { + this.store.release(); + } +} + +/** + * Stream every ballot once, projecting it into each race's compact store. + * Peak memory is the projections, not the ballot rows. + */ +export const projectBallots = async ( + races: Race[], + ballots: AsyncIterable, + hooks: ProjectionHooks = {}, +): Promise => { + const projections = races.map(race => new RaceProjection(race, hooks)); + for await (const ballot of ballots) { + for (let i = 0; i < projections.length; i++) { + const vote = ballot.votes.find(v => v.race_id === races[i].race_id); + if (vote) projections[i].addVote(vote); + } + } + return projections; +}; diff --git a/packages/backend/src/Tabulators/CompactVoteStore.ts b/packages/backend/src/Tabulators/CompactVoteStore.ts new file mode 100644 index 000000000..97a388911 --- /dev/null +++ b/packages/backend/src/Tabulators/CompactVoteStore.ts @@ -0,0 +1,181 @@ +import { OrderedVote, OrderedVoteMark } from "@equal-vote/star-vote-shared/domain_model/Vote"; +import { encodeOrderedVote } from "@equal-vote/star-vote-shared/domain_model/OrderedVoteCodec"; + +// Holds one race's marks for a whole election in flat typed arrays. +// +// It stores the same thing an OrderedVote does — a race's marks positionally +// against a RaceCandidateOrder, plus overvote_rank and has_duplicate_rank — but +// unrolled into a handful of big buffers instead of one small JS array per +// ballot. That's the difference between ~9 bytes and ~150 bytes per ballot, and +// between a few allocations and one per ballot for the GC to trace. +// +// Marks can't be packed into the values array alone: a mark is a number, an +// explicit null, or absent (the ballot has no entry for that candidate at all), +// and tabulation distinguishes all three. So each value gets a one-byte tag and +// the value slot only means anything when the tag says NUMBER. + +export const MARK_ABSENT = 0; +export const MARK_NULL = 1; +export const MARK_NUMBER = 2; + +export const RANK_UNSET = 0; +export const RANK_NULL = 1; +export const RANK_NUMBER = 2; + +export const DUPLICATE_UNSET = 0; +export const DUPLICATE_FALSE = 1; +export const DUPLICATE_TRUE = 2; +export const DUPLICATE_NULL = 3; + +const INITIAL_CAPACITY = 256; + +export class CompactVoteStore { + readonly candidateCount: number; + count = 0; + + // scratch row for the ballot being projected; reused so projecting a ballot + // allocates nothing + readonly rowValues: Float64Array; + readonly rowTags: Uint8Array; + + private capacity = 0; + private markValues = new Float64Array(0); + private markTags = new Uint8Array(0); + private overvoteValues = new Float64Array(0); + private overvoteTags = new Uint8Array(0); + private duplicateTags = new Uint8Array(0); + private released = false; + + constructor(candidateCount: number) { + this.candidateCount = candidateCount; + this.rowValues = new Float64Array(candidateCount); + this.rowTags = new Uint8Array(candidateCount); + } + + /** Clear the scratch row before projecting the next ballot. */ + startRow() { + this.rowTags.fill(MARK_ABSENT); + } + + // NOTE: a non-numeric mark (only reachable from corrupt ballot rows, since + // ballotValidation rejects them) is coerced to a number, NaN if unparseable, + // rather than surviving as-is. The old keyed-object path passed such a value + // straight into the tabulators' arithmetic, where a string mark turned sums + // into concatenation — coercing is no worse and keeps the store flat. + setMark(index: number, mark: number | null) { + if (mark === null) { + this.rowTags[index] = MARK_NULL; + } else { + this.rowTags[index] = MARK_NUMBER; + this.rowValues[index] = mark; + } + } + + markTagOf(index: number) { + return this.rowTags[index]; + } + + /** Commit the scratch row as one more ballot. */ + commitRow(overvote_rank: number | null | undefined, has_duplicate_rank: boolean | null | undefined) { + if (this.released) throw new Error('CompactVoteStore: write after release'); + this.grow(this.count + 1); + const base = this.count * this.candidateCount; + this.markValues.set(this.rowValues, base); + this.markTags.set(this.rowTags, base); + + if (overvote_rank === undefined) { + this.overvoteTags[this.count] = RANK_UNSET; + } else if (overvote_rank === null) { + this.overvoteTags[this.count] = RANK_NULL; + } else { + this.overvoteTags[this.count] = RANK_NUMBER; + this.overvoteValues[this.count] = overvote_rank; + } + + this.duplicateTags[this.count] = + has_duplicate_rank === undefined ? DUPLICATE_UNSET : + has_duplicate_rank === null ? DUPLICATE_NULL : + has_duplicate_rank ? DUPLICATE_TRUE : DUPLICATE_FALSE; + + this.count += 1; + } + + markTag(ballot: number, index: number) { + return this.markTags[ballot * this.candidateCount + index]; + } + + markValue(ballot: number, index: number) { + return this.markValues[ballot * this.candidateCount + index]; + } + + overvoteRank(ballot: number): number | null | undefined { + switch (this.overvoteTags[ballot]) { + case RANK_UNSET: return undefined; + case RANK_NULL: return null; + default: return this.overvoteValues[ballot]; + } + } + + hasDuplicateRank(ballot: number): boolean | null | undefined { + switch (this.duplicateTags[ballot]) { + case DUPLICATE_UNSET: return undefined; + case DUPLICATE_NULL: return null; + case DUPLICATE_TRUE: return true; + default: return false; + } + } + + /** + * The stored ballot in the shared OrderedVote layout (see OrderedVoteCodec) — + * the array-of-arrays form of exactly what these buffers hold. Lossy only for + * a null has_duplicate_rank, which the array layout can't express. + */ + toOrderedVote(ballot: number): OrderedVote { + const marks: OrderedVoteMark[] = new Array(this.candidateCount); + for (let i = 0; i < this.candidateCount; i++) { + const tag = this.markTag(ballot, i); + marks[i] = tag === MARK_ABSENT ? undefined : tag === MARK_NULL ? null : this.markValue(ballot, i); + } + return encodeOrderedVote( + marks, + this.overvoteRank(ballot) as number | undefined, + this.hasDuplicateRank(ballot) as boolean | undefined, + ); + } + + get isReleased() { + return this.released; + } + + /** Drop the buffers. `count` survives, but the marks can no longer be read. */ + release() { + this.released = true; + this.capacity = 0; + this.markValues = new Float64Array(0); + this.markTags = new Uint8Array(0); + this.overvoteValues = new Float64Array(0); + this.overvoteTags = new Uint8Array(0); + this.duplicateTags = new Uint8Array(0); + } + + private grow(needed: number) { + if (needed <= this.capacity) return; + const capacity = Math.max(INITIAL_CAPACITY, this.capacity * 2, needed); + const markValues = new Float64Array(capacity * this.candidateCount); + const markTags = new Uint8Array(capacity * this.candidateCount); + const overvoteValues = new Float64Array(capacity); + const overvoteTags = new Uint8Array(capacity); + const duplicateTags = new Uint8Array(capacity); + markValues.set(this.markValues); + markTags.set(this.markTags); + overvoteValues.set(this.overvoteValues); + overvoteTags.set(this.overvoteTags); + duplicateTags.set(this.duplicateTags); + this.markValues = markValues; + this.markTags = markTags; + this.overvoteValues = overvoteValues; + this.overvoteTags = overvoteTags; + this.duplicateTags = duplicateTags; + this.capacity = capacity; + } +} diff --git a/packages/backend/src/test/orderedVoteCodec.test.ts b/packages/backend/src/test/orderedVoteCodec.test.ts new file mode 100644 index 000000000..a5ef3ae6b --- /dev/null +++ b/packages/backend/src/test/orderedVoteCodec.test.ts @@ -0,0 +1,97 @@ +import { RaceCandidateOrder } from "@equal-vote/star-vote-shared/domain_model/Ballot"; +import { OrderedVote, Vote } from "@equal-vote/star-vote-shared/domain_model/Vote"; +import { + ORDERED_VOTE_TAIL_LENGTH, + OrderedVoteFormatError, + encodeOrderedVote, + orderedVoteHasDuplicateRank, + orderedVoteLength, + orderedVoteOvervoteRank, + orderedVotesToVotes, +} from "@equal-vote/star-vote-shared/domain_model/OrderedVoteCodec"; + +// The upload path (mapOrderedNewBallot) and the tabulation path (BallotProjection) +// both go through this codec, so it's pinned here against the exact behaviour the +// inline mapper in castVoteController had before it was extracted. + +const raceOrder: RaceCandidateOrder[] = [ + {race_id: 'r0', candidate_id_order: ['a', 'b', 'c']}, + {race_id: 'r1', candidate_id_order: ['d', 'e']}, +]; + +// what castVoteController used to do inline +const legacyMap = (orderedVotes: OrderedVote[], raceOrder: RaceCandidateOrder[]): Vote[] => + orderedVotes.map((vote, i) => ({ + race_id: raceOrder[i].race_id, + scores: vote.slice(0, -2).map((s, j) => ({ + candidate_id: raceOrder[i].candidate_id_order[j], + score: s, + })), + overvote_rank: vote.at(-2), + has_duplicate_rank: vote.at(-1) == 1, + })) as Vote[]; + +describe("OrderedVoteCodec", () => { + test("tail length matches the layout the mapper assumed", () => { + expect(ORDERED_VOTE_TAIL_LENGTH).toBe(2); + expect(orderedVoteLength(3)).toBe(5); + }); + + test("decodes the same votes the inline mapper produced", () => { + // exactly what the frontend uploader puts on the wire, including the + // nulls JSON.stringify writes for unmarked candidates and absent ranks + const orderedVotes: OrderedVote[] = [ + [5, null, 0, null, 0], + [1, 2, 3, 1], + ]; + expect(orderedVotesToVotes(orderedVotes, raceOrder)).toEqual(legacyMap(orderedVotes, raceOrder)); + expect(orderedVotesToVotes(orderedVotes, raceOrder)).toEqual([ + { + race_id: 'r0', + scores: [ + {candidate_id: 'a', score: 5}, + {candidate_id: 'b', score: null}, + {candidate_id: 'c', score: 0}, + ], + overvote_rank: null, + has_duplicate_rank: false, + }, + { + race_id: 'r1', + scores: [ + {candidate_id: 'd', score: 1}, + {candidate_id: 'e', score: 2}, + ], + overvote_rank: 3, + has_duplicate_rank: true, + }, + ]); + }); + + test("rejects a ballot with the wrong number of races", () => { + expect(() => orderedVotesToVotes([[1, 2, 3, 0, 0]], raceOrder)) + .toThrow(new OrderedVoteFormatError('Ballot contains different number of races than race_order: 1 != 2')); + }); + + test("rejects a race with the wrong number of candidates", () => { + expect(() => orderedVotesToVotes([[1, 2, 0, 0], [1, 2, 0, 0]], raceOrder)) + .toThrow(new OrderedVoteFormatError('Race 0 contains different number of candidates than race_order: 4 != 5')); + }); + + test("encode round-trips through decode", () => { + const encoded = encodeOrderedVote([5, null, undefined], 2, true); + expect(encoded).toEqual([5, null, undefined, 2, 1]); + expect(orderedVoteOvervoteRank(encoded)).toBe(2); + expect(orderedVoteHasDuplicateRank(encoded)).toBe(true); + }); + + test("an unset has_duplicate_rank stays unset, rather than becoming false", () => { + // JSON can't carry undefined, so this only happens for ordered votes we + // build in-process — where losing the distinction would mean inventing a + // field the source ballot never had + const encoded = encodeOrderedVote([1, 2], undefined, undefined); + expect(orderedVoteOvervoteRank(encoded)).toBeUndefined(); + expect(orderedVoteHasDuplicateRank(encoded)).toBeUndefined(); + expect(orderedVoteHasDuplicateRank(encodeOrderedVote([1, 2], undefined, false))).toBe(false); + }); +}); diff --git a/packages/frontend/src/components/UploadElections.tsx b/packages/frontend/src/components/UploadElections.tsx index 231d77642..6f865e05e 100644 --- a/packages/frontend/src/components/UploadElections.tsx +++ b/packages/frontend/src/components/UploadElections.tsx @@ -10,6 +10,7 @@ import { Candidate } from "@equal-vote/star-vote-shared/domain_model/Candidate"; import { Election, NewElection } from '@equal-vote/star-vote-shared/domain_model/Election'; import { useGetElections } from "~/hooks/useAPI"; import { OrderedNewBallot, RaceCandidateOrder } from "@equal-vote/star-vote-shared/domain_model/Ballot"; +import { encodeOrderedVote } from "@equal-vote/star-vote-shared/domain_model/OrderedVoteCodec"; import { inferElectionSettings } from "./ElectionSettingInference"; import { PrimaryButton, SecondaryButton } from "./styles"; import { makeDefaultElection } from "./ElectionForm/Wizard/Wizard"; @@ -169,7 +170,7 @@ const UploadElections = () => { delete subBallot.votes; return { ...subBallot, - orderedVotes: b.votes.map(v => [...v.scores.map(s => s.score), v.overvote_rank, v.has_duplicate_rank? 1 : 0]) + orderedVotes: b.votes.map(v => encodeOrderedVote(v.scores.map(s => s.score), v.overvote_rank, v.has_duplicate_rank ?? false)) } }); diff --git a/packages/shared/src/domain_model/OrderedVoteCodec.ts b/packages/shared/src/domain_model/OrderedVoteCodec.ts new file mode 100644 index 000000000..a529ba664 --- /dev/null +++ b/packages/shared/src/domain_model/OrderedVoteCodec.ts @@ -0,0 +1,76 @@ +import { RaceCandidateOrder } from "./Ballot"; +import { Score } from "./Score"; +import { OrderedVote, OrderedVoteMark, Vote } from "./Vote"; + +// An OrderedVote is one race's marks flattened into a positional array: +// +// [ ...one entry per candidate in RaceCandidateOrder.candidate_id_order, +// overvote_rank, +// has_duplicate_rank ] +// +// The candidate ids live once in the RaceCandidateOrder instead of once per +// ballot, which is what makes the format cheap enough to hold a whole election +// in memory. Bulk uploads send ballots in this shape and tabulation projects +// stored ballots back into it, so the layout is defined here, once. +export const ORDERED_VOTE_TAIL_LENGTH = 2; // overvote_rank, has_duplicate_rank + +export class OrderedVoteFormatError extends Error {} + +/** How long an OrderedVote must be for a race with this many candidates. */ +export const orderedVoteLength = (candidateCount: number) => candidateCount + ORDERED_VOTE_TAIL_LENGTH; + +export const orderedVoteMarks = (orderedVote: OrderedVote): OrderedVoteMark[] => + orderedVote.slice(0, -ORDERED_VOTE_TAIL_LENGTH); + +// null is what JSON.stringify writes for an absent overvote_rank, so uploaded +// ballots routinely carry it. It's passed through rather than normalized so +// stored ballots stay byte-identical to what the format has always produced +// (tabulation only tests overvote_rank for truthiness, where the two agree). +export const orderedVoteOvervoteRank = (orderedVote: OrderedVote) => + orderedVote.at(-2) as number | undefined; + +export const orderedVoteHasDuplicateRank = (orderedVote: OrderedVote) => { + const tail = orderedVote.at(-1); + // undefined only shows up in ordered votes we built in-process (JSON can't + // carry it); there it means "the source Vote didn't set the field" + return tail === undefined ? undefined : tail == 1; +}; + +export const encodeOrderedVote = ( + marks: readonly OrderedVoteMark[], + overvote_rank?: number, + has_duplicate_rank?: boolean, +): OrderedVote => [ + ...marks, + overvote_rank, + has_duplicate_rank === undefined ? undefined : (has_duplicate_rank ? 1 : 0), +]; + +/** + * Expand a ballot's ordered votes back into verbose Votes. + * Throws OrderedVoteFormatError when the ballot doesn't line up with raceOrder. + */ +export const orderedVotesToVotes = (orderedVotes: OrderedVote[], raceOrder: RaceCandidateOrder[]): Vote[] => { + if (orderedVotes.length != raceOrder.length) { + throw new OrderedVoteFormatError( + `Ballot contains different number of races than race_order: ${orderedVotes.length} != ${raceOrder.length}` + ); + } + return orderedVotes.map((orderedVote, i) => { + const expectedLength = orderedVoteLength(raceOrder[i].candidate_id_order.length); + if (orderedVote.length != expectedLength) { + throw new OrderedVoteFormatError( + `Race ${i} contains different number of candidates than race_order: ${orderedVote.length} != ${expectedLength}` + ); + } + return { + race_id: raceOrder[i].race_id, + scores: orderedVoteMarks(orderedVote).map((mark, j) => ({ + candidate_id: raceOrder[i].candidate_id_order[j], + score: mark + } as Score)), + overvote_rank: orderedVoteOvervoteRank(orderedVote), + has_duplicate_rank: orderedVoteHasDuplicateRank(orderedVote), + }; + }); +}; diff --git a/packages/shared/src/domain_model/Vote.ts b/packages/shared/src/domain_model/Vote.ts index 57ddd943a..535143b78 100644 --- a/packages/shared/src/domain_model/Vote.ts +++ b/packages/shared/src/domain_model/Vote.ts @@ -8,6 +8,14 @@ export interface Vote { has_duplicate_rank?: boolean; } +// One candidate's bubble value inside an OrderedVote: +// number — the score/rank the voter gave +// null — the voter left the candidate unmarked +// undefined — the ballot has no entry for this candidate at all +// null and undefined are not interchangeable during tabulation: a candidate the +// ballot never mentioned is treated differently from one left explicitly blank. +export type OrderedVoteMark = number | null | undefined; + // this format is used in bulk uploads where the race/candidate order is mapped in a separate structure -// the final number (if present) will represent overvote_rank -export type OrderedVote = number[]; \ No newline at end of file +// see OrderedVoteCodec for the layout (marks, then overvote_rank, then has_duplicate_rank) +export type OrderedVote = OrderedVoteMark[]; \ No newline at end of file From c35331986343480cddd6239f47a3ebb59aec9db3 Mon Sep 17 00:00:00 2001 From: Jackson Loper Date: Wed, 29 Jul 2026 12:09:44 -0400 Subject: [PATCH 2/7] Free the compact store as it's expanded, and stop boxing marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things kept peak memory higher than it needed to be. takeRawVotes built the whole rawVote[] before calling release(), so for the duration of that one call a race's compact store and its full expansion both sat in memory. The store now keeps marks in fixed-size blocks and consumeMarks drops each block as soon as it's read, so the overlap is the expansion plus one block instead of the expansion plus the whole store. Blocks also mean appending never copies — the old doubling buffer transiently held the old and new copies at once. Reading a Float64Array always yields a double, and V8 boxes a double stored into an object property as a HeapNumber. That was 310k HeapNumbers for a 31k-ballot race with 10 candidates, which more than doubled the expanded cvr (10.3 MB, against 5.2 MB for the same data built straight from JSON). Marks are almost always small integers, so hand those back as int32s and let V8 keep them as tagged small ints. Peak retained (heap + external), 31k ballots, 10 candidates/race: 1 race: 36.5 MB -> 5.3 MB (was 10.0 MB) 3 races: 70.5 MB -> 11.6 MB (was 16.3 MB) The compact store now costs less than the expansion it feeds, so it earns its place for single-race elections too, not just multi-race ones. Adds a projection test spanning three blocks — the block seam is exactly where an indexing mistake would attribute one ballot's marks to another, and both plausible off-by-one mutations there are caught by it. Co-Authored-By: Claude Opus 5 --- .../src/Tabulators/BallotProjection.test.ts | 19 ++++ .../src/Tabulators/BallotProjection.ts | 25 ++++-- .../src/Tabulators/CompactVoteStore.ts | 88 +++++++++++++++---- 3 files changed, 105 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/Tabulators/BallotProjection.test.ts b/packages/backend/src/Tabulators/BallotProjection.test.ts index e43d546d6..90579d140 100644 --- a/packages/backend/src/Tabulators/BallotProjection.test.ts +++ b/packages/backend/src/Tabulators/BallotProjection.test.ts @@ -9,6 +9,7 @@ import { trimLower } from "@equal-vote/star-vote-shared/domain_model/Util"; import { orderedVotesToVotes } from "@equal-vote/star-vote-shared/domain_model/OrderedVoteCodec"; import { BallotVotes } from "../Models/IBallotStore"; import { projectBallots } from "./BallotProjection"; +import { BLOCK_BALLOTS } from "./CompactVoteStore"; import { VotingMethods } from "./VotingMethodSelecter"; // The compact projection re-encodes the semantic content of every ballot, so a @@ -429,6 +430,24 @@ describe("BallotProjection matches the verbose projection", () => { expect(cvr).toEqual([{marks: {}, overvote_rank: undefined, has_duplicate_rank: undefined}]); }); + test("marks stay correct across block boundaries", async () => { + // the store keeps marks in BLOCK_BALLOTS-sized blocks and frees each one + // as it's consumed, so the block seam is exactly where an indexing + // mistake would put a ballot's marks on the wrong ballot + const race = makeRace({ + race_id: 'r0', + voting_method: 'STAR', + candidateNames: ['Allison', 'Bill', 'Carmen'], + enable_write_in: true, + write_in_candidates: WRITE_INS, + }); + // generated with slack: some of these ballots skip the race entirely, and + // it's the ballots that reach the store that have to span three blocks + const ballots = generateBallots(race, BLOCK_BALLOTS * 3, 55); + const {cvr} = await compareProjection(race, ballots); + expect(cvr.length).toBeGreaterThan(BLOCK_BALLOTS * 2); + }); + test("takeRawVotes releases the compact store", async () => { const race = makeRace({race_id: 'r0', voting_method: 'STAR', candidateNames: ['Allison']}); const ballots = generateBallots(race, 5, 7); diff --git a/packages/backend/src/Tabulators/BallotProjection.ts b/packages/backend/src/Tabulators/BallotProjection.ts index 8a37bd05c..6fb43151a 100644 --- a/packages/backend/src/Tabulators/BallotProjection.ts +++ b/packages/backend/src/Tabulators/BallotProjection.ts @@ -144,7 +144,8 @@ export class RaceProjection { /** * Expand this race's compact store into the tabulator's input and release * the store. Called one race at a time so only a single race's verbose - * tabulator input is ever alive. + * tabulator input is ever alive; consumeMarks frees the store block by block + * as it goes, so the two don't both sit at full size. */ takeRawVotes(): rawVote[] { const store = this.store; @@ -152,21 +153,21 @@ export class RaceProjection { const candidateIds = this.candidates.map(c => c.id); const candidateCount = store.candidateCount; const cvr: rawVote[] = new Array(store.count); - for (let b = 0; b < store.count; b++) { + store.consumeMarks((ballot, tags, values, offset) => { const marks: {[key: string]: number | null} = {}; for (let i = 0; i < candidateCount; i++) { - const tag = store.markTag(b, i); + const tag = tags[offset + i]; // MARK_ABSENT means the ballot had no entry for this candidate, which // is not the same as an explicit blank — leave the key out entirely if (tag === MARK_ABSENT) continue; - marks[candidateIds[i]] = tag === MARK_NULL ? null : store.markValue(b, i); + marks[candidateIds[i]] = tag === MARK_NULL ? null : unbox(values[offset + i]); } - cvr[b] = { + cvr[ballot] = { marks, - overvote_rank: store.overvoteRank(b) as number | undefined, - has_duplicate_rank: store.hasDuplicateRank(b) as boolean | undefined, + overvote_rank: store.overvoteRank(ballot) as number | undefined, + has_duplicate_rank: store.hasDuplicateRank(ballot) as boolean | undefined, }; - } + }); store.release(); return cvr; } @@ -177,6 +178,14 @@ export class RaceProjection { } } +// Reading a Float64Array always yields a double, and V8 boxes a double stored +// into an object property as a HeapNumber — 310k of them for a 31k-ballot race +// with 10 candidates, which doubles the size of the expanded cvr. Marks are +// almost always small integers, so hand those back as int32s, which V8 keeps as +// tagged small ints with no boxing. The value is unchanged either way; anything +// that isn't an exact int32 (a fraction, NaN, a huge number) falls through. +const unbox = (value: number) => ((value | 0) === value ? value | 0 : value); + /** * Stream every ballot once, projecting it into each race's compact store. * Peak memory is the projections, not the ballot rows. diff --git a/packages/backend/src/Tabulators/CompactVoteStore.ts b/packages/backend/src/Tabulators/CompactVoteStore.ts index 97a388911..89a56751b 100644 --- a/packages/backend/src/Tabulators/CompactVoteStore.ts +++ b/packages/backend/src/Tabulators/CompactVoteStore.ts @@ -6,13 +6,19 @@ import { encodeOrderedVote } from "@equal-vote/star-vote-shared/domain_model/Ord // It stores the same thing an OrderedVote does — a race's marks positionally // against a RaceCandidateOrder, plus overvote_rank and has_duplicate_rank — but // unrolled into a handful of big buffers instead of one small JS array per -// ballot. That's the difference between ~9 bytes and ~150 bytes per ballot, and -// between a few allocations and one per ballot for the GC to trace. +// ballot. That's the difference between ~100 and ~176 bytes per ballot, and +// between a handful of allocations and 62,000 per race for the GC to trace. // // Marks can't be packed into the values array alone: a mark is a number, an // explicit null, or absent (the ballot has no entry for that candidate at all), // and tabulation distinguishes all three. So each value gets a one-byte tag and // the value slot only means anything when the tag says NUMBER. +// +// The marks live in fixed-size blocks rather than one growable buffer, for two +// reasons: appending never has to copy (a doubling buffer transiently holds the +// old and new copies at once), and consumeMarks can drop each block as soon as +// it has been read, so expanding the store into the tabulator's input doesn't +// need room for both at full size. export const MARK_ABSENT = 0; export const MARK_NULL = 1; @@ -27,8 +33,19 @@ export const DUPLICATE_FALSE = 1; export const DUPLICATE_TRUE = 2; export const DUPLICATE_NULL = 3; +/** ballots per marks block; ~400KB per block at 10 candidates */ +export const BLOCK_BALLOTS = 4096; + const INITIAL_CAPACITY = 256; +/** + * Called once per ballot by consumeMarks. Reads mark `i` as + * `tags[offset + i]`, and its value (when the tag is MARK_NUMBER) as + * `values[offset + i]`. The raw buffers are handed over rather than a wrapper + * object so walking the store allocates nothing per ballot. + */ +export type MarkVisitor = (ballot: number, tags: Uint8Array, values: Float64Array, offset: number) => void; + export class CompactVoteStore { readonly candidateCount: number; count = 0; @@ -38,9 +55,12 @@ export class CompactVoteStore { readonly rowValues: Float64Array; readonly rowTags: Uint8Array; + // one entry per BLOCK_BALLOTS ballots; nulled out as they're consumed + private markValueBlocks: (Float64Array | null)[] = []; + private markTagBlocks: (Uint8Array | null)[] = []; + + // per-ballot, so ~9 bytes each — small enough to keep as plain growable buffers private capacity = 0; - private markValues = new Float64Array(0); - private markTags = new Uint8Array(0); private overvoteValues = new Float64Array(0); private overvoteTags = new Uint8Array(0); private duplicateTags = new Uint8Array(0); @@ -78,10 +98,16 @@ export class CompactVoteStore { /** Commit the scratch row as one more ballot. */ commitRow(overvote_rank: number | null | undefined, has_duplicate_rank: boolean | null | undefined) { if (this.released) throw new Error('CompactVoteStore: write after release'); - this.grow(this.count + 1); - const base = this.count * this.candidateCount; - this.markValues.set(this.rowValues, base); - this.markTags.set(this.rowTags, base); + this.growPerBallot(this.count + 1); + + const blockIndex = (this.count / BLOCK_BALLOTS) | 0; + if (blockIndex >= this.markValueBlocks.length) { + this.markValueBlocks.push(new Float64Array(BLOCK_BALLOTS * this.candidateCount)); + this.markTagBlocks.push(new Uint8Array(BLOCK_BALLOTS * this.candidateCount)); + } + const offset = (this.count % BLOCK_BALLOTS) * this.candidateCount; + this.markValueBlocks[blockIndex]!.set(this.rowValues, offset); + this.markTagBlocks[blockIndex]!.set(this.rowTags, offset); if (overvote_rank === undefined) { this.overvoteTags[this.count] = RANK_UNSET; @@ -100,12 +126,42 @@ export class CompactVoteStore { this.count += 1; } + /** + * Walk every ballot's marks in order, freeing each block as soon as it has + * been read. The store is empty afterwards: this is a move, not a read, so + * that expanding it into the tabulator's input never needs room for the + * whole store and the whole expansion at once. + */ + consumeMarks(visit: MarkVisitor) { + if (this.released) throw new Error('CompactVoteStore: read after release'); + const {candidateCount, count} = this; + for (let blockIndex = 0; blockIndex < this.markValueBlocks.length; blockIndex++) { + const values = this.markValueBlocks[blockIndex]!; + const tags = this.markTagBlocks[blockIndex]!; + const first = blockIndex * BLOCK_BALLOTS; + const last = Math.min(first + BLOCK_BALLOTS, count); + for (let ballot = first; ballot < last; ballot++) { + visit(ballot, tags, values, (ballot - first) * candidateCount); + } + // drop the block now that it's been read, so peak memory is the + // expansion plus one block rather than the expansion plus the store + this.markValueBlocks[blockIndex] = null; + this.markTagBlocks[blockIndex] = null; + } + this.markValueBlocks = []; + this.markTagBlocks = []; + } + markTag(ballot: number, index: number) { - return this.markTags[ballot * this.candidateCount + index]; + const block = this.markTagBlocks[(ballot / BLOCK_BALLOTS) | 0]; + if (!block) throw new Error('CompactVoteStore: marks already consumed'); + return block[(ballot % BLOCK_BALLOTS) * this.candidateCount + index]; } markValue(ballot: number, index: number) { - return this.markValues[ballot * this.candidateCount + index]; + const block = this.markValueBlocks[(ballot / BLOCK_BALLOTS) | 0]; + if (!block) throw new Error('CompactVoteStore: marks already consumed'); + return block[(ballot % BLOCK_BALLOTS) * this.candidateCount + index]; } overvoteRank(ballot: number): number | null | undefined { @@ -151,28 +207,22 @@ export class CompactVoteStore { release() { this.released = true; this.capacity = 0; - this.markValues = new Float64Array(0); - this.markTags = new Uint8Array(0); + this.markValueBlocks = []; + this.markTagBlocks = []; this.overvoteValues = new Float64Array(0); this.overvoteTags = new Uint8Array(0); this.duplicateTags = new Uint8Array(0); } - private grow(needed: number) { + private growPerBallot(needed: number) { if (needed <= this.capacity) return; const capacity = Math.max(INITIAL_CAPACITY, this.capacity * 2, needed); - const markValues = new Float64Array(capacity * this.candidateCount); - const markTags = new Uint8Array(capacity * this.candidateCount); const overvoteValues = new Float64Array(capacity); const overvoteTags = new Uint8Array(capacity); const duplicateTags = new Uint8Array(capacity); - markValues.set(this.markValues); - markTags.set(this.markTags); overvoteValues.set(this.overvoteValues); overvoteTags.set(this.overvoteTags); duplicateTags.set(this.duplicateTags); - this.markValues = markValues; - this.markTags = markTags; this.overvoteValues = overvoteValues; this.overvoteTags = overvoteTags; this.duplicateTags = duplicateTags; From 9afab2c434bafa2d7b564f171e76860d616cdc7a Mon Sep 17 00:00:00 2001 From: Jackson Loper Date: Wed, 29 Jul 2026 12:23:48 -0400 Subject: [PATCH 3/7] Review fixes: snapshot the candidate order the store is indexed by MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getElectionResultsController calls shuffleCandidatesForRandomTiebreak, which sorts `candidates` in place, and takeRawVotes was reading the mark-to-candidate mapping off that same array. The two calls happened to be in the only order that works. Swapping them attributes every ballot's marks to the wrong candidate — and the whole test suite still passed, which is exactly the silent miscount this projection is supposed to be guarded against. RaceProjection now snapshots candidateIds at construction and indexes the store by that, so callers can mutate `candidates` freely. The new test shuffles the list before expanding and checks the cvr against the reference projection; it fails against the pre-fix code. Also from the review: - CompactVoteStore rejects commitRow after consumeMarks. The blocks are handed out and nulled by then, so appending would push a fresh block at the wrong index and misplace the ballot. - The codec test now asserts the thrown error's class, not just its message. castVoteController maps OrderedVoteFormatError to a 400 via instanceof, so a break there would turn a bad upload into a 500. - startRow's comment claimed to clear the scratch row when it only clears the tags; explained why that's sufficient instead. Co-Authored-By: Claude Opus 5 --- .../src/Tabulators/BallotProjection.test.ts | 23 +++++++++++++++++++ .../src/Tabulators/BallotProjection.ts | 18 ++++++++++++--- .../src/Tabulators/CompactVoteStore.ts | 12 +++++++++- .../backend/src/test/orderedVoteCodec.test.ts | 3 +++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/Tabulators/BallotProjection.test.ts b/packages/backend/src/Tabulators/BallotProjection.test.ts index 90579d140..cc716911b 100644 --- a/packages/backend/src/Tabulators/BallotProjection.test.ts +++ b/packages/backend/src/Tabulators/BallotProjection.test.ts @@ -10,6 +10,7 @@ import { orderedVotesToVotes } from "@equal-vote/star-vote-shared/domain_model/O import { BallotVotes } from "../Models/IBallotStore"; import { projectBallots } from "./BallotProjection"; import { BLOCK_BALLOTS } from "./CompactVoteStore"; +import shuffleCandidatesForRandomTiebreak from "./shuffleCandidatesForRandomTiebreak"; import { VotingMethods } from "./VotingMethodSelecter"; // The compact projection re-encodes the semantic content of every ballot, so a @@ -448,6 +449,28 @@ describe("BallotProjection matches the verbose projection", () => { expect(cvr.length).toBeGreaterThan(BLOCK_BALLOTS * 2); }); + test("shuffling the candidate list doesn't disturb the projection", async () => { + // getElectionResultsController hands `candidates` to + // shuffleCandidatesForRandomTiebreak, which sorts it in place. If the + // store's positional order were read off that array rather than the + // snapshot, every ballot's marks would land on the wrong candidate — + // and nothing about the results would look obviously wrong. + const race = makeRace({ + race_id: 'r0', + voting_method: 'STAR', + candidateNames: ['Allison', 'Bill', 'Carmen', 'Doug', 'Elle'], + enable_write_in: true, + write_in_candidates: WRITE_INS, + }); + const ballots = generateBallots(race, 200, 77); + const reference = referenceProjection(race, ballots); + + const [projection] = await projectBallots([race], asStream(ballots)); + shuffleCandidatesForRandomTiebreak(new Date(), projection.candidates, projection.count, race.race_id); + expect(projection.candidates.map(c => c.id)).not.toEqual(projection.candidateIds); + expect(projection.takeRawVotes()).toEqual(reference.cvr); + }); + test("takeRawVotes releases the compact store", async () => { const race = makeRace({race_id: 'r0', voting_method: 'STAR', candidateNames: ['Allison']}); const ballots = generateBallots(race, 5, 7); diff --git a/packages/backend/src/Tabulators/BallotProjection.ts b/packages/backend/src/Tabulators/BallotProjection.ts index 6fb43151a..b8001ea31 100644 --- a/packages/backend/src/Tabulators/BallotProjection.ts +++ b/packages/backend/src/Tabulators/BallotProjection.ts @@ -32,8 +32,19 @@ export class RaceProjection { /** write-ins only participate when the race enables them AND has a write-in list */ readonly useWriteIns: boolean; readonly writeInCandidates: WriteInCandidate[]; - /** the candidate list this race is tabulated over: race candidates, then approved write-ins */ + /** + * The candidate list this race is tabulated over: race candidates, then + * approved write-ins. Callers mutate this — shuffleCandidatesForRandomTiebreak + * sorts it in place, and the tabulators write to its entries — so it must + * never be used to interpret the store. See candidateIds. + */ readonly candidates: candidate[]; + /** + * The order the store's marks are positional against, snapshotted at + * construction. Reading it from `candidates` instead would silently + * misattribute every ballot's marks once that array has been shuffled. + */ + readonly candidateIds: Uid[]; readonly store: CompactVoteStore; numUnprocessedWriteIns = 0; @@ -87,6 +98,7 @@ export class RaceProjection { }); }); + this.candidateIds = this.candidates.map(c => c.id); this.store = new CompactVoteStore(this.candidates.length); } @@ -97,7 +109,7 @@ export class RaceProjection { /** the candidate order the store's marks are positional against */ get candidateOrder(): RaceCandidateOrder { - return {race_id: this.race.race_id, candidate_id_order: this.candidates.map(c => c.id)}; + return {race_id: this.race.race_id, candidate_id_order: this.candidateIds}; } addVote(vote: Vote) { @@ -150,7 +162,7 @@ export class RaceProjection { takeRawVotes(): rawVote[] { const store = this.store; if (store.isReleased) throw new Error('RaceProjection: takeRawVotes after the store was released'); - const candidateIds = this.candidates.map(c => c.id); + const candidateIds = this.candidateIds; const candidateCount = store.candidateCount; const cvr: rawVote[] = new Array(store.count); store.consumeMarks((ballot, tags, values, offset) => { diff --git a/packages/backend/src/Tabulators/CompactVoteStore.ts b/packages/backend/src/Tabulators/CompactVoteStore.ts index 89a56751b..86bc34335 100644 --- a/packages/backend/src/Tabulators/CompactVoteStore.ts +++ b/packages/backend/src/Tabulators/CompactVoteStore.ts @@ -65,6 +65,7 @@ export class CompactVoteStore { private overvoteTags = new Uint8Array(0); private duplicateTags = new Uint8Array(0); private released = false; + private marksConsumed = false; constructor(candidateCount: number) { this.candidateCount = candidateCount; @@ -72,7 +73,11 @@ export class CompactVoteStore { this.rowTags = new Uint8Array(candidateCount); } - /** Clear the scratch row before projecting the next ballot. */ + /** + * Reset the scratch row before projecting the next ballot. Only the tags are + * cleared — a stale value can't be read back, because a slot's value is only + * ever consulted when this ballot's own setMark tagged it MARK_NUMBER. + */ startRow() { this.rowTags.fill(MARK_ABSENT); } @@ -98,6 +103,9 @@ export class CompactVoteStore { /** Commit the scratch row as one more ballot. */ commitRow(overvote_rank: number | null | undefined, has_duplicate_rank: boolean | null | undefined) { if (this.released) throw new Error('CompactVoteStore: write after release'); + // appending after the blocks were handed out would push a fresh block at + // the wrong index and silently drop or misplace the ballot + if (this.marksConsumed) throw new Error('CompactVoteStore: write after marks were consumed'); this.growPerBallot(this.count + 1); const blockIndex = (this.count / BLOCK_BALLOTS) | 0; @@ -134,6 +142,8 @@ export class CompactVoteStore { */ consumeMarks(visit: MarkVisitor) { if (this.released) throw new Error('CompactVoteStore: read after release'); + if (this.marksConsumed) throw new Error('CompactVoteStore: marks already consumed'); + this.marksConsumed = true; const {candidateCount, count} = this; for (let blockIndex = 0; blockIndex < this.markValueBlocks.length; blockIndex++) { const values = this.markValueBlocks[blockIndex]!; diff --git a/packages/backend/src/test/orderedVoteCodec.test.ts b/packages/backend/src/test/orderedVoteCodec.test.ts index a5ef3ae6b..77d7d82e0 100644 --- a/packages/backend/src/test/orderedVoteCodec.test.ts +++ b/packages/backend/src/test/orderedVoteCodec.test.ts @@ -71,6 +71,9 @@ describe("OrderedVoteCodec", () => { test("rejects a ballot with the wrong number of races", () => { expect(() => orderedVotesToVotes([[1, 2, 3, 0, 0]], raceOrder)) .toThrow(new OrderedVoteFormatError('Ballot contains different number of races than race_order: 1 != 2')); + // the class identity matters: castVoteController turns this into a 400 + // via instanceof, and would surface a 500 if it ever stopped matching + expect(() => orderedVotesToVotes([[1, 2, 3, 0, 0]], raceOrder)).toThrow(OrderedVoteFormatError); }); test("rejects a race with the wrong number of candidates", () => { From 72f32e3022d42299abacc42b08e203f31351c919 Mon Sep 17 00:00:00 2001 From: Jackson Loper Date: Wed, 29 Jul 2026 12:38:56 -0400 Subject: [PATCH 4/7] Pin the race/projection fan-out with a controller-level results test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tabulation had almost no coverage above the unit level. Stubbing the ballot stream to yield nothing failed exactly one test, and the controller could read the wrong race's projection with the whole suite still green — the multi-race and precinct tests only cover ballot submission and never fetch results, and the one test that does tabulate uses a single-race election. That left the fan-out unpinned: projectBallots builds one projection per race and the controller indexes them alongside election.races, and a mixup there yields plausible results for the wrong race rather than an obvious failure. Adds a two-race election that goes through submission and /API/ElectionResult, asserting each race's candidate list, winner and tally count. Both races reuse the same candidate ids on purpose, so a mixup produces marks that still resolve instead of an empty tally. Reading the wrong projection, or matching votes by position instead of race_id, each fail it. Co-Authored-By: Claude Opus 5 --- .../backend/src/test/multiRaceResults.test.ts | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 packages/backend/src/test/multiRaceResults.test.ts diff --git a/packages/backend/src/test/multiRaceResults.test.ts b/packages/backend/src/test/multiRaceResults.test.ts new file mode 100644 index 000000000..95cde5380 --- /dev/null +++ b/packages/backend/src/test/multiRaceResults.test.ts @@ -0,0 +1,141 @@ +require("dotenv").config(); + +import { Election } from "@equal-vote/star-vote-shared/domain_model/Election"; +import { NewBallot } from "@equal-vote/star-vote-shared/domain_model/Ballot"; +import { Race } from "@equal-vote/star-vote-shared/domain_model/Race"; +import { ElectionSettings } from "@equal-vote/star-vote-shared/domain_model/ElectionSettings"; +import testInputs from "./testInputs"; +import { TestHelper } from "./TestHelper"; + +// getElectionResultsController streams ballots once and fans them out into a +// per-race projection, then indexes those projections alongside election.races. +// The existing multi-race tests only cover ballot submission — they never fetch +// results — so nothing pinned that fan-out. A race/projection mixup produces +// plausible-looking results for the wrong race, which is the failure mode this +// whole change is supposed to be guarded against. +// +// Both races deliberately reuse the same candidate ids, so a mixup yields marks +// that still resolve rather than an obviously empty tally. + +const th = new TestHelper(); + +const waitForQueue = async () => (await th.eventQueue).waitUntilJobsFinished(); + +afterEach(() => { + jest.clearAllMocks(); + th.afterEach(); +}); + +const TwoRaceElection: Election = { + election_id: "0", + title: 'Two Race Election', + state: 'open', + frontend_url: '', + owner_id: 'Alice1234', + races: [ + { + race_id: 'race0', + title: 'Race Zero', + num_winners: 1, + voting_method: 'STAR', + candidates: [ + { candidate_id: '0', candidate_name: 'Alice' }, + { candidate_id: '1', candidate_name: 'Bob' }, + { candidate_id: '2', candidate_name: 'Cara' }, + ], + }, + { + race_id: 'race1', + title: 'Race One', + num_winners: 1, + voting_method: 'STAR', + candidates: [ + { candidate_id: '0', candidate_name: 'Dan' }, + { candidate_id: '1', candidate_name: 'Erin' }, + { candidate_id: '2', candidate_name: 'Fay' }, + ], + }, + ] as Race[], + settings: { + voter_access: 'open', + voter_authentication: {}, + public_results: true, + } as ElectionSettings, +} as Election; + +const scores = (a: number, b: number, c: number) => [ + { candidate_id: '0', score: a }, + { candidate_id: '1', score: b }, + { candidate_id: '2', score: c }, +]; + +// Alice wins race0 outright; Fay wins race1 outright. Only two of the five +// ballots vote in race1, so the per-race tally counts differ too — a projection +// read off the wrong race would get both the winner and the count wrong. +const BALLOTS: NewBallot[] = [ + { votes: [{ race_id: 'race0', scores: scores(5, 0, 1) }] }, + { votes: [{ race_id: 'race0', scores: scores(5, 0, 1) }] }, + { votes: [{ race_id: 'race0', scores: scores(5, 1, 0) }] }, + { votes: [ + { race_id: 'race0', scores: scores(5, 0, 2) }, + { race_id: 'race1', scores: scores(0, 1, 5) }, + ] }, + { votes: [ + { race_id: 'race0', scores: scores(4, 0, 1) }, + { race_id: 'race1', scores: scores(0, 2, 5) }, + ] }, +].map(b => b as NewBallot); + +describe("Multi Race Results", () => { + var election: Election; + + test("Create a two race election", async () => { + const response = await th.createElection(TwoRaceElection, testInputs.user1token); + expect(response.statusCode).toBe(200); + election = response.election; + expect(election.races.length).toBe(2); + th.testComplete(); + }); + + test("Submit ballots across both races", async () => { + for (const ballot of BALLOTS) { + const response = await th.submitBallot( + election.election_id, + { ...ballot, election_id: election.election_id } as NewBallot, + testInputs.user1token, + ); + expect(response.statusCode).toBe(200); + } + th.testComplete(); + }); + + test("each race is tabulated over its own ballots", async () => { + await waitForQueue(); + + const res = await th.getRequest( + `/API/ElectionResult/${election.election_id}`, + testInputs.user1token, + ); + expect(res.statusCode).toBe(200); + expect(res.body.results).toHaveLength(2); + + const [race0, race1] = res.body.results; + + // the candidate lists must not have crossed over + expect(race0.summaryData.candidates.map((c: any) => c.name).sort()) + .toEqual(['Alice', 'Bob', 'Cara']); + expect(race1.summaryData.candidates.map((c: any) => c.name).sort()) + .toEqual(['Dan', 'Erin', 'Fay']); + + // nor the marks + expect(race0.elected[0].name).toBe('Alice'); + expect(race1.elected[0].name).toBe('Fay'); + + // nor the per-race ballot counts: three ballots skipped race1 entirely + expect(race0.summaryData.nTallyVotes).toBe(5); + expect(race1.summaryData.nTallyVotes).toBe(2); + + th.testComplete(); + // the mock event queue drains roughly one ballot per second + }, 30000); +}); From aab8119d104fa9fee44a3904e7e1fa4b3f74d80b Mon Sep 17 00:00:00 2001 From: Jackson Loper Date: Wed, 19 Aug 2026 14:47:32 -0400 Subject: [PATCH 5/7] Use the existing tinyrand generator in the projection tests The test file had its own mulberry32 plus a hand-rolled Fisher-Yates, which duplicates what Tabulators/tinyrand.ts already provides (and what shuffleCandidatesForRandomTiebreak uses in production). Co-Authored-By: Claude Opus 5 --- .../src/Tabulators/BallotProjection.test.ts | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/backend/src/Tabulators/BallotProjection.test.ts b/packages/backend/src/Tabulators/BallotProjection.test.ts index cc716911b..689a307d0 100644 --- a/packages/backend/src/Tabulators/BallotProjection.test.ts +++ b/packages/backend/src/Tabulators/BallotProjection.test.ts @@ -11,6 +11,7 @@ import { BallotVotes } from "../Models/IBallotStore"; import { projectBallots } from "./BallotProjection"; import { BLOCK_BALLOTS } from "./CompactVoteStore"; import shuffleCandidatesForRandomTiebreak from "./shuffleCandidatesForRandomTiebreak"; +import { get as getTinyRand } from "./tinyrand"; import { VotingMethods } from "./VotingMethodSelecter"; // The compact projection re-encodes the semantic content of every ballot, so a @@ -110,21 +111,19 @@ const makeRace = (overrides: Partial & {race_id: string, voting_method: Vo write_in_candidates: overrides.write_in_candidates, } as Race); -// deterministic RNG so a failure is always reproducible -const mulberry32 = (seed: number) => () => { - seed = (seed + 0x6D2B79F5) | 0; - let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; -}; - -const shuffled = (items: T[], rand: () => number): T[] => { - const copy = [...items]; - for (let i = copy.length - 1; i > 0; i--) { - const j = Math.floor(rand() * (i + 1)); - [copy[i], copy[j]] = [copy[j], copy[i]]; - } - return copy; +// Deterministic RNG so a failure is always reproducible. This is the same +// generator the production tiebreak shuffler uses, so there's one seeded RNG +// in the backend rather than a hand-rolled second one here. +const makeRand = (seed: number) => { + const gen = getTinyRand(0, seed); + return { + next: () => gen._get() / 0x100000000, + shuffled: (items: T[]): T[] => { + const copy = [...items]; + gen.shuffle(copy); + return copy; + }, + }; }; /** @@ -135,7 +134,7 @@ const shuffled = (items: T[], rand: () => number): T[] => { * race entirely. */ const generateBallots = (race: Race, count: number, seed: number): BallotVotes[] => { - const rand = mulberry32(seed); + const {next: rand, shuffled} = makeRand(seed); const maxMark = ['IRV', 'STV', 'RankedRobin'].includes(race.voting_method) ? race.candidates.length : ['Approval', 'Plurality'].includes(race.voting_method) ? 1 : 5; const ballots: BallotVotes[] = []; @@ -173,7 +172,7 @@ const generateBallots = (race: Race, count: number, seed: number): BallotVotes[] } } - const vote: Vote = {race_id: race.race_id, scores: shuffled(scores, rand)}; + const vote: Vote = {race_id: race.race_id, scores: shuffled(scores)}; const overvoteRoll = rand(); if (overvoteRoll < 0.1) vote.overvote_rank = 1 + Math.floor(rand() * maxMark); const duplicateRoll = rand(); From 80a41428f076d7f1b52ad6b7091a94282053de47 Mon Sep 17 00:00:00 2001 From: Jackson Loper Date: Wed, 19 Aug 2026 15:46:01 -0400 Subject: [PATCH 6/7] Pin that no ballot is dropped at a block seam, at any stream length The store's final block is almost always partial, and takeRawVotes fills a pre-sized array by index, so a skipped ballot leaves a hole rather than a short array. Checks contents at every count straddling a seam. Co-Authored-By: Claude Opus 5 --- .../src/Tabulators/BallotProjection.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/backend/src/Tabulators/BallotProjection.test.ts b/packages/backend/src/Tabulators/BallotProjection.test.ts index 689a307d0..46b7c4898 100644 --- a/packages/backend/src/Tabulators/BallotProjection.test.ts +++ b/packages/backend/src/Tabulators/BallotProjection.test.ts @@ -448,6 +448,49 @@ describe("BallotProjection matches the verbose projection", () => { expect(cvr.length).toBeGreaterThan(BLOCK_BALLOTS * 2); }); + test("no ballot is lost at a block boundary, at any stream length", async () => { + // The store's last block is almost always partial, so "the stream ended + // mid-block" is the normal case rather than an edge case. takeRawVotes + // pre-sizes the output with `new Array(store.count)` and fills by index, + // which means a skipped ballot leaves a hole rather than a short array — + // cvr.length alone would not notice. Check the contents at every count + // straddling a block seam, plus the degenerate small ones. + const race = makeRace({race_id: 'r0', voting_method: 'STAR', candidateNames: ['Allison', 'Bill', 'Carmen']}); + const counts = new Set([0, 1, 2, 3]); + for (const k of [1, 2, 3]) for (const d of [-2, -1, 0, 1, 2]) counts.add(BLOCK_BALLOTS * k + d); + + for (const n of [...counts].sort((a, b) => a - b)) { + // every ballot marked distinctly so a dropped or duplicated one shows up + const ballots: BallotVotes[] = Array.from({length: n}, (_, i) => ({ + votes: [{ + race_id: 'r0', + scores: [ + {candidate_id: 'c-Allison', score: i % 6}, + {candidate_id: 'c-Bill', score: (i + 2) % 6}, + {candidate_id: 'c-Carmen', score: (i + 4) % 6}, + ], + overvote_rank: i % 7 === 0 ? (i % 5) + 1 : undefined, + }], + })); + const [projection] = await projectBallots([race], asStream(ballots)); + expect(projection.count).toBe(n); + const cvr = projection.takeRawVotes(); + expect(cvr.length).toBe(n); + // scanned by hand rather than with a per-ballot expect: at three + // blocks that would be ~100k matcher calls and dominate the suite + const problems: string[] = []; + for (let i = 0; i < n && problems.length < 3; i++) { + const got = cvr[i]; + if (got === undefined) { problems.push(`cvr[${i}] is a hole`); continue; } + const wantMarks = {'c-Allison': i % 6, 'c-Bill': (i + 2) % 6, 'c-Carmen': (i + 4) % 6}; + if (JSON.stringify(got.marks) !== JSON.stringify(wantMarks)) problems.push(`cvr[${i}].marks = ${JSON.stringify(got.marks)}`); + const wantRank = i % 7 === 0 ? (i % 5) + 1 : undefined; + if (got.overvote_rank !== wantRank) problems.push(`cvr[${i}].overvote_rank = ${got.overvote_rank}, want ${wantRank}`); + } + expect(`n=${n}: ${problems.join('; ')}`).toBe(`n=${n}: `); + } + }); + test("shuffling the candidate list doesn't disturb the projection", async () => { // getElectionResultsController hands `candidates` to // shuffleCandidatesForRandomTiebreak, which sorts it in place. If the From 188331067e4bf08f5b2356712455c3c755b9f434 Mon Sep 17 00:00:00 2001 From: Jackson Loper Date: Wed, 19 Aug 2026 15:56:22 -0400 Subject: [PATCH 7/7] Grow a race's first marks block instead of reserving it whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every race in the election holds a live store while the ballots stream, and the first commitRow allocated a full BLOCK_BALLOTS block regardless of how many ballots were cast. A 20-race election paid ~7 MB before counting a single ballot — a floor the pre-streaming path did not have. The first block now starts at 64 ballots and doubles up to a full block, which also right-sizes the trailing partial block of a large election. 20 races x 10 candidates, heapUsed + external held by the stores: 3 ballots 7.17 MB -> 0.27 MB 50 ballots 7.19 MB -> 0.27 MB 5000 ballots 15.75 MB -> 10.49 MB Co-Authored-By: Claude Opus 5 --- .../src/Tabulators/CompactVoteStore.ts | 51 +++++++++++++++++-- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/Tabulators/CompactVoteStore.ts b/packages/backend/src/Tabulators/CompactVoteStore.ts index 86bc34335..3769223d9 100644 --- a/packages/backend/src/Tabulators/CompactVoteStore.ts +++ b/packages/backend/src/Tabulators/CompactVoteStore.ts @@ -38,6 +38,17 @@ export const BLOCK_BALLOTS = 4096; const INITIAL_CAPACITY = 256; +/** + * Ballots the first block of a race is sized for. Blocks are addressed as + * `count / BLOCK_BALLOTS` either way; this only controls how much of that + * range is actually allocated up front, so a race with a handful of ballots + * doesn't reserve a full block. It doubles up to BLOCK_BALLOTS on demand. + * Every race in the election has a live store while the ballots stream, so + * without this a 20-race election paid a full block per race no matter how + * few ballots were cast. + */ +const INITIAL_BLOCK_BALLOTS = 64; + /** * Called once per ballot by consumeMarks. Reads mark `i` as * `tags[offset + i]`, and its value (when the tag is MARK_NUMBER) as @@ -58,6 +69,9 @@ export class CompactVoteStore { // one entry per BLOCK_BALLOTS ballots; nulled out as they're consumed private markValueBlocks: (Float64Array | null)[] = []; private markTagBlocks: (Uint8Array | null)[] = []; + // ballots each block is currently sized for; only ever below BLOCK_BALLOTS + // for a block that is still the last one (see ensureBlockCapacity) + private blockCapacity: number[] = []; // per-ballot, so ~9 bytes each — small enough to keep as plain growable buffers private capacity = 0; @@ -100,6 +114,33 @@ export class CompactVoteStore { return this.rowTags[index]; } + /** + * Make sure block `blockIndex` can hold `ballots` ballots, growing it by + * doubling (capped at BLOCK_BALLOTS) and copying what's already there. + * Only the newest block is ever short of BLOCK_BALLOTS, and it stops + * growing once it reaches a full block, so the copying is bounded by one + * block's worth of work per store. + */ + private ensureBlockCapacity(blockIndex: number, ballots: number) { + const have = this.blockCapacity[blockIndex] ?? 0; + if (have >= ballots) return; + + let capacity = have === 0 ? INITIAL_BLOCK_BALLOTS : have * 2; + while (capacity < ballots) capacity *= 2; + if (capacity > BLOCK_BALLOTS) capacity = BLOCK_BALLOTS; + + const values = new Float64Array(capacity * this.candidateCount); + const tags = new Uint8Array(capacity * this.candidateCount); + const priorValues = this.markValueBlocks[blockIndex]; + if (priorValues) { + values.set(priorValues); + tags.set(this.markTagBlocks[blockIndex]!); + } + this.markValueBlocks[blockIndex] = values; + this.markTagBlocks[blockIndex] = tags; + this.blockCapacity[blockIndex] = capacity; + } + /** Commit the scratch row as one more ballot. */ commitRow(overvote_rank: number | null | undefined, has_duplicate_rank: boolean | null | undefined) { if (this.released) throw new Error('CompactVoteStore: write after release'); @@ -109,11 +150,9 @@ export class CompactVoteStore { this.growPerBallot(this.count + 1); const blockIndex = (this.count / BLOCK_BALLOTS) | 0; - if (blockIndex >= this.markValueBlocks.length) { - this.markValueBlocks.push(new Float64Array(BLOCK_BALLOTS * this.candidateCount)); - this.markTagBlocks.push(new Uint8Array(BLOCK_BALLOTS * this.candidateCount)); - } - const offset = (this.count % BLOCK_BALLOTS) * this.candidateCount; + const ballotInBlock = this.count % BLOCK_BALLOTS; + this.ensureBlockCapacity(blockIndex, ballotInBlock + 1); + const offset = ballotInBlock * this.candidateCount; this.markValueBlocks[blockIndex]!.set(this.rowValues, offset); this.markTagBlocks[blockIndex]!.set(this.rowTags, offset); @@ -160,6 +199,7 @@ export class CompactVoteStore { } this.markValueBlocks = []; this.markTagBlocks = []; + this.blockCapacity = []; } markTag(ballot: number, index: number) { @@ -219,6 +259,7 @@ export class CompactVoteStore { this.capacity = 0; this.markValueBlocks = []; this.markTagBlocks = []; + this.blockCapacity = []; this.overvoteValues = new Float64Array(0); this.overvoteTags = new Uint8Array(0); this.duplicateTags = new Uint8Array(0);