From 28e3d57b0664f3c9eab7f76316679809ad4f40ca Mon Sep 17 00:00:00 2001 From: Adam Masiarek Date: Fri, 7 Aug 2026 20:17:12 -0400 Subject: [PATCH] Add an optional compact JSON export (format_version 2) The "Download JSON" button emits JSON.stringify({Election, Ballots, Results}) of the raw in-memory objects, which puts the tabulator's internal shape into a file people archive and parse: every candidate's O(n^2) votesPreferredOver / winsAgainst maps keyed by UUID (self-pairs included) repeated across elected/tied/other/summaryData/roundResults, Results in camelCase beside snake_case Election and Ballots, an ISO create_date beside an epoch-ms-string update_date, and no version field to tell consumers which shape they have. Issue #1420. This adds a second menu item rather than changing the first. The existing download is untouched and still byte-identical, so nothing consuming it breaks; "Download JSON (compact)" writes the new shape to a .v2.json filename. Promoting it to the default later is a one-line change in downloadJson. New packages/shared/src/utils/exportFormat.ts (buildElectionExport): candidates listed once, a deduped pairwise matrix keyed by name with self-pairs dropped, {id,name} refs, snake_case throughout, ISO-8601 timestamps, null/empty fields omitted, format_version: 2. Ballot scores are preserved verbatim including score: null, so a blank stays distinct from an explicit 0 and from a NOTA vote. Transform-only: no change to the tabulation engine, the results endpoint, or what the UI renders. --- .../backend/src/test/exportFormat.test.ts | 157 +++++++++++++ .../Election/Results/BallotDataExport.tsx | 20 ++ packages/shared/src/utils/exportFormat.ts | 214 ++++++++++++++++++ 3 files changed, 391 insertions(+) create mode 100644 packages/backend/src/test/exportFormat.test.ts create mode 100644 packages/shared/src/utils/exportFormat.ts diff --git a/packages/backend/src/test/exportFormat.test.ts b/packages/backend/src/test/exportFormat.test.ts new file mode 100644 index 000000000..41cd63034 --- /dev/null +++ b/packages/backend/src/test/exportFormat.test.ts @@ -0,0 +1,157 @@ +import { Star } from '../Tabulators/Star'; +import { mapMethodInputs } from './TestHelper'; +import { buildElectionExport } from '@equal-vote/star-vote-shared/utils/exportFormat'; + +// Builds a realistic STAR result and checks the v2 export shape. +describe('buildElectionExport (v2 export format)', () => { + const names = ['Allison', 'Bill', 'Carmen']; + const votes = [ + [5, 2, 1], + [5, 3, 0], + [4, 0, 5], + [3, 5, 5], + [5, 1, 4], + ]; + const results: any = Star(...mapMethodInputs(names, votes), 1); + + // Minimal election whose race candidates line up with the tabulator names. + const election: any = { + election_id: 'testelec', + title: 'Export format test', + state: 'closed', + create_date: '2026-07-05T13:41:39.541Z', + update_date: '1783258899541', // epoch-ms string — must normalize to ISO + owner_id: 'owner', + audit_ids: null, + public_archive_id: null, + head: true, + races: [ + { + race_id: 'r1', + title: 'Race 1', + voting_method: 'STAR', + num_winners: 1, + candidates: names.map((n) => ({ candidate_id: n, candidate_name: n })), + }, + ], + settings: { public_results: true }, + }; + + const ballots: any = votes.map((v, i) => ({ + ballot_id: `b${i}`, + election_id: 'testelec', + precinct: null, + votes: [ + { + race_id: 'r1', + scores: names.map((n, j) => ({ candidate_id: n, score: v[j] })), + }, + ], + })); + + const out: any = buildElectionExport(election, ballots, [results]); + const json = JSON.stringify(out); + + test('is versioned and self-describing', () => { + expect(out.format).toBe('bettervoting-export'); + expect(out.format_version).toBe(2); + expect(typeof out.exported_at).toBe('string'); + expect(Number.isNaN(Date.parse(out.exported_at))).toBe(false); + }); + + test('drops the O(n^2) pairwise maps from candidate objects', () => { + expect(json).not.toContain('votesPreferredOver'); + expect(json).not.toContain('winsAgainst'); + }); + + test('pairwise matrix is deduped with no self-pairs', () => { + const pw = out.results[0].pairwise; + for (const name of names) { + expect(pw[name]).toBeDefined(); + expect(pw[name][name]).toBeUndefined(); // no self-vs-self + } + // Allison beats Bill head-to-head in this ballot set + expect(pw['Allison']['Bill'].wins).toBe(true); + }); + + test('uses snake_case throughout the results section', () => { + const r = out.results[0]; + expect(r.voting_method).toBe('STAR'); + expect(r).toHaveProperty('tie_break_type'); + expect(r.summary).toHaveProperty('n_tally_votes'); + expect(r.summary).not.toHaveProperty('nTallyVotes'); + // candidate method field snake_cased + expect(r.candidates[0]).toHaveProperty('five_star_count'); + }); + + test('references candidates by both id and name', () => { + const elected = out.results[0].elected; + expect(Array.isArray(elected)).toBe(true); + expect(elected[0]).toHaveProperty('id'); + expect(elected[0]).toHaveProperty('name'); + expect(elected[0].name).toBe('Allison'); + }); + + test('ballot scores stay compact (id + score); names live on election.races', () => { + const score = out.ballots[0].votes[0].scores[0]; + expect(score).toHaveProperty('candidate_id'); + expect(score).toHaveProperty('score'); + expect(score).not.toHaveProperty('candidate_name'); // not repeated per row + // name is resolvable from the race candidate list + const cand = out.election.races[0].candidates.find( + (c: any) => c.candidate_id === score.candidate_id, + ); + expect(cand.candidate_name).toBeDefined(); + }); + + test('preserves a null score (abstention) distinct from an explicit 0', () => { + // null = "did not score this candidate"; must NOT be dropped or turned into 0. + const election2: any = { + election_id: 'e2', + title: 'null score test', + head: true, + races: [ + { + race_id: 'r1', + title: 'R', + voting_method: 'STAR', + num_winners: 1, + candidates: [ + { candidate_id: 'a', candidate_name: 'Amy' }, + { candidate_id: 'b', candidate_name: 'Bo' }, + ], + }, + ], + settings: {}, + }; + const ballots2: any = [ + { + ballot_id: 'x', + election_id: 'e2', + votes: [ + { + race_id: 'r1', + scores: [ + { candidate_id: 'a', score: 0 }, // flat zero + { candidate_id: 'b', score: null }, // abstained on Bo + ], + }, + ], + }, + ]; + const o: any = buildElectionExport(election2, ballots2, undefined); + const scores = o.ballots[0].votes[0].scores; + const a = scores.find((s: any) => s.candidate_id === 'a'); + const b = scores.find((s: any) => s.candidate_id === 'b'); + expect(a.score).toBe(0); + expect(b).toHaveProperty('score'); // key kept + expect(b.score).toBeNull(); // and it stays null, not dropped, not 0 + }); + + test('normalizes timestamps to ISO-8601 and omits null fields', () => { + expect(out.election.update_date).toBe(new Date(1783258899541).toISOString()); + expect(out.election.create_date).toBe('2026-07-05T13:41:39.541Z'); + expect(out.election).not.toHaveProperty('audit_ids'); // null omitted + expect(out.election).not.toHaveProperty('public_archive_id'); + }); +}); diff --git a/packages/frontend/src/components/Election/Results/BallotDataExport.tsx b/packages/frontend/src/components/Election/Results/BallotDataExport.tsx index 7759e41fb..24dbd19d1 100644 --- a/packages/frontend/src/components/Election/Results/BallotDataExport.tsx +++ b/packages/frontend/src/components/Election/Results/BallotDataExport.tsx @@ -3,9 +3,11 @@ import { ElectionResults } from '@equal-vote/star-vote-shared/domain_model/ITabu import MenuItem from "@mui/material/MenuItem"; import BorderAll from '@mui/icons-material/BorderAll'; import DataObject from '@mui/icons-material/DataObject'; +import Compress from '@mui/icons-material/Compress'; import { MenuButton } from '~/components/MenuButton'; import useAnonymizedBallots from '~/components/AnonymizedBallotsContextProvider'; import { Box } from '@mui/material'; +import { buildElectionExport } from '@equal-vote/star-vote-shared/utils/exportFormat'; interface Props { election: Election; @@ -84,6 +86,8 @@ export const BallotDataExport = ({ election, results }: Props) => { ); }; + // The existing export: the raw in-memory objects, unchanged. Anything already + // parsing this file keeps working byte-for-byte. const downloadJson = () => { const ballotObject = { Election: election, Ballots: ballots, ...(results && { Results: results }) }; triggerDownload( @@ -93,6 +97,18 @@ export const BallotDataExport = ({ election, results }: Props) => { ); }; + // The compact export (format_version 2): same data, without the tabulator's + // internal shape — see packages/shared/src/utils/exportFormat.ts. Offered + // alongside the original rather than replacing it, so no existing consumer + // breaks. Making it the default later is a one-line change here. + const downloadJsonV2 = () => { + triggerDownload( + JSON.stringify(buildElectionExport(election, ballots, results), null, 2), + 'application/json', + `Ballot Data - ${limit(election.title, 50)}-${election.election_id}.v2.json`, + ); + }; + return ( {/* A single MenuButton must stay mounted while the ballots load: swapping in a @@ -109,6 +125,10 @@ export const BallotDataExport = ({ election, results }: Props) => { Download JSON , + + + Download JSON (compact) + , ]} diff --git a/packages/shared/src/utils/exportFormat.ts b/packages/shared/src/utils/exportFormat.ts new file mode 100644 index 000000000..e9f05f91a --- /dev/null +++ b/packages/shared/src/utils/exportFormat.ts @@ -0,0 +1,214 @@ +// exportFormat.ts +// +// Builds the OPTIONAL compact "Ballot Data" JSON export (Election + Ballots + Results). +// +// This is offered ALONGSIDE the original download, not instead of it: the "Download JSON" +// menu item still emits `JSON.stringify({ Election, Ballots, Results })` byte-for-byte, so +// nothing consuming that file breaks. "Download JSON (compact)" emits the format below. +// +// The original export is the raw in-memory objects, which leaks the tabulator's internal +// shape into a file people archive and parse: +// - every candidate carried O(n^2) `votesPreferredOver` / `winsAgainst` maps keyed by +// UUID, including self-vs-self entries, duplicated across elected/tied/other/summary +// - Results used camelCase while Election/Ballots used snake_case +// - timestamps were inconsistent (ISO create_date vs epoch-ms-string update_date) +// - ballot scores referenced candidates by UUID only, forcing a join to read them +// +// This builds a versioned, self-describing v2 export: candidates listed once, a deduped +// pairwise matrix (self-pairs removed, keyed by name), elected/tied/other as name lists, +// snake_case keys throughout, ISO-8601 timestamps, null/empty fields omitted, and both +// candidate_id and candidate_name wherever a candidate is referenced. + +import { Election } from '../domain_model/Election'; +import { AnonymizedBallot } from '../domain_model/Ballot'; +import { ElectionResults } from '../domain_model/ITabulators'; + +export const EXPORT_FORMAT = 'bettervoting-export'; +export const EXPORT_FORMAT_VERSION = 2; + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +// camelCase -> snake_case for a single object key +const toSnake = (k: string): string => + k + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') + .toLowerCase(); + +// Deep-convert object keys to snake_case; values are left untouched. +const deepSnake = (v: any): any => { + if (Array.isArray(v)) return v.map(deepSnake); + if (v && typeof v === 'object') { + const out: any = {}; + for (const [k, val] of Object.entries(v)) out[toSnake(k)] = deepSnake(val); + return out; + } + return v; +}; + +// Recursively drop null / undefined values (keeps the file terse). +const omitEmpty = (v: any): any => { + if (Array.isArray(v)) return v.map(omitEmpty); + if (v && typeof v === 'object') { + const out: any = {}; + for (const [k, val] of Object.entries(v)) { + if (val === null || val === undefined) continue; + out[k] = omitEmpty(val); + } + return out; + } + return v; +}; + +// Normalize a timestamp (ISO string, epoch-ms number, or epoch-ms string) to ISO-8601. +const normalizeTimestamp = (v: any): string | undefined => { + if (v === null || v === undefined || v === '') return undefined; + const asNum = + typeof v === 'number' ? v : /^\d+$/.test(String(v)) ? Number(v) : NaN; + if (!Number.isNaN(asNum)) return new Date(asNum).toISOString(); + const d = new Date(v); + return Number.isNaN(d.getTime()) ? String(v) : d.toISOString(); +}; + +const cleanElection = (e: Election): any => { + const cleaned: any = omitEmpty({ ...e }); + if (cleaned.create_date) cleaned.create_date = normalizeTimestamp(cleaned.create_date); + if (cleaned.update_date) cleaned.update_date = normalizeTimestamp(cleaned.update_date); + return cleaned; +}; + +// Ballots stay compact (id + score). Candidate names are NOT repeated on every +// ballot row — that re-bloats large elections (e.g. 51 candidates x 100 ballots). +// Names are always resolvable from election.races[].candidates (and, when present, +// results[].candidates), so no information is lost. +// +// IMPORTANT: a score of `null` is MEANINGFUL — it means the voter did not score +// that candidate (an abstention on that candidate), which is distinct from an +// explicit `0` and from scoring the "None of the Above" (c-nota) candidate. So +// scores are preserved verbatim (including null); we do NOT run the null-omitting +// pass over ballot rows. Only genuinely-absent optional metadata is dropped. +const cleanBallots = (ballots: AnonymizedBallot[]): any[] => + (ballots ?? []).map((b: any) => { + const out: any = { ballot_id: b.ballot_id }; + if (b.precinct != null) out.precinct = b.precinct; + out.votes = (b.votes ?? []).map((v: any) => { + const vote: any = { race_id: v.race_id }; + if (v.overvote_rank != null) vote.overvote_rank = v.overvote_rank; + if (v.has_duplicate_rank != null) vote.has_duplicate_rank = v.has_duplicate_rank; + // Preserve every score exactly, including explicit `null` (= not scored). + vote.scores = (v.scores ?? []).map((s: any) => ({ + candidate_id: s.candidate_id, + score: s.score === undefined ? null : s.score, + })); + return vote; + }); + return out; + }); + +// Turn one race's tabulator result into the clean v2 shape. +const cleanResult = (r: any): any => { + const summaryCandidates: any[] = r.summaryData?.candidates ?? []; + + const idToName: Record = {}; + summaryCandidates.forEach((c) => { + idToName[c.id] = c.name; + }); + const nm = (id: string) => idToName[id] ?? id; + const refs = (arr: any[] | undefined) => + (arr ?? []).map((c: any) => ({ id: c.id, name: nm(c.id) })); + + // Candidates listed once, without the O(n^2) pairwise maps. + const candidates = summaryCandidates.map((c) => { + const { votesPreferredOver, winsAgainst, ...rest } = c; + return deepSnake(rest); + }); + + // Deduped pairwise matrix: self-pairs removed, keyed by candidate name. + const pairwise: Record> = {}; + summaryCandidates.forEach((c) => { + const row: Record = {}; + Object.keys(c.votesPreferredOver ?? {}).forEach((oid) => { + if (oid === c.id) return; // drop self-vs-self + row[nm(oid)] = { + prefer: c.votesPreferredOver[oid], + wins: !!c.winsAgainst?.[oid], + }; + }); + pairwise[nm(c.id)] = row; + }); + + const { candidates: _drop, ...summaryRest } = r.summaryData ?? {}; + const summary = deepSnake(summaryRest); + + const rounds = (r.roundResults ?? []).map((rr: any) => { + const out: any = { + winners: refs(rr.winners), + runner_up: refs(rr.runner_up), + tied: refs(rr.tied), + tie_break_type: rr.tieBreakType, + logs: rr.logs, + }; + if (rr.eliminated) out.eliminated = refs(rr.eliminated); + if (rr.exhaustedVoteCount !== undefined) out.exhausted_vote_count = rr.exhaustedVoteCount; + return omitEmpty(out); + }); + + // Pull off the fields handled explicitly; deepSnake whatever method-specific + // top-level fields remain (e.g. IRV exhaustedVoteCounts / nExhaustedViaOvervote, + // STAR_PR logs). + const { + summaryData, + roundResults, + elected, + tied, + other, + perm, + votingMethod, + tieBreakType, + writeInDiagnostics, + ...extra + } = r; + + return omitEmpty({ + voting_method: votingMethod, + elected: refs(elected), + tied: refs(tied), + other: refs(other), + tie_break_type: tieBreakType, + candidates, + pairwise, + rounds, + summary, + perm: perm ? perm.map((id: string) => ({ id, name: nm(id) })) : undefined, + write_in_diagnostics: writeInDiagnostics ? deepSnake(writeInDiagnostics) : undefined, + ...deepSnake(extra), + }); +}; + +export interface ElectionExport { + format: string; + format_version: number; + exported_at: string; + election: any; + ballots: any[]; + results?: any[]; +} + +/** + * Build the clean, versioned election export object. + * @param election the election config + * @param ballots anonymized ballots (may be undefined while loading) + * @param results per-race tabulation results (optional) + */ +export const buildElectionExport = ( + election: Election, + ballots: AnonymizedBallot[] | undefined, + results?: ElectionResults[], +): ElectionExport => ({ + format: EXPORT_FORMAT, + format_version: EXPORT_FORMAT_VERSION, + exported_at: new Date().toISOString(), + election: cleanElection(election), + ballots: cleanBallots(ballots ?? []), + ...(results ? { results: results.map(cleanResult) } : {}), +});