From 3fe54f4c23f7104fecea3bae502513e256b1a071 Mon Sep 17 00:00:00 2001 From: awdem Date: Thu, 23 Jul 2026 14:55:53 -0400 Subject: [PATCH 1/8] filter boundary reviews by public_visibility --- .../create_current_boundary_reviews_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py index 632c008..ccd0dd2 100644 --- a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py +++ b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py @@ -55,7 +55,7 @@ def export_sql(): JOIN organisations_organisation o ON o.id = obr.organisation_id JOIN organisations_organisationgeography og ON og.organisation_id = o.id WHERE - obr.id IN (963, 964) + obr.public_visibility != 'HIDDEN' ) SELECT r.slug, From 7f1de6b4f2ae6c1e6ef7a451b625141d79e69c67 Mon Sep 17 00:00:00 2001 From: awdem Date: Thu, 23 Jul 2026 14:56:41 -0400 Subject: [PATCH 2/8] add currency concept to boundary reviews this removes boundary reviews from the csv after the first elections to use their boundaries are no longer current. This mimics the get_current() property on the Election model in EE, except its selecting for "non-current" elections on or after the effective date of the boundary review. --- .../create_current_boundary_reviews_csv.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py index ccd0dd2..25867df 100644 --- a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py +++ b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py @@ -56,6 +56,18 @@ def export_sql(): JOIN organisations_organisationgeography og ON og.organisation_id = o.id WHERE obr.public_visibility != 'HIDDEN' + AND NOT EXISTS ( + SELECT + e.election_id, + e.poll_open_date + FROM + elections_election e + WHERE + e.organisation_id = o.id + AND e.poll_open_date BETWEEN obr.effective_date AND CURRENT_DATE - INTERVAL '20 days' + LIMIT + 1 + ) ) SELECT r.slug, From 44fdedfd0ee0448f945fb13cdc4c0ecd3fa7cf6b Mon Sep 17 00:00:00 2001 From: awdem Date: Mon, 27 Jul 2026 11:13:47 -0400 Subject: [PATCH 3/8] add related ballots to boundary changes pipeline --- cdk/queries/addresses_to_boundary_change.sql | 10 ++++--- .../create_current_boundary_reviews_csv.py | 27 ++++++++++++++++++- cdk/shared_components/tables.py | 1 + 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/cdk/queries/addresses_to_boundary_change.sql b/cdk/queries/addresses_to_boundary_change.sql index 77bca0c..d96e71b 100644 --- a/cdk/queries/addresses_to_boundary_change.sql +++ b/cdk/queries/addresses_to_boundary_change.sql @@ -5,6 +5,7 @@ UNLOAD ( division_slug, division_name, division_official_identifier, + division_related_ballots, boundary_review_id, division_type, division_boundary_wkt, @@ -43,6 +44,7 @@ UNLOAD ( nd.division_slug AS new_division_slug, nd.division_name AS new_division_name, nd.division_official_identifier AS new_division_official_identifier, + nd.division_related_ballots AS related_ballots, od.consultation_url, od.legislation_title, od.effective_date, @@ -110,8 +112,9 @@ UNLOAD ( a.organisation_name, a.organisation_official_name, a.organisation_gss, - bts.old_division_slug IS NOT NULL AS boundary_same, - nts.old_division_slug IS NOT NULL AS name_same + a.related_ballots AS related_ballots, + bts.old_division_slug IS NOT NULL AS boundary_same, + nts.old_division_slug IS NOT NULL AS name_same FROM addresses a LEFT JOIN boundaries_the_same bts ON a.old_division_slug = bts.old_division_slug @@ -128,7 +131,7 @@ UNLOAD ( division_type, boundary_review_id, MAP( - ARRAY['division_type', 'old_division_slug', 'old_division_name', 'old_division_official_identifier', 'old_divisionset_pmtiles_url', 'new_division_slug', 'new_division_name', 'new_division_official_identifier', 'new_divisionset_pmtiles_url', 'change_scenario'], + ARRAY['division_type', 'old_division_slug', 'old_division_name', 'old_division_official_identifier', 'old_divisionset_pmtiles_url', 'new_division_slug', 'new_division_name', 'new_division_official_identifier', 'new_divisionset_pmtiles_url', 'related_ballots', 'change_scenario'], ARRAY[ division_type, old_division_slug, @@ -139,6 +142,7 @@ UNLOAD ( new_division_name, new_division_official_identifier, new_divisionset_pmtiles_url, + related_ballots, CASE WHEN boundary_same AND name_same THEN 'NO_CHANGE' WHEN boundary_same AND NOT name_same THEN 'NAME_CHANGED' diff --git a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py index 25867df..f542d2d 100644 --- a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py +++ b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py @@ -49,7 +49,17 @@ def export_sql(): ds.end_date DESC NULLS LAST LIMIT 1 - ) AS old_divisionset_id + ) AS old_divisionset_id, + ( + SELECT + array_agg(e.election_id) + FROM + elections_election e + WHERE + e.organisation_id = o.id + AND e.group_type IS NULL + AND e.poll_open_date = obr.effective_date + ) as review_related_ballots FROM organisations_organisationboundaryreview obr JOIN organisations_organisation o ON o.id = obr.organisation_id @@ -87,6 +97,21 @@ def export_sql(): d.name AS division_name, d.official_identifier AS division_official_identifier, st_astext (dgs.geography) AS division_boundary_wkt, + CASE + WHEN ds.id = r.old_divisionset_id THEN NULL + WHEN ds.id = r.new_divisionset_id THEN COALESCE( + ( + SELECT + json_agg(b)::text + FROM + unnest(r.review_related_ballots) AS b + WHERE + b ~* d.slug + ), + '[]' + ) + ELSE NULL + END AS division_related_ballots, r.boundary_review_id, CASE WHEN ds.id = r.old_divisionset_id THEN 'old' diff --git a/cdk/shared_components/tables.py b/cdk/shared_components/tables.py index 77345f2..dc40185 100644 --- a/cdk/shared_components/tables.py +++ b/cdk/shared_components/tables.py @@ -116,6 +116,7 @@ "division_name": glue.Schema.STRING, "division_official_identifier": glue.Schema.STRING, "division_boundary_wkt": glue.Schema.STRING, + "division_related_ballots": glue.Schema.STRING, }, partition_keys=[ glue.Column( From 61f7a85d34186255f0e5c158e45241819c093197 Mon Sep 17 00:00:00 2001 From: awdem Date: Thu, 30 Jul 2026 13:18:09 -0400 Subject: [PATCH 4/8] add quality check for boundary changes csv --- cdk/stacks/current_boundary_changes.py | 70 ++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/cdk/stacks/current_boundary_changes.py b/cdk/stacks/current_boundary_changes.py index 2f3302b..0e66193 100644 --- a/cdk/stacks/current_boundary_changes.py +++ b/cdk/stacks/current_boundary_changes.py @@ -89,6 +89,10 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: current_boundary_changes ) + current_boundary_changes_csv_quality_check = ( + self.make_current_boundary_changes_csv_quality_check() + ) + boundary_review_pairs_map = self.make_boundary_review_pairs_map() make_addresses_to_boundary_change_partitions = ( @@ -142,6 +146,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: create_current_boundary_changes_csv_task ) .next(make_current_boundary_changes_partitions) + .next(current_boundary_changes_csv_quality_check) .next(boundary_review_pairs_map) .next(make_addresses_to_boundary_change_partitions) .next( @@ -240,6 +245,71 @@ def make_current_boundary_changes_csv_task(self) -> tasks.LambdaInvoke: ), ) + def make_current_boundary_changes_csv_quality_check( + self, + ) -> sfn.Chain: + division_ballots_query = tasks.LambdaInvoke( + self, + "Get divison ballots with more than one ballot", + lambda_function=self.athena_query_lambda, + payload=sfn.TaskInput.from_object( + { + "context": { + "table_name": current_boundary_changes.table_name + }, + "QueryString": "SELECT DISTINCT division_official_identifier, json_array_length(json_parse(division_related_ballots)) as division_ballot_count FROM {table_name} WHERE divisionset_generation = 'new' AND json_array_length(json_parse(division_related_ballots)) > 1;", + "blocking": True, + } + ), + ) + + # get results of above Athena query + get_division_ballots_query_result = tasks.AthenaGetQueryResults( + self, + "Get divison ballots with more than one ballot results", + query_execution_id="{% $states.input.Payload.queryExecutionId %}", + query_language=sfn.QueryLanguage.JSONATA, + ) + # drop the header row + remove_headers = sfn.Pass( + self, + "Drop header from division ballots query results", + parameters={ + "divisions_and_counts": sfn.JsonPath.string_at( + "$.ResultSet.Rows[1:]" + ), + }, + ) + count_results = sfn.Pass( + self, + "Count division ballots query results rows", + query_language=sfn.QueryLanguage.JSONATA, + outputs={ + "divisions_and_counts_length": "{% $count($states.input.divisions_and_counts) %}", + }, + ) + # check each division has exactly one ballot + check_results_count = ( + sfn.Choice(self, "Check No divisions have more than one ballot") + .when( + sfn.Condition.number_equals("$.divisions_and_counts_length", 0), + sfn.Pass( + self, + "No divisions have more than one ballot!", + ), + ) + .otherwise( + sfn.Fail(self, "Some divisions have more than one ballot :(") + ) + ) + + return ( + division_ballots_query.next(get_division_ballots_query_result) + .next(remove_headers) + .next(count_results) + .next(check_results_count.afterwards()) + ) + def make_partitions_task(self, table) -> tasks.LambdaInvoke: return MakePartitionsConstruct( self, From d7218fa80042f08f62aace4946543cd84544a174 Mon Sep 17 00:00:00 2001 From: awdem Date: Thu, 30 Jul 2026 13:24:38 -0400 Subject: [PATCH 5/8] get division related ballots using ballot -> division -> div set FKs doing it this way is more accurate and it means we no longer need to find the review related ballots in the review subquery --- .../create_current_boundary_reviews_csv.py | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py index f542d2d..0ab38f7 100644 --- a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py +++ b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py @@ -49,17 +49,7 @@ def export_sql(): ds.end_date DESC NULLS LAST LIMIT 1 - ) AS old_divisionset_id, - ( - SELECT - array_agg(e.election_id) - FROM - elections_election e - WHERE - e.organisation_id = o.id - AND e.group_type IS NULL - AND e.poll_open_date = obr.effective_date - ) as review_related_ballots + ) AS old_divisionset_id FROM organisations_organisationboundaryreview obr JOIN organisations_organisation o ON o.id = obr.organisation_id @@ -102,11 +92,13 @@ def export_sql(): WHEN ds.id = r.new_divisionset_id THEN COALESCE( ( SELECT - json_agg(b)::text + json_agg(e.election_id)::text FROM - unnest(r.review_related_ballots) AS b + elections_election AS e WHERE - b ~* d.slug + e.division_id = d.id + AND d.divisionset_id = r.new_divisionset_id + AND e.poll_open_date = r.effective_date ), '[]' ) From 7128088816dd0190b004039a59617e7b962948e2 Mon Sep 17 00:00:00 2001 From: awdem Date: Thu, 30 Jul 2026 14:06:32 -0400 Subject: [PATCH 6/8] filter ballots by current_status --- .../create_current_boundary_reviews_csv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py index 0ab38f7..f4bd2c6 100644 --- a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py +++ b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py @@ -96,7 +96,8 @@ def export_sql(): FROM elections_election AS e WHERE - e.division_id = d.id + e.current_status = 'Approved' + AND e.division_id = d.id AND d.divisionset_id = r.new_divisionset_id AND e.poll_open_date = r.effective_date ), From 321f1bdb71fcd5161312df596d7962ffecdd0f84 Mon Sep 17 00:00:00 2001 From: awdem Date: Thu, 30 Jul 2026 14:27:33 -0400 Subject: [PATCH 7/8] fix review filtering by currency --- .../create_current_boundary_reviews_csv.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py index f4bd2c6..a6a4e29 100644 --- a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py +++ b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py @@ -58,13 +58,15 @@ def export_sql(): obr.public_visibility != 'HIDDEN' AND NOT EXISTS ( SELECT - e.election_id, - e.poll_open_date + e.election_id FROM elections_election e WHERE - e.organisation_id = o.id - AND e.poll_open_date BETWEEN obr.effective_date AND CURRENT_DATE - INTERVAL '20 days' + e.current_status = 'Approved' + AND e.current IS NOT TRUE + AND e.organisation_id = o.id + AND e.poll_open_date >= obr.effective_date + AND e.poll_open_date <= CURRENT_DATE - INTERVAL '20 days' LIMIT 1 ) From a596705337ed2bde1b5ba05f1d9415cdba39bb75 Mon Sep 17 00:00:00 2001 From: awdem Date: Mon, 3 Aug 2026 07:14:56 -0400 Subject: [PATCH 8/8] remove cancelled ballots from query --- .../create_current_boundary_reviews_csv.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py index a6a4e29..224fbd1 100644 --- a/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py +++ b/cdk/shared_components/lambdas/create_boundary_changes_csv/create_current_boundary_reviews_csv.py @@ -63,6 +63,7 @@ def export_sql(): elections_election e WHERE e.current_status = 'Approved' + AND NOT e.cancelled AND e.current IS NOT TRUE AND e.organisation_id = o.id AND e.poll_open_date >= obr.effective_date @@ -99,6 +100,7 @@ def export_sql(): elections_election AS e WHERE e.current_status = 'Approved' + AND not e.cancelled AND e.division_id = d.id AND d.divisionset_id = r.new_divisionset_id AND e.poll_open_date = r.effective_date