From 4e771243c3b99a8cae1ecd2cf5cf3e4addd46048 Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Tue, 27 Jan 2026 14:37:55 +0100 Subject: [PATCH 01/10] Adds basic admin page --- app/cli.py | 2 +- app/db_utils.py | 1 - app/form_utils.py | 114 +++++++++++++++++++++++ app/routes.py | 186 +++++++++++++++----------------------- app/stembureaumanager.py | 2 +- app/templates/base.html | 37 ++++---- app/templates/beheer.html | 44 +++++++++ 7 files changed, 256 insertions(+), 130 deletions(-) create mode 100644 app/form_utils.py create mode 100644 app/templates/beheer.html diff --git a/app/cli.py b/app/cli.py index 8a264c1..5d003cf 100644 --- a/app/cli.py +++ b/app/cli.py @@ -2,10 +2,10 @@ from app.models import Gemeente, User, Gemeente_user, Election, BAG, add_user, db from app.ckan import ckan from app.email import send_invite, send_update +from app.form_utils import create_record, kieskringen from app.parser import UploadFileParser from app.published_monitor import PublishedMonitor from app.validator import Validator -from app.routes import create_record, kieskringen from app.db_utils import db_delete, db_delete_all, db_exec_all, db_exec_one, db_exec_one_optional from app.utils import get_gemeente, publish_gemeente_records, remove_id from app.stembureaumanager import StembureauManager diff --git a/app/db_utils.py b/app/db_utils.py index bff33c4..65a9526 100644 --- a/app/db_utils.py +++ b/app/db_utils.py @@ -1,6 +1,5 @@ from app.models import db from sqlalchemy import select, delete, func -from flask import current_app def db_exec_one(query): return db.session.execute(query).scalar_one() diff --git a/app/form_utils.py b/app/form_utils.py new file mode 100644 index 0000000..6714999 --- /dev/null +++ b/app/form_utils.py @@ -0,0 +1,114 @@ +import csv +from decimal import Decimal + +from app.db_utils import db_exec_first +from app.models import BAG +from flask import current_app + +kieskringen = [] +with open('app/data/kieskringen.csv') as IN: + reader = csv.reader(IN, delimiter=';') + # Skip header + next(reader) + kieskringen = list(reader) + + +def create_record(form, stemlokaal_id, gemeente, election): + ID = 'NLODS%sstembureaus%s%s' % ( + gemeente.gemeente_code, + current_app.config['CKAN_CURRENT_ELECTIONS'][election]['election_date'], + current_app.config['CKAN_CURRENT_ELECTIONS'][election]['election_number'] + ) + + kieskring_id = '' + hoofdstembureau = '' + if (election.startswith('gemeenteraadsverkiezingen') or + election.startswith('kiescollegeverkiezingen') or + election.startswith('eilandsraadsverkiezingen')): + kieskring_id = gemeente.gemeente_naam + hoofdstembureau = gemeente.gemeente_naam + elif (election.startswith('referendum') or + election.startswith('Tweede Kamerverkiezingen') or + election.startswith('Provinciale Statenverkiezingen')): + for row in kieskringen: + if row[2] == gemeente.gemeente_naam: + kieskring_id = row[0] + hoofdstembureau = row[1] + elif election.startswith('Europese Parlementsverkiezingen'): + kieskring_id = 'Nederland' + hoofdstembureau = 'Nederland' + + record = { + 'UUID': stemlokaal_id, + 'Gemeente': gemeente.gemeente_naam, + 'CBS gemeentecode': gemeente.gemeente_code, + 'Kieskring ID': kieskring_id, + 'Hoofdstembureau': hoofdstembureau, + 'ID': ID + } + + # Process the fields from the form + for f in form: + # Save the Verkiezingen by joining the list into a string + if f.label.text == 'Verkiezingen': + record[f.label.text] = ';'.join(f.data) + elif (f.type != 'SubmitField' and + f.type != 'CSRFTokenField' and f.type != 'RadioField'): + record[f.label.text[:62]] = f.data + + # prevent this field from being saved as it is not a real form field. + del record['Adres stembureau'] + + bag_nummer = record['BAG Nummeraanduiding ID'] + bag_record = db_exec_first(BAG, nummeraanduiding=bag_nummer) + + + if bag_record is not None: + bag_conversions = { + 'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw', + 'openbareruimte': 'Straatnaam', + 'huisnummer': 'Huisnummer', + 'huisletter': 'Huisletter', + 'huisnummertoevoeging': 'Huisnummertoevoeging', + 'postcode': 'Postcode', + 'woonplaats': 'Plaats', + 'lat': 'Latitude', + 'lon': 'Longitude', + 'x': 'X', + 'y': 'Y' + } + + for bag_field, record_field in bag_conversions.items(): + bag_field_value = getattr(bag_record, bag_field, None) + if bag_field_value is not None: + if isinstance(bag_field_value, Decimal): + # do not overwrite geocoordinates if they were otherwise specified + if not record.get(record_field): + record[record_field] = float(bag_field_value) + else: + record[record_field] = bag_field_value.encode( + 'utf-8' + ).decode() + else: + record[record_field] = None + + ## We stopped adding the wijk and buurt data as the data + ## supplied by CBS is not up to date enough as it is only + ## released once a year and many months after changes + ## have been made by the municipalities. + #wk_code, wk_naam, bu_code, bu_naam = find_buurt_and_wijk( + # bag_nummer, + # gemeente.gemeente_code, + # bag_record.lat, + # bag_record.lon + #) + #if wk_naam: + # record['Wijknaam'] = wk_naam + #if wk_code: + # record['CBS wijknummer'] = wk_code + #if bu_naam: + # record['Buurtnaam'] = bu_naam + #if bu_code: + # record['CBS buurtnummer'] = bu_code + + return record diff --git a/app/routes.py b/app/routes.py index 3b12ead..3959e24 100644 --- a/app/routes.py +++ b/app/routes.py @@ -3,11 +3,14 @@ import re from functools import wraps from datetime import datetime -from decimal import Decimal +from app.form_utils import create_record, kieskringen +from app.procura import ProcuraManager +from app.stembureaumanager import StembureauManager +from app.tsa import TSAManager from flask import ( render_template, request, redirect, url_for, flash, session, - jsonify, current_app + jsonify, current_app, has_request_context ) from markupsafe import Markup from flask_login import ( @@ -15,7 +18,7 @@ ) from werkzeug.utils import secure_filename -from sqlalchemy import or_, select, Integer +from sqlalchemy import or_, and_, select, Integer, func, case from sqlalchemy.sql.expression import cast from sqlalchemy.exc import OperationalError @@ -26,7 +29,7 @@ from app.parser import UploadFileParser from app.validator import Validator from app.email import send_password_reset_email -from app.models import Gemeente, User, Record, BAG, add_user, db +from app.models import Gemeente, Gemeente_user, User, Record, BAG, add_user, db from app.db_utils import db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional from app.utils import get_b64encoded_qr_image, get_gemeente, get_gemeente_by_id, get_gemeente_by_name, get_mysql_match_against_safe_string, remove_id from app.ckan import ckan @@ -136,12 +139,6 @@ with open('files/niet-deelnemende-gemeenten-2026-gr.csv') as IN: disclaimer_gemeenten = [x.strip() for x in IN.readlines()] -kieskringen = [] -with open('app/data/kieskringen.csv') as IN: - reader = csv.reader(IN, delimiter=';') - # Skip header - next(reader) - kieskringen = list(reader) # A list containing all gemeentenamen, used in the search box on the # homepage. Also allow for some alternative municipality names. @@ -159,6 +156,11 @@ ] + alternative_names +# Do not show the informational messages in base.html for these routes +skip_informational_messages_for = [ + '/beheer' +] + # Always allow admins to edit the data even if the deadline is passed def check_deadline_passed(): if current_user.admin: @@ -186,6 +188,15 @@ def get_stembureaus(elections, filters=None): return results.values() +def get_stembureaus_counts(resource_id): + all_records = ckan.get_records(resource_id) + records_hash = {} + for r in all_records: + gemeente_code = r['CBS gemeentecode'] + if not records_hash.get(gemeente_code): + records_hash[gemeente_code] = 0 + records_hash[gemeente_code] += 1 + return records_hash # Used to only retrieve the records that are needed on a page def _hydrate(record, minimal_type='default'): @@ -698,6 +709,60 @@ def verify_two_factor_auth(): return render_template("verify_2fa.html", form=form) + + @app.route( + "/beheer", + methods=['GET', 'POST'] + ) + @admin_login_required + def beheer(): + query = select(Gemeente.id, Gemeente.gemeente_code, Gemeente.gemeente_naam, \ + case((Gemeente.source == ProcuraManager.SOURCE_STRING, "Procura"), \ + (Gemeente.source == StembureauManager.SOURCE_STRING, "SBM"), \ + (Gemeente.source == TSAManager.SOURCE_STRING, "TSA"), \ + else_='').label('api'), \ + func.count(User.id).label('user_count')) \ + .select_from(Gemeente) \ + .join(Gemeente_user, isouter=True) \ + .join(User, and_(User.id == Gemeente_user.user_id, or_(User.admin == None, User.admin == False)), isouter=True) \ + .group_by(Gemeente.id) + gemeenten = db.session.execute(query).all() + + field_order = [ + 'ID', + 'Code', + 'Naam', + 'API', + 'Aantal stembureaus gepubliceerd', + 'Aantal stembureaus concept', + 'Aantal gebruikers' + ] + + resource_id = list(ckan.elections.values())[0]['draft_resource'] + draft_records_hash = get_stembureaus_counts(resource_id) + + resource_id = list(ckan.elections.values())[0]['publish_resource'] + published_records_hash = get_stembureaus_counts(resource_id) + + return render_template( + 'beheer.html', + gemeenten=gemeenten, + field_order=field_order, + draft_records_hash=draft_records_hash, + published_records_hash= published_records_hash + ) + + + @app.context_processor + def set_global_html_variable_values(): + if not has_request_context(): + show_informational_messages = True + else: + show_informational_messages = request.path not in skip_informational_messages_for + template_config = {'show_informational_messages': show_informational_messages} + return template_config + + @app.route("/gemeente-logout") @login_required def gemeente_logout(): @@ -1179,107 +1244,6 @@ def _format_verkiezingen_string(elections): return verkiezing_string -def create_record(form, stemlokaal_id, gemeente, election): - ID = 'NLODS%sstembureaus%s%s' % ( - gemeente.gemeente_code, - current_app.config['CKAN_CURRENT_ELECTIONS'][election]['election_date'], - current_app.config['CKAN_CURRENT_ELECTIONS'][election]['election_number'] - ) - - kieskring_id = '' - hoofdstembureau = '' - if (election.startswith('gemeenteraadsverkiezingen') or - election.startswith('kiescollegeverkiezingen') or - election.startswith('eilandsraadsverkiezingen')): - kieskring_id = gemeente.gemeente_naam - hoofdstembureau = gemeente.gemeente_naam - elif (election.startswith('referendum') or - election.startswith('Tweede Kamerverkiezingen') or - election.startswith('Provinciale Statenverkiezingen')): - for row in kieskringen: - if row[2] == gemeente.gemeente_naam: - kieskring_id = row[0] - hoofdstembureau = row[1] - elif election.startswith('Europese Parlementsverkiezingen'): - kieskring_id = 'Nederland' - hoofdstembureau = 'Nederland' - - record = { - 'UUID': stemlokaal_id, - 'Gemeente': gemeente.gemeente_naam, - 'CBS gemeentecode': gemeente.gemeente_code, - 'Kieskring ID': kieskring_id, - 'Hoofdstembureau': hoofdstembureau, - 'ID': ID - } - - # Process the fields from the form - for f in form: - # Save the Verkiezingen by joining the list into a string - if f.label.text == 'Verkiezingen': - record[f.label.text] = ';'.join(f.data) - elif (f.type != 'SubmitField' and - f.type != 'CSRFTokenField' and f.type != 'RadioField'): - record[f.label.text[:62]] = f.data - - # prevent this field from being saved as it is not a real form field. - del record['Adres stembureau'] - - bag_nummer = record['BAG Nummeraanduiding ID'] - bag_record = db_exec_first(BAG, nummeraanduiding=bag_nummer) - - - if bag_record is not None: - bag_conversions = { - 'verblijfsobjectgebruiksdoel': 'Gebruiksdoel van het gebouw', - 'openbareruimte': 'Straatnaam', - 'huisnummer': 'Huisnummer', - 'huisletter': 'Huisletter', - 'huisnummertoevoeging': 'Huisnummertoevoeging', - 'postcode': 'Postcode', - 'woonplaats': 'Plaats', - 'lat': 'Latitude', - 'lon': 'Longitude', - 'x': 'X', - 'y': 'Y' - } - - for bag_field, record_field in bag_conversions.items(): - bag_field_value = getattr(bag_record, bag_field, None) - if bag_field_value is not None: - if isinstance(bag_field_value, Decimal): - # do not overwrite geocoordinates if they were otherwise specified - if not record.get(record_field): - record[record_field] = float(bag_field_value) - else: - record[record_field] = bag_field_value.encode( - 'utf-8' - ).decode() - else: - record[record_field] = None - - ## We stopped adding the wijk and buurt data as the data - ## supplied by CBS is not up to date enough as it is only - ## released once a year and many months after changes - ## have been made by the municipalities. - #wk_code, wk_naam, bu_code, bu_naam = find_buurt_and_wijk( - # bag_nummer, - # gemeente.gemeente_code, - # bag_record.lat, - # bag_record.lon - #) - #if wk_naam: - # record['Wijknaam'] = wk_naam - #if wk_code: - # record['CBS wijknummer'] = wk_code - #if bu_naam: - # record['Buurtnaam'] = bu_naam - #if bu_code: - # record['CBS buurtnummer'] = bu_code - - return record - - # Converts a column number to a spreadsheet column string, e.g. 6 to F # and 124 to DT def _colnum2string(n): diff --git a/app/stembureaumanager.py b/app/stembureaumanager.py index 666e8f8..0c1c39a 100644 --- a/app/stembureaumanager.py +++ b/app/stembureaumanager.py @@ -1,3 +1,4 @@ +from app.form_utils import create_record from flask import current_app from datetime import datetime @@ -6,7 +7,6 @@ from app.email import send_email from app.parser import BaseParser, valid_headers from app.validator import Validator -from app.routes import create_record from app.utils import get_gemeente, publish_gemeente_records from urllib.parse import urljoin diff --git a/app/templates/base.html b/app/templates/base.html index 4c08a31..e24fe3b 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -64,6 +64,9 @@
  • Data
    • {% if current_user.is_authenticated %}
  • Dashboard
    • + {% if current_user.admin %} +
  • Beheer
    • + {% endif %}
  • Uitloggen
    • {% else %}
  • Inloggen
    • @@ -92,22 +95,24 @@
    {% include 'flashed-messages.html' %} - {# -
    - Deze website toont nu nog de stembureaus van de {{ config['PREVIOUS_ELECTION_TYPE'] }} van - {{ config['PREVIOUS_ELECTION_DATE_LONG'] }}. Voor de aankomende {{ config['ELECTION_TYPE'] }} van {{ config['ELECTION_DATE_LONG'] }} zullen alle stembureaus weer worden bijgewerkt. Gemeenten krijgen daarvoor begin januari een uitnodigingsmail. -
    - #} - {% set upcomingElection = "de " + config['ELECTION_TYPE'] + " van " + config['ELECTION_DATE_LONG'] %} -
    - {% if number_of_published_gemeenten %} -

    Op dit moment hebben {{ number_of_published_gemeenten }} van de in totaal {{ alle_gemeenten | length - 5 }} (bijzondere) gemeenten hun stembureaus voor {{ upcomingElection }} gepubliceerd. De rest wordt nog tot uiterlijk twee weken voor de verkiezingen toegevoegd.

    - {% else %} -

    Nog niet alle gemeenten hebben de stembureaus voor {{ upcomingElection }} aangeleverd. Deze worden nog tot uiterlijk twee weken voor de verkiezingen toegevoegd.

    - {% endif %} -

    Bericht voor de gemeenten die gebruik (willen) maken van het geautomatiseerd aanleveren van de stembureaugegevens via onze koppelingen met jullie stembureausoftware: wij hebben deze koppelingen nog niet aangezet. Wij verwachten deze uiterlijk 22 januari weer aangezet te hebben. Bekijk binnenkort dus of de gegevens (goed) zijn gepubliceerd op WaarIsMijnStemlokaal.nl.

    -
    - {# #} + {% if show_informational_messages %} + {# +
    + Deze website toont nu nog de stembureaus van de {{ config['PREVIOUS_ELECTION_TYPE'] }} van + {{ config['PREVIOUS_ELECTION_DATE_LONG'] }}. Voor de aankomende {{ config['ELECTION_TYPE'] }} van {{ config['ELECTION_DATE_LONG'] }} zullen alle stembureaus weer worden bijgewerkt. Gemeenten krijgen daarvoor begin januari een uitnodigingsmail. +
    + #} + {% set upcomingElection = "de " + config['ELECTION_TYPE'] + " van " + config['ELECTION_DATE_LONG'] %} +
    + {% if number_of_published_gemeenten %} +

    Op dit moment hebben {{ number_of_published_gemeenten }} van de in totaal {{ alle_gemeenten | length - 5 }} (bijzondere) gemeenten hun stembureaus voor {{ upcomingElection }} gepubliceerd. De rest wordt nog tot uiterlijk twee weken voor de verkiezingen toegevoegd.

    + {% else %} +

    Nog niet alle gemeenten hebben de stembureaus voor {{ upcomingElection }} aangeleverd. Deze worden nog tot uiterlijk twee weken voor de verkiezingen toegevoegd.

    + {% endif %} +

    Bericht voor de gemeenten die gebruik (willen) maken van het geautomatiseerd aanleveren van de stembureaugegevens via onze koppelingen met jullie stembureausoftware: wij hebben deze koppelingen nog niet aangezet. Wij verwachten deze uiterlijk 22 januari weer aangezet te hebben. Bekijk binnenkort dus of de gegevens (goed) zijn gepubliceerd op WaarIsMijnStemlokaal.nl.

    +
    + {# #} + {% endif %} {% block content %}{% endblock %}
    diff --git a/app/templates/beheer.html b/app/templates/beheer.html new file mode 100644 index 0000000..a645ce4 --- /dev/null +++ b/app/templates/beheer.html @@ -0,0 +1,44 @@ +{% extends "base.html" %} +{% block head %} + Waar is mijn stemlokaal - Beheer + + {{ super() }} +{% endblock %} + +{% block content %} +
    +

    Beheer

    + + + + + {% for field in field_order %} + + {% endfor %} + + + + {% for gemeente in gemeenten %} + + + + + + + + + {% endfor %} + +
    + {{ field }} +
    {{ gemeente.id }}{{ gemeente.gemeente_code }}{{ gemeente.gemeente_naam }}{{ gemeente.api }}{{ draft_records_hash.get(gemeente.gemeente_code, '') }}{{ published_records_hash.get(gemeente.gemeente_code, '') }}{{ gemeente.user_count }}
    +
    +{% endblock %} From 0e79e1acb00c0192376738cca5683af2a57a7891 Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Thu, 29 Jan 2026 09:55:13 +0100 Subject: [PATCH 02/10] Adds managing users (WIP, pending tests) --- app/forms.py | 27 ++++++++++ app/routes.py | 63 ++++++++++++++++++++-- app/templates/beheer.html | 7 ++- app/templates/gemeente-gebruikers.html | 75 ++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 app/templates/gemeente-gebruikers.html diff --git a/app/forms.py b/app/forms.py index 4ea6bd1..b09c1a1 100644 --- a/app/forms.py +++ b/app/forms.py @@ -76,6 +76,33 @@ def process_formdata(self, valuelist): self.data = value +class DeleteItemForm(FlaskForm): + hidden = HiddenField( + name="user_id", + id="user_id" + ) + + submit = SubmitField( + 'Verwijderen', + render_kw={ + 'class': 'btn btn-danger' + } + ) + + submit_one = SubmitField( + 'Verwijderen uit 1 gemeente', + render_kw={ + 'class': 'btn btn-danger' + } + ) + + submit_all = SubmitField( + 'Verwijderen uit alle gemeenten', + render_kw={ + 'class': 'btn btn-danger' + } + ) + class ResetPasswordRequestForm(FlaskForm): email = StringField('E-mailadres', validators=[DataRequired(), Email()]) submit = SubmitField( diff --git a/app/routes.py b/app/routes.py index 3959e24..ce03a1d 100644 --- a/app/routes.py +++ b/app/routes.py @@ -23,14 +23,14 @@ from sqlalchemy.exc import OperationalError from app.forms import ( - ResetPasswordRequestForm, ResetPasswordForm, LoginForm, EditForm, + DeleteItemForm, ResetPasswordRequestForm, ResetPasswordForm, LoginForm, EditForm, FileUploadForm, PubliceerForm, GemeenteSelectionForm, Setup2faForm, SignupForm, TwoFactorForm ) from app.parser import UploadFileParser from app.validator import Validator from app.email import send_password_reset_email from app.models import Gemeente, Gemeente_user, User, Record, BAG, add_user, db -from app.db_utils import db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional +from app.db_utils import db_delete, db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional from app.utils import get_b64encoded_qr_image, get_gemeente, get_gemeente_by_id, get_gemeente_by_name, get_mysql_match_against_safe_string, remove_id from app.ckan import ckan from time import sleep @@ -712,7 +712,7 @@ def verify_two_factor_auth(): @app.route( "/beheer", - methods=['GET', 'POST'] + methods=['GET'] ) @admin_login_required def beheer(): @@ -724,7 +724,7 @@ def beheer(): func.count(User.id).label('user_count')) \ .select_from(Gemeente) \ .join(Gemeente_user, isouter=True) \ - .join(User, and_(User.id == Gemeente_user.user_id, or_(User.admin == None, User.admin == False)), isouter=True) \ + .join(User, and_(User.id == Gemeente_user.user_id, User.admin == False), isouter=True) \ .group_by(Gemeente.id) gemeenten = db.session.execute(query).all() @@ -752,6 +752,61 @@ def beheer(): published_records_hash= published_records_hash ) + @app.route("/gemeente//gebruikers", methods=['GET', 'POST']) + @admin_login_required + def gemeente_gebruikers(gemeente_id = None, user_id = None): + gemeente = get_gemeente_by_id(gemeente_id) + if not gemeente: + return redirect('/') + + # Get ids of users + subquery = select(User.id) \ + .join(Gemeente_user) \ + .where(and_(Gemeente_user.gemeente_id == gemeente_id, User.admin == False)) + + # Now get the users and also the list of municipalities + query = select(User.id, User.email, func.group_concat(Gemeente.gemeente_naam).label("gemeenten")) \ + .join(Gemeente_user, Gemeente_user.user_id == User.id) \ + .join(Gemeente, and_(Gemeente.id == Gemeente_user.gemeente_id, Gemeente.id != gemeente_id), isouter=True) \ + .where(User.id.in_(subquery)).group_by(User.id) + users = db.session.execute(query).all() + + delete_form = DeleteItemForm() + if custom_form_validate_on_submit(delete_form): + user_id = int(request.form.get('user_id')) + if user_id: + user = next(iter(list(filter(lambda u: u.id == user_id, users))), None) + print(user) + # user = db_exec_one_optional(User, id=user_id) + if user: + if user.gemeenten: + if request.form.get('submit_all'): + pass + else: + pass + else: + pass + + + pass + # db_delete(Gemeente_user, user_id=user.id) + # db_delete(User, id=user.id) + # db.session.commit() + + field_order = [ + 'ID', + 'E-mailadres', + 'Andere gemeenten' + ] + + return render_template( + 'gemeente-gebruikers.html', + gemeente=gemeente, + field_order=field_order, + users=users, + delete_form=delete_form + ) + @app.context_processor def set_global_html_variable_values(): diff --git a/app/templates/beheer.html b/app/templates/beheer.html index a645ce4..97b65e4 100644 --- a/app/templates/beheer.html +++ b/app/templates/beheer.html @@ -1,7 +1,6 @@ {% extends "base.html" %} {% block head %} Waar is mijn stemlokaal - Beheer - {{ super() }} {% endblock %} @@ -36,7 +35,11 @@

    Beheer

    {{ gemeente.api }} {{ draft_records_hash.get(gemeente.gemeente_code, '') }} {{ published_records_hash.get(gemeente.gemeente_code, '') }} - {{ gemeente.user_count }} + {% if gemeente.user_count > 0 %} + {{ gemeente.user_count }} + {% else %} + {{ gemeente.user_count }} + {% endif %} {% endfor %} diff --git a/app/templates/gemeente-gebruikers.html b/app/templates/gemeente-gebruikers.html new file mode 100644 index 0000000..fefecb8 --- /dev/null +++ b/app/templates/gemeente-gebruikers.html @@ -0,0 +1,75 @@ +{% extends "base.html" %} +{% block head %} + Waar is mijn stemlokaal - Gebruikers voor gemeente {{ gemeente.gemeente_naam }} + {{ super() }} +{% endblock %} + +{% block content %} +
    +

    Gebruikers voor gemeente {{ gemeente.gemeente_naam }}

    + + + + + + {% for field in field_order %} + + {% endfor %} + + + + {% for user in users %} + + + + + + + + {% endfor %} + +
    + {{ field }} +
    + verwijderen + {{ user.id }}{{ user.email }}{{ user.gemeenten or '' }}
    +
    +{% endblock %} From 22c5ec2f75643adba74b49167a4c4886b08bf3b7 Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Thu, 29 Jan 2026 09:56:43 +0100 Subject: [PATCH 03/10] Adds separate mysql docker container for testing --- app/__init__.py | 12 ++++++++++-- docker/docker-compose-dev.yml | 16 ++++++++++++++++ test_config.py | 7 +++++++ tests/__init__.py | 16 +++++++++++++++- tests/db/bag.sql | 32 ++++++++++++++++++++++++++++++++ tests/db/bagadres-tests.csv | 2 ++ tests/db/test_bag.sql | 7 +++++++ tests/test_forms.py | 2 -- 8 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 test_config.py create mode 100755 tests/db/bag.sql create mode 100644 tests/db/bagadres-tests.csv create mode 100644 tests/db/test_bag.sql diff --git a/app/__init__.py b/app/__init__.py index ffe7bc0..b421853 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -14,9 +14,13 @@ from flask import Flask from flask_babel import Babel -def create_app(): +def create_app(config_class=None): app = Flask(__name__) - app.config.from_object(Config) + if config_class: + app.config.from_object(config_class) + else: + app.config.from_object(Config) + app.config["SQLALCHEMY_ENGINES"] = { "default": { "url": app.config['SQLALCHEMY_DATABASE_URI'], @@ -27,6 +31,7 @@ def create_app(): from app.models import db db.init_app(app) + from app.email import mail mail.init_app(app) from app.ckan import ckan @@ -103,6 +108,9 @@ def make_shell_context(): # Create the MySQL tables if they don't exist from app.models import create_all create_all() + if app.config["TESTING"]: + from tests import insert_db_test_records + insert_db_test_records(db) from .routes import create_routes create_routes(app) diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml index 867572c..6d2808b 100644 --- a/docker/docker-compose-dev.yml +++ b/docker/docker-compose-dev.yml @@ -11,6 +11,20 @@ services: context: . dockerfile: Dockerfile-app-dev restart: "no" + mysql-tests: + image: mysql:8.0 + environment: + - MYSQL_DATABASE=stembureaus_tests + - MYSQL_ROOT_PASSWORD=wims_tests + command: --local_infile=1 --innodb_ft_min_token_size=1 --innodb-ft-enable-stopword=OFF --general-log=1 --general-log-file=/var/log/mysql/general-tests-log.log + networks: + - stm + volumes: + - stm-mysql-tests-volume:/var/lib/mysql + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + timeout: 2s + retries: 5 mysql: restart: "no" nodejs: @@ -18,3 +32,5 @@ services: networks: stm: nginx-load-balancer: +volumes: + stm-mysql-tests-volume: diff --git a/test_config.py b/test_config.py new file mode 100644 index 0000000..80c1b19 --- /dev/null +++ b/test_config.py @@ -0,0 +1,7 @@ + +from config import Config + +class TestConfig(Config): + TESTING = True + SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://root:wims_tests@mysql-tests:3306/stembureaus_tests?local_infile=True' + WTF_CSRF_ENABLED = False diff --git a/tests/__init__.py b/tests/__init__.py index 0a23b5a..4d25291 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,17 @@ from app import create_app +from config import basedir +from test_config import TestConfig +from sqlalchemy import text + +def insert_db_test_records(db): + with open(f"{basedir}/tests/db/bag.sql") as file: + queries = file.read().split(";") + for query in queries: + db.session.execute(text(query)) + with open(f"{basedir}/tests/db/test_bag.sql") as file: + query = text(file.read()) + db.session.execute(query) + db.session.commit() + +app = create_app(TestConfig) -app = create_app() diff --git a/tests/db/bag.sql b/tests/db/bag.sql new file mode 100755 index 0000000..71cc18f --- /dev/null +++ b/tests/db/bag.sql @@ -0,0 +1,32 @@ +CREATE DATABASE IF NOT EXISTS `stembureaus_tests`; +CREATE TABLE IF NOT EXISTS `stembureaus_tests`.`bag` ( + openbareruimte VARCHAR(255), + huisnummer varchar(5), + huisletter varchar(5), + huisnummertoevoeging varchar(5), + postcode varchar(6), + woonplaats varchar(255), + gemeente varchar(255), + provincie varchar(255), + nummeraanduiding varchar(24) primary key, + verblijfsobjectgebruiksdoel varchar(255), + oppervlakteverblijfsobject varchar(10), + verblijfsobjectstatus varchar(255), + object_id varchar(24), + object_type varchar(10), + nevenadres varchar(1), + pandid varchar(24), + pandstatus varchar(255), + pandbouwjaar varchar(20), + x DECIMAL(25,9), + y DECIMAL(25,9), + lon decimal(24, 16), + lat decimal(24, 16), + verkorteopenbareruimte varchar(255), + index fullidx (gemeente, openbareruimte, huisnummer, huisnummertoevoeging, woonplaats), + index postcode (postcode), + fulltext(openbareruimte), + index bag_nummeraanduiding (nummeraanduiding), + index bag_object_id (object_id), + index bag_pandid (pandid) +) CHARACTER SET=utf8 \ No newline at end of file diff --git a/tests/db/bagadres-tests.csv b/tests/db/bagadres-tests.csv new file mode 100644 index 0000000..05591ad --- /dev/null +++ b/tests/db/bagadres-tests.csv @@ -0,0 +1,2 @@ +openbareruimte;huisnummer;huisletter;huisnummertoevoeging;postcode;woonplaats;gemeente;provincie;nummeraanduiding;verblijfsobjectgebruiksdoel;oppervlakteverblijfsobject;verblijfsobjectstatus;object_id;object_type;nevenadres;pandid;pandstatus;pandbouwjaar;x;y;lon;lat;verkorteopenbareruimte +Spui;70;;;2511BT;'s-Gravenhage;'s-Gravenhage;Zuid-Holland;0518200000747446;kantoorfunctie;63425;Verblijfsobject in gebruik;0518010000747448;VBO;f;0518100000275247;Pand in gebruik;1995;81611.373;454909.349;4.31663961;52.07759119; diff --git a/tests/db/test_bag.sql b/tests/db/test_bag.sql new file mode 100644 index 0000000..d6ef3f8 --- /dev/null +++ b/tests/db/test_bag.sql @@ -0,0 +1,7 @@ +LOAD DATA LOCAL INFILE "tests/db/bagadres-tests.csv" +INTO TABLE `stembureaus_tests`.`bag` +COLUMNS TERMINATED BY ';' +OPTIONALLY ENCLOSED BY '"' +ESCAPED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; diff --git a/tests/test_forms.py b/tests/test_forms.py index 6cb7bc4..609ddd1 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -9,10 +9,8 @@ from tests.record_to_test import record_to_test - class TestEditForm(unittest.TestCase): def test_good(self): - app.config['WTF_CSRF_ENABLED'] = False with app.test_request_context('/'): r = Record(**record_to_test(app.config["ELECTION_DATE"])) form = EditForm(MultiDict(r.record)) From 09949a4f9816612a4a4b30d9f4724c6bdac9ad78 Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Thu, 29 Jan 2026 12:39:24 +0100 Subject: [PATCH 04/10] Refactors existing tests to enable DB manipulations --- app/__init__.py | 3 - app/forms.py | 2 +- app/models.py | 8 +- config.py.example | 2 +- test_config.py | 1 + tests/__init__.py | 4 - tests/base_test_class.py | 25 +++++ tests/test_forms.py | 12 +-- tests/test_parser.py | 214 +++++++++++++++++++-------------------- tests/test_validator.py | 39 ++++--- 10 files changed, 164 insertions(+), 146 deletions(-) create mode 100644 tests/base_test_class.py diff --git a/app/__init__.py b/app/__init__.py index b421853..b9f1863 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -108,9 +108,6 @@ def make_shell_context(): # Create the MySQL tables if they don't exist from app.models import create_all create_all() - if app.config["TESTING"]: - from tests import insert_db_test_records - insert_db_test_records(db) from .routes import create_routes create_routes(app) diff --git a/app/forms.py b/app/forms.py index b09c1a1..5472636 100644 --- a/app/forms.py +++ b/app/forms.py @@ -1,7 +1,7 @@ from datetime import datetime -from app.db_utils import db_exec_all, db_exec_one_optional from app.models import BAG +from app.db_utils import db_exec_all, db_exec_one_optional from flask import current_app from flask_wtf import FlaskForm from flask_wtf.file import FileField, FileRequired, FileAllowed diff --git a/app/models.py b/app/models.py index 4336445..45af485 100644 --- a/app/models.py +++ b/app/models.py @@ -9,7 +9,7 @@ from flask_login import UserMixin, LoginManager from werkzeug.security import generate_password_hash, check_password_hash from sqlalchemy import ForeignKey, String, DECIMAL, select -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, close_all_sessions from flask_sqlalchemy_lite import SQLAlchemy from typing import List import jwt @@ -28,6 +28,12 @@ class Base(DeclarativeBase): def create_all(): Base.metadata.create_all(db.engine) +def drop_all(app): + if not app.config['TESTING']: + return + + close_all_sessions() + Base.metadata.drop_all(db.engine) # Association table for the many-to-many relationship # between Gemeente and User diff --git a/config.py.example b/config.py.example index b6c7cc0..4118da0 100644 --- a/config.py.example +++ b/config.py.example @@ -27,7 +27,7 @@ class Config(object): SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://root:@mysql:3306/stembureaus' SQLALCHEMY_TRACK_MODIFICATIONS = False - SQLALCHEMY_ECHO = False # Set to True to see SQL statements created + SQLALCHEMY_ECHO = False # True to see SQL statements created; False to hide them # If 'False', allows gemeenten to add stembureaus. Set to 'True' once the # election day has passed as gemeenten are not allowed to add/edit diff --git a/test_config.py b/test_config.py index 80c1b19..3e5fca5 100644 --- a/test_config.py +++ b/test_config.py @@ -5,3 +5,4 @@ class TestConfig(Config): TESTING = True SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://root:wims_tests@mysql-tests:3306/stembureaus_tests?local_infile=True' WTF_CSRF_ENABLED = False + SQLALCHEMY_ECHO = False \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index 4d25291..ef1ed5a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +1,4 @@ -from app import create_app from config import basedir -from test_config import TestConfig from sqlalchemy import text def insert_db_test_records(db): @@ -13,5 +11,3 @@ def insert_db_test_records(db): db.session.execute(query) db.session.commit() -app = create_app(TestConfig) - diff --git a/tests/base_test_class.py b/tests/base_test_class.py new file mode 100644 index 0000000..2e48420 --- /dev/null +++ b/tests/base_test_class.py @@ -0,0 +1,25 @@ +import unittest + +from test_config import TestConfig + +class BaseTestClass(unittest.TestCase): + AFFECTS_DB = False + + def setUp(self): + from app.models import db, create_all, drop_all + from app import create_app + self.app = create_app(TestConfig) + self.appctx = self.app.app_context() + self.appctx.push() + + if self.AFFECTS_DB: + drop_all(self.app) + create_all() + from tests import insert_db_test_records + insert_db_test_records(db) + + def tearDown(self): + self.appctx.pop() + self.app = None + self.appctx = None + diff --git a/tests/test_forms.py b/tests/test_forms.py index 609ddd1..bf6ba29 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -1,18 +1,16 @@ #!/usr/bin/env python -import unittest - -from tests import app +from tests.base_test_class import BaseTestClass from werkzeug.datastructures import MultiDict -from app.forms import EditForm from app.models import Record from tests.record_to_test import record_to_test -class TestEditForm(unittest.TestCase): +class TestEditForm(BaseTestClass): def test_good(self): - with app.test_request_context('/'): - r = Record(**record_to_test(app.config["ELECTION_DATE"])) + from app.forms import EditForm + with self.app.test_request_context('/'): + r = Record(**record_to_test(self.app.config["ELECTION_DATE"])) form = EditForm(MultiDict(r.record)) result = form.validate() for field, errors in form.errors.items(): diff --git a/tests/test_parser.py b/tests/test_parser.py index 6371e51..8c59c44 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,97 +1,10 @@ #!/usr/bin/env python import os -import unittest import pyexcel -from tests import app - -from app.parser import BaseParser, UploadFileParser - -test_record1 = { - 'nummer_stembureau': 517, - 'naam_stembureau': 'Stadhuis', - 'type_stembureau': 'regulier', - 'website_locatie': ( - 'https://www.denhaag.nl/nl/contact-met-de-gemeente/stadhuis-den-haag/' - ), - 'bag_nummeraanduiding_id': '0518200000747446', - 'extra_adresaanduiding': 'Ingang aan achterkant gebouw', - 'x': '81611', - 'y': '454909', - 'latitude': '52.0775912', - 'longitude': '4.3166395', - 'openingstijd': '2026-03-18T07:30:00', - 'sluitingstijd': '2026-03-18T21:00:00', - 'toegankelijk_voor_mensen_met_een_lichamelijke_beperking': 'ja', - 'toegankelijke_ov_halte': 'ja', - 'toilet': 'ja, toegankelijk toilet', - 'host': 'ja', - 'geleidelijnen': 'buiten en binnen', - 'stemmal_met_audio_ondersteuning': 'ja', - 'kandidatenlijst_in_braille': 'ja', - 'kandidatenlijst_met_grote_letters': 'ja', - 'gebarentolk_ngt': 'op locatie', - 'gebarentalig_stembureaulid_ngt': 'ja', - 'akoestiek_geschikt_voor_slechthorenden': 'ja', - 'prikkelarm': 'ja', - 'prokkelduo': 'ja', - 'extra_toegankelijkheidsinformatie': ( - 'Dit stembureau is ingericht voor kwetsbare mensen, stembureau is ' - 'volledig toegankelijk voor mensen met een lichamelijke beperking er ' - 'is echter geen gehandicaptenparkeerplaats, gebarentolk op locatie ' - '(NGT) is aanwezig van 10:00-12:00 en 16:00-18:00, oefenstembureau' - ), - 'overige_informatie': '', - 'tellocatie': 'ja', - 'contactgegevens_gemeente': ( - 'Unit Verkiezingen, verkiezingen@denhaag.nl 070-3534488 Gemeente Den ' - 'Haag Publiekszaken/Unit Verkiezingen Postbus 84008 2508 AA Den Haag' - ), - 'verkiezingswebsite_gemeente': 'https://www.denhaag.nl/nl/verkiezingen/' -} - -# If there are 'waterschapsverkiezingen', add the 'Verkiezingen' field -# to test_record1 -if [x for x in app.config['CKAN_CURRENT_ELECTIONS'] if 'waterschapsverkiezingen' in x]: - test_record1['verkiezingen'] = 'waterschapsverkiezingen voor Delfland' - -test_record2 = { - 'nummer_stembureau': 516, - 'naam_stembureau': 'Stadhuis', - 'type_stembureau': 'bijzonder', - 'website_locatie': ( - 'https://www.denhaag.nl/nl/contact-met-de-gemeente/stadhuis-den-haag/' - ), - 'bag_nummeraanduiding_id': '0518200000747446', - 'extra_adresaanduiding': '', - 'x': '81611', - 'y': '454909', - 'latitude': '52.0775912', - 'longitude': '4.3166395', - 'openingstijd': '2026-03-18T02:30:00', - 'sluitingstijd': '2026-03-18T20:00:00', - 'toegankelijk_voor_mensen_met_een_lichamelijke_beperking': 'nee', - 'toegankelijke_ov_halte': 'nee', - 'toilet': 'nee', - 'host': 'nee', - 'geleidelijnen': 'nee', - 'stemmal_met_audio_ondersteuning': 'nee', - 'kandidatenlijst_in_braille': 'nee', - 'kandidatenlijst_met_grote_letters': 'nee', - 'gebarentolk_ngt': 'nee', - 'gebarentalig_stembureaulid_ngt': 'nee', - 'akoestiek_geschikt_voor_slechthorenden': 'nee', - 'prikkelarm': 'nee', - 'prokkelduo': 'nee', - 'extra_toegankelijkheidsinformatie': '', - 'overige_informatie': '', - 'tellocatie': 'nee', - 'contactgegevens_gemeente': ( - 'Unit Verkiezingen, verkiezingen@denhaag.nl 070-3534488 Gemeente Den ' - 'Haag Publiekszaken/Unit Verkiezingen Postbus 84008 2508 AA Den Haag' - ), - 'verkiezingswebsite_gemeente': 'https://www.denhaag.nl/nl/verkiezingen/' -} + +from tests.base_test_class import BaseTestClass + accepted_headers = [ 'nummer_stembureau', @@ -126,23 +39,8 @@ 'verkiezingswebsite_gemeente' ] -# If there are 'waterschapsverkiezingen', add the 'Verkiezingen' field -# to test_record2 -if [x for x in app.config['CKAN_CURRENT_ELECTIONS'] if 'waterschapsverkiezingen' in x]: - test_record2['verkiezingen'] = '' - - -class TestBaseParser(unittest.TestCase): - def setUp(self): - self.parser = BaseParser() - - def test_parse(self): - with self.assertRaises(NotImplementedError): - self.parser.parse('/dev/null') - - # From https://gist.github.com/twolfson/13f5f5784f67fd49b245 -class BaseTestParsing(unittest.TestCase): +class BaseTestParsing(BaseTestClass): file_name = '' @classmethod @@ -156,19 +54,21 @@ def setUpOverride(self, *args, **kwargs): cls.setUp = setUpOverride def setUp(self): + super().setUp() + from app.parser import UploadFileParser self.parser = UploadFileParser() self.file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), self.file_name) self.parser._set_parser(self.file_path) - self.records = [test_record1, test_record2] + self.records = [self.get_test_record1(), self.get_test_record2()] self.accepted_headers = accepted_headers def get_headers_good_impl(self): - with app.test_request_context('/'): + with self.app.test_request_context('/'): sh = pyexcel.get_array(file_name = self.file_path, sheet_name='Attributen') headers = self.parser.parser._get_headers(sh) # If there are 'waterschapsverkiezingen', add the 'Verkiezingen' field # to the accepted_headers - if [x for x in app.config['CKAN_CURRENT_ELECTIONS'] if 'waterschapsverkiezingen' in x]: + if [x for x in self.app.config['CKAN_CURRENT_ELECTIONS'] if 'waterschapsverkiezingen' in x]: self.accepted_headers += ['verkiezingen'] self.assertListEqual(headers, self.accepted_headers) @@ -178,6 +78,102 @@ def get_rows_good_impl(self): self.assertDictEqual(rows[0], self.records[0]) self.assertDictEqual(rows[1], self.records[1]) + def get_test_record1(self): + test_record1 = { + 'nummer_stembureau': 517, + 'naam_stembureau': 'Stadhuis', + 'type_stembureau': 'regulier', + 'website_locatie': ( + 'https://www.denhaag.nl/nl/contact-met-de-gemeente/stadhuis-den-haag/' + ), + 'bag_nummeraanduiding_id': '0518200000747446', + 'extra_adresaanduiding': 'Ingang aan achterkant gebouw', + 'x': '81611', + 'y': '454909', + 'latitude': '52.0775912', + 'longitude': '4.3166395', + 'openingstijd': '2026-03-18T07:30:00', + 'sluitingstijd': '2026-03-18T21:00:00', + 'toegankelijk_voor_mensen_met_een_lichamelijke_beperking': 'ja', + 'toegankelijke_ov_halte': 'ja', + 'toilet': 'ja, toegankelijk toilet', + 'host': 'ja', + 'geleidelijnen': 'buiten en binnen', + 'stemmal_met_audio_ondersteuning': 'ja', + 'kandidatenlijst_in_braille': 'ja', + 'kandidatenlijst_met_grote_letters': 'ja', + 'gebarentolk_ngt': 'op locatie', + 'gebarentalig_stembureaulid_ngt': 'ja', + 'akoestiek_geschikt_voor_slechthorenden': 'ja', + 'prikkelarm': 'ja', + 'prokkelduo': 'ja', + 'extra_toegankelijkheidsinformatie': ( + 'Dit stembureau is ingericht voor kwetsbare mensen, stembureau is ' + 'volledig toegankelijk voor mensen met een lichamelijke beperking er ' + 'is echter geen gehandicaptenparkeerplaats, gebarentolk op locatie ' + '(NGT) is aanwezig van 10:00-12:00 en 16:00-18:00, oefenstembureau' + ), + 'overige_informatie': '', + 'tellocatie': 'ja', + 'contactgegevens_gemeente': ( + 'Unit Verkiezingen, verkiezingen@denhaag.nl 070-3534488 Gemeente Den ' + 'Haag Publiekszaken/Unit Verkiezingen Postbus 84008 2508 AA Den Haag' + ), + 'verkiezingswebsite_gemeente': 'https://www.denhaag.nl/nl/verkiezingen/' + } + + # If there are 'waterschapsverkiezingen', add the 'Verkiezingen' field + # to test_record1 + if [x for x in self.app.config['CKAN_CURRENT_ELECTIONS'] if 'waterschapsverkiezingen' in x]: + test_record1['verkiezingen'] = 'waterschapsverkiezingen voor Delfland' + + return test_record1 + + def get_test_record2(self): + test_record2 = { + 'nummer_stembureau': 516, + 'naam_stembureau': 'Stadhuis', + 'type_stembureau': 'bijzonder', + 'website_locatie': ( + 'https://www.denhaag.nl/nl/contact-met-de-gemeente/stadhuis-den-haag/' + ), + 'bag_nummeraanduiding_id': '0518200000747446', + 'extra_adresaanduiding': '', + 'x': '81611', + 'y': '454909', + 'latitude': '52.0775912', + 'longitude': '4.3166395', + 'openingstijd': '2026-03-18T02:30:00', + 'sluitingstijd': '2026-03-18T20:00:00', + 'toegankelijk_voor_mensen_met_een_lichamelijke_beperking': 'nee', + 'toegankelijke_ov_halte': 'nee', + 'toilet': 'nee', + 'host': 'nee', + 'geleidelijnen': 'nee', + 'stemmal_met_audio_ondersteuning': 'nee', + 'kandidatenlijst_in_braille': 'nee', + 'kandidatenlijst_met_grote_letters': 'nee', + 'gebarentolk_ngt': 'nee', + 'gebarentalig_stembureaulid_ngt': 'nee', + 'akoestiek_geschikt_voor_slechthorenden': 'nee', + 'prikkelarm': 'nee', + 'prokkelduo': 'nee', + 'extra_toegankelijkheidsinformatie': '', + 'overige_informatie': '', + 'tellocatie': 'nee', + 'contactgegevens_gemeente': ( + 'Unit Verkiezingen, verkiezingen@denhaag.nl 070-3534488 Gemeente Den ' + 'Haag Publiekszaken/Unit Verkiezingen Postbus 84008 2508 AA Den Haag' + ), + 'verkiezingswebsite_gemeente': 'https://www.denhaag.nl/nl/verkiezingen/' + } + + # If there are 'waterschapsverkiezingen', add the 'Verkiezingen' field + # to test_record2 + if [x for x in self.app.config['CKAN_CURRENT_ELECTIONS'] if 'waterschapsverkiezingen' in x]: + test_record2['verkiezingen'] = '' + + return test_record2 class TestXlsxParsing(BaseTestParsing): file_name = 'data/waarismijnstemlokaal.nl_invulformulier.xlsx' diff --git a/tests/test_validator.py b/tests/test_validator.py index bbe5c24..c8d6fa5 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,36 +1,34 @@ #!/usr/bin/env python -import unittest - -from tests import app - -from app.validator import Validator, RecordValidator from app.models import Record +from tests.base_test_class import BaseTestClass from tests.record_to_test import record_to_test -class TestRecordValidator(unittest.TestCase): +class TestRecordValidator(BaseTestClass): def setUp(self): + super().setUp() + from app.validator import RecordValidator self.record_validator = RecordValidator() - with app.app_context(): - self.test_record = Record(**record_to_test(app.config["ELECTION_DATE"])) + self.test_record = Record(**record_to_test(self.app.config["ELECTION_DATE"])) def test_parse(self): - with app.test_request_context('/'): + with self.app.test_request_context('/'): result, errors, form = self.record_validator.validate( record=self.test_record.record ) self.assertEqual(result, True) -class TestValidator(unittest.TestCase): +class TestValidator(BaseTestClass): def setUp(self): + super().setUp() + from app.validator import Validator self.validator = Validator() - with app.app_context(): - test_rec = record_to_test(app.config["ELECTION_DATE"]) - self.test_records = [ - Record(**x).record for x in [test_rec, test_rec] - ] + test_rec = record_to_test(self.app.config["ELECTION_DATE"]) + self.test_records = [ + Record(**x).record for x in [test_rec, test_rec] + ] def test_parse_empty(self): results = self.validator.validate() @@ -38,7 +36,7 @@ def test_parse_empty(self): self.assertEqual(results['results'], {}) def test_parse_one(self): - with app.test_request_context('/'): + with self.app.test_request_context('/'): results = self.validator.validate( records=self.test_records) self.assertEqual(results['no_errors'], True) @@ -46,14 +44,15 @@ def test_parse_one(self): #self.assertEqual(results['results'], {}) -class TestClosingTimeValidation(unittest.TestCase): +class TestClosingTimeValidation(BaseTestClass): def setUp(self): + super().setUp() + from app.validator import RecordValidator self.record_validator = RecordValidator() - with app.app_context(): - self.test_record = Record(**record_to_test(app.config["ELECTION_DATE"], closing_time='21:01:00')) + self.test_record = Record(**record_to_test(self.app.config["ELECTION_DATE"], closing_time='21:01:00')) def test_parse(self): - with app.test_request_context('/'): + with self.app.test_request_context('/'): result, errors, form = self.record_validator.validate( record=self.test_record.record ) From b084b35202805474035b1fcfd1c8092fb8047625 Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Thu, 29 Jan 2026 15:34:25 +0100 Subject: [PATCH 05/10] Removes dropping tables during tests and switches to transactions instead --- app/models.py | 6 ------ tests/__init__.py | 16 ++++++++++------ tests/base_test_class.py | 18 ++++++++++++++---- tests/test_forms.py | 2 ++ tests/test_validator.py | 6 ++++++ 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/app/models.py b/app/models.py index 45af485..c35a0c8 100644 --- a/app/models.py +++ b/app/models.py @@ -28,12 +28,6 @@ class Base(DeclarativeBase): def create_all(): Base.metadata.create_all(db.engine) -def drop_all(app): - if not app.config['TESTING']: - return - - close_all_sessions() - Base.metadata.drop_all(db.engine) # Association table for the many-to-many relationship # between Gemeente and User diff --git a/tests/__init__.py b/tests/__init__.py index ef1ed5a..2db720e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,12 +2,16 @@ from sqlalchemy import text def insert_db_test_records(db): - with open(f"{basedir}/tests/db/bag.sql") as file: - queries = file.read().split(";") - for query in queries: - db.session.execute(text(query)) + # Note that we are currently not reading/executing tests/db/bag.sql here. + # It somehow leads to a `SAVEPOINT sa_savepoint_1 does not exist` error, + # which is somehow caused by the use of `test_isolation` in `BaseTestClass`. + # The tests succeed because the `bag` table is also created via `models.py`. + # with open(f"{basedir}/tests/db/bag.sql") as file: + # queries = file.read().split(";") + # for query in queries: + # db.session.execute(text(query)) + # db.session.commit() with open(f"{basedir}/tests/db/test_bag.sql") as file: query = text(file.read()) db.session.execute(query) - db.session.commit() - + db.session.commit() \ No newline at end of file diff --git a/tests/base_test_class.py b/tests/base_test_class.py index 2e48420..35a0823 100644 --- a/tests/base_test_class.py +++ b/tests/base_test_class.py @@ -1,3 +1,4 @@ +import contextlib import unittest from test_config import TestConfig @@ -5,16 +6,26 @@ class BaseTestClass(unittest.TestCase): AFFECTS_DB = False + @contextlib.contextmanager + def start_transaction(self): + from app.models import db + try: + with db.test_isolation(): + yield + finally: + pass + def setUp(self): - from app.models import db, create_all, drop_all from app import create_app self.app = create_app(TestConfig) self.appctx = self.app.app_context() self.appctx.push() if self.AFFECTS_DB: - drop_all(self.app) - create_all() + self.transaction = self.start_transaction() + self.enterContext(self.transaction) + + from app.models import db from tests import insert_db_test_records insert_db_test_records(db) @@ -22,4 +33,3 @@ def tearDown(self): self.appctx.pop() self.app = None self.appctx = None - diff --git a/tests/test_forms.py b/tests/test_forms.py index bf6ba29..fef7964 100644 --- a/tests/test_forms.py +++ b/tests/test_forms.py @@ -7,6 +7,8 @@ from tests.record_to_test import record_to_test class TestEditForm(BaseTestClass): + AFFECTS_DB = True + def test_good(self): from app.forms import EditForm with self.app.test_request_context('/'): diff --git a/tests/test_validator.py b/tests/test_validator.py index c8d6fa5..9c0ed4c 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -6,6 +6,8 @@ class TestRecordValidator(BaseTestClass): + AFFECTS_DB = True + def setUp(self): super().setUp() from app.validator import RecordValidator @@ -21,6 +23,8 @@ def test_parse(self): class TestValidator(BaseTestClass): + AFFECTS_DB = True + def setUp(self): super().setUp() from app.validator import Validator @@ -45,6 +49,8 @@ def test_parse_one(self): class TestClosingTimeValidation(BaseTestClass): + AFFECTS_DB = True + def setUp(self): super().setUp() from app.validator import RecordValidator From a98ea7774c8928087b9e6c5ad8d3dad243bf7d1e Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Thu, 29 Jan 2026 16:40:04 +0100 Subject: [PATCH 06/10] Adds tests for removing user from gemeente(n) --- app/db_utils.py | 3 ++ app/utils.py | 14 ++++++-- tests/test_utils.py | 78 +++++++++++++++++++++++++++++++++++++++++++++ tests/utils.py | 48 ++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 tests/test_utils.py create mode 100644 tests/utils.py diff --git a/app/db_utils.py b/app/db_utils.py index 65a9526..e1b2088 100644 --- a/app/db_utils.py +++ b/app/db_utils.py @@ -53,3 +53,6 @@ def db_delete(klass, **kwargs): def db_delete_all(klass): return db.session.execute(delete(klass)).rowcount + +def db_commit(): + db.session.commit() diff --git a/app/utils.py b/app/utils.py index 898a120..11cd464 100644 --- a/app/utils.py +++ b/app/utils.py @@ -2,7 +2,7 @@ import os from io import BytesIO -from app.db_utils import db_exec_all, db_exec_one +from app.db_utils import db_commit, db_delete, db_exec_all, db_exec_one import fiona import shapely import shapely.geometry @@ -11,7 +11,7 @@ from sqlalchemy import select from base64 import b64encode -from app.models import Gemeente +from app.models import Gemeente, Gemeente_user, User from app.ckan import ckan @@ -72,6 +72,16 @@ def publish_gemeente_records(gemeente_code): ckan.publish(election, current_gemeente.gemeente_code, temp_gemeente_draft_records) +def remove_user_from_gemeente(user, gemeente): + db_delete(Gemeente_user, user_id=user.id, gemeente_id=gemeente.id) + db_commit() + + +def remove_user(user): + db_delete(Gemeente_user, user_id=user.id) + db_delete(User, id=user.id) + db_commit() + def get_shapes(shape_file): shapes = [] with fiona.open(shape_file) as shape_records: diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..a6e3592 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,78 @@ +from tests.base_test_class import BaseTestClass + +class TestRemovingUserConnectedToOneGemeente(BaseTestClass): + AFFECTS_DB = True + gemeente_code='GM0518' + + def setUp(self): + super().setUp() + from tests.utils import add_gemeente + self.gemeente = add_gemeente(self, gemeente_code=self.gemeente_code) + + from tests.utils import add_user + self.user1 = add_user(self, self.gemeente, "testuser1@openstate.eu") + self.user2 = add_user(self, self.gemeente, "testuser2@openstate.eu") + + + def test_remove(self): + from tests.utils import get_gemeente_user, get_user + from app.utils import remove_user, get_gemeente + + remove_user(self.user1) + + self.assertIsNotNone(get_gemeente(self.gemeente_code)) + self.assertIsNotNone(get_user(self.user2.email)) + self.assertIsNotNone(get_gemeente_user(self.user2, self.gemeente)) + self.assertIsNone(get_user(self.user1.email)) + self.assertIsNone(get_gemeente_user(self.user1, self.gemeente)) + + +class TestRemovingUserConnectedToMultipleGemeenten(BaseTestClass): + AFFECTS_DB = True + gemeente_code1='GM0518' # 's-Gravenhage + gemeente_code2='GM0106' # Assen + + def setUp(self): + super().setUp() + from tests.utils import add_gemeente, add_user_to_gemeente + self.gemeente1 = add_gemeente(self, gemeente_code=self.gemeente_code1) + self.gemeente2 = add_gemeente(self, gemeente_code=self.gemeente_code2, gemeente_naam="Assen") + + from tests.utils import add_user + self.user1 = add_user(self, self.gemeente1, "testuser1@openstate.eu") + add_user_to_gemeente(self, self.user1, self.gemeente2) + self.user2 = add_user(self, self.gemeente1, "testuser2@openstate.eu") + self.user3 = add_user(self, self.gemeente2, "testuser3@openstate.eu") + + + def test_remove_from_one_gemeente(self): + from tests.utils import get_gemeente_user, get_user + from app.utils import remove_user_from_gemeente, get_gemeente + + remove_user_from_gemeente(self.user1, self.gemeente1) + + self.assertIsNotNone(get_gemeente(self.gemeente_code1)) + self.assertIsNotNone(get_gemeente(self.gemeente_code2)) + self.assertIsNotNone(get_user(self.user2.email)) + self.assertIsNotNone(get_gemeente_user(self.user2, self.gemeente1)) + self.assertIsNotNone(get_user(self.user3.email)) + self.assertIsNotNone(get_gemeente_user(self.user3, self.gemeente2)) + self.assertIsNotNone(get_user(self.user1.email)) + self.assertIsNotNone(get_gemeente_user(self.user1, self.gemeente2)) + self.assertIsNone(get_gemeente_user(self.user1, self.gemeente1)) + + def test_remove_from_all_gemeenten(self): + from tests.utils import get_gemeente_user, get_user + from app.utils import remove_user, get_gemeente + + remove_user(self.user1) + + self.assertIsNotNone(get_gemeente(self.gemeente_code1)) + self.assertIsNotNone(get_gemeente(self.gemeente_code2)) + self.assertIsNotNone(get_user(self.user2.email)) + self.assertIsNotNone(get_gemeente_user(self.user2, self.gemeente1)) + self.assertIsNotNone(get_user(self.user3.email)) + self.assertIsNotNone(get_gemeente_user(self.user3, self.gemeente2)) + self.assertIsNone(get_user(self.user1.email)) + self.assertIsNone(get_gemeente_user(self.user1, self.gemeente1)) + self.assertIsNone(get_gemeente_user(self.user1, self.gemeente2)) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..eb3d5e8 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,48 @@ +import os + +from app.db_utils import db_exec_one_optional +from app.models import Gemeente_user, User, db, Gemeente +from app.utils import get_gemeente + +def add_gemeente(testcase, gemeente_code='GM0518', gemeente_naam="'s-Gravenhage"): + gemeente = Gemeente(gemeente_naam=gemeente_naam, gemeente_code=gemeente_code) + db.session.add(gemeente) + db.session.commit() + + testcase.assertIsNotNone(get_gemeente(gemeente_code)) + + return gemeente + + +def add_user(testcase, gemeente, email): + user = User(email=email) + user.set_password(str(os.urandom(24))) + db.session.add(user) + db.session.commit() + + gemeente_user = Gemeente_user(gemeente_id=gemeente.id, user_id=user.id) + db.session.add(gemeente_user) + db.session.commit() + + testcase.assertIsNotNone(get_user(email)) + testcase.assertIsNotNone(get_gemeente_user(user, gemeente)) + + return user + + +def add_user_to_gemeente(testcase, user, gemeente): + gemeente_user = Gemeente_user(gemeente_id=gemeente.id, user_id=user.id) + db.session.add(gemeente_user) + db.session.commit() + + testcase.assertIsNotNone(get_gemeente_user(user, gemeente)) + + +def get_user(email): + user = db_exec_one_optional(User, email=email) + return user + + +def get_gemeente_user(user, gemeente): + gemeente_user = db_exec_one_optional(Gemeente_user, user_id=user.id, gemeente_id=gemeente.id) + return gemeente_user From a41a94e133b287b1c8607aa2779d4aa5afcd5b90 Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Thu, 29 Jan 2026 17:25:00 +0100 Subject: [PATCH 07/10] Finishes removing users --- app/forms.py | 4 ++-- app/routes.py | 48 +++++++++++++++++++++++------------------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/app/forms.py b/app/forms.py index 5472636..404303c 100644 --- a/app/forms.py +++ b/app/forms.py @@ -76,7 +76,7 @@ def process_formdata(self, valuelist): self.data = value -class DeleteItemForm(FlaskForm): +class DeleteUserForm(FlaskForm): hidden = HiddenField( name="user_id", id="user_id" @@ -90,7 +90,7 @@ class DeleteItemForm(FlaskForm): ) submit_one = SubmitField( - 'Verwijderen uit 1 gemeente', + 'Verwijderen uit deze gemeente', render_kw={ 'class': 'btn btn-danger' } diff --git a/app/routes.py b/app/routes.py index ce03a1d..47c285f 100644 --- a/app/routes.py +++ b/app/routes.py @@ -23,15 +23,15 @@ from sqlalchemy.exc import OperationalError from app.forms import ( - DeleteItemForm, ResetPasswordRequestForm, ResetPasswordForm, LoginForm, EditForm, + DeleteUserForm, ResetPasswordRequestForm, ResetPasswordForm, LoginForm, EditForm, FileUploadForm, PubliceerForm, GemeenteSelectionForm, Setup2faForm, SignupForm, TwoFactorForm ) from app.parser import UploadFileParser from app.validator import Validator from app.email import send_password_reset_email from app.models import Gemeente, Gemeente_user, User, Record, BAG, add_user, db -from app.db_utils import db_delete, db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional -from app.utils import get_b64encoded_qr_image, get_gemeente, get_gemeente_by_id, get_gemeente_by_name, get_mysql_match_against_safe_string, remove_id +from app.db_utils import db_exec_all, db_exec_first, db_exec_one, db_exec_one_optional +from app.utils import get_b64encoded_qr_image, get_gemeente, get_gemeente_by_id, get_gemeente_by_name, get_mysql_match_against_safe_string, remove_id, remove_user, remove_user_from_gemeente from app.ckan import ckan from time import sleep import uuid @@ -759,6 +759,26 @@ def gemeente_gebruikers(gemeente_id = None, user_id = None): if not gemeente: return redirect('/') + # Do we have to remove users or connections to gemeenten? + delete_form = DeleteUserForm() + if custom_form_validate_on_submit(delete_form): + user_id = int(request.form.get('user_id')) + user = db_exec_one(select(User).filter_by(id=user_id)) + + gemeenten_query = select(Gemeente) \ + .join(Gemeente_user) \ + .where(Gemeente_user.user_id == user.id) + gemeenten = db.session.execute(gemeenten_query).scalars().all() + + if len(gemeenten) == 1 and request.form.get('submit') or \ + len(gemeenten) > 1 and request.form.get('submit_all'): + remove_user(user) + flash(f'Gebruiker {user.email} is verwijderd') + elif len(gemeenten) > 1 and request.form.get('submit_one'): + remove_user_from_gemeente(user, gemeente) + flash(f'Gebruiker {user.email} is niet meer verbonden aan gemeente {gemeente.gemeente_naam}') + + # Get ids of users subquery = select(User.id) \ .join(Gemeente_user) \ @@ -771,28 +791,6 @@ def gemeente_gebruikers(gemeente_id = None, user_id = None): .where(User.id.in_(subquery)).group_by(User.id) users = db.session.execute(query).all() - delete_form = DeleteItemForm() - if custom_form_validate_on_submit(delete_form): - user_id = int(request.form.get('user_id')) - if user_id: - user = next(iter(list(filter(lambda u: u.id == user_id, users))), None) - print(user) - # user = db_exec_one_optional(User, id=user_id) - if user: - if user.gemeenten: - if request.form.get('submit_all'): - pass - else: - pass - else: - pass - - - pass - # db_delete(Gemeente_user, user_id=user.id) - # db_delete(User, id=user.id) - # db.session.commit() - field_order = [ 'ID', 'E-mailadres', From de9f3fa3acfa9758dddbbb5324d541a5f1d44a54 Mon Sep 17 00:00:00 2001 From: Rob van Dijk <697696+robvandijk@users.noreply.github.com> Date: Mon, 2 Feb 2026 09:49:43 +0100 Subject: [PATCH 08/10] Refactors deleting stembureau to use form POST --- app/forms.py | 15 +++++++++++++++ app/routes.py | 15 ++++++++++----- app/templates/gemeente-stemlokalen-overzicht.html | 7 ++++++- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/app/forms.py b/app/forms.py index 404303c..8043faf 100644 --- a/app/forms.py +++ b/app/forms.py @@ -103,6 +103,21 @@ class DeleteUserForm(FlaskForm): } ) + +class DeleteStembureauForm(FlaskForm): + hidden = HiddenField( + name="stemlokaal_id", + id="stemlokaal_id" + ) + + submit = SubmitField( + 'Verwijderen', + render_kw={ + 'class': 'btn btn-danger' + } + ) + + class ResetPasswordRequestForm(FlaskForm): email = StringField('E-mailadres', validators=[DataRequired(), Email()]) submit = SubmitField( diff --git a/app/routes.py b/app/routes.py index 47c285f..3bb9980 100644 --- a/app/routes.py +++ b/app/routes.py @@ -23,7 +23,7 @@ from sqlalchemy.exc import OperationalError from app.forms import ( - DeleteUserForm, ResetPasswordRequestForm, ResetPasswordForm, LoginForm, EditForm, + DeleteStembureauForm, DeleteUserForm, ResetPasswordRequestForm, ResetPasswordForm, LoginForm, EditForm, FileUploadForm, PubliceerForm, GemeenteSelectionForm, Setup2faForm, SignupForm, TwoFactorForm ) from app.parser import UploadFileParser @@ -1076,6 +1076,7 @@ def gemeente_stemlokalen_overzicht(): remove_id(gemeente_draft_records) publish_form = PubliceerForm() + delete_form = DeleteStembureauForm() # Publiceren if custom_form_validate_on_submit(publish_form): @@ -1114,6 +1115,7 @@ def gemeente_stemlokalen_overzicht(): draft_records=gemeente_draft_records, field_order=field_order, publish_form=publish_form, + delete_form=delete_form, disable_publish_form=disable_publish_form, upload_deadline_passed=check_deadline_passed(), editing_disabled=editing_disabled @@ -1244,11 +1246,11 @@ def gemeente_stemlokalen_edit(stemlokaal_id=None): @app.route( - "/gemeente-stemlokaal-delete/", - methods=['GET', 'POST'] + "/gemeente-stemlokaal-delete", + methods=['POST'] ) @ensure_2fa_verification - def gemeente_stemlokaal_delete(stemlokaal_id=None): + def gemeente_stemlokaal_delete(): # Select a gemeente if none is currently selected if not 'selected_gemeente_code' in session: return redirect(url_for('gemeente_selectie')) @@ -1256,7 +1258,10 @@ def gemeente_stemlokaal_delete(stemlokaal_id=None): gemeente = get_gemeente(session['selected_gemeente_code']) elections = gemeente.elections - if stemlokaal_id: + delete_form = DeleteStembureauForm() + if custom_form_validate_on_submit(delete_form): + stemlokaal_id = request.form.get('stemlokaal_id') + for election in [x.verkiezing for x in elections]: ckan.delete_records( ckan.elections[election]['draft_resource'], diff --git a/app/templates/gemeente-stemlokalen-overzicht.html b/app/templates/gemeente-stemlokalen-overzicht.html index d941e34..2a49d97 100644 --- a/app/templates/gemeente-stemlokalen-overzicht.html +++ b/app/templates/gemeente-stemlokalen-overzicht.html @@ -94,7 +94,12 @@