From 5d4dd2efe6257c8696cfc019c2f0da17b4c00083 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:21:09 +0000 Subject: [PATCH 01/45] Initial plan From ec03f05f9614eeaddde7db9e738bdab2a6ef92ad Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:32:38 +0000 Subject: [PATCH 02/45] Move data ingestion logic to job queue (ingestion queue) Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/d8f1f856-844d-4274-a34b-36e33e6d8f69 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- documentation/host/queues.rst | 7 +++- flexmeasures/api/common/utils/api_utils.py | 22 +++++++++- flexmeasures/app.py | 1 + flexmeasures/cli/jobs.py | 4 +- flexmeasures/data/services/data_ingestion.py | 44 ++++++++++++++++++++ 5 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 flexmeasures/data/services/data_ingestion.py diff --git a/documentation/host/queues.rst b/documentation/host/queues.rst index ac489a2b69..58219cd80c 100644 --- a/documentation/host/queues.rst +++ b/documentation/host/queues.rst @@ -24,7 +24,7 @@ Here is how to run one worker for each kind of job (in separate terminals): .. code-block:: bash - $ flexmeasures jobs run-worker --name our-only-worker --queue forecasting|scheduling + $ flexmeasures jobs run-worker --name our-only-worker --queue forecasting|scheduling|ingestion Running multiple workers in parallel might be a great idea. @@ -32,6 +32,7 @@ Running multiple workers in parallel might be a great idea. $ flexmeasures jobs run-worker --name forecaster --queue forecasting $ flexmeasures jobs run-worker --name scheduler --queue scheduling + $ flexmeasures jobs run-worker --name ingester --queue ingestion You can also clear the job queues: @@ -39,10 +40,14 @@ You can also clear the job queues: $ flexmeasures jobs clear-queue --queue forecasting $ flexmeasures jobs clear-queue --queue scheduling + $ flexmeasures jobs clear-queue --queue ingestion When the main FlexMeasures process runs (e.g. by ``flexmeasures run``\ ), the queues of forecasting and scheduling jobs can be visited at ``http://localhost:5000/tasks/forecasting`` and ``http://localhost:5000/tasks/schedules``\ , respectively (by admins). +.. note:: + The ``ingestion`` queue is used for sensor data posted via the API. If no worker is connected to this queue, data is processed synchronously (in the web process) with a warning logged. Running a dedicated ingestion worker is recommended in production to keep API responses fast when large amounts of data are posted. + Inspect the queue and jobs diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 5b70551ef3..c984579a4b 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -138,7 +138,27 @@ def save_and_enqueue( forecasting_jobs: list[Job] | None = None, save_changed_beliefs_only: bool = True, ) -> ResponseTuple: - # Attempt to save + from rq import Worker + + from flexmeasures.data.services.data_ingestion import add_beliefs_to_database + + ingestion_queue = current_app.queues.get("ingestion") + if ingestion_queue is not None: + workers = Worker.all(queue=ingestion_queue) + if workers: + ingestion_queue.enqueue( + add_beliefs_to_database, + data, + forecasting_jobs=forecasting_jobs, + save_changed_beliefs_only=save_changed_beliefs_only, + ) + return request_processed() + else: + current_app.logger.warning( + "No workers connected to the ingestion queue. Processing sensor data directly." + ) + + # Attempt to save directly (fallback when no ingestion workers are available) status = save_to_db(data, save_changed_beliefs_only=save_changed_beliefs_only) db.session.commit() diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 538d503eed..c574f92129 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -110,6 +110,7 @@ def create( # noqa C901 app.queues = dict( forecasting=Queue(connection=redis_conn, name="forecasting"), scheduling=Queue(connection=redis_conn, name="scheduling"), + ingestion=Queue(connection=redis_conn, name="ingestion"), # reporting=Queue(connection=redis_conn, name="reporting"), # labelling=Queue(connection=redis_conn, name="labelling"), # alerting=Queue(connection=redis_conn, name="alerting"), diff --git a/flexmeasures/cli/jobs.py b/flexmeasures/cli/jobs.py index bea4ce8bdd..e9132f605e 100644 --- a/flexmeasures/cli/jobs.py +++ b/flexmeasures/cli/jobs.py @@ -292,7 +292,7 @@ def run_job(job_id: str): "--queue", default=None, required=True, - help="State which queue(s) to work on (using '|' as separator), e.g. 'forecasting', 'scheduling' or 'forecasting|scheduling'.", + help="State which queue(s) to work on (using '|' as separator), e.g. 'forecasting', 'scheduling', 'ingestion' or 'forecasting|scheduling'.", ) @click.option( "--name", @@ -302,7 +302,7 @@ def run_job(job_id: str): ) def run_worker(queue: str, name: str | None): """ - Start a worker process for forecasting and/or scheduling jobs. + Start a worker process for forecasting, scheduling and/or ingestion jobs. We use the app context to find out which redis queues to use. """ diff --git a/flexmeasures/data/services/data_ingestion.py b/flexmeasures/data/services/data_ingestion.py new file mode 100644 index 0000000000..3c0a1735dc --- /dev/null +++ b/flexmeasures/data/services/data_ingestion.py @@ -0,0 +1,44 @@ +""" +Logic around data ingestion (jobs) +""" + +from __future__ import annotations + +from flask import current_app +from rq.job import Job +import timely_beliefs as tb + +from flexmeasures.data import db +from flexmeasures.data.utils import save_to_db + + +def add_beliefs_to_database( + data: tb.BeliefsDataFrame | list[tb.BeliefsDataFrame], + forecasting_jobs: list[Job] | None = None, + save_changed_beliefs_only: bool = True, +) -> str: + """Save sensor data to the database and optionally enqueue forecasting jobs. + + This function is intended to be called as an RQ job by an ingestion queue worker, + but can also be called directly (e.g. as a fallback when no workers are available). + + :param data: BeliefsDataFrame (or list thereof) to be saved. + :param forecasting_jobs: Optional list of forecasting Jobs to enqueue after saving. + :param save_changed_beliefs_only: If True, skip saving beliefs whose value hasn't changed. + :returns: Status string, one of: + - 'success' + - 'success_with_unchanged_beliefs_skipped' + - 'success_but_nothing_new' + """ + status = save_to_db(data, save_changed_beliefs_only=save_changed_beliefs_only) + db.session.commit() + + # Only enqueue forecasting jobs upon successfully saving new data + if status[:7] == "success" and status != "success_but_nothing_new": + if forecasting_jobs is not None: + [ + current_app.queues["forecasting"].enqueue_job(job) + for job in forecasting_jobs + ] + + return status From ea634dbebb2a5ac07af0a6ec5435c7b76d1a7d86 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:35:26 +0000 Subject: [PATCH 03/45] Fix code review comments: use for loop and move imports to module level Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/d8f1f856-844d-4274-a34b-36e33e6d8f69 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/api/common/utils/api_utils.py | 6 ++---- flexmeasures/data/services/data_ingestion.py | 4 +--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index c984579a4b..1f1facae5c 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -8,12 +8,14 @@ from werkzeug.exceptions import Forbidden, Unauthorized from numpy import array from psycopg2.errors import UniqueViolation +from rq import Worker from rq.job import Job, JobStatus, NoSuchJobError from sqlalchemy import select from sqlalchemy.exc import IntegrityError from flexmeasures.data import db from flexmeasures.data.models.user import Account +from flexmeasures.data.services.data_ingestion import add_beliefs_to_database from flexmeasures.data.utils import save_to_db from flexmeasures.auth.policy import check_access from flexmeasures.api.common.responses import ( @@ -138,10 +140,6 @@ def save_and_enqueue( forecasting_jobs: list[Job] | None = None, save_changed_beliefs_only: bool = True, ) -> ResponseTuple: - from rq import Worker - - from flexmeasures.data.services.data_ingestion import add_beliefs_to_database - ingestion_queue = current_app.queues.get("ingestion") if ingestion_queue is not None: workers = Worker.all(queue=ingestion_queue) diff --git a/flexmeasures/data/services/data_ingestion.py b/flexmeasures/data/services/data_ingestion.py index 3c0a1735dc..74a9c7cec3 100644 --- a/flexmeasures/data/services/data_ingestion.py +++ b/flexmeasures/data/services/data_ingestion.py @@ -36,9 +36,7 @@ def add_beliefs_to_database( # Only enqueue forecasting jobs upon successfully saving new data if status[:7] == "success" and status != "success_but_nothing_new": if forecasting_jobs is not None: - [ + for job in forecasting_jobs: current_app.queues["forecasting"].enqueue_job(job) - for job in forecasting_jobs - ] return status From 3a98da82dab886704a5b31c3f43c3b00afb9979e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:36:06 +0000 Subject: [PATCH 04/45] Add warning when ingestion queue is not configured Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/a05e0a24-6919-4080-a466-cbaf6a8369de Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- documentation/host/queues.rst | 2 +- flexmeasures/api/common/utils/api_utils.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/documentation/host/queues.rst b/documentation/host/queues.rst index 58219cd80c..f5b8dcf69a 100644 --- a/documentation/host/queues.rst +++ b/documentation/host/queues.rst @@ -46,7 +46,7 @@ You can also clear the job queues: When the main FlexMeasures process runs (e.g. by ``flexmeasures run``\ ), the queues of forecasting and scheduling jobs can be visited at ``http://localhost:5000/tasks/forecasting`` and ``http://localhost:5000/tasks/schedules``\ , respectively (by admins). .. note:: - The ``ingestion`` queue is used for sensor data posted via the API. If no worker is connected to this queue, data is processed synchronously (in the web process) with a warning logged. Running a dedicated ingestion worker is recommended in production to keep API responses fast when large amounts of data are posted. + The ``ingestion`` queue is used for sensor data posted via the API. If the queue is not configured, or if no worker is connected to it, data is processed synchronously (in the web process) with a warning logged. Running a dedicated ingestion worker is recommended in production to keep API responses fast when large amounts of data are posted. diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 1f1facae5c..52c5fdfb49 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -141,7 +141,11 @@ def save_and_enqueue( save_changed_beliefs_only: bool = True, ) -> ResponseTuple: ingestion_queue = current_app.queues.get("ingestion") - if ingestion_queue is not None: + if ingestion_queue is None: + current_app.logger.warning( + "No ingestion queue configured. Processing sensor data directly." + ) + else: workers = Worker.all(queue=ingestion_queue) if workers: ingestion_queue.enqueue( @@ -156,7 +160,7 @@ def save_and_enqueue( "No workers connected to the ingestion queue. Processing sensor data directly." ) - # Attempt to save directly (fallback when no ingestion workers are available) + # Attempt to save directly (fallback when no ingestion queue or workers are available) status = save_to_db(data, save_changed_beliefs_only=save_changed_beliefs_only) db.session.commit() From eb0f8869de12a3bbcf73f669c15913043065d129 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:41:36 +0000 Subject: [PATCH 05/45] docs: add changelog entry and upgrade notice for ingestion queue warning (PR #2101) Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/2383b9c6-1167-413c-9cf0-e31f2a9db9f5 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- documentation/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 553b87a653..85dd0a1f6f 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,6 +9,8 @@ v0.32.0 | April XX, 2026 .. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). +.. note:: It is recommended to add a worker to the ``ingestion`` queue (or configure existing workers to handle it, as well), so that sensor data posted via the API is processed asynchronously. Without this, data is processed synchronously in the web process, with a warning logged [see `PR #2101 `_]. + New features ------------- * Support inferring ``soc-at-start`` from configured ``state-of-charge`` sources and fail early when those values are stale or missing near schedule start [see `PR #2026 `_] @@ -24,6 +26,7 @@ New features Infrastructure / Support ---------------------- +* Log a warning when posting sensor data without an ingestion queue configured, or without workers connected to that queue [see `PR #2101 `_] * Support coupling data sources to accounts, and preserve user ID and account ID references in audit logs and data sources for traceability and compliance [see `PR #2058 `_] * Stop creating new toy assets when restarting the docker-compose stack [see `PR #2018 `_] * Migrate from ``pip`` to ``uv`` for dependency management, and from ``make`` to ``poe`` [see `PR #1973 `_] From 5d5b921cc7010d8ee6e352bef7aaf7c05fcd6238 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Fri, 17 Apr 2026 13:04:37 +0100 Subject: [PATCH 06/45] chore: remvoe wrong log entry Signed-off-by: joshuaunity --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index df7c2c80a0..53988faa15 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -41,10 +41,10 @@ New features * Separate the ``StorageScheduler``'s tie-breaking preference for a full :abbr:`SoC (state of charge)` from its reported energy costs [see `PR #2023 `_ and `PR #2108 `_] * Improve asset graph hover interaction with a vertical ruler across subcharts, while keeping hover dots for easier visual tracking [see `PR #2079 `_] * Improve asset audit log messages for JSON field edits (especially ``sensors_to_show`` and nested flex-config values) [see `PR #2055 `_] +* Infrastructure / Support ---------------------- -* Log a warning when posting sensor data without an ingestion queue configured, or without workers connected to that queue [see `PR #2101 `_] * Support coupling data sources to accounts, and preserve user ID and account ID references in audit logs and data sources for traceability and compliance [see `PR #2058 `_] * Stop creating new toy assets when restarting the docker-compose stack [see `PR #2018 `_] * Migrate from ``pip`` to ``uv`` for dependency management, and from ``make`` to ``poe`` [see `PR #1973 `_] From be2d019e91e8f6290f81afb9924932d0f45b76ce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:23:45 +0000 Subject: [PATCH 07/45] docs: add ingestion queue changelog entry and upgrade notice to v0.33.0 Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/3eae55cb-3902-4be8-84f7-76436f33cdd6 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- documentation/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 53988faa15..f4d4dcff17 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -6,11 +6,14 @@ FlexMeasures Changelog v0.33.0 | May XX, 2026 ============================ +.. note:: It is recommended to add a worker to the ``ingestion`` queue (or configure existing workers to handle it, as well), so that sensor data posted via the API is processed asynchronously [see `PR #2101 `_]. + New features ------------- Infrastructure / Support ---------------------- +* Move sensor data ingestion to a job queue for improved performance when POSTing large amounts of data to the sensor data API [see `PR #2101 `_] Bugfixes ----------- @@ -23,8 +26,6 @@ v0.32.0 | April 15, 2026 .. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). -.. note:: It is recommended to add a worker to the ``ingestion`` queue (or configure existing workers to handle it, as well), so that sensor data posted via the API is processed asynchronously. Without this, data is processed synchronously in the web process, with a warning logged [see `PR #2101 `_]. - New features ------------- * Upgrade graph modal to support flex-config references in plots [see `PR #1926 `_ and `PR #1904 `_] From 0bc6580c209e1480f404ec579659f691a5efaa8a Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 20 Apr 2026 10:12:52 +0100 Subject: [PATCH 08/45] chore: remove duplicate changelog entry Signed-off-by: joshuaunity --- documentation/changelog.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 56a4982ac0..efb207d527 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -53,8 +53,6 @@ Infrastructure / Support * Support coupling data sources to accounts, and preserve user ID and account ID references in audit logs and data sources for traceability and compliance [see `PR #2058 `_] * Stop creating new toy assets when restarting the docker-compose stack [see `PR #2018 `_] * Migrate from ``pip`` to ``uv`` for dependency management, and from ``make`` to ``poe`` [see `PR #1973 `_] -* Stop creating new toy assets when restarting the docker-compose stack [see `PR #2018 `_] -* Migrate from ``pip`` to ``uv`` for dependency management, and from ``make`` to ``poe`` [see `PR #1973 `_] * Improve contact information to get in touch with the FlexMeasures community [see `PR #2022 `_] * Expand audit logging for password life-cycle: password_reset, password_changed and reset_password_instructions_sent events [see `PR #2036 `_] * Upgraded some dependencies for security reasons [see `PR #2037 `_] From 842eae6b1ae29244a79bd4a305fe37bacecc86e5 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 20 Apr 2026 10:28:35 +0100 Subject: [PATCH 09/45] fix: update job queue command to include ingestion Signed-off-by: joshuaunity --- documentation/host/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/host/installation.rst b/documentation/host/installation.rst index 2a32705eab..1c7ab6183b 100644 --- a/documentation/host/installation.rst +++ b/documentation/host/installation.rst @@ -385,7 +385,7 @@ Then, start workers in a console (or some other method to keep a long-running pr .. code-block:: bash - $ flexmeasures jobs run-worker --queue "scheduling|forecasting" + $ flexmeasures jobs run-worker --queue "scheduling|forecasting|ingestion" You can go to `http://localhost:5000/tasks/` and see the state of job queues and find individual jobs (and investigate why they failed, for instance). From c811cf0fc3529d3890bdecba1b8423daa339ef2b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:29:38 +0000 Subject: [PATCH 10/45] docs: add ingestion queue to worker examples in forecasting_scheduling docs Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/c290f889-38c7-4f48-b96b-72f918051281 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- documentation/tut/forecasting_scheduling.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index f69d028177..952faf87f7 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -28,6 +28,7 @@ Start to run one worker for each kind of job (in a separate terminal): $ flexmeasures jobs run-worker --queue forecasting $ flexmeasures jobs run-worker --queue scheduling + $ flexmeasures jobs run-worker --queue ingestion You can also clear the job queues: @@ -36,6 +37,7 @@ You can also clear the job queues: $ flexmeasures jobs clear-queue --queue forecasting $ flexmeasures jobs clear-queue --queue scheduling + $ flexmeasures jobs clear-queue --queue ingestion When the main FlexMeasures process runs (e.g. by ``flexmeasures run``), the queues of forecasting and scheduling jobs can be visited at ``http://localhost:5000/tasks/forecasting`` and ``http://localhost:5000/tasks/schedules``, respectively (by admins). From 7d275a9762ef5158f161a5744522abf60da55a46 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 21 Apr 2026 10:57:08 +0100 Subject: [PATCH 11/45] refactor: update data ingestion logic to enqueue forecasting jobs Signed-off-by: joshuaunity --- flexmeasures/api/common/utils/api_utils.py | 6 ++++-- flexmeasures/data/services/data_ingestion.py | 2 +- flexmeasures/ui/static/openapi-specs.json | 3 +-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 52c5fdfb49..fa3a972257 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -15,7 +15,9 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account -from flexmeasures.data.services.data_ingestion import add_beliefs_to_database +from flexmeasures.data.services.data_ingestion import ( + add_beliefs_to_db_and_enqueue_forecasting_jobs, +) from flexmeasures.data.utils import save_to_db from flexmeasures.auth.policy import check_access from flexmeasures.api.common.responses import ( @@ -149,7 +151,7 @@ def save_and_enqueue( workers = Worker.all(queue=ingestion_queue) if workers: ingestion_queue.enqueue( - add_beliefs_to_database, + add_beliefs_to_db_and_enqueue_forecasting_jobs, data, forecasting_jobs=forecasting_jobs, save_changed_beliefs_only=save_changed_beliefs_only, diff --git a/flexmeasures/data/services/data_ingestion.py b/flexmeasures/data/services/data_ingestion.py index 74a9c7cec3..3bea3cd1bc 100644 --- a/flexmeasures/data/services/data_ingestion.py +++ b/flexmeasures/data/services/data_ingestion.py @@ -12,7 +12,7 @@ from flexmeasures.data.utils import save_to_db -def add_beliefs_to_database( +def add_beliefs_to_db_and_enqueue_forecasting_jobs( data: tb.BeliefsDataFrame | list[tb.BeliefsDataFrame], forecasting_jobs: list[Job] | None = None, save_changed_beliefs_only: bool = True, diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index df43b48806..c4e15112b1 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "0.32.0" + "version": "0.31.0" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -4116,7 +4116,6 @@ }, "/api/v3_0": {}, "/api/v3_0/sensors/data": {}, - "/api/v3_0/docs/dist/{filename}": {}, "/api/v3_0/docs/{path}": {}, "/api/dev/sensor/{id}": {}, "/api/dev/sensor/{id}/chart": {}, From acfb1446faca043593f7aea0e47d1101601a28c1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:54:37 +0000 Subject: [PATCH 12/45] refactor: eliminate duplicated save/enqueue logic in save_and_enqueue fallback Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/a4ce9ca2-10ca-4304-8df3-9cb99864f075 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/api/common/utils/api_utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 79bfd61526..2c6f7bce91 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -25,7 +25,6 @@ ) from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.utils import save_to_db from flexmeasures.auth.policy import check_access from flexmeasures.api.common.responses import ( invalid_replacement, @@ -171,12 +170,11 @@ def save_and_enqueue( ) # Attempt to save directly (fallback when no ingestion queue or workers are available) - status = save_to_db(data, save_changed_beliefs_only=save_changed_beliefs_only) - db.session.commit() - - # Only enqueue forecasting jobs upon successfully saving new data - if status[:7] == "success" and status != "success_but_nothing_new": - enqueue_forecasting_jobs(forecasting_jobs) + status = add_beliefs_to_db_and_enqueue_forecasting_jobs( + data, + forecasting_jobs=forecasting_jobs, + save_changed_beliefs_only=save_changed_beliefs_only, + ) # Pick a response if status == "success": From aff5b0ba22e9988d263da2f85107cf293dc6f58f Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 22 Apr 2026 14:52:29 +0100 Subject: [PATCH 13/45] refactor: enhance comparison logic for unchanged beliefs to support job queue deserialization Signed-off-by: joshuaunity --- flexmeasures/data/services/time_series.py | 44 ++++++++++++++++------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index a2c77c811b..53d654ce6b 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -147,7 +147,11 @@ def _drop_unchanged_beliefs_compared_to_db( """ source = bdf.lineage.sources[0] # unique source belief_time = bdf.lineage.belief_times[0] # unique belief time - bdf_db_from_source = bdf_db[bdf_db.sources == source] + # Compare by ID rather than object identity: the candidate bdf may have been + # deserialized from an RQ job queue (pickled in a different process), so its + # DataSource objects are detached and won't be identical to the freshly-loaded + # ones in bdf_db even when they represent the same DB row. + bdf_db_from_source = bdf_db[bdf_db.sources.map(lambda s: s.id) == source.id] if bdf_db_from_source.empty: return bdf # Use .max() rather than searchsorted: the result is correct regardless of @@ -161,21 +165,35 @@ def _drop_unchanged_beliefs_compared_to_db( previous_most_recent_beliefs = bdf_db_from_source[ bdf_db_from_source.belief_times == most_recent_bt ] - compare_fields = ["event_start", "source", "cumulative_probability", "event_value"] - a = bdf.reset_index().set_index(compare_fields) - b = previous_most_recent_beliefs.reset_index().set_index(compare_fields) - bdf = a.drop( - b.index, - errors="ignore", - axis=0, - ) + # Use source_id (integer) instead of source (object) for cross-session comparison. + # When the candidate bdf was deserialized from an RQ job queue, its DataSource + # objects are detached instances from a different Python process. pandas index + # comparison falls back to identity, so deserialized and freshly-loaded objects + # that represent the same DB row are never considered equal. Comparing by .id + # avoids this. + a_df = bdf.reset_index() + a_df["source_id"] = a_df["source"].map(lambda s: s.id) + b_df = previous_most_recent_beliefs.reset_index() + b_df["source_id"] = b_df["source"].map(lambda s: s.id) + + compare_fields = [ + "event_start", + "source_id", + "cumulative_probability", + "event_value", + ] + a = a_df.set_index(compare_fields) + b = b_df.set_index(compare_fields) + dropped = a.drop(b.index, errors="ignore", axis=0) # Keep whole probabilistic beliefs, not just the parts that changed - c = bdf.reset_index().set_index(["event_start", "source"]) - d = a.reset_index().set_index(["event_start", "source"]) + c = dropped.reset_index().set_index(["event_start", "source_id"]) + d = a_df.set_index(["event_start", "source_id"]) bdf = d[d.index.isin(c.index)] - bdf = bdf.reset_index().set_index( - ["event_start", "belief_time", "source", "cumulative_probability"] + bdf = ( + bdf.reset_index() + .drop(columns=["source_id"], errors="ignore") + .set_index(["event_start", "belief_time", "source", "cumulative_probability"]) ) return bdf From 974cb78c64ea4ee13b56c19c460bbaa5cf7116eb Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Fri, 24 Apr 2026 08:52:29 +0100 Subject: [PATCH 14/45] feat: implement serialization and deserialization for ingestion data, update job enqueue logic Signed-off-by: joshuaunity --- flexmeasures/api/common/utils/api_utils.py | 11 +- flexmeasures/data/services/data_ingestion.py | 114 +++++++++++++++++- .../data/tests/test_data_ingestion.py | 91 ++++++++++++++ 3 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 flexmeasures/data/tests/test_data_ingestion.py diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 2c6f7bce91..87136721c7 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -22,6 +22,7 @@ from flexmeasures.data.models.user import Account from flexmeasures.data.services.data_ingestion import ( add_beliefs_to_db_and_enqueue_forecasting_jobs, + serialize_ingestion_data, ) from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor @@ -157,10 +158,16 @@ def save_and_enqueue( else: workers = Worker.all(queue=ingestion_queue) if workers: + serialized_data = serialize_ingestion_data(data) + forecasting_job_ids = ( + [job.id for job in forecasting_jobs] + if forecasting_jobs is not None + else None + ) ingestion_queue.enqueue( add_beliefs_to_db_and_enqueue_forecasting_jobs, - data, - forecasting_jobs=forecasting_jobs, + serialized_data=serialized_data, + forecasting_job_ids=forecasting_job_ids, save_changed_beliefs_only=save_changed_beliefs_only, ) return request_processed() diff --git a/flexmeasures/data/services/data_ingestion.py b/flexmeasures/data/services/data_ingestion.py index 3bea3cd1bc..9ac52b73ea 100644 --- a/flexmeasures/data/services/data_ingestion.py +++ b/flexmeasures/data/services/data_ingestion.py @@ -4,17 +4,110 @@ from __future__ import annotations +from collections.abc import Sequence + from flask import current_app +import pandas as pd from rq.job import Job +from rq.job import NoSuchJobError +from sqlalchemy import select import timely_beliefs as tb from flexmeasures.data import db +from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.utils import save_to_db -def add_beliefs_to_db_and_enqueue_forecasting_jobs( +def serialize_ingestion_data( data: tb.BeliefsDataFrame | list[tb.BeliefsDataFrame], +) -> list[dict]: + """Serialize beliefs data to primitive types suitable for queue kwargs. + + The returned payload intentionally avoids ORM instances (Sensor/DataSource), + which can break across process boundaries when pickled by RQ. + """ + + bdfs: list[tb.BeliefsDataFrame] + if isinstance(data, list): + bdfs = data + else: + bdfs = [data] + + payload: list[dict] = [] + for bdf in bdfs: + # Normalize timing representation to belief_time for stable serialization. + bdf = bdf.convert_index_from_belief_horizon_to_time() + serialized_rows: list[dict] = [] + for belief in bdf.reset_index().itertuples(index=False): + serialized_rows.append( + { + "event_start": belief.event_start.isoformat(), + "belief_time": belief.belief_time.isoformat(), + "source_id": belief.source.id, + "cumulative_probability": float(belief.cumulative_probability), + "event_value": ( + None + if pd.isna(belief.event_value) + else float(belief.event_value) + ), + } + ) + + payload.append( + { + "sensor_id": bdf.sensor.id, + "beliefs": serialized_rows, + } + ) + return payload + + +def deserialize_ingestion_data(payload: Sequence[dict]) -> list[tb.BeliefsDataFrame]: + """Deserialize queue-safe ingestion payload back into BeliefsDataFrames.""" + + bdfs: list[tb.BeliefsDataFrame] = [] + for item in payload: + sensor = db.session.get(Sensor, item["sensor_id"]) + if sensor is None: + raise ValueError(f"No such sensor: {item['sensor_id']}") + + belief_rows = item.get("beliefs", []) + if not belief_rows: + bdfs.append(tb.BeliefsDataFrame(sensor=sensor)) + continue + + source_ids = sorted({row["source_id"] for row in belief_rows}) + sources = db.session.scalars( + select(DataSource).filter(DataSource.id.in_(source_ids)) + ).all() + source_map = {source.id: source for source in sources} + + beliefs: list[TimedBelief] = [] + for row in belief_rows: + source = source_map.get(row["source_id"]) + if source is None: + raise ValueError(f"No such source: {row['source_id']}") + beliefs.append( + TimedBelief( + sensor=sensor, + source=source, + event_start=pd.Timestamp(row["event_start"]), + belief_time=pd.Timestamp(row["belief_time"]), + cumulative_probability=row["cumulative_probability"], + event_value=row["event_value"], + ) + ) + bdfs.append(tb.BeliefsDataFrame(beliefs)) + + return bdfs + + +def add_beliefs_to_db_and_enqueue_forecasting_jobs( + data: tb.BeliefsDataFrame | list[tb.BeliefsDataFrame] | None = None, + serialized_data: list[dict] | None = None, forecasting_jobs: list[Job] | None = None, + forecasting_job_ids: list[str] | None = None, save_changed_beliefs_only: bool = True, ) -> str: """Save sensor data to the database and optionally enqueue forecasting jobs. @@ -23,13 +116,20 @@ def add_beliefs_to_db_and_enqueue_forecasting_jobs( but can also be called directly (e.g. as a fallback when no workers are available). :param data: BeliefsDataFrame (or list thereof) to be saved. + :param serialized_data: Queue-safe payload containing only primitive types. :param forecasting_jobs: Optional list of forecasting Jobs to enqueue after saving. + :param forecasting_job_ids: Optional list of forecasting Job ids to enqueue after saving. :param save_changed_beliefs_only: If True, skip saving beliefs whose value hasn't changed. :returns: Status string, one of: - 'success' - 'success_with_unchanged_beliefs_skipped' - 'success_but_nothing_new' """ + if serialized_data is not None: + data = deserialize_ingestion_data(serialized_data) + if data is None: + raise ValueError("Expected either data or serialized_data.") + status = save_to_db(data, save_changed_beliefs_only=save_changed_beliefs_only) db.session.commit() @@ -38,5 +138,17 @@ def add_beliefs_to_db_and_enqueue_forecasting_jobs( if forecasting_jobs is not None: for job in forecasting_jobs: current_app.queues["forecasting"].enqueue_job(job) + if forecasting_job_ids is not None: + connection = current_app.queues["forecasting"].connection + for job_id in forecasting_job_ids: + try: + job = Job.fetch(job_id, connection=connection) + except NoSuchJobError: + current_app.logger.warning( + "Forecasting job %s no longer exists; skipping enqueue.", + job_id, + ) + continue + current_app.queues["forecasting"].enqueue_job(job) return status diff --git a/flexmeasures/data/tests/test_data_ingestion.py b/flexmeasures/data/tests/test_data_ingestion.py new file mode 100644 index 0000000000..389e3e4e2b --- /dev/null +++ b/flexmeasures/data/tests/test_data_ingestion.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +import pandas.testing as pdt + +from flexmeasures.data.services.data_ingestion import ( + add_beliefs_to_db_and_enqueue_forecasting_jobs, + deserialize_ingestion_data, + serialize_ingestion_data, +) +from flexmeasures.tests.utils import get_test_sensor + + +def _to_comparable_df(bdf): + df = bdf.convert_index_from_belief_horizon_to_time().reset_index() + df["source_id"] = df["source"].map(lambda s: s.id) + return ( + df[ + [ + "event_start", + "belief_time", + "source_id", + "cumulative_probability", + "event_value", + ] + ] + .sort_values( + [ + "event_start", + "belief_time", + "source_id", + "cumulative_probability", + ] + ) + .reset_index(drop=True) + ) + + +def _is_primitive_payload(value) -> bool: + if value is None or isinstance(value, (str, int, float, bool)): + return True + if isinstance(value, Mapping): + return all( + isinstance(k, str) and _is_primitive_payload(v) for k, v in value.items() + ) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return all(_is_primitive_payload(v) for v in value) + return False + + +def test_serialize_ingestion_data_uses_primitive_types(setup_beliefs, db): + sensor = get_test_sensor(db) + bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ + :2 + ] + + payload = serialize_ingestion_data(bdf) + + assert _is_primitive_payload(payload) + + +def test_ingestion_data_roundtrip_preserves_beliefs(setup_beliefs, db): + sensor = get_test_sensor(db) + bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ + :2 + ] + + payload = serialize_ingestion_data(bdf) + restored = deserialize_ingestion_data(payload) + + assert len(restored) == 1 + pdt.assert_frame_equal( + _to_comparable_df(restored[0]), + _to_comparable_df(bdf), + check_dtype=False, + ) + + +def test_ingestion_service_accepts_serialized_data(setup_beliefs, db): + sensor = get_test_sensor(db) + bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ + :1 + ] + + status = add_beliefs_to_db_and_enqueue_forecasting_jobs( + serialized_data=serialize_ingestion_data(bdf), + save_changed_beliefs_only=True, + ) + + assert status == "success_but_nothing_new" From 6380541ad2c41c1686b79512dce2af9ee66acd8d Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 29 Apr 2026 09:19:59 +0100 Subject: [PATCH 15/45] refactor: improve cross-session comparison logic by using source_id for detached ORM instances Signed-off-by: joshuaunity --- flexmeasures/data/services/time_series.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index 53d654ce6b..a16ee6c3b1 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -165,12 +165,10 @@ def _drop_unchanged_beliefs_compared_to_db( previous_most_recent_beliefs = bdf_db_from_source[ bdf_db_from_source.belief_times == most_recent_bt ] - # Use source_id (integer) instead of source (object) for cross-session comparison. - # When the candidate bdf was deserialized from an RQ job queue, its DataSource - # objects are detached instances from a different Python process. pandas index - # comparison falls back to identity, so deserialized and freshly-loaded objects - # that represent the same DB row are never considered equal. Comparing by .id - # avoids this. + # Use source_id (integer) instead of source (object) for robust cross-session + # comparison. Detached ORM instances (for example after serialization boundaries + # or different session lifecycles) may represent the same DB row but still fail + # object-identity based comparison in pandas indices. a_df = bdf.reset_index() a_df["source_id"] = a_df["source"].map(lambda s: s.id) b_df = previous_most_recent_beliefs.reset_index() From da441ef4fe82d7a61d07b51f3376467b6c68f7b0 Mon Sep 17 00:00:00 2001 From: JDev <45713692+joshuaunity@users.noreply.github.com> Date: Mon, 4 May 2026 08:40:58 +0100 Subject: [PATCH 16/45] Update API version to 0.32.0 Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/ui/static/openapi-specs.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index da1361860e..8948f7e126 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "0.31.0" + "version": "0.32.0" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -5964,4 +5964,4 @@ } } } -} \ No newline at end of file +} From 513a81ecc32e9da985fa6c9cd676ec1b4e83296b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 May 2026 11:12:27 +0000 Subject: [PATCH 17/45] fix: pass keep_only_most_recent_belief=True to resample_events in file upload to prevent O(N^2) hang when belief_time_measured_instantly=True Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/d2ae7c51-7c2a-4d24-b2aa-1ddcd84ee3f8 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- .../api/v3_0/tests/test_sensors_api.py | 39 +++++++++++++++++++ flexmeasures/data/schemas/sensors.py | 11 +++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index f22cc7f5c2..744feccc9c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -341,6 +341,45 @@ def test_upload_csv_file(client, db, setup_api_test_data, sensor_name, requestin ) +@pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) +def test_upload_csv_file_measured_instantly_with_resampling( + client, db, setup_api_test_data, requesting_user +): + """Test that uploading data with belief-time-measured-instantly=on and resampling needed + completes correctly and does not hang (regression test for O(N^2) slow track in resample_events). + + The "some gas sensor" has 10-minute resolution. We upload 5-minute data, triggering + downsampling. With belief_time_measured_instantly=True, each event gets a unique belief_time + which previously caused the slow track in resample_events with O(N^2) memory usage. + """ + auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") + # 5-minute resolution data -> needs downsampling to sensor's 10-minute resolution + csv_content = """event_start,event_value +2022-12-16T05:00:00Z,10 +2022-12-16T05:05:00Z,20 +2022-12-16T05:10:00Z,30 +2022-12-16T05:15:00Z,40 +2022-12-16T05:20:00Z,50 +2022-12-16T05:25:00Z,60 +""" + sensor = setup_api_test_data["some gas sensor"] + file = (io.BytesIO(csv_content.encode("utf-8")), "test.csv") + + data = { + "uploaded-files": file, + "belief-time-measured-instantly": "on", + } + + response = client.post( + url_for("SensorAPI:upload_data", id=sensor.id), + data=data, + content_type="multipart/form-data", + headers={"Authorization": auth_token}, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + + @pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) def test_upload_excel_file(client, requesting_user): import openpyxl diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index af73d93ae1..1510021f5b 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -728,9 +728,16 @@ def post_load(self, fields, **kwargs): is_stock_unit(from_unit) for is_stock_unit in known_stock_unit_validators ): - bdf = bdf.resample_events(sensor.event_resolution, method="sum") + bdf = bdf.resample_events( + sensor.event_resolution, + method="sum", + keep_only_most_recent_belief=True, + ) else: - bdf = bdf.resample_events(sensor.event_resolution) + bdf = bdf.resample_events( + sensor.event_resolution, + keep_only_most_recent_belief=True, + ) dfs.append(bdf) except Exception as e: error_message = ( From 0dd844f183fb564cf7bb310cdb25ffeb39061df9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 May 2026 11:14:15 +0000 Subject: [PATCH 18/45] fix: improve test docstring and remove debug print from new upload test Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/d2ae7c51-7c2a-4d24-b2aa-1ddcd84ee3f8 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensors_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 744feccc9c..9a1300129b 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -345,12 +345,13 @@ def test_upload_csv_file(client, db, setup_api_test_data, sensor_name, requestin def test_upload_csv_file_measured_instantly_with_resampling( client, db, setup_api_test_data, requesting_user ): - """Test that uploading data with belief-time-measured-instantly=on and resampling needed - completes correctly and does not hang (regression test for O(N^2) slow track in resample_events). + """Regression test: uploading data with belief-time-measured-instantly=on and resampling + needed should complete with a 200 status and not trigger the O(N²) slow path in + resample_events that previously caused server hangs and OOM crashes. The "some gas sensor" has 10-minute resolution. We upload 5-minute data, triggering downsampling. With belief_time_measured_instantly=True, each event gets a unique belief_time - which previously caused the slow track in resample_events with O(N^2) memory usage. + which previously caused the slow track in resample_events with O(N²) memory usage. """ auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") # 5-minute resolution data -> needs downsampling to sensor's 10-minute resolution @@ -376,7 +377,6 @@ def test_upload_csv_file_measured_instantly_with_resampling( content_type="multipart/form-data", headers={"Authorization": auth_token}, ) - print("Server responded with:\n%s" % response.json) assert response.status_code == 200 From 588f0eabfdb3b184b1682cb35348fe52e872bc52 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 May 2026 12:23:01 +0000 Subject: [PATCH 19/45] fix: add db.engine.dispose() to ingestion job function to fix forked worker crashes Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/fa726915-c7de-4fb8-9135-f0d7dcd2a51d Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- documentation/changelog.rst | 1 + flexmeasures/data/services/data_ingestion.py | 3 +++ .../data/tests/test_data_ingestion.py | 25 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index de919b0526..9de699fb8e 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -29,6 +29,7 @@ Bugfixes * Make the sensor page load much faster for sensors with lots of data, by avoiding to load statistics over all of its history by default [see `PR #2129 `_] * Return a clear validation error (instead of a server ZeroDivisionError) when posting instantaneous (0-minute) data to non-instantaneous sensors via ``[POST] /sensors/(id)/data`` [see `PR #2116 `_] * Fix asset context page for asset names containing apostrophes [see `PR #2117 `_] +* Fix ingestion queue worker crashing with a database error when the worker process is forked (Linux), by calling ``db.engine.dispose()`` at the start of the ingestion job function, matching the existing pattern used by the scheduling worker [see `PR #2101 `_] v0.32.1 | May XX, 2026 diff --git a/flexmeasures/data/services/data_ingestion.py b/flexmeasures/data/services/data_ingestion.py index 9ac52b73ea..41f6530b14 100644 --- a/flexmeasures/data/services/data_ingestion.py +++ b/flexmeasures/data/services/data_ingestion.py @@ -125,6 +125,9 @@ def add_beliefs_to_db_and_enqueue_forecasting_jobs( - 'success_with_unchanged_beliefs_skipped' - 'success_but_nothing_new' """ + # https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork + db.engine.dispose() + if serialized_data is not None: data = deserialize_ingestion_data(serialized_data) if data is None: diff --git a/flexmeasures/data/tests/test_data_ingestion.py b/flexmeasures/data/tests/test_data_ingestion.py index 389e3e4e2b..349339b31e 100644 --- a/flexmeasures/data/tests/test_data_ingestion.py +++ b/flexmeasures/data/tests/test_data_ingestion.py @@ -9,7 +9,9 @@ deserialize_ingestion_data, serialize_ingestion_data, ) +from flexmeasures.data.tests.utils import exception_reporter from flexmeasures.tests.utils import get_test_sensor +from flexmeasures.utils.job_utils import work_on_rq def _to_comparable_df(bdf): @@ -89,3 +91,26 @@ def test_ingestion_service_accepts_serialized_data(setup_beliefs, db): ) assert status == "success_but_nothing_new" + + +def test_ingestion_job_succeeds_via_rq_worker(app, setup_beliefs, db): + """Regression test: ingestion job must succeed when processed by an RQ worker. + + Without db.engine.dispose() the forked worker inherits stale SQLAlchemy + connections from the parent process, causing the job to fail. + """ + sensor = get_test_sensor(db) + bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ + :1 + ] + serialized = serialize_ingestion_data(bdf) + + app.queues["ingestion"].enqueue( + add_beliefs_to_db_and_enqueue_forecasting_jobs, + serialized_data=serialized, + save_changed_beliefs_only=True, + ) + + work_on_rq(app.queues["ingestion"], exc_handler=exception_reporter) + + assert app.queues["ingestion"].failed_job_registry.count == 0 From 3c8d7767309535ed1ddbcf52ae472df27249e18c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 May 2026 12:24:02 +0000 Subject: [PATCH 20/45] fix: address code review comments - clearer test comment and simpler changelog entry Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/fa726915-c7de-4fb8-9135-f0d7dcd2a51d Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- documentation/changelog.rst | 2 +- flexmeasures/data/tests/test_data_ingestion.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9de699fb8e..6a36212281 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -29,7 +29,7 @@ Bugfixes * Make the sensor page load much faster for sensors with lots of data, by avoiding to load statistics over all of its history by default [see `PR #2129 `_] * Return a clear validation error (instead of a server ZeroDivisionError) when posting instantaneous (0-minute) data to non-instantaneous sensors via ``[POST] /sensors/(id)/data`` [see `PR #2116 `_] * Fix asset context page for asset names containing apostrophes [see `PR #2117 `_] -* Fix ingestion queue worker crashing with a database error when the worker process is forked (Linux), by calling ``db.engine.dispose()`` at the start of the ingestion job function, matching the existing pattern used by the scheduling worker [see `PR #2101 `_] +* Fix ingestion jobs crashing when processed by an RQ worker due to stale database connections inherited by the forked worker process [see `PR #2101 `_] v0.32.1 | May XX, 2026 diff --git a/flexmeasures/data/tests/test_data_ingestion.py b/flexmeasures/data/tests/test_data_ingestion.py index 349339b31e..383157ea56 100644 --- a/flexmeasures/data/tests/test_data_ingestion.py +++ b/flexmeasures/data/tests/test_data_ingestion.py @@ -100,6 +100,7 @@ def test_ingestion_job_succeeds_via_rq_worker(app, setup_beliefs, db): connections from the parent process, causing the job to fail. """ sensor = get_test_sensor(db) + # A single belief is sufficient to exercise the worker's DB connection handling. bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ :1 ] From 719ac8f253ea4d6f2c1ad752d009362f16599537 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 4 May 2026 14:00:50 +0100 Subject: [PATCH 21/45] fix: enhance timezone handling in data ingestion serialization and deserialization Signed-off-by: joshuaunity --- flexmeasures/data/services/data_ingestion.py | 18 ++++++++--- .../data/tests/test_data_ingestion.py | 32 +++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/services/data_ingestion.py b/flexmeasures/data/services/data_ingestion.py index 41f6530b14..b8f6dfea95 100644 --- a/flexmeasures/data/services/data_ingestion.py +++ b/flexmeasures/data/services/data_ingestion.py @@ -19,6 +19,16 @@ from flexmeasures.data.utils import save_to_db +def _to_utc_iso(dt) -> str: + """Serialize a datetime-like value as an ISO string in UTC.""" + ts = pd.Timestamp(dt) + if ts.tzinfo is None: + ts = ts.tz_localize("UTC") + else: + ts = ts.tz_convert("UTC") + return ts.isoformat() + + def serialize_ingestion_data( data: tb.BeliefsDataFrame | list[tb.BeliefsDataFrame], ) -> list[dict]: @@ -42,8 +52,8 @@ def serialize_ingestion_data( for belief in bdf.reset_index().itertuples(index=False): serialized_rows.append( { - "event_start": belief.event_start.isoformat(), - "belief_time": belief.belief_time.isoformat(), + "event_start": _to_utc_iso(belief.event_start), + "belief_time": _to_utc_iso(belief.belief_time), "source_id": belief.source.id, "cumulative_probability": float(belief.cumulative_probability), "event_value": ( @@ -92,8 +102,8 @@ def deserialize_ingestion_data(payload: Sequence[dict]) -> list[tb.BeliefsDataFr TimedBelief( sensor=sensor, source=source, - event_start=pd.Timestamp(row["event_start"]), - belief_time=pd.Timestamp(row["belief_time"]), + event_start=pd.to_datetime(row["event_start"], utc=True), + belief_time=pd.to_datetime(row["belief_time"], utc=True), cumulative_probability=row["cumulative_probability"], event_value=row["event_value"], ) diff --git a/flexmeasures/data/tests/test_data_ingestion.py b/flexmeasures/data/tests/test_data_ingestion.py index 383157ea56..7eeae611d5 100644 --- a/flexmeasures/data/tests/test_data_ingestion.py +++ b/flexmeasures/data/tests/test_data_ingestion.py @@ -115,3 +115,35 @@ def test_ingestion_job_succeeds_via_rq_worker(app, setup_beliefs, db): work_on_rq(app.queues["ingestion"], exc_handler=exception_reporter) assert app.queues["ingestion"].failed_job_registry.count == 0 + + +def test_deserialize_ingestion_data_handles_mixed_timezone_offsets(setup_beliefs, db): + sensor = get_test_sensor(db) + source = sensor.search_beliefs(source="ENTSO-E").lineage.sources[0] + payload = [ + { + "sensor_id": sensor.id, + "beliefs": [ + { + "event_start": "2021-03-28T01:00:00+01:00", + "belief_time": "2021-03-27T12:00:00+01:00", + "source_id": source.id, + "cumulative_probability": 0.5, + "event_value": 21.0, + }, + { + "event_start": "2021-03-28T03:00:00+02:00", + "belief_time": "2021-03-27T13:00:00+02:00", + "source_id": source.id, + "cumulative_probability": 0.5, + "event_value": 22.0, + }, + ], + } + ] + + restored = deserialize_ingestion_data(payload) + + assert len(restored) == 1 + assert restored[0].event_starts.tz is not None + assert str(restored[0].event_starts.tz) in ("UTC", "UTC+00:00") From 3b41915c30c0b4e531797bc1c394683967fd75c3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 May 2026 13:04:05 +0000 Subject: [PATCH 22/45] revert: undo db.engine.dispose() ingestion fix and related changes Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- documentation/changelog.rst | 1 - flexmeasures/data/services/data_ingestion.py | 21 ++----- .../data/tests/test_data_ingestion.py | 58 ------------------- 3 files changed, 4 insertions(+), 76 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 6a36212281..de919b0526 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -29,7 +29,6 @@ Bugfixes * Make the sensor page load much faster for sensors with lots of data, by avoiding to load statistics over all of its history by default [see `PR #2129 `_] * Return a clear validation error (instead of a server ZeroDivisionError) when posting instantaneous (0-minute) data to non-instantaneous sensors via ``[POST] /sensors/(id)/data`` [see `PR #2116 `_] * Fix asset context page for asset names containing apostrophes [see `PR #2117 `_] -* Fix ingestion jobs crashing when processed by an RQ worker due to stale database connections inherited by the forked worker process [see `PR #2101 `_] v0.32.1 | May XX, 2026 diff --git a/flexmeasures/data/services/data_ingestion.py b/flexmeasures/data/services/data_ingestion.py index b8f6dfea95..9ac52b73ea 100644 --- a/flexmeasures/data/services/data_ingestion.py +++ b/flexmeasures/data/services/data_ingestion.py @@ -19,16 +19,6 @@ from flexmeasures.data.utils import save_to_db -def _to_utc_iso(dt) -> str: - """Serialize a datetime-like value as an ISO string in UTC.""" - ts = pd.Timestamp(dt) - if ts.tzinfo is None: - ts = ts.tz_localize("UTC") - else: - ts = ts.tz_convert("UTC") - return ts.isoformat() - - def serialize_ingestion_data( data: tb.BeliefsDataFrame | list[tb.BeliefsDataFrame], ) -> list[dict]: @@ -52,8 +42,8 @@ def serialize_ingestion_data( for belief in bdf.reset_index().itertuples(index=False): serialized_rows.append( { - "event_start": _to_utc_iso(belief.event_start), - "belief_time": _to_utc_iso(belief.belief_time), + "event_start": belief.event_start.isoformat(), + "belief_time": belief.belief_time.isoformat(), "source_id": belief.source.id, "cumulative_probability": float(belief.cumulative_probability), "event_value": ( @@ -102,8 +92,8 @@ def deserialize_ingestion_data(payload: Sequence[dict]) -> list[tb.BeliefsDataFr TimedBelief( sensor=sensor, source=source, - event_start=pd.to_datetime(row["event_start"], utc=True), - belief_time=pd.to_datetime(row["belief_time"], utc=True), + event_start=pd.Timestamp(row["event_start"]), + belief_time=pd.Timestamp(row["belief_time"]), cumulative_probability=row["cumulative_probability"], event_value=row["event_value"], ) @@ -135,9 +125,6 @@ def add_beliefs_to_db_and_enqueue_forecasting_jobs( - 'success_with_unchanged_beliefs_skipped' - 'success_but_nothing_new' """ - # https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork - db.engine.dispose() - if serialized_data is not None: data = deserialize_ingestion_data(serialized_data) if data is None: diff --git a/flexmeasures/data/tests/test_data_ingestion.py b/flexmeasures/data/tests/test_data_ingestion.py index 7eeae611d5..389e3e4e2b 100644 --- a/flexmeasures/data/tests/test_data_ingestion.py +++ b/flexmeasures/data/tests/test_data_ingestion.py @@ -9,9 +9,7 @@ deserialize_ingestion_data, serialize_ingestion_data, ) -from flexmeasures.data.tests.utils import exception_reporter from flexmeasures.tests.utils import get_test_sensor -from flexmeasures.utils.job_utils import work_on_rq def _to_comparable_df(bdf): @@ -91,59 +89,3 @@ def test_ingestion_service_accepts_serialized_data(setup_beliefs, db): ) assert status == "success_but_nothing_new" - - -def test_ingestion_job_succeeds_via_rq_worker(app, setup_beliefs, db): - """Regression test: ingestion job must succeed when processed by an RQ worker. - - Without db.engine.dispose() the forked worker inherits stale SQLAlchemy - connections from the parent process, causing the job to fail. - """ - sensor = get_test_sensor(db) - # A single belief is sufficient to exercise the worker's DB connection handling. - bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ - :1 - ] - serialized = serialize_ingestion_data(bdf) - - app.queues["ingestion"].enqueue( - add_beliefs_to_db_and_enqueue_forecasting_jobs, - serialized_data=serialized, - save_changed_beliefs_only=True, - ) - - work_on_rq(app.queues["ingestion"], exc_handler=exception_reporter) - - assert app.queues["ingestion"].failed_job_registry.count == 0 - - -def test_deserialize_ingestion_data_handles_mixed_timezone_offsets(setup_beliefs, db): - sensor = get_test_sensor(db) - source = sensor.search_beliefs(source="ENTSO-E").lineage.sources[0] - payload = [ - { - "sensor_id": sensor.id, - "beliefs": [ - { - "event_start": "2021-03-28T01:00:00+01:00", - "belief_time": "2021-03-27T12:00:00+01:00", - "source_id": source.id, - "cumulative_probability": 0.5, - "event_value": 21.0, - }, - { - "event_start": "2021-03-28T03:00:00+02:00", - "belief_time": "2021-03-27T13:00:00+02:00", - "source_id": source.id, - "cumulative_probability": 0.5, - "event_value": 22.0, - }, - ], - } - ] - - restored = deserialize_ingestion_data(payload) - - assert len(restored) == 1 - assert restored[0].event_starts.tz is not None - assert str(restored[0].event_starts.tz) in ("UTC", "UTC+00:00") From 7df88bfe602e2e41fef64ebcddefd8d6abf68812 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 May 2026 17:36:28 +0000 Subject: [PATCH 23/45] fix: update test_fetch_sensor_stats expected values to account for new upload test beliefs Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/75646f7a-4568-4776-bb93-addceab7e930 Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensors_api.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 9a1300129b..5bfac6814b 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -686,9 +686,12 @@ def test_fetch_sensor_stats( assert record["Min value"] assert record["Max value"] if source == "Test Admin User (ID: 7)": - sum_values = 162.0 - count_values = 36 - mean_value = 4.5 + # 36 values from CSV/Excel uploads (upsampled from 1H to 10min) + # + 3 values from test_upload_csv_file_measured_instantly_with_resampling + # (downsampled from 5min to 10min: values 15, 35, 55) + sum_values = 267.0 + count_values = 39 + mean_value = 267.0 / 39 elif source == "Test Supplier User (ID: 6)": sum_values = 275.1 count_values = 3 From 706caf34c6d7b361bcadff096fd26b5ed9f821e2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 6 May 2026 10:30:48 +0000 Subject: [PATCH 24/45] fix: remove unused event_start variable in time_series.py (flake8 F841) Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/7299051d-f916-4c5c-9baf-713ddb67a59d Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/data/services/time_series.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index dc4977fff4..a16ee6c3b1 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -146,7 +146,6 @@ def _drop_unchanged_beliefs_compared_to_db( It is preferable to call the public function drop_unchanged_beliefs instead. """ source = bdf.lineage.sources[0] # unique source - event_start = bdf.event_starts[0] # unique event_start belief_time = bdf.lineage.belief_times[0] # unique belief time # Compare by ID rather than object identity: the candidate bdf may have been # deserialized from an RQ job queue (pickled in a different process), so its From 3402684dcad95fca30beee103d6e38f528ff0532 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 May 2026 09:37:11 +0000 Subject: [PATCH 25/45] fix: filter bdf_db by event_start in _drop_unchanged_beliefs_compared_to_db Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/fa0a6da6-6ccd-4416-b0ba-8da634d946be Co-authored-by: joshuaunity <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/data/services/time_series.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index a16ee6c3b1..06eab92fd1 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -146,12 +146,19 @@ def _drop_unchanged_beliefs_compared_to_db( It is preferable to call the public function drop_unchanged_beliefs instead. """ source = bdf.lineage.sources[0] # unique source + event_start = bdf.event_starts[0] # unique event_start belief_time = bdf.lineage.belief_times[0] # unique belief time # Compare by ID rather than object identity: the candidate bdf may have been # deserialized from an RQ job queue (pickled in a different process), so its # DataSource objects are detached and won't be identical to the freshly-loaded # ones in bdf_db even when they represent the same DB row. - bdf_db_from_source = bdf_db[bdf_db.sources.map(lambda s: s.id) == source.id] + # Also filter by event_start: bdf_db may contain beliefs for multiple event_starts, + # and we must not let a newer belief_time from a different event_start contaminate + # the most-recent-belief-time lookup for this candidate's event_start. + bdf_db_from_source = bdf_db[ + (bdf_db.sources.map(lambda s: s.id) == source.id) + & (bdf_db.event_starts == event_start) + ] if bdf_db_from_source.empty: return bdf # Use .max() rather than searchsorted: the result is correct regardless of From 2e052f64a4b64986626b78eacb1ccb11cf3debcf Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 12 May 2026 08:51:02 +0100 Subject: [PATCH 26/45] chore: modify chnagelog Signed-off-by: joshuaunity --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index d5426277ea..fb38cd1271 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -13,7 +13,6 @@ New features * Added API and UI support for copying assets and their subtrees [see `PR #2017 `_ and `PR #2120 `_] * Improve UX after deleting a child asset through the UI [see `PR #2119 `_] * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_] -* Clean up stale sensor references from ``flex-config`` and ``sensors_to_show`` when deleting a sensor, using JSONB queries to find affected assets before pruning those references [see `PR #2106 `_] * Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] * Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors//data`` [see `PR #2127 `_] @@ -42,6 +41,7 @@ Bugfixes * Fix a bug where toast messages in flex-config modal are broken due to unexpected JSON structure [see `PR #2124 `_] * Fix asset context page for asset names containing apostrophes [see `PR #2117 `_] * Fix removal of unchanged beliefs by comparing per event [see `PR #2150 `_] +* Clean up stale sensor references from ``flex-config`` and ``sensors_to_show`` when deleting a sensor, using JSONB queries to find affected assets before pruning those references [see `PR #2106 `_] v0.32.0 | April 15, 2026 From 06f86c9fd1511e006124300afdd75329c23bab7c Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 12 May 2026 16:24:40 +0100 Subject: [PATCH 27/45] feat: implement job status polling for background data processing Signed-off-by: joshuaunity --- flexmeasures/api/common/utils/api_utils.py | 6 +- flexmeasures/ui/static/js/ui-utils.js | 87 +++++++++++++++++++ flexmeasures/ui/static/openapi-specs.json | 2 +- .../ui/templates/includes/graphs.html | 21 ++++- 4 files changed, 109 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 87136721c7..376acb538a 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -164,13 +164,15 @@ def save_and_enqueue( if forecasting_jobs is not None else None ) - ingestion_queue.enqueue( + job = ingestion_queue.enqueue( add_beliefs_to_db_and_enqueue_forecasting_jobs, serialized_data=serialized_data, forecasting_job_ids=forecasting_job_ids, save_changed_beliefs_only=save_changed_beliefs_only, ) - return request_processed() + response, code = request_processed() + response["job_id"] = job.id + return response, code else: current_app.logger.warning( "No workers connected to the ingestion queue. Processing sensor data directly." diff --git a/flexmeasures/ui/static/js/ui-utils.js b/flexmeasures/ui/static/js/ui-utils.js index f6c1e00306..23baa9b113 100644 --- a/flexmeasures/ui/static/js/ui-utils.js +++ b/flexmeasures/ui/static/js/ui-utils.js @@ -479,3 +479,90 @@ export function initDeleteAssetButton() { ); }); } + +/** + * Poll a background job until it reaches a terminal state (finished or failed). + * + * Calls GET /api/v3_0/jobs/ every `intervalMs` milliseconds and updates + * a toast notification as the status changes. Polling stops automatically on + * success, failure, or network error, and also when the optional AbortSignal + * fires (useful for component/page teardown). + * + * @param {string} jobUuid - UUID returned by the upload endpoint. + * @param {object} [options] + * @param {number} [options.intervalMs=3000] - Polling interval in ms. + * @param {string} [options.processingMessage="Processing…"] - Toast text while queued/started. + * @param {string} [options.successMessage="Job completed successfully."] - Toast text on finish. + * @param {string} [options.errorMessage] - Override toast text on failure (defaults to the + * server message when absent). + * @param {AbortSignal} [options.signal] - Abort polling externally (e.g. page unload). + * @param {function} [options.onFinished] - Called with the full response JSON on finish. + * @param {function} [options.onFailed] - Called with the full response JSON on failure. + * @returns {function} stopPolling - Call to cancel polling manually. + */ +export function pollJobStatus(jobUuid, options = {}) { + const { + intervalMs = 3000, + processingMessage = "Processing\u2026", + successMessage = "Job completed successfully.", + errorMessage = null, + signal = null, + onFinished = null, + onFailed = null, + } = options; + + const url = `${apiBasePath}/api/v3_0/jobs/${encodeURIComponent(jobUuid)}`; + + // Show an initial "processing" info toast right away. + showToast(processingMessage, "info"); + + let stopped = false; + + function stop() { + stopped = true; + clearInterval(intervalId); + } + + // Honour an external AbortSignal (e.g. page navigation / component unmount). + if (signal) { + signal.addEventListener("abort", stop, { once: true }); + } + + async function poll() { + if (stopped) return; + + let data; + try { + const response = await fetch(url, { credentials: "same-origin" }); + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + data = await response.json(); + } catch (err) { + stop(); + showToast("Could not reach job status endpoint: " + err.message, "error"); + if (onFailed) onFailed(null); + return; + } + + const status = (data.status || "").toUpperCase(); + + if (status === "FINISHED") { + stop(); + showToast(successMessage, "success"); + if (onFinished) onFinished(data); + } else if (status === "FAILED" || status === "STOPPED" || status === "CANCELED") { + stop(); + const msg = errorMessage || data.message || "Job failed."; + showToast(msg, "error"); + if (onFailed) onFailed(data); + } + // QUEUED / STARTED / DEFERRED / SCHEDULED → keep polling, toast already shown. + } + + // Poll immediately on first tick, then on each interval. + const intervalId = setInterval(poll, intervalMs); + poll(); + + return stop; +} diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 5c65719b14..9688fd10de 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -6187,4 +6187,4 @@ } } } -} +} \ No newline at end of file diff --git a/flexmeasures/ui/templates/includes/graphs.html b/flexmeasures/ui/templates/includes/graphs.html index 62164b7903..71a15296cd 100644 --- a/flexmeasures/ui/templates/includes/graphs.html +++ b/flexmeasures/ui/templates/includes/graphs.html @@ -23,7 +23,7 @@ // Import local js (the FM version is used for cache-busting, causing the browser to fetch the updated version from the server) import { convertToCSV } from "{{ url_for('flexmeasures_ui.static', filename='js/data-utils.js') }}?v={{ flexmeasures_version }}"; - import {apiBasePath} from "{{ url_for('flexmeasures_ui.static', filename='js/ui-utils.js') }}?v={{ flexmeasures_version }}"; + import { apiBasePath, pollJobStatus } from "{{ url_for('flexmeasures_ui.static', filename='js/ui-utils.js') }}?v={{ flexmeasures_version }}"; import { subtract, computeSimulationRanges, lastNMonths, encodeUrlQuery, getOffsetBetweenTimezonesForDate, toIsoStringWithOffset } from "{{ url_for('flexmeasures_ui.static', filename='js/daterange-utils.js') }}?v={{ flexmeasures_version }}"; import { partition, updateBeliefs, beliefTimedelta, setAbortableTimeout } from "{{ url_for('flexmeasures_ui.static', filename='js/replay-utils.js') }}?v={{ flexmeasures_version }}"; import { decompressChartData, checkDSTTransitions, checkSourceMasking } from "{{ url_for('flexmeasures_ui.static', filename='js/chart-data-utils.js') }}?v={{ flexmeasures_version }}"; @@ -172,7 +172,6 @@ .then(async function (response) { if (response.ok) { const data = await response.json(); - showToast(data.message, "success"); // Update date range picker.setDateRange( storeStartDate, @@ -180,8 +179,22 @@ ); uploadSpinner.classList.add('d-none'); upload.classList.remove('d-none'); - // Reload the chart so newly-uploaded data is immediately visible - document.dispatchEvent(new CustomEvent('newDataAvailable')); + + if (data.job_id) { + // Track the background ingestion job and refresh when done. + pollJobStatus(data.job_id, { + processingMessage: "Upload received. Processing data in the background\u2026", + successMessage: "Data processed successfully.", + onFinished: () => { + document.dispatchEvent(new CustomEvent('newDataAvailable')); + loadSensorStats({{ sensor.id }}, "", "", true); + }, + }); + } else { + // No queued job: data was saved synchronously. + showToast(data.message, "success"); + document.dispatchEvent(new CustomEvent('newDataAvailable')); + } } else { const data = await response.json(); const messageKeys = Object.keys(data.message); From 3c4d71780b68802f22de16b79337ec9d93481a71 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 13 May 2026 13:33:44 +0100 Subject: [PATCH 28/45] feat: enhance job status polling with in-progress notifications Signed-off-by: joshuaunity --- flexmeasures/ui/static/js/ui-utils.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flexmeasures/ui/static/js/ui-utils.js b/flexmeasures/ui/static/js/ui-utils.js index 23baa9b113..4b28d051d2 100644 --- a/flexmeasures/ui/static/js/ui-utils.js +++ b/flexmeasures/ui/static/js/ui-utils.js @@ -556,8 +556,15 @@ export function pollJobStatus(jobUuid, options = {}) { const msg = errorMessage || data.message || "Job failed."; showToast(msg, "error"); if (onFailed) onFailed(data); + } else if (status === "STARTED") { // This is the shown status when ingestion is in progress + // show a inprogress message + const inProgressMessage = data.message || "Job is in progress."; + showToast(inProgressMessage, "info"); + } else { + // QUEUED / STARTED / DEFERRED / SCHEDULED → keep polling, show status updates + const statusToast = data.message || `Job status: ${status}`; + console.log(`[pollJobStatus] Still processing: ${statusToast}`); } - // QUEUED / STARTED / DEFERRED / SCHEDULED → keep polling, toast already shown. } // Poll immediately on first tick, then on each interval. From 38c81d04dce2099972dad2e75041fd377f4e3b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 13 May 2026 15:35:12 +0200 Subject: [PATCH 29/45] move all work to the jobs, including resampling. Just a thin acceptance layer is left in the API endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/changelog.rst | 2 +- documentation/configuration.rst | 8 + documentation/host/queues.rst | 2 +- flexmeasures/api/__init__.py | 3 + flexmeasures/api/common/responses.py | 23 ++- .../api/common/schemas/sensor_data.py | 33 +++- flexmeasures/api/common/utils/api_utils.py | 51 +++++- flexmeasures/api/common/utils/args_parsing.py | 29 +++ flexmeasures/api/v3_0/sensors.py | 68 +++++-- .../api/v3_0/tests/test_sensor_data.py | 94 +++++++++- .../api/v3_0/tests/test_sensors_api.py | 64 ++++++- flexmeasures/data/schemas/sensors.py | 18 +- flexmeasures/data/services/data_ingestion.py | 169 ++++++++---------- .../data/tests/test_data_ingestion.py | 75 +------- flexmeasures/ui/static/openapi-specs.json | 18 +- flexmeasures/utils/config_defaults.py | 1 + 16 files changed, 461 insertions(+), 197 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index f8366f3900..7c7163a17d 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -21,7 +21,7 @@ New features Infrastructure / Support ---------------------- -* Move sensor data ingestion to a job queue for improved performance when POSTing large amounts of data to the sensor data API [see `PR #2101 `_] +* Move sensor data ingestion to a job queue for improved performance when POSTing large amounts of data to the sensor data API, returning a ``202 Accepted`` response with a job status URL when queued [see `PR #2101 `_] * Remove legacy rolling viewpoint forecasting code and utilities after migrating to fixed-point forecasting [see `PR #2082 `_] * Upgraded dependencies [see `PR #2114 `_ and `PR #2148 `_] * Run ``flexmeasures jobs run-worker`` with RQ's embedded scheduler on by default so jobs created with ``enqueue_in`` are promoted from the scheduled registry when due; pass ``--without-scheduler`` to disable [see `PR #2112 `_] diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 7202f94fef..1b823aa585 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -347,6 +347,14 @@ Set a negative value to persist forever. Default: ``3600`` +FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Maximum request body size for sensor data posted to the sensor data API, both for JSON data and file uploads. +Set to ``None`` to disable this FlexMeasures-specific limit. + +Default: ``3 * 1024 * 1024`` + .. _datasource_config: FLEXMEASURES_DEFAULT_DATASOURCE diff --git a/documentation/host/queues.rst b/documentation/host/queues.rst index f5b8dcf69a..5454061615 100644 --- a/documentation/host/queues.rst +++ b/documentation/host/queues.rst @@ -46,7 +46,7 @@ You can also clear the job queues: When the main FlexMeasures process runs (e.g. by ``flexmeasures run``\ ), the queues of forecasting and scheduling jobs can be visited at ``http://localhost:5000/tasks/forecasting`` and ``http://localhost:5000/tasks/schedules``\ , respectively (by admins). .. note:: - The ``ingestion`` queue is used for sensor data posted via the API. If the queue is not configured, or if no worker is connected to it, data is processed synchronously (in the web process) with a warning logged. Running a dedicated ingestion worker is recommended in production to keep API responses fast when large amounts of data are posted. + The ``ingestion`` queue is used for sensor data posted via the API. If the queue is not configured, or if no worker is connected to it, data is processed synchronously (in the web process) with a warning logged. Running a dedicated ingestion worker is recommended in production to keep API responses fast when large amounts of data are posted. When ingestion is queued, the API returns ``202 Accepted`` with a job status URL. diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index f7afec5463..c4c8a445a9 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -17,9 +17,11 @@ from flexmeasures.data.models.user import User from flexmeasures.api.common.utils.args_parsing import ( validation_error_handler, + request_entity_too_large_handler, ) from flexmeasures.api.common.responses import invalid_sender from flexmeasures.data.schemas.utils import FMValidationError +from werkzeug.exceptions import RequestEntityTooLarge from flexmeasures.api.v3_0.users import AuthRequestSchema # The api blueprint. It is registered with the Flask app (see app.py) @@ -150,6 +152,7 @@ def register_at(app: Flask): # handle API specific errors app.register_error_handler(FMValidationError, validation_error_handler) + app.register_error_handler(RequestEntityTooLarge, request_entity_too_large_handler) app.register_error_handler(IntegrityError, catch_timed_belief_replacements) app.unauthorized_handler_api = invalid_sender diff --git a/flexmeasures/api/common/responses.py b/flexmeasures/api/common/responses.py index a59ce62762..5b161ffa27 100644 --- a/flexmeasures/api/common/responses.py +++ b/flexmeasures/api/common/responses.py @@ -3,6 +3,8 @@ import inflect from functools import wraps +from flask import url_for + from flexmeasures.auth.error_handling import FORBIDDEN_MSG, FORBIDDEN_STATUS_CODE p = inflect.engine() @@ -62,7 +64,7 @@ def already_received_and_successfully_processed(message: str) -> ResponseTuple: @BaseMessage( - "Some of the data represents a replacement, which is reserved for customized servers. If you are hosting FlexMeasures, you can enable replacements by setting FLEXMEASURES_ALLOW_DATA_OVERWRITE=True in the configuration settings. Alternatively, update the prior in your request." + "Some of the data represents a replacement: existing values would be changed for the same sensor, source, event timestamp and belief time. This is reserved for customized servers. If you are hosting FlexMeasures, you can enable replacements by setting FLEXMEASURES_ALLOW_DATA_OVERWRITE=True in the configuration settings. Alternatively, submit the data with a different prior, or check whether the same file was already uploaded with different values." ) def invalid_replacement(message: str) -> ResponseTuple: return ( @@ -378,6 +380,25 @@ def request_processed(message: str) -> ResponseTuple: return dict(status="PROCESSED", message=message), 200 +def request_accepted_for_processing( + job_id: str, + message: str = "Request has been accepted for processing.", +) -> ResponseTuple: + return ( + dict( + status="ACCEPTED", + message=message, + job_monitor_url=url_for("JobAPI:get_job_status", uuid=job_id), + job_id=job_id, + ), + 202, + ) + + +def request_too_large(message: str) -> ResponseTuple: + return dict(result="Rejected", status="PAYLOAD_TOO_LARGE", message=message), 413 + + def pluralize(usef_role_name: str) -> str: """Adding a trailing 's' works well for USEF roles.""" return "%ss" % usef_role_name diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 1d77f694c4..61f6ba6b98 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -12,6 +12,7 @@ from flexmeasures.data import ma from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.user import User from flexmeasures.api.common.schemas.sensors import ( SensorEntityAddressField, SensorIdField, @@ -314,6 +315,10 @@ class PostSensorDataSchema(SensorDataDescriptionSchema): This schema includes data (values) and still describes it. """ + def __init__(self, *args, source_user: User | None = None, **kwargs): + super().__init__(*args, **kwargs) + self.source_user = source_user + values = PolyField( deserialization_schema_selector=select_schema_to_ensure_list_of_floats, serialization_schema_selector=select_schema_to_ensure_list_of_floats, @@ -455,12 +460,11 @@ def possibly_upsample_values(data): ) return data - @staticmethod - def load_bdf(sensor_data: dict) -> BeliefsDataFrame: + def load_bdf(self, sensor_data: dict) -> BeliefsDataFrame: """ Turn the de-serialized and validated data into a BeliefsDataFrame. """ - source = get_or_create_source(current_user) + source = get_or_create_source(self.source_user or current_user) num_values = len(sensor_data["values"]) event_resolution = sensor_data["duration"] / num_values start = sensor_data["start"] @@ -498,6 +502,29 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: ) +class PostSensorDataRequestSchema(PostSensorDataSchema): + """Validate posted sensor data without building a BeliefsDataFrame.""" + + @post_load() + def post_load_sequence(self, data: dict, **kwargs) -> dict: + sensor_data = { + "values": data["values"], + "start": datetime_isoformat(data["start"]), + "duration": duration_isoformat(data["duration"]), + "unit": data["unit"], + } + if "prior" in data: + sensor_data["prior"] = datetime_isoformat(data["prior"]) + elif "horizon" in data: + sensor_data["horizon"] = duration_isoformat(data["horizon"]) + else: + # Preserve request-time semantics when processing happens later in a worker. + sensor_data["prior"] = datetime_isoformat(server_now()) + if "type" in data: + sensor_data["type"] = data["type"] + return dict(sensor=data["sensor"], sensor_data=sensor_data) + + class GetSensorDataSchemaEntityAddress(GetSensorDataSchema): """DEPRECATED, only here to support deprecated endpoints""" diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 87136721c7..a8fd3f983e 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -22,7 +22,6 @@ from flexmeasures.data.models.user import Account from flexmeasures.data.services.data_ingestion import ( add_beliefs_to_db_and_enqueue_forecasting_jobs, - serialize_ingestion_data, ) from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor @@ -31,6 +30,7 @@ invalid_replacement, ResponseTuple, request_processed, + request_accepted_for_processing, already_received_and_successfully_processed, ) from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema @@ -149,6 +149,32 @@ def save_and_enqueue( data: BeliefsDataFrame | list[BeliefsDataFrame], forecasting_jobs: list[Job] | None = None, save_changed_beliefs_only: bool = True, +) -> ResponseTuple: + status = add_beliefs_to_db_and_enqueue_forecasting_jobs( + data, + forecasting_jobs=forecasting_jobs, + save_changed_beliefs_only=save_changed_beliefs_only, + ) + + # Pick a response + if status == "success": + return request_processed() + elif status in ( + "success_with_unchanged_beliefs_skipped", + "success_but_nothing_new", + ): + return already_received_and_successfully_processed() + return invalid_replacement() + + +def enqueue_sensor_data_ingestion( + sensor_id: int, + user_id: int, + sensor_data: dict | None = None, + uploaded_files: list[dict] | None = None, + upload_data: dict | None = None, + forecasting_jobs: list[Job] | None = None, + save_changed_beliefs_only: bool = True, ) -> ResponseTuple: ingestion_queue = current_app.queues.get("ingestion") if ingestion_queue is None: @@ -158,32 +184,41 @@ def save_and_enqueue( else: workers = Worker.all(queue=ingestion_queue) if workers: - serialized_data = serialize_ingestion_data(data) forecasting_job_ids = ( [job.id for job in forecasting_jobs] if forecasting_jobs is not None else None ) - ingestion_queue.enqueue( + job = ingestion_queue.enqueue( add_beliefs_to_db_and_enqueue_forecasting_jobs, - serialized_data=serialized_data, + sensor_id=sensor_id, + user_id=user_id, + sensor_data=sensor_data, + uploaded_files=uploaded_files, + upload_data=upload_data, forecasting_job_ids=forecasting_job_ids, save_changed_beliefs_only=save_changed_beliefs_only, + meta={"sensor_id": sensor_id}, + ) + return request_accepted_for_processing( + job.id, + "Sensor data has been accepted for processing.", ) - return request_processed() else: current_app.logger.warning( "No workers connected to the ingestion queue. Processing sensor data directly." ) - # Attempt to save directly (fallback when no ingestion queue or workers are available) status = add_beliefs_to_db_and_enqueue_forecasting_jobs( - data, + sensor_id=sensor_id, + user_id=user_id, + sensor_data=sensor_data, + uploaded_files=uploaded_files, + upload_data=upload_data, forecasting_jobs=forecasting_jobs, save_changed_beliefs_only=save_changed_beliefs_only, ) - # Pick a response if status == "success": return request_processed() elif status in ( diff --git a/flexmeasures/api/common/utils/args_parsing.py b/flexmeasures/api/common/utils/args_parsing.py index 9665f42708..b9c84e006b 100644 --- a/flexmeasures/api/common/utils/args_parsing.py +++ b/flexmeasures/api/common/utils/args_parsing.py @@ -1,11 +1,14 @@ from flask import jsonify +from flask import current_app from flask import Request from flask_json import JsonError from webargs import ValidationError from webargs.flaskparser import parser from webargs.multidictproxy import MultiDictProxy from werkzeug.datastructures import MultiDict +from werkzeug.exceptions import RequestEntityTooLarge +from flexmeasures.api.common.responses import request_too_large from flexmeasures.data.schemas.utils import FMValidationError """ @@ -45,6 +48,25 @@ def validation_error_handler(error: FMValidationError): return response +def request_entity_too_large_handler(error: RequestEntityTooLarge): + response_data, status_code = request_too_large(error.description) + response = jsonify(response_data) + response.status_code = status_code + return response + + +def _enforce_request_size_limit(request: Request, config_key: str): + max_size = current_app.config.get(config_key) + if max_size is None or request.content_length is None: + return + if request.content_length > max_size: + raise RequestEntityTooLarge( + description=( + f"Request body exceeds the configured limit of {max_size} bytes." + ) + ) + + @parser.location_loader("args_and_json") def load_data(request, schema): """ @@ -99,6 +121,8 @@ def combined_sensor_data_upload(request: Request, schema): MultiDictProxy: A proxy object wrapping the merged data from path parameters and uploaded files. """ + _enforce_request_size_limit(request, "FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES") + data = MultiDict(request.view_args) data.update(request.files) belief_time = request.form.get("belief-time-measured-instantly") @@ -132,6 +156,11 @@ def combined_sensor_data_description(request: Request, schema): MultiDictProxy: A proxy object wrapping the merged data from path parameters, URL and/or uploaded json. """ + if request.method == "POST": + _enforce_request_size_limit( + request, "FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES" + ) + # combine data data = MultiDict(request.view_args) data.update(request.args) # Url (GET) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 5ed0413dde..a11b1d6b26 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -15,7 +15,6 @@ from marshmallow import fields, Schema, ValidationError import marshmallow.validate as validate from rq.job import Job, JobStatus, NoSuchJobError -import timely_beliefs as tb from webargs.flaskparser import use_args, use_kwargs from sqlalchemy import delete, select, or_ @@ -33,12 +32,13 @@ GetSensorDataSchema, GetSensorDataQuerySchema, PostSensorDataSchema, + PostSensorDataRequestSchema, ) from flexmeasures.api.common.schemas.sensors import SensorId # noqa F401 from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.utils.api_utils import ( job_status_description, - save_and_enqueue, + enqueue_sensor_data_ingestion, ) from flexmeasures.auth.policy import check_access from flexmeasures.auth.decorators import permission_required_for_context @@ -55,6 +55,7 @@ SensorSchema, SensorIdField, SensorDataFileSchema, + SensorDataFileRequestSchema, ) from flexmeasures.data.schemas.times import ( AwareDateTimeField, @@ -447,19 +448,23 @@ def index( @route("/data/upload", methods=["POST"]) @use_args( - SensorDataFileSchema(), location="combined_sensor_data_upload", as_kwargs=True + SensorDataFileRequestSchema(), + location="combined_sensor_data_upload", + as_kwargs=True, ) @permission_required_for_context( "create-children", - ctx_arg_name="data", - ctx_loader=lambda data: data[0].sensor if data else None, + ctx_arg_name="sensor", + ctx_loader=lambda sensor: sensor, pass_ctx_to_loader=True, ) def upload_data( self, - data: list[tb.BeliefsDataFrame], + sensor: Sensor, + uploaded_files: list, filenames: list[str], unit: str | None = None, + belief_time_measured_instantly: bool | None = None, **kwargs, ): """ @@ -476,6 +481,8 @@ def upload_data( The resolution of the data has to match the sensor's required resolution, but FlexMeasures will attempt to upsample lower resolutions. The list of values may include null values. + The request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES + (3 MiB by default). security: - ApiKeyAuth: [] @@ -492,6 +499,8 @@ def upload_data( uploaded-files: contentType: application/octet-stream responses: + 202: + description: ACCEPTED 200: description: PROCESSED content: @@ -546,33 +555,54 @@ def upload_data( description: UNAUTHORIZED 403: description: INVALID_SENDER + 413: + description: PAYLOAD_TOO_LARGE 422: description: UNPROCESSABLE_ENTITY tags: - Sensors """ - sensor = data[0].sensor - AssetAuditLog.add_record( sensor.generic_asset, f"Data from {join_words_into_a_list(filenames)} uploaded to sensor '{sensor.name}': {sensor.id}", ) - response, code = save_and_enqueue(data) + files_for_job = [] + for file in uploaded_files: + files_for_job.append( + dict( + filename=file.filename, + content_type=file.content_type, + content=file.read(), + ) + ) + upload_data = { + "belief-time-measured-instantly": ( + "on" if belief_time_measured_instantly else "off" + ), + } + if unit is not None: + upload_data["unit"] = unit + response, code = enqueue_sensor_data_ingestion( + sensor_id=sensor.id, + user_id=current_user.id, + uploaded_files=files_for_job, + upload_data=upload_data, + ) return response, code @route("//data", methods=["POST"]) @use_args( - PostSensorDataSchema(), + PostSensorDataRequestSchema(), location="combined_sensor_data_description", as_kwargs=True, ) @permission_required_for_context( "create-children", - ctx_arg_name="bdf", - ctx_loader=lambda bdf: bdf.sensor, + ctx_arg_name="sensor", + ctx_loader=lambda sensor: sensor, pass_ctx_to_loader=True, ) - def post_data(self, id: int, bdf: tb.BeliefsDataFrame): + def post_data(self, id: int, sensor: Sensor, sensor_data: dict): """ .. :quickref: Data; Post sensor data --- @@ -589,6 +619,8 @@ def post_data(self, id: int, bdf: tb.BeliefsDataFrame): The resolution of the data has to match the sensor's required resolution, but FlexMeasures will attempt to upsample lower resolutions. The list of values may include null values. + The request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES + (3 MiB by default). security: - ApiAuthKey: [] @@ -610,6 +642,8 @@ def post_data(self, id: int, bdf: tb.BeliefsDataFrame): "duration": "PT1H" "unit": "m³/h" responses: + 202: + description: ACCEPTED 200: description: PROCESSED 400: @@ -618,12 +652,18 @@ def post_data(self, id: int, bdf: tb.BeliefsDataFrame): description: UNAUTHORIZED 403: description: INVALID_SENDER + 413: + description: PAYLOAD_TOO_LARGE 422: description: UNPROCESSABLE_ENTITY tags: - Sensors """ - response, code = save_and_enqueue(bdf) + response, code = enqueue_sensor_data_ingestion( + sensor_id=sensor.id, + user_id=current_user.id, + sensor_data=sensor_data, + ) return response, code @route("//data", methods=["GET"]) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 873ecc182c..fbc490a766 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import timedelta -from flask import url_for +from flask import current_app, url_for import pytest from sqlalchemy import event from sqlalchemy.engine import Engine @@ -11,6 +11,10 @@ from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor +def _fake_ingestion_worker(queue): + return [object()] + + @pytest.mark.parametrize( "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True ) @@ -278,6 +282,64 @@ def test_post_sensor_data_bad_auth( assert post_data_response.status_code == status_code +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_post_sensor_data_returns_accepted_job( + client, + setup_api_test_data, + requesting_user, + monkeypatch, +): + monkeypatch.setattr( + "flexmeasures.api.common.utils.api_utils.Worker.all", + _fake_ingestion_worker, + ) + current_app.queues["ingestion"].empty() + post_data = make_sensor_data_request_for_gas_sensor() + sensor = setup_api_test_data["some gas sensor"] + + response = client.post( + url_for("SensorAPI:post_data", id=sensor.id), + json=post_data, + ) + + assert response.status_code == 202 + assert response.json["status"] == "ACCEPTED" + assert response.json["job_monitor_url"] == url_for( + "JobAPI:get_job_status", uuid=response.json["job_id"] + ) + job = current_app.queues["ingestion"].fetch_job(response.json["job_id"]) + assert job.kwargs["sensor_id"] == sensor.id + assert job.kwargs["sensor_data"] == post_data + assert "data" not in job.kwargs + + +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_post_sensor_data_rejects_large_json( + client, + setup_api_test_data, + requesting_user, + monkeypatch, +): + monkeypatch.setitem( + current_app.config, + "FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES", + 1, + ) + sensor = setup_api_test_data["some gas sensor"] + + response = client.post( + url_for("SensorAPI:post_data", id=sensor.id), + json=make_sensor_data_request_for_gas_sensor(), + ) + + assert response.status_code == 413 + assert response.json["status"] == "PAYLOAD_TOO_LARGE" + + @pytest.mark.parametrize( "request_field, new_value, error_field, error_text", [ @@ -307,7 +369,13 @@ def test_post_invalid_sensor_data( error_field, error_text, requesting_user, + monkeypatch, ): + monkeypatch.setattr( + "flexmeasures.api.common.utils.api_utils.Worker.all", + _fake_ingestion_worker, + ) + current_app.queues["ingestion"].empty() post_data = make_sensor_data_request_for_gas_sensor() sensor = setup_api_test_data["some gas sensor"] post_data[request_field] = new_value @@ -322,6 +390,30 @@ def test_post_invalid_sensor_data( error_text in response.json["message"]["combined_sensor_data_description"][error_field][0] ) + assert current_app.queues["ingestion"].count == 0 + + +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_post_sensor_data_rejects_unknown_sensor_before_queueing( + client, + requesting_user, + monkeypatch, +): + monkeypatch.setattr( + "flexmeasures.api.common.utils.api_utils.Worker.all", + _fake_ingestion_worker, + ) + current_app.queues["ingestion"].empty() + + response = client.post( + url_for("SensorAPI:post_data", id=999999), + json=make_sensor_data_request_for_gas_sensor(), + ) + + assert response.status_code == 404 + assert current_app.queues["ingestion"].count == 0 @pytest.mark.parametrize( diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 5bfac6814b..d8d033ca16 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -5,7 +5,7 @@ import io import json -from flask import url_for +from flask import current_app, url_for from sqlalchemy import select, func from flexmeasures.data.models.time_series import TimedBelief @@ -341,6 +341,68 @@ def test_upload_csv_file(client, db, setup_api_test_data, sensor_name, requestin ) +@pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) +def test_upload_csv_file_returns_accepted_job( + client, setup_api_test_data, requesting_user, monkeypatch +): + monkeypatch.setattr( + "flexmeasures.api.common.utils.api_utils.Worker.all", + lambda queue: [object()], + ) + current_app.queues["ingestion"].empty() + auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") + csv_content = """event_start,event_value +2022-12-16T05:11:00Z,4 +""" + sensor = setup_api_test_data["some gas sensor"] + file = (io.BytesIO(csv_content.encode("utf-8")), "test.csv") + + response = client.post( + url_for("SensorAPI:upload_data", id=sensor.id), + data={"uploaded-files": file}, + content_type="multipart/form-data", + headers={"Authorization": auth_token}, + ) + + assert response.status_code == 202 + assert response.json["status"] == "ACCEPTED" + assert response.json["job_monitor_url"] == url_for( + "JobAPI:get_job_status", uuid=response.json["job_id"] + ) + job = current_app.queues["ingestion"].fetch_job(response.json["job_id"]) + assert job.kwargs["sensor_id"] == sensor.id + assert job.kwargs["uploaded_files"][0]["filename"] == "test.csv" + assert job.kwargs["uploaded_files"][0]["content"] == csv_content.encode("utf-8") + assert "data" not in job.kwargs + + +@pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) +def test_upload_csv_file_rejects_large_upload( + client, setup_api_test_data, requesting_user, monkeypatch +): + monkeypatch.setitem( + current_app.config, + "FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES", + 1, + ) + auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") + sensor = setup_api_test_data["some gas sensor"] + file = ( + io.BytesIO(b"event_start,event_value\n2022-12-16T05:11:00Z,4\n"), + "test.csv", + ) + + response = client.post( + url_for("SensorAPI:upload_data", id=sensor.id), + data={"uploaded-files": file}, + content_type="multipart/form-data", + headers={"Authorization": auth_token}, + ) + + assert response.status_code == 413 + assert response.json["status"] == "PAYLOAD_TOO_LARGE" + + @pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) def test_upload_csv_file_measured_instantly_with_resampling( client, db, setup_api_test_data, requesting_user diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 1510021f5b..2ceeb2ac0c 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -32,12 +32,14 @@ from flexmeasures.data import ma, db from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.user import User from flexmeasures.data.schemas.utils import ( FMValidationError, MarshmallowClickMixin, with_appcontext_if_needed, convert_to_quantity, ) +from flexmeasures.data.services.data_sources import get_or_create_source from flexmeasures.utils.time_utils import get_timezone from flexmeasures.utils.unit_utils import ( is_valid_unit, @@ -606,6 +608,10 @@ class SensorDataFileDescriptionSchema(Schema): class SensorDataFileSchema(SensorDataFileDescriptionSchema): sensor = SensorIdField(data_key="id") + def __init__(self, *args, source_user: User | None = None, **kwargs): + super().__init__(*args, **kwargs) + self.source_user = source_user + _valid_content_types = { "text/csv", "text/plain", @@ -677,7 +683,7 @@ def post_load(self, fields, **kwargs): bdf = tb.read_csv( file, sensor, - source=current_user.data_source[0], + source=get_or_create_source(self.source_user or current_user), belief_time=( pd.Timestamp.utcnow() if not belief_time_measured_instantly @@ -754,6 +760,16 @@ def post_load(self, fields, **kwargs): return fields +class SensorDataFileRequestSchema(SensorDataFileSchema): + """Validate a sensor data upload without parsing or resampling its files.""" + + @post_load + def post_load(self, fields, **kwargs): + files: list[FileStorage] = fields["uploaded_files"] + fields["filenames"] = [file.filename for file in files] + return fields + + class QuantitySchema(Schema): """Represents a quantity string like '1 EUR/MWh'.""" diff --git a/flexmeasures/data/services/data_ingestion.py b/flexmeasures/data/services/data_ingestion.py index 9ac52b73ea..39a28610e7 100644 --- a/flexmeasures/data/services/data_ingestion.py +++ b/flexmeasures/data/services/data_ingestion.py @@ -4,108 +4,82 @@ from __future__ import annotations -from collections.abc import Sequence +from io import BytesIO from flask import current_app -import pandas as pd from rq.job import Job from rq.job import NoSuchJobError -from sqlalchemy import select import timely_beliefs as tb +from werkzeug.datastructures import FileStorage from flexmeasures.data import db -from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.models.time_series import Sensor, TimedBelief +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.user import User from flexmeasures.data.utils import save_to_db -def serialize_ingestion_data( - data: tb.BeliefsDataFrame | list[tb.BeliefsDataFrame], -) -> list[dict]: - """Serialize beliefs data to primitive types suitable for queue kwargs. +def _get_ingestion_context(sensor_id: int, user_id: int) -> tuple[Sensor, User]: + sensor = db.session.get(Sensor, sensor_id) + if sensor is None: + raise ValueError(f"No such sensor: {sensor_id}") + user = db.session.get(User, user_id) + if user is None: + raise ValueError(f"No such user: {user_id}") + return sensor, user - The returned payload intentionally avoids ORM instances (Sensor/DataSource), - which can break across process boundaries when pickled by RQ. - """ - bdfs: list[tb.BeliefsDataFrame] - if isinstance(data, list): - bdfs = data - else: - bdfs = [data] - - payload: list[dict] = [] - for bdf in bdfs: - # Normalize timing representation to belief_time for stable serialization. - bdf = bdf.convert_index_from_belief_horizon_to_time() - serialized_rows: list[dict] = [] - for belief in bdf.reset_index().itertuples(index=False): - serialized_rows.append( - { - "event_start": belief.event_start.isoformat(), - "belief_time": belief.belief_time.isoformat(), - "source_id": belief.source.id, - "cumulative_probability": float(belief.cumulative_probability), - "event_value": ( - None - if pd.isna(belief.event_value) - else float(belief.event_value) - ), - } - ) - - payload.append( - { - "sensor_id": bdf.sensor.id, - "beliefs": serialized_rows, - } - ) - return payload - - -def deserialize_ingestion_data(payload: Sequence[dict]) -> list[tb.BeliefsDataFrame]: - """Deserialize queue-safe ingestion payload back into BeliefsDataFrames.""" - - bdfs: list[tb.BeliefsDataFrame] = [] - for item in payload: - sensor = db.session.get(Sensor, item["sensor_id"]) - if sensor is None: - raise ValueError(f"No such sensor: {item['sensor_id']}") - - belief_rows = item.get("beliefs", []) - if not belief_rows: - bdfs.append(tb.BeliefsDataFrame(sensor=sensor)) - continue - - source_ids = sorted({row["source_id"] for row in belief_rows}) - sources = db.session.scalars( - select(DataSource).filter(DataSource.id.in_(source_ids)) - ).all() - source_map = {source.id: source for source in sources} - - beliefs: list[TimedBelief] = [] - for row in belief_rows: - source = source_map.get(row["source_id"]) - if source is None: - raise ValueError(f"No such source: {row['source_id']}") - beliefs.append( - TimedBelief( - sensor=sensor, - source=source, - event_start=pd.Timestamp(row["event_start"]), - belief_time=pd.Timestamp(row["belief_time"]), - cumulative_probability=row["cumulative_probability"], - event_value=row["event_value"], - ) - ) - bdfs.append(tb.BeliefsDataFrame(beliefs)) - - return bdfs +def _load_json_sensor_data( + sensor_id: int, + user_id: int, + sensor_data: dict, +) -> tb.BeliefsDataFrame: + """Validate and transform raw JSON sensor data into a BeliefsDataFrame.""" + + from flexmeasures.api.common.schemas.sensor_data import PostSensorDataSchema + + _sensor, user = _get_ingestion_context(sensor_id, user_id) + payload = dict(sensor_data) + payload.pop("id", None) + payload["sensor"] = sensor_id + return PostSensorDataSchema(source_user=user).load(payload)["bdf"] + + +def _file_storage_from_payload(file_payload: dict) -> FileStorage: + stream = BytesIO(file_payload["content"]) + stream.name = file_payload["filename"] + return FileStorage( + stream=stream, + filename=file_payload["filename"], + content_type=file_payload["content_type"], + ) + + +def _load_uploaded_sensor_data( + sensor_id: int, + user_id: int, + uploaded_files: list[dict], + upload_data: dict, +) -> list[tb.BeliefsDataFrame]: + """Validate and transform raw uploaded files into BeliefsDataFrames.""" + + from flexmeasures.data.schemas.sensors import SensorDataFileSchema + + _sensor, user = _get_ingestion_context(sensor_id, user_id) + payload = dict(upload_data) + payload["id"] = sensor_id + payload["uploaded-files"] = [ + _file_storage_from_payload(file_payload) for file_payload in uploaded_files + ] + return SensorDataFileSchema(source_user=user).load(payload)["data"] def add_beliefs_to_db_and_enqueue_forecasting_jobs( data: tb.BeliefsDataFrame | list[tb.BeliefsDataFrame] | None = None, - serialized_data: list[dict] | None = None, + sensor_id: int | None = None, + user_id: int | None = None, + sensor_data: dict | None = None, + uploaded_files: list[dict] | None = None, + upload_data: dict | None = None, forecasting_jobs: list[Job] | None = None, forecasting_job_ids: list[str] | None = None, save_changed_beliefs_only: bool = True, @@ -116,7 +90,11 @@ def add_beliefs_to_db_and_enqueue_forecasting_jobs( but can also be called directly (e.g. as a fallback when no workers are available). :param data: BeliefsDataFrame (or list thereof) to be saved. - :param serialized_data: Queue-safe payload containing only primitive types. + :param sensor_id: Sensor ID for raw JSON or file ingestion. + :param user_id: User ID used to resolve the source of raw ingested data. + :param sensor_data: Raw JSON payload from the sensor data endpoint. + :param uploaded_files: Uploaded file contents and metadata. + :param upload_data: Raw form payload from the sensor data upload endpoint. :param forecasting_jobs: Optional list of forecasting Jobs to enqueue after saving. :param forecasting_job_ids: Optional list of forecasting Job ids to enqueue after saving. :param save_changed_beliefs_only: If True, skip saving beliefs whose value hasn't changed. @@ -125,10 +103,21 @@ def add_beliefs_to_db_and_enqueue_forecasting_jobs( - 'success_with_unchanged_beliefs_skipped' - 'success_but_nothing_new' """ - if serialized_data is not None: - data = deserialize_ingestion_data(serialized_data) + if sensor_data is not None: + if sensor_id is None or user_id is None: + raise ValueError("Expected sensor_id and user_id for raw sensor data.") + data = _load_json_sensor_data(sensor_id, user_id, sensor_data) + elif uploaded_files is not None: + if sensor_id is None or user_id is None: + raise ValueError("Expected sensor_id and user_id for uploaded sensor data.") + data = _load_uploaded_sensor_data( + sensor_id, + user_id, + uploaded_files, + upload_data or {}, + ) if data is None: - raise ValueError("Expected either data or serialized_data.") + raise ValueError("Expected data, sensor_data, or uploaded_files.") status = save_to_db(data, save_changed_beliefs_only=save_changed_beliefs_only) db.session.commit() diff --git a/flexmeasures/data/tests/test_data_ingestion.py b/flexmeasures/data/tests/test_data_ingestion.py index 389e3e4e2b..7d5b46a04a 100644 --- a/flexmeasures/data/tests/test_data_ingestion.py +++ b/flexmeasures/data/tests/test_data_ingestion.py @@ -1,90 +1,19 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence - -import pandas.testing as pdt - from flexmeasures.data.services.data_ingestion import ( add_beliefs_to_db_and_enqueue_forecasting_jobs, - deserialize_ingestion_data, - serialize_ingestion_data, ) from flexmeasures.tests.utils import get_test_sensor -def _to_comparable_df(bdf): - df = bdf.convert_index_from_belief_horizon_to_time().reset_index() - df["source_id"] = df["source"].map(lambda s: s.id) - return ( - df[ - [ - "event_start", - "belief_time", - "source_id", - "cumulative_probability", - "event_value", - ] - ] - .sort_values( - [ - "event_start", - "belief_time", - "source_id", - "cumulative_probability", - ] - ) - .reset_index(drop=True) - ) - - -def _is_primitive_payload(value) -> bool: - if value is None or isinstance(value, (str, int, float, bool)): - return True - if isinstance(value, Mapping): - return all( - isinstance(k, str) and _is_primitive_payload(v) for k, v in value.items() - ) - if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): - return all(_is_primitive_payload(v) for v in value) - return False - - -def test_serialize_ingestion_data_uses_primitive_types(setup_beliefs, db): - sensor = get_test_sensor(db) - bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ - :2 - ] - - payload = serialize_ingestion_data(bdf) - - assert _is_primitive_payload(payload) - - -def test_ingestion_data_roundtrip_preserves_beliefs(setup_beliefs, db): - sensor = get_test_sensor(db) - bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ - :2 - ] - - payload = serialize_ingestion_data(bdf) - restored = deserialize_ingestion_data(payload) - - assert len(restored) == 1 - pdt.assert_frame_equal( - _to_comparable_df(restored[0]), - _to_comparable_df(bdf), - check_dtype=False, - ) - - -def test_ingestion_service_accepts_serialized_data(setup_beliefs, db): +def test_ingestion_service_accepts_beliefs_data_frame(setup_beliefs, db): sensor = get_test_sensor(db) bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False).iloc[ :1 ] status = add_beliefs_to_db_and_enqueue_forecasting_jobs( - serialized_data=serialize_ingestion_data(bdf), + data=bdf, save_changed_beliefs_only=True, ) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 5c65719b14..c3707c333b 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -500,7 +500,7 @@ }, "post": { "summary": "Post sensor data", - "description": "Send data values via JSON, where the duration and number of values determine the resolution.\n\nThe example request posts four values for a duration of one hour, where the first\nevent start is at the given start time, and subsequent events start in 15 minute intervals throughout the one hour duration.\n\nThe sensor is the one with ID=1.\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to upsample lower resolutions.\nThe list of values may include null values.\n", + "description": "Send data values via JSON, where the duration and number of values determine the resolution.\n\nThe example request posts four values for a duration of one hour, where the first\nevent start is at the given start time, and subsequent events start in 15 minute intervals throughout the one hour duration.\n\nThe sensor is the one with ID=1.\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to upsample lower resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", "security": [ { "ApiAuthKey": [] @@ -542,6 +542,9 @@ } }, "responses": { + "202": { + "description": "ACCEPTED" + }, "200": { "description": "PROCESSED" }, @@ -554,6 +557,9 @@ "403": { "description": "INVALID_SENDER" }, + "413": { + "description": "PAYLOAD_TOO_LARGE" + }, "422": { "description": "UNPROCESSABLE_ENTITY" } @@ -1537,7 +1543,7 @@ "/api/v3_0/sensors/{id}/data/upload": { "post": { "summary": "Upload sensor data by file", - "description": "The file should have columns for a timestamp (event_start) and a value (event_value).\nThe timestamp should be in ISO 8601 format.\nThe value should be a numeric value.\n\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to upsample lower resolutions.\nThe list of values may include null values.\n", + "description": "The file should have columns for a timestamp (event_start) and a value (event_value).\nThe timestamp should be in ISO 8601 format.\nThe value should be a numeric value.\n\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to upsample lower resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", "security": [ { "ApiKeyAuth": [] @@ -1568,6 +1574,9 @@ } }, "responses": { + "202": { + "description": "ACCEPTED" + }, "200": { "description": "PROCESSED", "content": { @@ -1639,6 +1648,9 @@ "403": { "description": "INVALID_SENDER" }, + "413": { + "description": "PAYLOAD_TOO_LARGE" + }, "422": { "description": "UNPROCESSABLE_ENTITY" } @@ -6187,4 +6199,4 @@ } } } -} +} \ No newline at end of file diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 9c1d63cc50..f22b3e1427 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -155,6 +155,7 @@ class Config(object): FLEXMEASURES_JOB_CACHE_TTL: int = ( 3600 # Time to live for the job caching keys in seconds. Set a negative timedelta to persist forever. ) + FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES: int | None = 3 * 1024 * 1024 FLEXMEASURES_TASK_CHECK_AUTH_TOKEN: str | None = None FLEXMEASURES_REDIS_URL: str = "localhost" FLEXMEASURES_REDIS_PORT: int = 6379 From 37c28234f32cb4fa57e1846f352282d9d0de41ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 13 May 2026 15:37:15 +0200 Subject: [PATCH 30/45] also log that data has been saved successfully, the logs are not clear otherwise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/utils.py b/flexmeasures/data/utils.py index 55c1ce2263..97ae3176ef 100644 --- a/flexmeasures/data/utils.py +++ b/flexmeasures/data/utils.py @@ -134,7 +134,7 @@ def save_to_db( # No state changes among the beliefs continue - current_app.logger.info("SAVING TO DB...") + current_app.logger.info("SAVING DATA ...") TimedBelief.add_to_session( session=db.session, beliefs_data_frame=timed_values, @@ -143,6 +143,7 @@ def save_to_db( "FLEXMEASURES_ALLOW_DATA_OVERWRITE", False ), ) + current_app.logger.info("SAVED DATA TO DB.") values_saved += len(timed_values) # Flush to bring up potential unique violations (due to attempting to replace beliefs) db.session.flush() From 8efd483784d3e3c7a4a4823faf85c3612fbda005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 13 May 2026 22:16:07 +0200 Subject: [PATCH 31/45] Improve poling and Toasting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/utils.py | 3 +- flexmeasures/ui/static/js/ui-utils.js | 3 + .../ui/templates/includes/graphs.html | 89 +++++++++++-------- .../ui/templates/includes/toasts.html | 35 ++++---- flexmeasures/ui/templates/sensors/index.html | 9 +- flexmeasures/utils/config_defaults.py | 2 +- 6 files changed, 79 insertions(+), 62 deletions(-) diff --git a/flexmeasures/data/utils.py b/flexmeasures/data/utils.py index 55c1ce2263..848aa735cf 100644 --- a/flexmeasures/data/utils.py +++ b/flexmeasures/data/utils.py @@ -134,7 +134,7 @@ def save_to_db( # No state changes among the beliefs continue - current_app.logger.info("SAVING TO DB...") + current_app.logger.info("SAVING TO DB ...") TimedBelief.add_to_session( session=db.session, beliefs_data_frame=timed_values, @@ -144,6 +144,7 @@ def save_to_db( ), ) values_saved += len(timed_values) + current_app.logger.info(f"SAVED {len(timed_values)} TO DB.") # Flush to bring up potential unique violations (due to attempting to replace beliefs) db.session.flush() diff --git a/flexmeasures/ui/static/js/ui-utils.js b/flexmeasures/ui/static/js/ui-utils.js index 4b28d051d2..799f675f56 100644 --- a/flexmeasures/ui/static/js/ui-utils.js +++ b/flexmeasures/ui/static/js/ui-utils.js @@ -496,6 +496,7 @@ export function initDeleteAssetButton() { * @param {string} [options.errorMessage] - Override toast text on failure (defaults to the * server message when absent). * @param {AbortSignal} [options.signal] - Abort polling externally (e.g. page unload). + * @param {function} [options.onStatus] - Called with the full response JSON on each poll. * @param {function} [options.onFinished] - Called with the full response JSON on finish. * @param {function} [options.onFailed] - Called with the full response JSON on failure. * @returns {function} stopPolling - Call to cancel polling manually. @@ -507,6 +508,7 @@ export function pollJobStatus(jobUuid, options = {}) { successMessage = "Job completed successfully.", errorMessage = null, signal = null, + onStatus = null, onFinished = null, onFailed = null, } = options; @@ -546,6 +548,7 @@ export function pollJobStatus(jobUuid, options = {}) { } const status = (data.status || "").toUpperCase(); + if (onStatus) onStatus(data); if (status === "FINISHED") { stop(); diff --git a/flexmeasures/ui/templates/includes/graphs.html b/flexmeasures/ui/templates/includes/graphs.html index 71a15296cd..603edd85cd 100644 --- a/flexmeasures/ui/templates/includes/graphs.html +++ b/flexmeasures/ui/templates/includes/graphs.html @@ -161,9 +161,14 @@ {% if active_page == "sensors" and user_can_update_sensor %} let upload = document.querySelector('#submitUpload'); let uploadSpinner = document.querySelector("#spinner-upload-sensor-data"); + function finishUploadStatus() { + uploadSpinner.classList.add('d-none'); + upload.disabled = false; + } upload.addEventListener('click', function (e) { - upload.classList.add('d-none'); + upload.disabled = true; uploadSpinner.classList.remove('d-none'); + showToast("Uploading data...", "info"); var formData = new FormData(document.getElementById('uploadForm')); fetch(apiBasePath + '/api/v3_0/sensors/' + {{ sensor.id }} + '/data/upload', { method: 'POST', @@ -172,69 +177,75 @@ .then(async function (response) { if (response.ok) { const data = await response.json(); - // Update date range - picker.setDateRange( - storeStartDate, - subtract(storeEndDate, 1), - ); - uploadSpinner.classList.add('d-none'); - upload.classList.remove('d-none'); - - if (data.job_id) { + if (response.status === 202 && data.job_id) { + const jobLink = data.job_monitor_url ? ` Monitor job` : ""; + showToast(`${data.message}${jobLink}`, "success", {delay: 10000}); // Track the background ingestion job and refresh when done. pollJobStatus(data.job_id, { processingMessage: "Upload received. Processing data in the background\u2026", successMessage: "Data processed successfully.", onFinished: () => { + finishUploadStatus(); document.dispatchEvent(new CustomEvent('newDataAvailable')); loadSensorStats({{ sensor.id }}, "", "", true); }, + onFailed: () => { + finishUploadStatus(); + }, }); } else { // No queued job: data was saved synchronously. showToast(data.message, "success"); + // Update date range + picker.setDateRange( + storeStartDate, + subtract(storeEndDate, 1), + ); document.dispatchEvent(new CustomEvent('newDataAvailable')); + finishUploadStatus(); } } else { const data = await response.json(); - const messageKeys = Object.keys(data.message); - messageKeys.forEach(mKey => { - if (data.message[mKey] === undefined) { - showToast(data.message, "error"); - } - else { - for (const [key, value] of Object.entries(data.message[mKey])) { - if (typeof value === 'string') { - showToast(value, "error"); - } else if (Array.isArray(value)) { - value.forEach(element => { - showToast(element, "error"); - }); - } else if (typeof value === 'object' && value !== null) { - Object.values(value).forEach(element => { - if (Array.isArray(element)) { - element.forEach(item => showToast(item, "error")); - } else { + if (typeof data.message === "string") { + showToast(data.message, "error"); + } else { + const messageKeys = Object.keys(data.message); + messageKeys.forEach(mKey => { + if (data.message[mKey] === undefined) { + showToast(data.message, "error"); + } + else { + for (const [key, value] of Object.entries(data.message[mKey])) { + if (typeof value === 'string') { + showToast(value, "error"); + } else if (Array.isArray(value)) { + value.forEach(element => { showToast(element, "error"); - } - }); - } else { - // fallback for unexpected types - showToast(String(value), "error"); + }); + } else if (typeof value === 'object' && value !== null) { + Object.values(value).forEach(element => { + if (Array.isArray(element)) { + element.forEach(item => showToast(item, "error")); + } else { + showToast(element, "error"); + } + }); + } else { + // fallback for unexpected types + showToast(String(value), "error"); + } } } - } - }); - uploadSpinner.classList.add('d-none'); - upload.classList.remove('d-none'); + }); + } + finishUploadStatus(); } }) .catch(function (error) { // Network or unexpected error console.error("Upload error:", error); showToast("An error occurred: " + error.message, "error"); - uploadSpinner.classList.add('d-none'); - upload.classList.remove('d-none'); + finishUploadStatus(); }); }); {% endif %} diff --git a/flexmeasures/ui/templates/includes/toasts.html b/flexmeasures/ui/templates/includes/toasts.html index d934f5fa0d..92c9ffc145 100644 --- a/flexmeasures/ui/templates/includes/toasts.html +++ b/flexmeasures/ui/templates/includes/toasts.html @@ -31,14 +31,6 @@ }); } - function showAllToasts() { - const toastElements = document.querySelectorAll(".toast"); - toastElements.forEach((toast) => { - const toastInstance = new bootstrap.Toast(toast); - toastInstance.show(); - }); - } - function maybeHideCloseToastBtn() { const remainingToasts = toastStack.querySelectorAll(".toast"); if (remainingToasts.length === 0) { @@ -55,27 +47,30 @@ * GLOBAL FUNCTION: showToast * Attached to window so it can be called from anywhere. */ - window.showToast = function(message, type, { highlightDuplicates = true, showDuplicateCount = true } = {}) { + window.showToast = function(message, type, { highlightDuplicates = true, showDuplicateCount = true, delay = null } = {}) { let colorClass; let colorStyle = ""; let title; - let delay; + let toastDelay; // Determine the type of toast if (type == "error") { - delay = 10000; + toastDelay = 10000; colorClass = "bg-danger"; title = "Error"; } else if (type == "success") { - delay = 2000; + toastDelay = 2000; colorClass = "bg-success"; title = "Success"; } else { - delay = 5000; + toastDelay = 5000; // JINJA VARIABLE WORKS HERE NOW: colorStyle = "background-color: {{ primary_color | default('#007bff') }};"; title = "Info"; } + if (delay !== null) { + toastDelay = delay; + } // Search for duplicate const existingToasts = toastStack.querySelectorAll(".toast"); @@ -107,7 +102,7 @@ const toast = document.createElement("div"); toast.classList.add("toast", "mb-1"); toast.setAttribute("data-bs-autohide", "true"); - toast.setAttribute("data-bs-delay", delay); + toast.setAttribute("data-bs-delay", toastDelay); toast.setAttribute("role", "alert"); toast.setAttribute("aria-live", "assertive"); toast.setAttribute("aria-atomic", "true"); @@ -118,14 +113,16 @@ ${title} -
- ${message} -
+
`; + const toastBody = toast.querySelector(".toast-body"); + toastBody.dataset.originalMessage = message; + toastBody.innerHTML = message; toastStack.insertAdjacentElement("afterbegin", toast); closeToastBtn.style.display = "block"; - showAllToasts(); + const toastInstance = new bootstrap.Toast(toast); + toastInstance.show(); // Cleanup listeners const handleClose = () => { @@ -139,4 +136,4 @@ }; })(); - \ No newline at end of file + diff --git a/flexmeasures/ui/templates/sensors/index.html b/flexmeasures/ui/templates/sensors/index.html index 3f23b2f1d7..d1145b6606 100644 --- a/flexmeasures/ui/templates/sensors/index.html +++ b/flexmeasures/ui/templates/sensors/index.html @@ -73,7 +73,12 @@

Upload {{ sensor.name }} data

- The resolution does not have to be the sensor resolution ― FlexMeasures will attempt to convert your data to the sensor resolution. Not all resolutions fit. What works best: Upsampling. - Data can have gaps, but FlexMeasures still needs to be able to guess the frequency. - Duplicates will be removed. -- For the unit of the data (e.g. kW), see help text on the unit selection below." +- For the unit of the data (e.g. kW), see help text on the unit selection below. +{% if config.FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES is not none -%} +- Uploads are limited to {{ "%.1f"|format(config.FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES / 1024 / 1024) }} MiB per request. +{%- else -%} +- This server has no FlexMeasures-specific upload size limit configured. +{%- endif %}" style="color: white; cursor: pointer;" > @@ -138,7 +143,7 @@

Upload {{ sensor.name }} data

style="margin-top: 20px; float: right; border: 1px solid var(--light-gray);"> Upload file -
- Select all data + Select all sensor data
+ {% endif %} + {% if user_can_create_children_sensor %}
Upload data
@@ -153,9 +155,6 @@

Upload {{ sensor.name }} data

- {% endif %} - - {% if user_can_create_children_sensor %}
Forecast
From 3002a1969af72e093b4b8f8f479667f974f3180f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 14 May 2026 22:44:11 +0200 Subject: [PATCH 38/45] refactor: utils work on any AuthModelMixin, not just on Asset Signed-off-by: F.N. Claessen --- flexmeasures/ui/views/assets/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flexmeasures/ui/views/assets/utils.py b/flexmeasures/ui/views/assets/utils.py index 72c5d3d036..fb6b1bbfa8 100644 --- a/flexmeasures/ui/views/assets/utils.py +++ b/flexmeasures/ui/views/assets/utils.py @@ -4,7 +4,7 @@ from flask_security import current_user from werkzeug.exceptions import NotFound -from flexmeasures.auth.policy import check_access +from flexmeasures.auth.policy import AuthModelMixin, check_access from flexmeasures.data import db from flexmeasures import Asset from flexmeasures.data.models.generic_assets import GenericAsset @@ -32,25 +32,25 @@ def user_can_create_assets(account: Account | None = None) -> bool: return True -def user_can_create_children(asset: GenericAsset) -> bool: +def user_can_create_children(context: AuthModelMixin) -> bool: try: - check_access(asset, "create-children") + check_access(context, "create-children") except Exception: return False return True -def user_can_delete(asset: GenericAsset) -> bool: +def user_can_delete(context: AuthModelMixin) -> bool: try: - check_access(asset, "delete") + check_access(context, "delete") except Exception: return False return True -def user_can_update(asset: GenericAsset) -> bool: +def user_can_update(context: AuthModelMixin) -> bool: try: - check_access(asset, "update") + check_access(context, "update") except Exception: return False return True From 7ea92ad98d2b3f085012d19e8b96decf96eda75b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 14 May 2026 22:52:02 +0200 Subject: [PATCH 39/45] refactor: move auth utils to module that is not asset specific Signed-off-by: F.N. Claessen --- flexmeasures/ui/utils/auth_utils.py | 40 ++++++++++++++++++++++++ flexmeasures/ui/views/assets/__init__.py | 2 +- flexmeasures/ui/views/assets/utils.py | 39 +---------------------- flexmeasures/ui/views/assets/views.py | 6 ++-- flexmeasures/ui/views/sensors.py | 12 +++---- 5 files changed, 52 insertions(+), 47 deletions(-) create mode 100644 flexmeasures/ui/utils/auth_utils.py diff --git a/flexmeasures/ui/utils/auth_utils.py b/flexmeasures/ui/utils/auth_utils.py new file mode 100644 index 0000000000..935e447ee5 --- /dev/null +++ b/flexmeasures/ui/utils/auth_utils.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from flask_login import current_user + +from flexmeasures import Account +from flexmeasures.auth.policy import check_access, AuthModelMixin + + +def user_can_create_assets(account: Account | None = None) -> bool: + if account is None: + account = current_user.account + try: + check_access(account, "create-children") + except Exception: + return False + return True + + +def user_can_create_children(context: AuthModelMixin) -> bool: + try: + check_access(context, "create-children") + except Exception: + return False + return True + + +def user_can_delete(context: AuthModelMixin) -> bool: + try: + check_access(context, "delete") + except Exception: + return False + return True + + +def user_can_update(context: AuthModelMixin) -> bool: + try: + check_access(context, "update") + except Exception: + return False + return True diff --git a/flexmeasures/ui/views/assets/__init__.py b/flexmeasures/ui/views/assets/__init__.py index 242e9633de..ba4e2f532c 100644 --- a/flexmeasures/ui/views/assets/__init__.py +++ b/flexmeasures/ui/views/assets/__init__.py @@ -1,5 +1,5 @@ from flexmeasures.ui.views.assets.forms import AssetForm, NewAssetForm # noqa: F401 -from flexmeasures.ui.views.assets.utils import ( # noqa: F401 +from flexmeasures.ui.utils.auth_utils import ( # noqa: F401 user_can_create_assets, user_can_delete, user_can_update, diff --git a/flexmeasures/ui/views/assets/utils.py b/flexmeasures/ui/views/assets/utils.py index fb6b1bbfa8..b7bfdbda22 100644 --- a/flexmeasures/ui/views/assets/utils.py +++ b/flexmeasures/ui/views/assets/utils.py @@ -1,14 +1,11 @@ from __future__ import annotations from flask import url_for -from flask_security import current_user from werkzeug.exceptions import NotFound -from flexmeasures.auth.policy import AuthModelMixin, check_access -from flexmeasures.data import db from flexmeasures import Asset +from flexmeasures.data import db from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.models.user import Account from flexmeasures.ui.utils.view_utils import svg_asset_icon_name @@ -22,40 +19,6 @@ def get_asset_by_id_or_raise_notfound(asset_id: str) -> GenericAsset: return asset -def user_can_create_assets(account: Account | None = None) -> bool: - if account is None: - account = current_user.account - try: - check_access(account, "create-children") - except Exception: - return False - return True - - -def user_can_create_children(context: AuthModelMixin) -> bool: - try: - check_access(context, "create-children") - except Exception: - return False - return True - - -def user_can_delete(context: AuthModelMixin) -> bool: - try: - check_access(context, "delete") - except Exception: - return False - return True - - -def user_can_update(context: AuthModelMixin) -> bool: - try: - check_access(context, "update") - except Exception: - return False - return True - - def serialize_asset(asset: Asset, is_head=False) -> dict: serialized_asset = { "name": asset.name, diff --git a/flexmeasures/ui/views/assets/views.py b/flexmeasures/ui/views/assets/views.py index ea04664539..384661fa6e 100644 --- a/flexmeasures/ui/views/assets/views.py +++ b/flexmeasures/ui/views/assets/views.py @@ -30,12 +30,14 @@ ATTRIBUTES_FIELD_LABEL, ATTRIBUTES_FIELD_DESCRIPTION, ) -from flexmeasures.ui.views.assets.utils import ( - get_asset_by_id_or_raise_notfound, +from flexmeasures.ui.utils.auth_utils import ( user_can_create_assets, user_can_create_children, user_can_delete, user_can_update, +) +from flexmeasures.ui.views.assets.utils import ( + get_asset_by_id_or_raise_notfound, get_list_assets_chart, add_child_asset, ) diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index 40c2f5bfbe..8cb473b368 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -10,16 +10,16 @@ from flexmeasures.data.schemas import StartEndTimeSchema from flexmeasures.data.services.timerange import get_timerange from flexmeasures import Sensor -from flexmeasures.ui.utils.view_utils import ( - render_flexmeasures_template, - available_units, -) -from flexmeasures.ui.utils.breadcrumb_utils import get_breadcrumb_info -from flexmeasures.ui.views.assets.utils import ( +from flexmeasures.ui.utils.auth_utils import ( user_can_create_children, user_can_delete, user_can_update, ) +from flexmeasures.ui.utils.breadcrumb_utils import get_breadcrumb_info +from flexmeasures.ui.utils.view_utils import ( + render_flexmeasures_template, + available_units, +) from flexmeasures.ui.views import ( ATTRIBUTES_FIELD_LABEL, ATTRIBUTES_FIELD_DESCRIPTION, From fef8dacca877ca84e504a640fbf338c11e0e056c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 14 May 2026 23:28:59 +0200 Subject: [PATCH 40/45] remove Toast about job acceptance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/ui/templates/includes/graphs.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/flexmeasures/ui/templates/includes/graphs.html b/flexmeasures/ui/templates/includes/graphs.html index 7cfb71a1bc..cde70a3344 100644 --- a/flexmeasures/ui/templates/includes/graphs.html +++ b/flexmeasures/ui/templates/includes/graphs.html @@ -181,8 +181,6 @@ if (response.ok) { const data = await response.json(); if (response.status === 202 && data.job_id) { - const jobLink = data.job_monitor_url ? ` Monitor job` : ""; - showToast(`${data.message}${jobLink}`, "success", {delay: 10000}); // Track the background ingestion job and refresh when done. pollJobStatus(data.job_id, { processingMessage: "Upload received. Processing data in the background\u2026", From dced65986cec0c9eb018f7f4c24725c700c23c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 14 May 2026 23:56:16 +0200 Subject: [PATCH 41/45] in uploaded data, we (keep) allow(ing) multiple beliefs per event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/schemas/sensors.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 2ceeb2ac0c..70b624fe18 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -737,12 +737,10 @@ def post_load(self, fields, **kwargs): bdf = bdf.resample_events( sensor.event_resolution, method="sum", - keep_only_most_recent_belief=True, ) else: bdf = bdf.resample_events( sensor.event_resolution, - keep_only_most_recent_belief=True, ) dfs.append(bdf) except Exception as e: From 38169949e061f6d42a3a8402914cdd2ff6bdfa80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 14 May 2026 23:58:20 +0200 Subject: [PATCH 42/45] add comment on default ingestion limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/utils/config_defaults.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 5fdd7228ba..e61b92fbe2 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -155,7 +155,9 @@ class Config(object): FLEXMEASURES_JOB_CACHE_TTL: int = ( 3600 # Time to live for the job caching keys in seconds. Set a negative timedelta to persist forever. ) - FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES: int | None = 3.3 * 1024 * 1024 + FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES: int | None = ( + 3.1 * 1024 * 1024 + ) # up to 3MB are allowed per request FLEXMEASURES_TASK_CHECK_AUTH_TOKEN: str | None = None FLEXMEASURES_REDIS_URL: str = "localhost" FLEXMEASURES_REDIS_PORT: int = 6379 From 1bd5ac8c36bc7c352f0cba01c6ae9f7c168b0a3c Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 19 May 2026 14:15:34 +0100 Subject: [PATCH 43/45] feat: adding TTL to ingestion jobs Signed-off-by: joshuaunity --- flexmeasures/api/common/utils/api_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 689e7a06e0..7bef373bfa 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -210,6 +210,14 @@ def process_sensor_data_ingestion( forecasting_job_ids=forecasting_job_ids, save_changed_beliefs_only=save_changed_beliefs_only, meta={"sensor_id": sensor_id}, + ttl=current_app.config.get( + "FLEXMEASURES_JOB_TTL", timedelta(-1) + ).total_seconds(), + result_ttl=int( + current_app.config.get( + "FLEXMEASURES_JOB_TTL", timedelta(-1) + ).total_seconds() + ), ) return request_accepted_for_processing( job.id, From 9328aaf0d0085559008f409e8ffaf358913e20a9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 11:45:34 +0200 Subject: [PATCH 44/45] docs: add comment explaining the TTL choice for ingestion Signed-off-by: F.N. Claessen --- flexmeasures/api/common/utils/api_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 7bef373bfa..9633ed0e2a 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -213,6 +213,7 @@ def process_sensor_data_ingestion( ttl=current_app.config.get( "FLEXMEASURES_JOB_TTL", timedelta(-1) ).total_seconds(), + # No need to keep ingestion results for the FLEXMEASURES_PLANNING_TTL result_ttl=int( current_app.config.get( "FLEXMEASURES_JOB_TTL", timedelta(-1) From 4d55596eedab8ce5dc56d985dab8203881038d4c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 13:20:44 +0200 Subject: [PATCH 45/45] fix: put back filters for the sake of UX (speed) Related revisions: - 513a81ecc32e9da985fa6c9cd676ec1b4e83296b - dced65986cec0c9eb018f7f4c24725c700c23c74 Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 70b624fe18..2ceeb2ac0c 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -737,10 +737,12 @@ def post_load(self, fields, **kwargs): bdf = bdf.resample_events( sensor.event_resolution, method="sum", + keep_only_most_recent_belief=True, ) else: bdf = bdf.resample_events( sensor.event_resolution, + keep_only_most_recent_belief=True, ) dfs.append(bdf) except Exception as e: