From 670f32ef34519c7a0cf5ff1af0c04e8fee0017a5 Mon Sep 17 00:00:00 2001 From: knx-ai Date: Sun, 30 Aug 2026 23:39:04 +0200 Subject: [PATCH 1/2] feat: KNX device programming, knxproj support, and GUI improvements Add xknx-download: programs applications and individual addresses into real KNX devices over a live bus (Load State Machine, chunked memory/property writes, group communication tables, master reset, function properties, APDU-length negotiation, device mask guard) with a read-only preflight. Verified on real hardware. Add knxproj import/export with per-folder signing to xknx-project, ship the KNX project schema in xknx-models, and extend the parameter encoder (byte order, enum binary values, color/date/time/raw-data). knx-gui: knxproj open/save, Test Before Programming, online catalog browsing, keyring and monitor plugins, and assorted improvements. --- .gitignore | 1 + README.md | 19 +- apps/knx-gui/.gitignore | 1 + apps/knx-gui/pyproject.toml | 2 + apps/knx-gui/src/knx_gui/concurrency.py | 38 + apps/knx-gui/src/knx_gui/device.py | 5 + apps/knx-gui/src/knx_gui/dpt.py | 19 + .../knx_gui/locales/de/LC_MESSAGES/knx_gui.mo | Bin 1080 -> 1143 bytes .../knx_gui/locales/de/LC_MESSAGES/knx_gui.po | 3 + apps/knx-gui/src/knx_gui/main.py | 389 +- .../catalog/locales/de/LC_MESSAGES/catalog.mo | Bin 442 -> 699 bytes .../catalog/locales/de/LC_MESSAGES/catalog.po | 12 + .../catalog/locales/nl/LC_MESSAGES/catalog.mo | Bin 422 -> 683 bytes .../catalog/locales/nl/LC_MESSAGES/catalog.po | 12 + .../knx_gui/plugins/catalog/online_catalog.py | 138 + .../src/knx_gui/plugins/catalog/plugin.py | 12 + .../src/knx_gui/plugins/catalog/service.py | 35 +- .../src/knx_gui/plugins/catalog/strings.py | 16 + .../knx_gui/plugins/catalog/tests/__init__.py | 1 + .../catalog/tests/test_online_catalog.py | 100 + .../src/knx_gui/plugins/catalog/ui/panel.py | 67 + .../src/knx_gui/plugins/connection/plugin.py | 30 + .../src/knx_gui/plugins/connection/service.py | 203 +- .../plugins/connection/tests/__init__.py | 1 + .../connection/tests/test_connection_guard.py | 67 + .../src/knx_gui/plugins/keyring/__init__.py | 3 + .../src/knx_gui/plugins/keyring/plugin.py | 36 + .../src/knx_gui/plugins/keyring/service.py | 43 + .../src/knx_gui/plugins/keyring/strings.py | 65 + .../knx-gui/src/knx_gui/plugins/keyring/ui.py | 179 + .../src/knx_gui/plugins/logger/strings.py | 4 + apps/knx-gui/src/knx_gui/plugins/logger/ui.py | 42 +- .../src/knx_gui/plugins/monitor/__init__.py | 3 + .../src/knx_gui/plugins/monitor/plugin.py | 39 + .../src/knx_gui/plugins/monitor/service.py | 115 + .../src/knx_gui/plugins/monitor/strings.py | 45 + .../knx-gui/src/knx_gui/plugins/monitor/ui.py | 136 + .../knx_gui/plugins/node_editor/strings.py | 4 + .../plugins/project/knxproj_manufacturer.py | 138 + .../project/locales/de/LC_MESSAGES/project.mo | Bin 2838 -> 3149 bytes .../project/locales/de/LC_MESSAGES/project.po | 3 + .../src/knx_gui/plugins/project/plugin.py | 217 +- .../src/knx_gui/plugins/project/service.py | 195 +- .../src/knx_gui/plugins/project/strings.py | 278 + .../knx_gui/plugins/project/ui/__init__.py | 6 + .../src/knx_gui/plugins/project/ui/_filter.py | 23 + .../knx_gui/plugins/project/ui/configure.py | 221 +- .../src/knx_gui/plugins/project/ui/devices.py | 67 +- .../plugins/project/ui/group_addresses.py | 247 + .../plugins/project/ui/preflight_result.py | 285 + .../plugins/project/ui/project_info.py | 43 + .../src/knx_gui/plugins/project/ui/spaces.py | 151 + apps/knx-gui/src/knx_gui/programming.py | 113 + apps/knx-gui/src/knx_gui/settings.py | 45 + apps/knx-gui/src/knx_gui/strings/__init__.py | 80 + apps/knx-gui/src/knx_gui/widgets/__init__.py | 2 + .../knx_gui/widgets/group_objects_widgets.py | 144 + .../src/xknxmono/catalog/core/hardware.py | 24 + .../src/xknxmono/catalog/core/service.py | 6 + packages/download/CHANGELOG.md | 33 + packages/download/README.md | 147 + packages/download/pyproject.toml | 19 + .../src/xknxmono/download/__init__.py | 90 + .../src/xknxmono/download/commissioning.py | 46 + .../download/src/xknxmono/download/crc.py | 34 + .../src/xknxmono/download/download.py | 194 + .../download/src/xknxmono/download/errors.py | 23 + .../download/src/xknxmono/download/gaps.py | 85 + .../xknxmono/download/group_communication.py | 68 + .../download/src/xknxmono/download/image.py | 520 ++ .../src/xknxmono/download/load_state.py | 187 + .../download/src/xknxmono/download/merge.py | 116 + .../src/xknxmono/download/preflight.py | 148 + .../src/xknxmono/download/procedure.py | 928 +++ .../src/xknxmono/download/programmer.py | 454 ++ .../src/xknxmono/download/project_data.py | 166 + .../download/src/xknxmono/download/py.typed | 0 .../src/xknxmono/download/resources.py | 156 + .../download/src/xknxmono/download/scope.py | 66 + .../download/src/xknxmono/download/tables.py | 212 + .../src/xknxmono/download/tables_systemb.py | 124 + packages/download/tests/__init__.py | 0 packages/download/tests/conftest.py | 172 + packages/download/tests/test_commissioning.py | 46 + packages/download/tests/test_crc.py | 34 + packages/download/tests/test_gaps.py | 77 + .../tests/test_group_communication.py | 59 + packages/download/tests/test_image.py | 82 + packages/download/tests/test_load_state.py | 116 + packages/download/tests/test_merge.py | 102 + packages/download/tests/test_preflight.py | 257 + packages/download/tests/test_procedure.py | 709 ++ packages/download/tests/test_programmer.py | 154 + packages/download/tests/test_resources.py | 78 + packages/download/tests/test_scope.py | 42 + packages/download/tests/test_tables.py | 103 + .../download/tests/test_tables_systemb.py | 99 + .../src/xknxmono/models/schemas/.gitignore | 3 + .../src/xknxmono/models/schemas/README.md | 20 + .../models/schemas/knx_project_v23.xsd | 5997 +++++++++++++++++ .../src/xknxmono/product/parser_v2/dynamic.py | 12 + .../src/xknxmono/product/parser_v2/encode.py | 444 +- .../parser_v2/nodes/com_object_ref_ref.py | 9 + .../product/parser_v2/ui/com_object.py | 4 + .../tests/parser_v2/test_encode_types.py | 228 + packages/project/CHANGELOG.md | 3 +- packages/project/pyproject.toml | 1 + .../project/src/xknxmono/project/__init__.py | 4 +- .../src/xknxmono/project/core/__init__.py | 4 +- .../src/xknxmono/project/core/events.py | 32 + .../xknxmono/project/core/knxproj_export.py | 332 + .../xknxmono/project/core/knxproj_import.py | 364 + .../xknxmono/project/core/knxproj_signing.py | 71 + .../src/xknxmono/project/core/service.py | 182 + .../project/src/xknxmono/project/models.py | 98 + packages/project/tests/test_knxproj_export.py | 144 + packages/project/tests/test_knxproj_import.py | 393 ++ .../project/tests/test_knxproj_signing.py | 41 + packages/project/tests/test_project_core.py | 9 + pyproject.toml | 2 + tools/gen_mask_resources.py | 58 + uv.lock | 94 +- 122 files changed, 18456 insertions(+), 212 deletions(-) create mode 100644 apps/knx-gui/src/knx_gui/concurrency.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/catalog/online_catalog.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/catalog/tests/__init__.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/catalog/tests/test_online_catalog.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/connection/tests/__init__.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/connection/tests/test_connection_guard.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/keyring/__init__.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/keyring/plugin.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/keyring/service.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/keyring/strings.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/keyring/ui.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/monitor/__init__.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/monitor/plugin.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/monitor/service.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/monitor/strings.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/monitor/ui.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/project/knxproj_manufacturer.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/project/ui/_filter.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/project/ui/group_addresses.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/project/ui/preflight_result.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/project/ui/project_info.py create mode 100644 apps/knx-gui/src/knx_gui/plugins/project/ui/spaces.py create mode 100644 apps/knx-gui/src/knx_gui/programming.py create mode 100644 apps/knx-gui/src/knx_gui/settings.py create mode 100644 apps/knx-gui/src/knx_gui/widgets/group_objects_widgets.py create mode 100644 packages/download/CHANGELOG.md create mode 100644 packages/download/README.md create mode 100644 packages/download/pyproject.toml create mode 100644 packages/download/src/xknxmono/download/__init__.py create mode 100644 packages/download/src/xknxmono/download/commissioning.py create mode 100644 packages/download/src/xknxmono/download/crc.py create mode 100644 packages/download/src/xknxmono/download/download.py create mode 100644 packages/download/src/xknxmono/download/errors.py create mode 100644 packages/download/src/xknxmono/download/gaps.py create mode 100644 packages/download/src/xknxmono/download/group_communication.py create mode 100644 packages/download/src/xknxmono/download/image.py create mode 100644 packages/download/src/xknxmono/download/load_state.py create mode 100644 packages/download/src/xknxmono/download/merge.py create mode 100644 packages/download/src/xknxmono/download/preflight.py create mode 100644 packages/download/src/xknxmono/download/procedure.py create mode 100644 packages/download/src/xknxmono/download/programmer.py create mode 100644 packages/download/src/xknxmono/download/project_data.py create mode 100644 packages/download/src/xknxmono/download/py.typed create mode 100644 packages/download/src/xknxmono/download/resources.py create mode 100644 packages/download/src/xknxmono/download/scope.py create mode 100644 packages/download/src/xknxmono/download/tables.py create mode 100644 packages/download/src/xknxmono/download/tables_systemb.py create mode 100644 packages/download/tests/__init__.py create mode 100644 packages/download/tests/conftest.py create mode 100644 packages/download/tests/test_commissioning.py create mode 100644 packages/download/tests/test_crc.py create mode 100644 packages/download/tests/test_gaps.py create mode 100644 packages/download/tests/test_group_communication.py create mode 100644 packages/download/tests/test_image.py create mode 100644 packages/download/tests/test_load_state.py create mode 100644 packages/download/tests/test_merge.py create mode 100644 packages/download/tests/test_preflight.py create mode 100644 packages/download/tests/test_procedure.py create mode 100644 packages/download/tests/test_programmer.py create mode 100644 packages/download/tests/test_resources.py create mode 100644 packages/download/tests/test_scope.py create mode 100644 packages/download/tests/test_tables.py create mode 100644 packages/download/tests/test_tables_systemb.py create mode 100644 packages/models/src/xknxmono/models/schemas/README.md create mode 100644 packages/models/src/xknxmono/models/schemas/knx_project_v23.xsd create mode 100644 packages/product/tests/parser_v2/test_encode_types.py create mode 100644 packages/project/src/xknxmono/project/core/knxproj_export.py create mode 100644 packages/project/src/xknxmono/project/core/knxproj_import.py create mode 100644 packages/project/src/xknxmono/project/core/knxproj_signing.py create mode 100644 packages/project/tests/test_knxproj_export.py create mode 100644 packages/project/tests/test_knxproj_import.py create mode 100644 packages/project/tests/test_knxproj_signing.py create mode 100644 tools/gen_mask_resources.py diff --git a/.gitignore b/.gitignore index 607edbd9..83a6f7c8 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ htmlcov/ .DS_Store NodeEditor.json XKNX_Toolkit.ini +.references scan_repeats.py diff --git a/README.md b/README.md index bf7dec6d..e393468c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # XKNX Toolkit > [!WARNING] -> **Alpha, experimental software.** XKNX Toolkit is not intended for end users and comes with no stability or safety guarantees — expect breaking changes, rough edges, and bugs. It's mostly useful for developers experimenting with the [xknx](https://github.com/XKNX/xknx) library. It cannot program devices, and no support is offered to end users. Contributions are welcome. Large parts of this project were built using LLMs. +> **Alpha, experimental software.** XKNX Toolkit is not intended for end users and comes with no stability or safety guarantees — expect breaking changes, rough edges, and bugs. It's mostly useful for developers experimenting with the [xknx](https://github.com/XKNX/xknx) library. It can program real KNX devices (see below), but that path is experimental — always review the change set with a preflight first and keep a way to restore the device. No support is offered to end users. Contributions are welcome. Large parts of this project were built using LLMs. XKNX Toolkit is a desktop application and set of Python libraries for working with [KNX](https://www.knx.org/) home and building automation installations — browsing product catalogs, editing installation projects, and talking to real or simulated KNX devices. @@ -15,16 +15,20 @@ Devices and group addresses are laid out as nodes on a canvas. Communication obj ### Project management -Devices are organized by area/line/segment, matching standard KNX topology. Each device's parameters, com object flags, load procedures, and raw memory layout can be inspected and edited directly. +Devices are organized by area/line/segment, matching standard KNX topology. Each device's parameters, com object flags, load procedures, and raw memory layout can be inspected and edited directly. Projects import from and export to ETS `.knxproj` archives: the export bundles the referenced manufacturer data (verbatim from the original `.knxprod` archives) and a merged `knx_master.xml`, so applications resolve from the archive alone. The project can be browsed like ETS through the device topology, the group address tree with assignments, and the building/space tree with functions. ### Product catalog -Import `.knxprod` archives to build up a searchable catalog of manufacturers, hardware, and application programs, independent of any single project. Catalog entries can be dragged into a project as new devices. +Import `.knxprod` archives to build up a searchable catalog of manufacturers, hardware, and application programs, independent of any single project. Catalog entries can be dragged into a project as new devices. The GUI also connects to the **KNX online catalog** (the same anonymous service ETS uses — no account, no license check on the service side) and can browse and refresh the manufacturer list, cached locally so it stays available offline; downloading the actual products from the online catalog is not implemented yet. ### Real KNX connections Connect to a real KNX interface over tunneling (TCP/UDP) or routing (multicast), with automatic gateway discovery or manual IP entry. +### Programming real devices + +The toolkit can commission a physical KNX device end-to-end — the job ETS does on "Download" — without ETS. It assembles the device's memory image from the product database and your parameter and group-address choices, then executes the application's Load Procedure over a live bus: driving each loadable part's Load State Machine, writing memory and properties, laying down the group communication tables, and restarting the device. It also programs a virgin device's individual address. A read-only **preflight** reads back every location a write would touch and shows the exact diff before anything is written. **Test Before Programming** runs that preflight and reports per memory segment and property whether the generated image matches what is already programmed on the device (with the current and planned bytes exportable as a report), guarding against writing a wrongly generated image. Programming requires a live connection; every connection-dependent action (test, program, sending frames) is refused with a visible "no KNX connection" notice when the bus is not linked. This is a vendor-independent implementation derived from the KNX Standard v3.0.0, verified on real hardware; it lives in the `xknx-download` package and can be used without the GUI. + ### Virtual devices and proxy — test without hardware No KNX interface on hand? XKNX Toolkit can stand in for one: @@ -45,9 +49,13 @@ Only running from source is supported for now — this is developer-focused soft ```bash uv sync -uv run python -m knx_gui.main +uv run --package knx-gui python -m knx_gui.main ``` +`knx-gui` is a workspace member that the root project does not depend on, so a plain `uv sync` does +not install it (or its `imgui-bundle` dependency). The `--package knx-gui` flag runs it in — and +installs it — from the workspace. + ## Packages The application is built on a set of standalone, typed Python libraries (the `xknxmono` namespace) that can also be used independently of the GUI: @@ -59,9 +67,10 @@ The application is built on a set of standalone, typed Python libraries (the `xk | `xknx-catalog` | `xknxmono.catalog` | Product catalog built from imported `.knxprod` archives | | `xknx-project` | `xknxmono.project` | Project state management for KNX installations | | `xknx-keyring` | `xknxmono.keyring` | Parses and serializes KNX keyring XML (KNX IP Secure keys) | +| `xknx-download` | `xknxmono.download` | Programs applications and individual addresses into real KNX devices | ```bash -pip install xknx-models xknx-product xknx-catalog xknx-project xknx-keyring +pip install xknx-models xknx-product xknx-catalog xknx-project xknx-keyring xknx-download ``` ## Development diff --git a/apps/knx-gui/.gitignore b/apps/knx-gui/.gitignore index 158e1fd6..c5b86dcb 100644 --- a/apps/knx-gui/.gitignore +++ b/apps/knx-gui/.gitignore @@ -3,5 +3,6 @@ *.xknx knxprod/ NodeEditor.json +online_catalog_manufacturers.json .venv/ internal/ diff --git a/apps/knx-gui/pyproject.toml b/apps/knx-gui/pyproject.toml index 9a7a63ff..af820453 100644 --- a/apps/knx-gui/pyproject.toml +++ b/apps/knx-gui/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "structlog>=24.0.0", "xknx>=3.18.0", "xknx-catalog", + "xknx-download", "xknx-models", "xknx-product", "xknx-project", @@ -31,6 +32,7 @@ packages = ["src/knx_gui"] [tool.uv.sources] xknx-catalog = { workspace = true } +xknx-download = { workspace = true } xknx-models = { workspace = true } xknx-product = { workspace = true } xknx-project = { workspace = true } diff --git a/apps/knx-gui/src/knx_gui/concurrency.py b/apps/knx-gui/src/knx_gui/concurrency.py new file mode 100644 index 00000000..3a90a92b --- /dev/null +++ b/apps/knx-gui/src/knx_gui/concurrency.py @@ -0,0 +1,38 @@ +"""A tiny guard for service reads during a background import. + +A long ``.knxproj`` import runs on a worker thread that mutates the catalog and project databases. +The GUI panels read those same services every frame on the UI thread. To keep the UI responsive +without racing the writer, the importer holds a shared *re-entrant* lock for the whole import, and +every per-frame service read is wrapped with :func:`io_guarded`: it acquires the lock without +blocking and, if the importer holds it, returns an empty placeholder instead of touching the +database. Because the lock is re-entrant, the importing thread itself (which already holds it) still +reads real data while building the project view. +""" + +from __future__ import annotations + +from collections.abc import Callable +from functools import wraps +from typing import Any + + +def io_guarded[T]( + default_factory: Callable[[], T], +) -> Callable[[Callable[..., T]], Callable[..., T]]: + """Return the wrapped read's result, or ``default_factory()`` if a background import is running. + + The instance must expose a re-entrant ``self._io_lock`` (``threading.RLock``).""" + + def decorator(fn: Callable[..., T]) -> Callable[..., T]: + @wraps(fn) + def wrapper(self: Any, *args: Any, **kwargs: Any) -> T: + if not self._io_lock.acquire(blocking=False): + return default_factory() + try: + return fn(self, *args, **kwargs) + finally: + self._io_lock.release() + + return wrapper + + return decorator diff --git a/apps/knx-gui/src/knx_gui/device.py b/apps/knx-gui/src/knx_gui/device.py index 5c66f767..6f1c02e3 100644 --- a/apps/knx-gui/src/knx_gui/device.py +++ b/apps/knx-gui/src/knx_gui/device.py @@ -306,3 +306,8 @@ def find_com_object(self, co_id: str) -> ComObject | None: if co.id == co_id: return co return None + + @property + def dynamic_ui(self) -> DynamicUI | None: + """The live evaluator holding this device's current parameter state.""" + return self._dynamic_ui diff --git a/apps/knx-gui/src/knx_gui/dpt.py b/apps/knx-gui/src/knx_gui/dpt.py index 1fdef763..4817d7ff 100644 --- a/apps/knx-gui/src/knx_gui/dpt.py +++ b/apps/knx-gui/src/knx_gui/dpt.py @@ -43,6 +43,25 @@ def lookup_or_make_dpt(code: str | None) -> DPT: return DPT(major, minor, f"DPT {major}.{minor:03d}", code) +@lru_cache(maxsize=512) +def transcoder_for(dpt_str: str | None) -> type[DPTBase] | None: + """Resolve an xknx transcoder from a project DPT string (``"DPST-1-1"`` / ``"DPT-5"``). + + The project stores ETS-token DPTs, but xknx's ``parse_transcoder`` wants dotted ``"1.1"`` (or + ``"DPT-1"`` for main-only). We convert and fall back to the main-only transcoder.""" + if not dpt_str: + return None + parts = dpt_str.split("-") + if parts[0] == "DPST" and len(parts) >= 3: + transcoder = DPTBase.parse_transcoder(f"{parts[1]}.{parts[2]}") + if transcoder is not None: + return transcoder + return DPTBase.parse_transcoder(f"DPT-{parts[1]}") + if parts[0] == "DPT" and len(parts) >= 2: + return DPTBase.parse_transcoder(f"DPT-{parts[1]}") + return DPTBase.parse_transcoder(dpt_str) + + DPT_MAJOR_COLORS: dict[int, imgui.ImVec4] = { 1: imgui.ImVec4(0.9, 0.3, 0.3, 1.0), 2: imgui.ImVec4(0.9, 0.5, 0.5, 1.0), diff --git a/apps/knx-gui/src/knx_gui/locales/de/LC_MESSAGES/knx_gui.mo b/apps/knx-gui/src/knx_gui/locales/de/LC_MESSAGES/knx_gui.mo index d619bf5576247ee223002458733a2d2de10181b3..afbfa85576d9a9b62877bee290cca242db5d1d55 100644 GIT binary patch delta 577 zcmXxhF-rqM5QgEsB$`W1)Ff&Yf~K$#L2MMU(L%&R2o@3~ot{UyW&>)l39{Sk8EcH#`5;jg4bB<{?T%g|Y3f1`ys?kTx<10?! z2hQLZj^V)N({!x{DmaA;sB88X>Fsn{)nlyP(eF%c5!ei6uOG5;Gjb(2pJuV!NFY!f-e36E)I3* z>gM3?WOg?@h@*a=>){J;KDk`--g}pyXc}cjT)s78lWdc3}`76-d$`m-^DfT;j&p~N8D`j-~?-Uf$MmMDwzB6ogY7XpHS~VV*_7s zAK#I}R;O#tw@{6DF~U7u!UH$X%m)6!Ay%0RZqK6Bayd(YPf>|>$~I72MQI`WK3!?6?7k?nqYIT=n5(}=D1&?TWY#ySo=S; e8J&PaJGcIDX#F(Z8J(wj5}r=3!$FW2rTahX$|WfP diff --git a/apps/knx-gui/src/knx_gui/locales/de/LC_MESSAGES/knx_gui.po b/apps/knx-gui/src/knx_gui/locales/de/LC_MESSAGES/knx_gui.po index 818e575e..a4dbbe0d 100644 --- a/apps/knx-gui/src/knx_gui/locales/de/LC_MESSAGES/knx_gui.po +++ b/apps/knx-gui/src/knx_gui/locales/de/LC_MESSAGES/knx_gui.po @@ -66,3 +66,6 @@ msgstr "XKNX Projekt speichern" msgid "XKNX project (*.xknx)" msgstr "XKNX Projekt (*.xknx)" + +msgid "No KNX connection" +msgstr "Keine KNX-Verbindung" diff --git a/apps/knx-gui/src/knx_gui/main.py b/apps/knx-gui/src/knx_gui/main.py index 7d146887..558aafb8 100644 --- a/apps/knx-gui/src/knx_gui/main.py +++ b/apps/knx-gui/src/knx_gui/main.py @@ -1,3 +1,6 @@ +import threading +import time +from collections.abc import Callable from pathlib import Path from typing import Any @@ -7,20 +10,26 @@ from imgui_bundle import hello_imgui, imgui from imgui_bundle import portable_file_dialogs as pfd +from sqlalchemy.exc import SQLAlchemyError +from xknxproject.exceptions import InvalidPasswordException, XknxProjectException from knx_gui.plugins.base import API_VERSION, Logger, PanelDefinition, PluginAPI from knx_gui.plugins.cat import CatPlugin from knx_gui.plugins.catalog import CatalogPlugin, CatalogService from knx_gui.plugins.connection import ConnectionPlugin from knx_gui.plugins.connection.service import ConnectionService +from knx_gui.plugins.keyring import KeyringPlugin from knx_gui.plugins.logger import LoggerPlugin, LogService +from knx_gui.plugins.monitor import MonitorPlugin from knx_gui.plugins.network import NetworkPlugin from knx_gui.plugins.node_editor import NodeEditorPlugin from knx_gui.plugins.project import ProjectPlugin, ProjectService +from knx_gui.plugins.project.knxproj_manufacturer import collect_manufacturer_bundle from knx_gui.plugins.proxy import ProxyPlugin from knx_gui.plugins.virtual import VirtualPlugin from knx_gui.strings import S, set_locale from xknxmono.product.errors import ArchiveError +from xknxmono.project import export_knxproj class KnxGuiApp: @@ -31,6 +40,24 @@ def __init__(self, catalog_path: Path) -> None: self._save_file_dialog: pfd.save_file | None = None self._open_project_dialog: pfd.open_file | None = None self._save_project_dialog: pfd.save_file | None = None + self._import_knxproj_save_dialog: pfd.save_file | None = None + self._export_knxproj_dialog: pfd.save_file | None = None + self._import_knxproj_source: str | None = None + self._import_knxproj_dest: str | None = None + self._password_prompt_requested = False + self._import_password = "" + self._import_password_error: str | None = None + # Background import (keeps the UI responsive during the slow parse/catalog/device build). + self._import_thread: threading.Thread | None = None + self._import_pw: str | None = None + self._import_needs_password = False + # Generic background worker (catalog load, project open) sharing the progress modal. + self._bg_thread: threading.Thread | None = None + # Shared progress-modal state, driven by both the importer and generic background ops. + self._progress_running = False + self._progress_requested = False + self._progress_started_at = 0.0 + self._progress_text = "" self._project_service = ProjectService(self._catalog_service) self._connection_service = ConnectionService() @@ -48,6 +75,8 @@ def __init__(self, catalog_path: Path) -> None: self._connection_plugin = ConnectionPlugin(self._plugin_api) self._proxy_plugin = ProxyPlugin(self._plugin_api) self._network_plugin = NetworkPlugin(self._plugin_api) + self._monitor_plugin = MonitorPlugin(self._plugin_api) + self._keyring_plugin = KeyringPlugin(self._plugin_api) self._virtual_plugin = VirtualPlugin(self._plugin_api) self._node_editor_plugin = NodeEditorPlugin(self._plugin_api) self._project_plugin = ProjectPlugin( @@ -64,9 +93,13 @@ def __init__(self, catalog_path: Path) -> None: self._connection_plugin, self._proxy_plugin, self._network_plugin, + self._monitor_plugin, + self._keyring_plugin, self._virtual_plugin, - self._node_editor_plugin, + # project before node_editor so the ETS-like Editor is the default MainDockSpace tab and + # the node graph sits behind it (still available as a tab). self._project_plugin, + self._node_editor_plugin, self._logger_plugin, ] @@ -90,17 +123,245 @@ def _new_project(self) -> None: ) def _open_project(self) -> None: + # Accept .knxproj here too: _do_open_project routes ETS archives through the importer. self._open_project_dialog = pfd.open_file( S.FILE_DIALOG_PROJECT_TITLE, "", - [S.FILE_DIALOG_PROJECT_FILTER, "*.xknx", S.FILE_DIALOG_ALL_FILES, "*"], + [ + S.FILE_DIALOG_OPEN_FILTER, + "*.xknx *.knxproj", + S.FILE_DIALOG_PROJECT_FILTER, + "*.xknx", + S.FILE_DIALOG_KNXPROJ_FILTER, + "*.knxproj", + S.FILE_DIALOG_ALL_FILES, + "*", + ], + ) + + def _export_knxproj(self) -> None: + if not self._project_service.is_open: + return + default = "project.knxproj" + if self._project_service.path is not None: + default = self._project_service.path.with_suffix(".knxproj").name + self._export_knxproj_dialog = pfd.save_file( + S.FILE_DIALOG_KNXPROJ_SAVE_TITLE, + default, + [S.FILE_DIALOG_KNXPROJ_FILTER, "*.knxproj", S.FILE_DIALOG_ALL_FILES, "*"], + ) + + def _do_export_knxproj(self, dest: str) -> None: + source = self._project_service.path + if source is None: + return + extra_files: dict[str, bytes] = {} + master_xml: bytes | None = None + try: + bundle = collect_manufacturer_bundle( + self._project_service.program_refs(), self._catalog_service + ) + extra_files, master_xml = bundle.extra_files, bundle.master_xml + self._log.info( + "manufacturer bundle collected", + manufacturers=len(bundle.resolved_manufacturers), + files=len(extra_files), + skipped=len(bundle.skipped_refs), + ) + except ( + Exception + ) as e: # best effort: export the structure even if bundling fails + self._log.warning( + "manufacturer bundle failed", error=f"{type(e).__name__}: {e}" + ) + try: + export_knxproj( + source, Path(dest), extra_files=extra_files, master_xml=master_xml + ) + self._log.info("project exported", path=dest) + except (OSError, ValueError) as e: + self._log.error( + "export failed", path=dest, error=f"{type(e).__name__}: {e}" + ) + + def _do_import_knxproj(self, source: str, dest: str) -> None: + self._import_knxproj_source = source + self._import_knxproj_dest = dest + self._start_import(None) + + def _start_import(self, password: str | None) -> None: + """Run the import on a worker thread so the UI stays responsive (the facade holds the shared + lock, so per-frame reads bail to empty placeholders while it runs).""" + source, dest = self._import_knxproj_source, self._import_knxproj_dest + if source is None or dest is None or self._import_thread is not None: + return + self._log.info("importing knxproj", source=source, dest=dest) + self._import_pw = password + self._import_needs_password = False + self._begin_progress(S.IMPORT_PROGRESS_TEXT) + self._import_thread = threading.Thread( + target=self._run_import, args=(source, dest, password), daemon=True + ) + self._import_thread.start() + + def _run_import(self, source: str, dest: str, password: str | None) -> None: + # Worker thread: only touches services + logging (never imgui). Outcome is read in + # _poll_import on the UI thread once the thread has finished. + self._import_needs_password = self._try_import_knxproj(source, dest, password) + + def _poll_import(self) -> None: + thread = self._import_thread + if thread is None or thread.is_alive(): + return + self._import_thread = None + self._progress_running = False + if self._import_needs_password: + # Wrong password on retry (a password was supplied); otherwise the first prompt. + self._import_password_error = ( + S.IMPORT_PASSWORD_WRONG if self._import_pw is not None else None + ) + self._import_password = "" + self._password_prompt_requested = True + else: + self._clear_import_prompt() + + def _begin_progress(self, text: str) -> None: + self._progress_text = text + self._progress_running = True + self._progress_requested = True + self._progress_started_at = time.time() + + def _run_bg(self, text: str, fn: "Callable[[], None]") -> None: + """Run ``fn`` on a worker thread behind the progress modal. ``fn`` must hold the shared IO + lock while it writes so per-frame UI reads bail (see ProjectService/CatalogService).""" + if self._bg_thread is not None or self._import_thread is not None: + return + self._begin_progress(text) + + def worker() -> None: + try: + fn() + except Exception as e: + self._log.error( + "background task failed", error=f"{type(e).__name__}: {e}" + ) + + self._bg_thread = threading.Thread(target=worker, daemon=True) + self._bg_thread.start() + + def _poll_bg(self) -> None: + if self._bg_thread is not None and not self._bg_thread.is_alive(): + self._bg_thread = None + self._progress_running = False + + def _render_progress_modal(self) -> None: + if self._progress_requested: + imgui.open_popup(S.PROGRESS_TITLE) + self._progress_requested = False + imgui.set_next_window_size(imgui.ImVec2(360.0, 0.0), imgui.Cond_.appearing) + if not imgui.begin_popup_modal( + S.PROGRESS_TITLE, None, imgui.WindowFlags_.always_auto_resize + )[0]: + return + imgui.text_wrapped(self._progress_text) + elapsed = time.time() - self._progress_started_at + dots = "." * (int(elapsed * 2) % 4) + imgui.text_disabled(f"{elapsed:0.0f}s {dots}") + if not self._progress_running: + imgui.close_current_popup() + imgui.end_popup() + + def _try_import_knxproj(self, source: str, dest: str, password: str | None) -> bool: + """Attempt the import. Returns ``True`` if a password is required (or wrong) and the caller + should prompt; ``False`` on success or on any other failure (which is logged).""" + try: + self._project_service.import_knxproj( + Path(source), Path(dest), password=password + ) + except InvalidPasswordException: + return True + except XknxProjectException as e: + self._log.error("knxproj import failed", source=source, error=str(e)) + except Exception as e: + # Runs on a worker thread: never let an unexpected error kill the thread silently, or + # the UI would wait on a spinner that never clears. Log and report as a plain failure. + self._log.error( + "knxproj import error", source=source, error=f"{type(e).__name__}: {e}" + ) + return False + + def _render_import_password_modal(self) -> None: + if self._password_prompt_requested: + imgui.open_popup(S.IMPORT_PASSWORD_TITLE) + self._password_prompt_requested = False + imgui.set_next_window_size(imgui.ImVec2(420, 0), imgui.Cond_.appearing) + if not imgui.begin_popup_modal( + S.IMPORT_PASSWORD_TITLE, None, imgui.WindowFlags_.always_auto_resize + )[0]: + return + imgui.text_wrapped(S.IMPORT_PASSWORD_PROMPT) + imgui.set_next_item_width(-1) + submitted, self._import_password = imgui.input_text( + "##import-password", + self._import_password, + imgui.InputTextFlags_.password | imgui.InputTextFlags_.enter_returns_true, ) + if self._import_password_error: + imgui.text_colored( + imgui.ImVec4(0.9, 0.4, 0.4, 1.0), self._import_password_error + ) + imgui.spacing() + btn_w = imgui.ImVec2(120, 0) + confirm = imgui.button(S.BTN_OK, btn_w) or submitted + imgui.same_line() + cancel = imgui.button(S.BTN_CANCEL, btn_w) + source, dest = self._import_knxproj_source, self._import_knxproj_dest + if confirm and source is not None and dest is not None: + # Retry on a worker thread; _poll_import re-opens this modal if the password was wrong. + imgui.close_current_popup() + self._start_import(self._import_password) + elif cancel: + self._clear_import_prompt() + imgui.close_current_popup() + imgui.end_popup() + + def _clear_import_prompt(self) -> None: + self._import_knxproj_source = None + self._import_knxproj_dest = None + self._import_password = "" + self._import_password_error = None def _do_new_project(self, path: str) -> None: self._project_service.new(Path(path)) def _do_open_project(self, path: str) -> None: - self._project_service.open(Path(path)) + # A .knxproj is an ETS archive, not an .xknx SQLite document. If one is picked here (a common + # mix-up), route it through the importer instead of trying to open it as a database. + if path.lower().endswith(".knxproj"): + self._prompt_import_dest(path) + return + + def worker() -> None: + try: + self._project_service.open(Path(path)) + except (ValueError, SQLAlchemyError) as e: + # A missing/stale/corrupt file must not take down the app (e.g. the demo project + # opened at startup, or a bad file picked via "Open Project"). + self._log.error("could not open project", path=path, error=str(e)) + + # Open on a worker thread behind the spinner: building a large project's device view is slow. + self._run_bg(S.PROGRESS_OPEN_PROJECT, worker) + + def _prompt_import_dest(self, source: str) -> None: + """Remember the .knxproj source and ask where to save the imported .xknx project.""" + self._import_knxproj_source = source + # Pass only the default file name, not a full path: on macOS a path with "/" in the save + # dialog's name field gets mangled into ":"-separated segments. + self._import_knxproj_save_dialog = pfd.save_file( + S.FILE_DIALOG_PROJECT_SAVE_TITLE, + Path(source).with_suffix(".xknx").name, + [S.FILE_DIALOG_PROJECT_FILTER, "*.xknx", S.FILE_DIALOG_ALL_FILES, "*"], + ) def _undo(self) -> None: self._project_service.undo() @@ -115,6 +376,9 @@ def _can_redo(self) -> bool: return self._project_service.is_open and self._project_service.can_redo() def _poll_dialogs(self) -> None: + self._poll_import() + self._poll_bg() + if self._open_file_dialog is not None and self._open_file_dialog.ready(): result = self._open_file_dialog.result() self._open_file_dialog = None @@ -133,6 +397,26 @@ def _poll_dialogs(self) -> None: if result: self._do_open_project(result[0]) + if ( + self._export_knxproj_dialog is not None + and self._export_knxproj_dialog.ready() + ): + result = self._export_knxproj_dialog.result() + self._export_knxproj_dialog = None + if result: + self._do_export_knxproj(result) + + if ( + self._import_knxproj_save_dialog is not None + and self._import_knxproj_save_dialog.ready() + ): + result = self._import_knxproj_save_dialog.result() + self._import_knxproj_save_dialog = None + source = self._import_knxproj_source + self._import_knxproj_source = None + if result and source is not None: + self._do_import_knxproj(source, result) + def _handle_shortcuts(self) -> None: io = imgui.get_io() if (io.key_ctrl or io.key_super) and imgui.is_key_pressed(imgui.Key.z): @@ -144,27 +428,102 @@ def _handle_shortcuts(self) -> None: self._redo() def _load_knxprod(self, path: str) -> None: - self._log.info("loading knxprod", path=path) - try: - added = self._catalog_service.import_knxprod(Path(path)) - if added: - self._log.info("added applications to catalog", count=len(added)) - else: - self._log.info("no new applications", path=path) - except ArchiveError as e: - self._log.error("archive error", path=path, error=str(e)) - except (OSError, ValueError) as e: - self._log.error("import error", path=path, error=f"{type(e).__name__}: {e}") + def worker() -> None: + # Hold the catalog lock so per-frame catalog reads bail while it writes (see io_guarded). + with self._catalog_service.io_lock: + self._log.info("loading knxprod", path=path) + try: + added = self._catalog_service.import_knxprod(Path(path)) + if added: + self._log.info( + "added applications to catalog", count=len(added) + ) + else: + self._log.info("no new applications", path=path) + except ArchiveError as e: + self._log.error("archive error", path=path, error=str(e)) + except (OSError, ValueError) as e: + self._log.error( + "import error", path=path, error=f"{type(e).__name__}: {e}" + ) + + self._run_bg(S.PROGRESS_LOAD_KNXPROD, worker) def gui_status_bar(self) -> None: self._connection_plugin.render_status_indicator() + # ETS-like indicator: a bus operation (programming/testing) in progress. + busy = self._connection_service.busy_operation + if busy is not None: + kind, address = busy + if kind == "program": + text = S.STATUS_PROGRAMMING.format(address=address) + prog = self._connection_service.busy_progress + if prog is not None: + text += f" ({prog[0]}/{prog[1]})" + else: + text = S.STATUS_TESTING.format(address=address) + imgui.same_line() + imgui.text_disabled(" | ") + imgui.same_line() + imgui.push_style_color(imgui.Col_.text, imgui.ImVec4(0.95, 0.75, 0.2, 1.0)) + imgui.text(text) + imgui.pop_style_color() + elif self._connection_service.not_connected_notice(): + # A connection-requiring feature (program/test/send) was just refused: + # flash the reason so the user sees *why* nothing happened. + imgui.same_line() + imgui.text_disabled(" | ") + imgui.same_line() + imgui.push_style_color(imgui.Col_.text, imgui.ImVec4(0.9, 0.35, 0.35, 1.0)) + imgui.text(S.STATUS_NO_CONNECTION) + imgui.pop_style_color() + else: + # Briefly show the last programming outcome after the busy indicator clears. + notice = self._connection_service.program_notice() + if notice is not None: + ok = notice + color = ( + imgui.ImVec4(0.4, 0.85, 0.45, 1.0) + if ok + else imgui.ImVec4(0.9, 0.35, 0.35, 1.0) + ) + imgui.same_line() + imgui.text_disabled(" | ") + imgui.same_line() + imgui.push_style_color(imgui.Col_.text, color) + imgui.text(S.STATUS_PROGRAM_DONE if ok else S.STATUS_PROGRAM_FAILED) + imgui.pop_style_color() + + # Current project summary. + imgui.same_line() + imgui.text_disabled(" | ") + imgui.same_line() + if self._project_service.is_open: + meta = self._project_service.get_project_metadata() + name = meta.name if meta and meta.name else None + if not name and self._project_service.path is not None: + name = self._project_service.path.name + imgui.text_disabled( + S.STATUS_PROJECT.format( + name=name or "?", + devices=len(self._project_service.devices), + gas=len(self._project_service.group_addresses), + ) + ) + else: + imgui.text_disabled(S.STATUS_NO_PROJECT) + def gui_menu(self) -> None: if imgui.begin_menu(S.MENU_FILE): if imgui.menu_item(S.MENU_NEW_PROJECT, "", False)[0]: self._new_project() if imgui.menu_item(S.MENU_OPEN_PROJECT, "", False)[0]: self._open_project() + if imgui.menu_item( + S.MENU_EXPORT_KNXPROJ, "", False, self._project_service.is_open + )[0]: + self._export_knxproj() imgui.separator() if imgui.menu_item(S.MENU_LOAD_KNXPROD, "", False)[0]: self._open_file_dialog = pfd.open_file( @@ -204,6 +563,8 @@ def gui_menu(self) -> None: def render_overlays(self) -> None: self._cat_plugin.render() self._project_plugin.render_overlays() + self._render_progress_modal() + self._render_import_password_modal() def get_all_panels(self) -> list[PanelDefinition]: panels: list[PanelDefinition] = [] diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/locales/de/LC_MESSAGES/catalog.mo b/apps/knx-gui/src/knx_gui/plugins/catalog/locales/de/LC_MESSAGES/catalog.mo index f569c163571979b4d6bad7b958f8aa9240b7fe09..15865cd1d60e2eec91999de224a65d5af1038b2a 100644 GIT binary patch literal 699 zcmYk2!DCXSN-1J_+fB*&=bf*51?OAI)9-@&^5G~bjz3> zun(sEXW&C{(#9X#IJEIMFx6jzFTtPS9{2}r{B9fb6mcKi1z&)D@B{c5v|#c-15@31 za2vb=VbQ!h`D{Sx+=K2z_`N!J@U;cqO*ez{EYzYGl!YaQ*=NqQ8k&h!>qB9t)_KlJ z1NmTcvN4;xFKnhL$*HTfW$9gJ)dd-#_6V7r@6&l{b$-!=P6}FDsZVUCb>v9iRXQGz z&3o}qn7Vdt2(Q$gBq&9;+1$=No37peMwL0TQq;EKeag8v38E}IIzB6Lq5M^!m)<&g zP4I{n>zhx<)6uKmdg{ZlT-M?i&FiZ)|3Z0a4%_(;nt^E%1jZyTn=2LUF`&>XM_~1= z136{MYbgdSLnNJGMfm;pCVRaO)Y3bSm9qC7(;|4iiBZ@1W)of5cIFrvPng-b>biC^ F{{ZYEvhx4{ delta 178 zcmdnZx{JB~o)F7a1|VPqVi_Rz0b*_-t^r~YSOLVMK)e!&*@1XJ5Q_ovaUce%zX!y8 zK>QnsO@Ww;k%2)TNIL*&kop85tpubqfV3Qt?grBQKzbFB1|l^ERv-rkK#pL6a+shr OGl-t}Hf{1I#`OR?D-e(`I48;pFevEocRD&^PD~isupf0X;!qqCOb21kQu8 z{wMey?9~2u?L+Oa!PtKTZh^O8?0*2ig3tB;xsS%I!mon!;2yXP9@Rbr*Wk~3Mt6hQrqpeIrh$a zq3OEXVpiNvJ}4E!>3|a_X1ebGL8UolpH-a64tdw=1h3+C-Hq+md#S}Yv8&ST$E4CHhb*2(wsh4{n25&rhD@utJVuuNH(Jb!2O*AjK;I04w delta 178 zcmZ3@x{SI0o)F7a1|VPqVi_Rz0b*_-t^r~YSOLVMK)e!&*@1XJ5Q_ovaUce%zX!y8 zK>QnsO@Ww;5hCvZq&b27a3BrRp9rMofOI*K<_FT#fHV-PF|YzTFaUA{3zWkIrI|tW M#J6daH!;oz01bl>4gdfE diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/locales/nl/LC_MESSAGES/catalog.po b/apps/knx-gui/src/knx_gui/plugins/catalog/locales/nl/LC_MESSAGES/catalog.po index 379186c8..98493a19 100644 --- a/apps/knx-gui/src/knx_gui/plugins/catalog/locales/nl/LC_MESSAGES/catalog.po +++ b/apps/knx-gui/src/knx_gui/plugins/catalog/locales/nl/LC_MESSAGES/catalog.po @@ -18,3 +18,15 @@ msgstr "{count} applicatie(s) gevonden" msgid "({count} com objects)" msgstr "({count} com objecten)" + +msgid "Online Catalog" +msgstr "Onlinecatalogus" + +msgid "Loading manufacturer list..." +msgstr "Fabrikantenlijst laden..." + +msgid "Online catalog not reachable" +msgstr "Onlinecatalogus niet bereikbaar" + +msgid "{count} manufacturers" +msgstr "{count} fabrikanten" diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/online_catalog.py b/apps/knx-gui/src/knx_gui/plugins/catalog/online_catalog.py new file mode 100644 index 00000000..5c22127d --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/catalog/online_catalog.py @@ -0,0 +1,138 @@ +"""Client for the anonymous KNX online catalog service. + +It is a plain HTTP file-style REST service with no authentication: + +- ``GET {base}/Download/Manufacturers`` -> XML, one ```` per manufacturer +- the manufacturer *names* come from the public master data file + (``https://update.knx.org/data/XML/project-23/knx_master.xml``), the same file ETS ships + and our ``.knxproj`` export already uses + +For now the GUI only needs the manufacturer list; product index / ``.knxprod`` download is +deliberately not implemented yet. +""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_BASE_URL = "https://onlinecatalog.knx.org" +MASTER_DATA_URL = "https://update.knx.org/data/XML/project-23/knx_master.xml" +_CACHE_FILE = "online_catalog_manufacturers.json" +_TIMEOUT_SECONDS = 30.0 + + +class OnlineCatalogError(Exception): + """Raised when the online catalog service cannot be reached or returns bad data.""" + + +@dataclass(frozen=True) +class OnlineManufacturer: + id: int + name: str + + +def _local_name(tag: object) -> str: + """The element's tag without its XML namespace (both feeds use namespaces).""" + return str(tag).rsplit("}", 1)[-1] + + +def parse_manufacturer_ids(xml_bytes: bytes) -> list[int]: + """Parse the ``Download/Manufacturers`` XML into a sorted list of manufacturer ids.""" + root = ET.fromstring(xml_bytes) + ids = [ + int(e.text) + for e in root + if _local_name(e.tag) == "unsignedShort" and (e.text or "").strip() + ] + if not ids: + raise OnlineCatalogError("manufacturer list is empty") + return sorted(ids) + + +def parse_manufacturer_names(xml_bytes: bytes) -> dict[int, str]: + """Parse ``knx_master.xml`` into ``{manufacturer_id: name}``.""" + root = ET.fromstring(xml_bytes) + names: dict[int, str] = {} + for element in root.iter(): + if _local_name(element.tag) != "Manufacturer": + continue + # The numeric KNX manufacturer id is ``KnxManufacturerId``; ``Id`` is ``M-xxxx``. + mid = element.get("KnxManufacturerId") or ( + element.get("Id") or "" + ).removeprefix("M-") + if not mid.isdigit(): + continue + names[int(mid)] = element.get("Name") or f"M-{int(mid):04d}" + if not names: + raise OnlineCatalogError("master data contains no manufacturers") + return names + + +def _http_get(url: str) -> bytes: + request = urllib.request.Request(url, headers={"User-Agent": "xknxtoolkit/0.1"}) + try: + with urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) as response: + return response.read() + except (urllib.error.URLError, TimeoutError, OSError) as exc: + reason = getattr(exc, "reason", None) or exc + raise OnlineCatalogError(f"{url}: {reason}") from exc + + +class OnlineCatalogClient: + """Fetches the manufacturer list, with a small on-disk cache. + + The cache lives next to the catalog database. ``cached_manufacturers`` is a pure read + (for per-frame UI use); ``refresh_manufacturers`` hits the network and replaces the cache. + """ + + def __init__(self, cache_dir: Path, base_url: str = DEFAULT_BASE_URL) -> None: + self._cache_path = cache_dir / _CACHE_FILE + self._base_url = base_url + self._lock = threading.Lock() + + def _load_cache(self) -> list[OnlineManufacturer] | None: + try: + raw = json.loads(self._cache_path.read_text(encoding="utf-8")) + return [OnlineManufacturer(item["id"], item["name"]) for item in raw] + except (OSError, ValueError, KeyError): + return None + + def _save_cache(self, manufacturers: list[OnlineManufacturer]) -> None: + try: + self._cache_path.parent.mkdir(parents=True, exist_ok=True) + self._cache_path.write_text( + json.dumps( + [{"id": m.id, "name": m.name} for m in manufacturers], + ensure_ascii=False, + ), + encoding="utf-8", + ) + except OSError: + pass # a missing cache only costs a re-download, never a failure + + def cached_manufacturers(self) -> list[OnlineManufacturer] | None: + """Return the manufacturer list from the cache, or None; never touches the network.""" + with self._lock: + return self._load_cache() + + def refresh_manufacturers(self) -> list[OnlineManufacturer]: + """Download the manufacturer list and replace the cache.""" + with self._lock: + return self._fetch_locked() + + def _fetch_locked(self) -> list[OnlineManufacturer]: + ids = parse_manufacturer_ids( + _http_get(f"{self._base_url}/Download/Manufacturers") + ) + names = parse_manufacturer_names(_http_get(MASTER_DATA_URL)) + manufacturers = [ + OnlineManufacturer(mid, names.get(mid, f"M-{mid:04d}")) for mid in ids + ] + self._save_cache(manufacturers) + return manufacturers diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/plugin.py b/apps/knx-gui/src/knx_gui/plugins/catalog/plugin.py index acb906d3..829ba360 100644 --- a/apps/knx-gui/src/knx_gui/plugins/catalog/plugin.py +++ b/apps/knx-gui/src/knx_gui/plugins/catalog/plugin.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING from knx_gui.plugins.base import Logger, PanelDefinition, PluginAPI +from knx_gui.plugins.catalog.online_catalog import OnlineCatalogError from knx_gui.plugins.catalog.strings import S from knx_gui.plugins.catalog.ui import CatalogPanel @@ -19,6 +20,8 @@ def __init__(self, api: PluginAPI) -> None: self._panel = CatalogPanel( get_products=api.catalog.get_products, on_select=self._on_select, + get_online_manufacturers=api.catalog.online_manufacturers, + on_online_refresh=self._refresh_online, ) self._panels = [ PanelDefinition( @@ -51,6 +54,15 @@ def _on_select(self, product: ProductSummary) -> None: if device_id: self._log.info("device added", name=app.name, id=device_id) + def _refresh_online(self) -> None: + """Fetch the online manufacturer list (called on a worker thread).""" + try: + manufacturers = self._api.catalog.refresh_online_manufacturers() + except OnlineCatalogError as exc: + self._log.error("online catalog refresh failed", error=str(exc)) + raise + self._log.info("online catalog manufacturers loaded", count=len(manufacturers)) + @property def panels(self) -> list[PanelDefinition]: return self._panels diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/service.py b/apps/knx-gui/src/knx_gui/plugins/catalog/service.py index d4fbb945..cca23196 100644 --- a/apps/knx-gui/src/knx_gui/plugins/catalog/service.py +++ b/apps/knx-gui/src/knx_gui/plugins/catalog/service.py @@ -7,21 +7,48 @@ from __future__ import annotations +import threading from pathlib import Path from typing import TYPE_CHECKING +from knx_gui.concurrency import io_guarded + if TYPE_CHECKING: + from knx_gui.plugins.catalog.online_catalog import OnlineManufacturer from xknxmono.catalog import ProductSummary from xknxmono.product import Application class CatalogService: - def __init__(self, catalog_path: Path) -> None: + def __init__( + self, catalog_path: Path, io_lock: threading.RLock | None = None + ) -> None: + from knx_gui.plugins.catalog.online_catalog import OnlineCatalogClient from xknxmono.catalog import CatalogService as _CatalogService self._service = _CatalogService(catalog_path) self._products: list[ProductSummary] | None = None + # Shared with the project service so a background import can hold both while it writes. + self._io_lock = io_lock or threading.RLock() + # Manufacturer list from the KNX online catalog service (cached next to the db). + self._online_client = OnlineCatalogClient(catalog_path.parent) + + def online_manufacturers(self) -> list[OnlineManufacturer] | None: + """The cached online manufacturer list, or None when the cache is empty. + + Never touches the network: the panel reads this every frame.""" + return self._online_client.cached_manufacturers() + def refresh_online_manufacturers(self) -> list[OnlineManufacturer]: + """Download the manufacturer list now; raises OnlineCatalogError on failure.""" + return self._online_client.refresh_manufacturers() + + @property + def io_lock(self) -> threading.RLock: + """The re-entrant lock a background import holds while writing catalog + project data.""" + return self._io_lock + + @io_guarded(list) def get_products(self) -> list[ProductSummary]: """Product-centric browse entries — each carries the product/program refs add_device needs.""" if self._products is None: @@ -36,8 +63,14 @@ def import_knxprod(self, path: Path) -> list[str]: after = {p.product_ref_id for p in self.get_products()} return sorted(after - before) + @io_guarded(lambda: None) def get_application(self, application_id: str) -> Application | None: return self._service.get_application(application_id) + @io_guarded(lambda: None) + def get_program_source(self, program_id: str) -> tuple[str, str] | None: + """Return ``(knxprod_path, manufacturer_id)`` for a hardware program id, or ``None``.""" + return self._service.get_program_source(program_id) + def refresh(self) -> None: self._products = None diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/strings.py b/apps/knx-gui/src/knx_gui/plugins/catalog/strings.py index 058d66ce..986496eb 100644 --- a/apps/knx-gui/src/knx_gui/plugins/catalog/strings.py +++ b/apps/knx-gui/src/knx_gui/plugins/catalog/strings.py @@ -29,5 +29,21 @@ def ARCHIVE_FOUND_APPS(self) -> str: def ARCHIVE_COM_OBJECTS(self) -> str: return _("({count} com objects)") + @property + def BTN_ONLINE_CATALOG(self) -> str: + return _("Online Catalog") + + @property + def ONLINE_LOADING(self) -> str: + return _("Loading manufacturer list...") + + @property + def ONLINE_FAILED(self) -> str: + return _("Online catalog not reachable") + + @property + def ONLINE_COUNT(self) -> str: + return _("{count} manufacturers") + S = CatalogStrings() diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/tests/__init__.py b/apps/knx-gui/src/knx_gui/plugins/catalog/tests/__init__.py new file mode 100644 index 00000000..cc9f4a83 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/catalog/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the catalog plugin (online catalog client + panel state).""" diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/tests/test_online_catalog.py b/apps/knx-gui/src/knx_gui/plugins/catalog/tests/test_online_catalog.py new file mode 100644 index 00000000..c5687a8f --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/catalog/tests/test_online_catalog.py @@ -0,0 +1,100 @@ +"""Tests for the online catalog client (parsers, cache) and the panel fetch state.""" + +import json + +import pytest + +from knx_gui.plugins.catalog.online_catalog import ( + OnlineCatalogClient, + OnlineCatalogError, + OnlineManufacturer, + parse_manufacturer_ids, + parse_manufacturer_names, +) + +_MANUFACTURERS_XML = ( + b'' + b'' + b"2" + b"1" + b"10" + b"" +) + +_MASTER_XML = ( + b'' + b'' + b"" + b'' + b'' + b"" + b"" + b"" +) + + +class TestParse: + def test_manufacturer_ids_sorted_and_namespaced(self) -> None: + assert parse_manufacturer_ids(_MANUFACTURERS_XML) == [1, 2, 10] + + def test_manufacturer_ids_empty_raises(self) -> None: + with pytest.raises(OnlineCatalogError): + parse_manufacturer_ids(b"") + + def test_manufacturer_names(self) -> None: + assert parse_manufacturer_names(_MASTER_XML) == {1: "Siemens", 2: "ABB"} + + def test_manufacturer_names_empty_raises(self) -> None: + with pytest.raises(OnlineCatalogError): + parse_manufacturer_names(b"") + + +class TestClient: + def test_cache_roundtrip(self, tmp_path) -> None: + client = OnlineCatalogClient(tmp_path) + assert client.cached_manufacturers() is None + + client._save_cache([OnlineManufacturer(1, "Siemens")]) + assert [m.name for m in client.cached_manufacturers()] == ["Siemens"] + + def test_cache_ignores_corrupt_file(self, tmp_path) -> None: + (tmp_path / "online_catalog_manufacturers.json").write_text("not json", "utf-8") + assert OnlineCatalogClient(tmp_path).cached_manufacturers() is None + + def test_cache_survives_reload(self, tmp_path) -> None: + OnlineCatalogClient(tmp_path)._save_cache( + [OnlineManufacturer(7, "Berker"), OnlineManufacturer(1, "Siemens")] + ) + reloaded = OnlineCatalogClient(tmp_path).cached_manufacturers() + assert [m.id for m in reloaded] == [7, 1] + assert json.loads( + (tmp_path / "online_catalog_manufacturers.json").read_text("utf-8") + ) == [{"id": 7, "name": "Berker"}, {"id": 1, "name": "Siemens"}] + + +class TestPanelOnlineState: + """The panel's fetch flag logic (imgui rendering is exercised by the app itself).""" + + def test_refresh_flag_lifecycle(self, tmp_path) -> None: + from knx_gui.plugins.catalog.ui.panel import CatalogPanel + + panel = CatalogPanel( + get_products=list, + on_select=lambda product: None, + get_online_manufacturers=lambda: None, + on_online_refresh=lambda: None, + ) + assert not panel._online_shown + assert not panel._online_loading + + panel._fetch_online() # success path sets shown, clears loading + assert panel._online_shown + assert not panel._online_loading + + def failing_refresh() -> None: + raise OnlineCatalogError("no network") + + panel._on_online_refresh = failing_refresh + panel._fetch_online() # error path keeps the list hidden, clears loading + assert panel._online_error + assert panel._online_shown # a previous successful list stays visible diff --git a/apps/knx-gui/src/knx_gui/plugins/catalog/ui/panel.py b/apps/knx-gui/src/knx_gui/plugins/catalog/ui/panel.py index bb855ff2..926da067 100644 --- a/apps/knx-gui/src/knx_gui/plugins/catalog/ui/panel.py +++ b/apps/knx-gui/src/knx_gui/plugins/catalog/ui/panel.py @@ -1,11 +1,15 @@ from __future__ import annotations +import threading from collections.abc import Callable from typing import TYPE_CHECKING from imgui_bundle import imgui +from knx_gui.plugins.catalog.strings import S + if TYPE_CHECKING: + from knx_gui.plugins.catalog.online_catalog import OnlineManufacturer from xknxmono.catalog import ProductSummary @@ -18,12 +22,25 @@ def __init__( self, get_products: Callable[[], list[ProductSummary]], on_select: Callable[[ProductSummary], None], + get_online_manufacturers: Callable[[], list[OnlineManufacturer] | None] + | None = None, + on_online_refresh: Callable[[], None] | None = None, ) -> None: self._get_products = get_products self._on_select = on_select self._search: str = "" + # Online catalog (manufacturer list from the KNX service). + self._get_online_manufacturers = get_online_manufacturers + self._on_online_refresh = on_online_refresh + self._online_loading = False + self._online_error = False + self._online_shown = False def render(self) -> None: + if self._get_online_manufacturers is not None: + self._render_online_catalog() + imgui.separator() + imgui.set_next_item_width(-1) _, self._search = imgui.input_text_with_hint( "##catalog_search", "Search...", self._search @@ -72,3 +89,53 @@ def render(self) -> None: if imgui.is_item_clicked() and imgui.is_mouse_double_clicked(0): self._on_select(product) imgui.tree_pop() + + def _render_online_catalog(self) -> None: + """Online manufacturer list from the KNX catalog service (button + tree). + + ``get_online_manufacturers`` is called every frame; it must return the cached list + (or None) without touching the network. The fetch itself runs on a worker thread.""" + online = ( + self._get_online_manufacturers() + if (self._get_online_manufacturers is not None) + else None + ) + imgui.set_next_item_width(160.0) + if imgui.button(S.BTN_ONLINE_CATALOG): + self._start_online_refresh() + if self._online_loading: + imgui.same_line() + imgui.text_disabled(S.ONLINE_LOADING) + elif self._online_error: + imgui.same_line() + imgui.text_colored(S.ONLINE_FAILED, 1.0, 0.4, 0.4) + + if online is None or not self._online_shown: + return + leaf_flags = ( + imgui.TreeNodeFlags_.leaf + | imgui.TreeNodeFlags_.no_tree_push_on_open + | imgui.TreeNodeFlags_.span_avail_width + ) + imgui.text_disabled(S.ONLINE_COUNT.format(count=len(online))) + for mfr in online: + imgui.tree_node_ex( + f"{mfr.name} (M-{mfr.id:04d})##online_mfr_{mfr.id}", leaf_flags + ) + + def _start_online_refresh(self) -> None: + """Fetch the manufacturer list off the UI thread (urllib blocks).""" + if self._online_loading or self._on_online_refresh is None: + return + self._online_loading = True + self._online_error = False + threading.Thread(target=self._fetch_online, daemon=True).start() + + def _fetch_online(self) -> None: + try: + self._on_online_refresh() + self._online_shown = True + except Exception: + self._online_error = True + finally: + self._online_loading = False diff --git a/apps/knx-gui/src/knx_gui/plugins/connection/plugin.py b/apps/knx-gui/src/knx_gui/plugins/connection/plugin.py index fde29e09..99d61b57 100644 --- a/apps/knx-gui/src/knx_gui/plugins/connection/plugin.py +++ b/apps/knx-gui/src/knx_gui/plugins/connection/plugin.py @@ -1,4 +1,5 @@ import asyncio +import contextlib import math import threading from collections.abc import Coroutine @@ -16,6 +17,9 @@ from knx_gui.plugins.base import Logger, PanelDefinition, PluginAPI from knx_gui.plugins.connection.interface import ObservableKNXIPInterfaceThreaded from knx_gui.plugins.connection.strings import S +from knx_gui.settings import load_settings, save_settings + +_SETTINGS = "connection" class ConnectionState(Enum): @@ -40,6 +44,7 @@ def __init__(self, api: PluginAPI) -> None: self._multicast_group: str = DEFAULT_MCAST_GRP self._selected_gateway: GatewayDescriptor | None = None self._panels: list[PanelDefinition] = [] + self._load_saved_settings() self._xknx: XKNX | None = None self._interface: ObservableKNXIPInterfaceThreaded | None = None @@ -85,6 +90,29 @@ def _run_async(self, coro: Coroutine[Any, Any, None]) -> None: loop = self._ensure_async_loop() asyncio.run_coroutine_threadsafe(coro, loop) + def _load_saved_settings(self) -> None: + data = load_settings(_SETTINGS) + ip = data.get("controller_ip") + if isinstance(ip, str) and ip: + self._controller_ip = ip + mcast = data.get("multicast_group") + if isinstance(mcast, str) and mcast: + self._multicast_group = mcast + ctype = data.get("connection_type") + if ctype is not None: + with contextlib.suppress(ValueError): + self._connection_type = ConnectionType(ctype) + + def _save_current_settings(self) -> None: + save_settings( + _SETTINGS, + { + "controller_ip": self._controller_ip, + "multicast_group": self._multicast_group, + "connection_type": self._connection_type.value, + }, + ) + def connect(self) -> None: """Connect using the manually entered IP (always tunneling).""" if self._state in (ConnectionState.CONNECTING, ConnectionState.CONNECTED): @@ -93,6 +121,7 @@ def connect(self) -> None: self._selected_gateway = None self._state = ConnectionState.CONNECTING self._error_message = None + self._save_current_settings() self._run_async(self._connect_async()) def connect_to_gateway(self, gateway: GatewayDescriptor) -> None: @@ -110,6 +139,7 @@ def connect_to_gateway(self, gateway: GatewayDescriptor) -> None: self._selected_gateway = gateway self._state = ConnectionState.CONNECTING self._error_message = None + self._save_current_settings() self._run_async(self._connect_async()) @property diff --git a/apps/knx-gui/src/knx_gui/plugins/connection/service.py b/apps/knx-gui/src/knx_gui/plugins/connection/service.py index 9a7f634b..f875a679 100644 --- a/apps/knx-gui/src/knx_gui/plugins/connection/service.py +++ b/apps/knx-gui/src/knx_gui/plugins/connection/service.py @@ -1,11 +1,15 @@ from __future__ import annotations import asyncio +import functools +import threading +import time from collections.abc import Callable, Coroutine from concurrent.futures import Future from typing import TYPE_CHECKING, Any from xknx.cemi import CEMIFrame +from xknx.exceptions import ManagementConnectionError from xknx.management.procedures import ( nm_individual_address_read, nm_individual_address_serial_number_write, @@ -18,6 +22,8 @@ from knx_gui.device import Device from knx_gui.net import TelegramSource from knx_gui.plugins.base import Logger + from xknxmono.download.image import GroupCommunication + from xknxmono.download.scope import DownloadScope class ConnectionService: @@ -27,10 +33,66 @@ def __init__(self) -> None: self._connected_listeners: list[Callable[[], None]] = [] self._xknx: XKNX | None = None self._loop: asyncio.AbstractEventLoop | None = None + # Current point-to-point bus operation, so the UI can show an ETS-like + # "programming/testing in progress" indicator. Set/cleared from worker threads. + self._busy_lock = threading.Lock() + self._busy: tuple[str, str] | None = None + self._busy_progress: tuple[int, int] | None = None + self._program_notice: tuple[bool, float] | None = None + # Timestamp of the last connection-requiring call made while disconnected, + # so the status bar can show a brief "no connection" notice. + self._not_connected_notice: float | None = None + + @property + def busy_operation(self) -> tuple[str, str] | None: + """``(kind, address)`` of the running bus op (``kind`` is ``program``/``test``), or ``None``.""" + with self._busy_lock: + return self._busy + + @property + def busy_progress(self) -> tuple[int, int] | None: + """``(done, total)`` load-control progress of the running download, or ``None``.""" + with self._busy_lock: + return self._busy_progress + + def _set_busy(self, kind: str, address: str) -> None: + with self._busy_lock: + self._busy = (kind, address) + self._busy_progress = None + + def _clear_busy(self) -> None: + with self._busy_lock: + self._busy = None + self._busy_progress = None + + def _on_program_progress(self, done: int, total: int) -> None: + with self._busy_lock: + self._busy_progress = (done, total) def set_logger(self, log: Logger) -> None: self._log = log + def not_connected(self, op: str) -> bool: + """Guard for every connection-requiring call: log + a status-bar notice if no KNX link. + + Returns ``True`` when there is no connection (the caller must abort).""" + if self._xknx is not None: + return False + self._log.error(f"{op} failed: no KNX connection") + with self._busy_lock: + self._not_connected_notice = time.monotonic() + return True + + def not_connected_notice(self, max_age: float = 6.0) -> bool: + """Whether a recent call was refused for lack of a connection (for the status bar).""" + with self._busy_lock: + stamp = self._not_connected_notice + return ( + stamp is not None + and self._xknx is None + and time.monotonic() - stamp <= max_age + ) + def add_raw_cemi_listener( self, callback: Callable[[bytes, TelegramSource], None] ) -> None: @@ -75,8 +137,7 @@ def xknx(self) -> XKNX | None: return self._xknx def send_cemi(self, raw_cemi: bytes) -> Future[Any] | None: - if self._xknx is None: - self._log.warning("send_cemi called while disconnected") + if self.not_connected("send_cemi"): return None self._log.debug("send_cemi", hex=raw_cemi.hex(" ")) try: @@ -116,18 +177,14 @@ def _log_send_cemi_result(self, future: Future[Any]) -> None: self._log.debug("send_cemi ok") def read_programming_mode_devices(self, timeout: float = 3.0) -> Future[Any] | None: - if self._xknx is None: - self._log.warning("read_programming_mode_devices called while disconnected") + if self.not_connected("read_programming_mode_devices"): return None return self.run_async(nm_individual_address_read(self._xknx, timeout=timeout)) def assign_individual_address_by_serial( self, serial: bytes, address: str ) -> Future[Any] | None: - if self._xknx is None: - self._log.warning( - "assign_individual_address_by_serial called while disconnected" - ) + if self.not_connected("assign_individual_address_by_serial"): return None self._log.debug( "Assigning individual address by serial", @@ -139,8 +196,7 @@ def assign_individual_address_by_serial( ) def assign_individual_address(self, address: str) -> Future[Any] | None: - if self._xknx is None: - self._log.warning("assign_individual_address called while disconnected") + if self.not_connected("assign_individual_address"): return None self._log.debug("Assigning individual address", address=address) return self.run_async(nm_individual_address_write(self._xknx, address)) @@ -155,6 +211,133 @@ def assign_individual_address_for_device( return None return self.assign_individual_address(device.individual_address) + def program_device( + self, + device: Device, + scope: DownloadScope | None = None, + group_communication: GroupCommunication | None = None, + ) -> Future[Any] | None: + """Download the device's configured application/parameters onto the bus. + + ``scope`` selects a full or partial download (defaults to full). + ``group_communication`` supplies the address/association tables. + """ + if self.not_connected("program_device"): + return None + if not device.individual_address: + self._log.warning("Device has no individual address", device=device.name) + return None + from knx_gui.programming import download_device + from xknxmono.download.scope import DownloadScope + + scope = scope or DownloadScope.FULL + self._log.info( + "Programming device", + device=device.name, + address=device.individual_address, + scope=scope.name, + ) + self._set_busy("program", device.individual_address) + future = self.run_async( + download_device( + self._xknx, + device, + scope, + group_communication, + progress=self._on_program_progress, + ) + ) + if future is not None: + future.add_done_callback(self._log_program_result) + else: + self._clear_busy() + return future + + def _log_program_result(self, future: Future[Any]) -> None: + self._clear_busy() + if future.cancelled(): + return + exc = future.exception() + if exc is not None: + self._log.error("Programming failed", error=str(exc)) + self._set_program_notice(False) + else: + self._log.info("Programming complete") + self._set_program_notice(True) + + def _set_program_notice(self, ok: bool) -> None: + with self._busy_lock: + self._program_notice = (ok, time.monotonic()) + + def program_notice(self, max_age: float = 6.0) -> bool | None: + """Recent programming outcome (``True`` ok, ``False`` failed) within ``max_age`` s, else ``None``.""" + with self._busy_lock: + notice = self._program_notice + if notice is None or time.monotonic() - notice[1] > max_age: + return None + return notice[0] + + def evaluate_device( + self, + device: Device, + scope: DownloadScope | None = None, + group_communication: GroupCommunication | None = None, + ) -> Future[Any] | None: + """Dry run: read the device and log what programming it would change.""" + if self.not_connected("evaluate_device"): + return None + if not device.individual_address: + self._log.warning("Device has no individual address", device=device.name) + return None + from knx_gui.programming import eval_device + from xknxmono.download.scope import DownloadScope + + scope = scope or DownloadScope.FULL + self._log.info( + "Testing device before programming", device=device.name, scope=scope.name + ) + self._set_busy("test", device.individual_address) + future = self.run_async( + eval_device(self._xknx, device, scope, group_communication) + ) + if future is not None: + future.add_done_callback( + functools.partial( + self._log_evaluate_result, address=device.individual_address + ) + ) + else: + self._clear_busy() + return future + + def _log_evaluate_result( + self, future: Future[Any], address: str | None = None + ) -> None: + self._clear_busy() + if future.cancelled(): + return + exc = future.exception() + if exc is not None: + self._log.error("Evaluation failed", address=address, error=str(exc)) + if isinstance(exc, ManagementConnectionError): + # A dry run still reads the live device over a point-to-point connection. + # No ACK means nothing answered at that individual address on the bus. + self._log.error( + "Device did not respond on the bus — check it is powered and " + "reachable from this interface (line/coupler) at the address", + address=address, + ) + return + report = future.result() + self._log.info( + "Pre-flight result", + changed_bytes=report.total_changed_bytes, + segments=len(report.changed_segments), + properties=len(report.changed_properties), + ) + for line in report.summary().splitlines(): + self._log.info(line.strip()) + def run_async(self, coro: Coroutine[Any, Any, Any]) -> Future[Any] | None: if self._loop is None: coro.close() diff --git a/apps/knx-gui/src/knx_gui/plugins/connection/tests/__init__.py b/apps/knx-gui/src/knx_gui/plugins/connection/tests/__init__.py new file mode 100644 index 00000000..09d10730 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/connection/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the connection plugin service (connection guards).""" diff --git a/apps/knx-gui/src/knx_gui/plugins/connection/tests/test_connection_guard.py b/apps/knx-gui/src/knx_gui/plugins/connection/tests/test_connection_guard.py new file mode 100644 index 00000000..b180dc44 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/connection/tests/test_connection_guard.py @@ -0,0 +1,67 @@ +"""Tests for the no-connection guard shared by all connection-requiring features.""" + +import time + +import pytest + +from knx_gui.plugins.connection.service import ConnectionService + + +class _FakeLogger: + def __init__(self) -> None: + self.errors: list[str] = [] + + def error(self, event: str, **kwargs: object) -> None: + self.errors.append(event) + + def debug(self, event: str, **kwargs: object) -> None: + pass + + def info(self, event: str, **kwargs: object) -> None: + pass + + def warning(self, event: str, **kwargs: object) -> None: + pass + + +@pytest.fixture +def service() -> ConnectionService: + svc = ConnectionService() + svc.set_logger(_FakeLogger()) # type: ignore[arg-type] + return svc + + +class TestNotConnectedGuard: + def test_refused_when_disconnected(self, service: ConnectionService) -> None: + assert service.not_connected("program_device") is True + assert service.not_connected_notice() is True + assert service.not_connected_notice(max_age=0.0) is False # not already aged out + + def test_allowed_when_connected(self, service: ConnectionService) -> None: + service.set_connection(object(), None) # any non-None xknx counts as connected + assert service.not_connected("program_device") is False + assert service.not_connected_notice() is False + + def test_notice_requires_a_live_rejection(self, service: ConnectionService) -> None: + service.set_connection(object(), None) + service.not_connected("program_device") # logged but no notice: connected + assert service.not_connected_notice() is False + service.set_connection(None, None) # dropped the link afterwards + assert service.not_connected_notice() is False # ...but nothing was refused + + def test_notice_covers_send_cemi(self, service: ConnectionService) -> None: + assert service.send_cemi(b"\x01\x23") is None + assert service.not_connected_notice() is True + + def test_notice_covers_eval_and_program(self, service: ConnectionService) -> None: + assert service.evaluate_device(device=None) is None # type: ignore[arg-type] + assert service.program_device(device=None) is None # type: ignore[arg-type] + assert service.not_connected_notice() is True + + def test_notice_clears_after_max_age( + self, service: ConnectionService, monkeypatch: pytest.MonkeyPatch + ) -> None: + service.not_connected("send_cemi") + now = time.monotonic() + monkeypatch.setattr(time, "monotonic", lambda: now + 6.1) + assert service.not_connected_notice(max_age=6.0) is False diff --git a/apps/knx-gui/src/knx_gui/plugins/keyring/__init__.py b/apps/knx-gui/src/knx_gui/plugins/keyring/__init__.py new file mode 100644 index 00000000..7c58a7b5 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/keyring/__init__.py @@ -0,0 +1,3 @@ +from knx_gui.plugins.keyring.plugin import KeyringPlugin + +__all__ = ["KeyringPlugin"] diff --git a/apps/knx-gui/src/knx_gui/plugins/keyring/plugin.py b/apps/knx-gui/src/knx_gui/plugins/keyring/plugin.py new file mode 100644 index 00000000..8a285098 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/keyring/plugin.py @@ -0,0 +1,36 @@ +"""Keyring / KNX Secure plugin: import and browse a ``.knxkeys`` keyring.""" + +from knx_gui.plugins.base import Logger, PanelDefinition, PluginAPI +from knx_gui.plugins.keyring.service import KeyringService +from knx_gui.plugins.keyring.strings import S +from knx_gui.plugins.keyring.ui import KeyringPanel + + +class KeyringPlugin: + name = "keyring" + + def __init__(self, api: PluginAPI) -> None: + self._api = api + self._service = KeyringService() + self._service.set_logger(Logger(api.log, "keyring")) + self._panel = KeyringPanel( + service=self._service, + get_group_addresses=lambda: api.project.group_addresses, + ) + + @property + def panels(self) -> list[PanelDefinition]: + return [ + PanelDefinition( + name="keyring", + label=S.PANEL_KEYRING, + dock="LeftSpace", + render=self._panel.render, + ) + ] + + def on_load(self) -> None: + pass + + def on_unload(self) -> None: + pass diff --git a/apps/knx-gui/src/knx_gui/plugins/keyring/service.py b/apps/knx-gui/src/knx_gui/plugins/keyring/service.py new file mode 100644 index 00000000..5409eefc --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/keyring/service.py @@ -0,0 +1,43 @@ +"""Keyring service: load a password-protected ETS ``.knxkeys`` keyring and hold it in memory. + +Uses xknx's own keyring loader (which decrypts), not the toolkit's plaintext-only ``xknx-keyring`` +package. The keyring is runtime-only state (never persisted into the project document).""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from xknx.secure.keyring import Keyring, sync_load_keyring + +if TYPE_CHECKING: + from knx_gui.plugins.base import Logger + + +class KeyringService: + def __init__(self) -> None: + self._log: Logger + self._keyring: Keyring | None = None + self._path: Path | None = None + + def set_logger(self, log: Logger) -> None: + self._log = log + + @property + def keyring(self) -> Keyring | None: + return self._keyring + + @property + def path(self) -> Path | None: + return self._path + + def load(self, path: Path, password: str) -> None: + """Decrypt and load a keyring. Raises on a wrong password / invalid file.""" + keyring = sync_load_keyring(path, password) + self._keyring = keyring + self._path = path + self._log.info("keyring loaded", path=str(path)) + + def clear(self) -> None: + self._keyring = None + self._path = None diff --git a/apps/knx-gui/src/knx_gui/plugins/keyring/strings.py b/apps/knx-gui/src/knx_gui/plugins/keyring/strings.py new file mode 100644 index 00000000..95f830d1 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/keyring/strings.py @@ -0,0 +1,65 @@ +"""Keyring plugin strings.""" + +from pathlib import Path + +from knx_gui.strings import create_translator + +_locale_dir = Path(__file__).parent / "locales" +_ = create_translator("keyring", _locale_dir) + + +class KeyringStrings: + @property + def PANEL_KEYRING(self) -> str: + return _("Keyring") + + @property + def KEYRING_EMPTY(self) -> str: + return _("No keyring loaded") + + @property + def KEYRING_IMPORT(self) -> str: + return _("Import keyring…") + + @property + def KEYRING_FILTER(self) -> str: + return _("KNX keyring (*.knxkeys)") + + @property + def ALL_FILES(self) -> str: + return _("All files") + + @property + def KEYRING_LOAD(self) -> str: + return _("Load") + + @property + def BTN_CANCEL(self) -> str: + return _("Cancel") + + @property + def KEYRING_CLOSE(self) -> str: + return _("Close") + + @property + def KEYRING_HEADER(self) -> str: + return _("{project} — created by {by}") + + @property + def KEYRING_BACKBONE(self) -> str: + return _("Backbone") + + @property + def KEYRING_INTERFACES(self) -> str: + return _("Interfaces ({count})") + + @property + def KEYRING_GROUP_ADDRESSES(self) -> str: + return _("Group address keys ({count})") + + @property + def KEYRING_DEVICES(self) -> str: + return _("Devices ({count})") + + +S = KeyringStrings() diff --git a/apps/knx-gui/src/knx_gui/plugins/keyring/ui.py b/apps/knx-gui/src/knx_gui/plugins/keyring/ui.py new file mode 100644 index 00000000..3c57dd89 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/keyring/ui.py @@ -0,0 +1,179 @@ +"""Keyring / KNX Secure panel: import a ``.knxkeys`` keyring (with password) and browse its +decrypted contents — backbone, tunnel interfaces, data-secure group-address keys, device keys.""" + +from collections.abc import Callable +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from imgui_bundle import imgui +from imgui_bundle import portable_file_dialogs as pfd + +from knx_gui.plugins.keyring.strings import S + +if TYPE_CHECKING: + from knx_gui.plugins.keyring.service import KeyringService + from knx_gui.plugins.project.service import _GroupAddress + + +def _key_preview(value: Any) -> str: + if not value: + return "-" + raw = value.hex() if isinstance(value, bytes | bytearray) else str(value) + return f"{raw[:8]}…" if len(raw) > 8 else raw + + +class KeyringPanel: + def __init__( + self, + service: "KeyringService", + get_group_addresses: "Callable[[], list[_GroupAddress]]", + ) -> None: + self._service = service + self._get_group_addresses = get_group_addresses + self._dialog: pfd.open_file | None = None + self._pending_path: str | None = None + self._password = "" + self._error: str | None = None + + def render(self) -> None: + self._poll_dialog() + if self._service.keyring is None: + self._render_import() + return + self._render_contents() + + # --- import ------------------------------------------------------------ + + def _poll_dialog(self) -> None: + if self._dialog is not None and self._dialog.ready(): + result = self._dialog.result() + self._dialog = None + if result: + self._pending_path = result[0] + self._password = "" + self._error = None + + def _render_import(self) -> None: + if self._pending_path is None: + imgui.text_disabled(S.KEYRING_EMPTY) + if imgui.button(S.KEYRING_IMPORT): + self._dialog = pfd.open_file( + S.KEYRING_IMPORT, + "", + [S.KEYRING_FILTER, "*.knxkeys", S.ALL_FILES, "*"], + ) + return + imgui.text(Path(self._pending_path).name) + imgui.set_next_item_width(240.0) + submitted, self._password = imgui.input_text( + "##keyring_pw", + self._password, + imgui.InputTextFlags_.password | imgui.InputTextFlags_.enter_returns_true, + ) + if self._error: + imgui.text_colored(imgui.ImVec4(0.9, 0.4, 0.4, 1.0), self._error) + if (imgui.button(S.KEYRING_LOAD) or submitted) and self._pending_path: + try: + self._service.load(Path(self._pending_path), self._password) + self._pending_path = None + self._password = "" + self._error = None + except ( + Exception + ) as e: # xknx raises InvalidSecureConfiguration on bad password + self._error = f"{type(e).__name__}: {e}" + imgui.same_line() + if imgui.button(S.BTN_CANCEL): + self._pending_path = None + self._password = "" + self._error = None + + # --- contents ---------------------------------------------------------- + + def _render_contents(self) -> None: + keyring = self._service.keyring + assert keyring is not None + if imgui.button(S.KEYRING_CLOSE): + self._service.clear() + return + imgui.same_line() + imgui.text_disabled( + S.KEYRING_HEADER.format( + project=keyring.project_name or "?", by=keyring.created_by or "?" + ) + ) + + if keyring.backbone is not None and imgui.collapsing_header( + S.KEYRING_BACKBONE, imgui.TreeNodeFlags_.default_open + ): + bb = keyring.backbone + imgui.text_disabled( + f"{bb.multicast_address} key {_key_preview(bb.decrypted_key)}" + ) + + self._render_interfaces(keyring.interfaces) + self._render_group_addresses(keyring.group_addresses) + self._render_devices(keyring.devices) + + def _render_interfaces(self, interfaces: list[Any]) -> None: + if not interfaces or not imgui.collapsing_header( + S.KEYRING_INTERFACES.format(count=len(interfaces)) + ): + return + flags = imgui.TableFlags_.borders_inner | imgui.TableFlags_.sizing_stretch_prop + if imgui.begin_table("##kr_ifaces", 4, flags): + for header in ("Type", "Address", "Host", "User"): + imgui.table_setup_column(header) + imgui.table_headers_row() + for iface in interfaces: + imgui.table_next_row() + imgui.table_set_column_index(0) + imgui.text(str(getattr(iface, "type", "") or "")) + imgui.table_set_column_index(1) + imgui.text(str(iface.individual_address or "")) + imgui.table_set_column_index(2) + imgui.text_disabled(str(iface.host or "")) + imgui.table_set_column_index(3) + imgui.text_disabled(str(iface.user_id or "")) + imgui.end_table() + + def _render_group_addresses(self, group_addresses: list[Any]) -> None: + if not group_addresses or not imgui.collapsing_header( + S.KEYRING_GROUP_ADDRESSES.format(count=len(group_addresses)) + ): + return + secure = {g.address for g in self._get_group_addresses() if g.data_secure} + flags = imgui.TableFlags_.borders_inner | imgui.TableFlags_.sizing_stretch_prop + if imgui.begin_table("##kr_gas", 3, flags): + for header in ("Address", "Key", "In project"): + imgui.table_setup_column(header) + imgui.table_headers_row() + for ga in group_addresses: + addr = str(ga.address) + imgui.table_next_row() + imgui.table_set_column_index(0) + imgui.text(addr) + imgui.table_set_column_index(1) + imgui.text_disabled(_key_preview(ga.decrypted_key)) + imgui.table_set_column_index(2) + if addr in secure: + imgui.text("✓") + imgui.end_table() + + def _render_devices(self, devices: list[Any]) -> None: + if not devices or not imgui.collapsing_header( + S.KEYRING_DEVICES.format(count=len(devices)) + ): + return + flags = imgui.TableFlags_.borders_inner | imgui.TableFlags_.sizing_stretch_prop + if imgui.begin_table("##kr_devices", 2, flags): + for header in ("Address", "Tool key"): + imgui.table_setup_column(header) + imgui.table_headers_row() + for dev in devices: + imgui.table_next_row() + imgui.table_set_column_index(0) + imgui.text(str(dev.individual_address or "")) + imgui.table_set_column_index(1) + imgui.text_disabled(_key_preview(dev.decrypted_tool_key)) + imgui.end_table() diff --git a/apps/knx-gui/src/knx_gui/plugins/logger/strings.py b/apps/knx-gui/src/knx_gui/plugins/logger/strings.py index 47ef3957..ef1cacc6 100644 --- a/apps/knx-gui/src/knx_gui/plugins/logger/strings.py +++ b/apps/knx-gui/src/knx_gui/plugins/logger/strings.py @@ -31,5 +31,9 @@ def COL_PLUGIN(self) -> str: def COL_MESSAGE(self) -> str: return _("Message") + @property + def COPY_LOG(self) -> str: + return _("Copy Log") + S = LoggerStrings() diff --git a/apps/knx-gui/src/knx_gui/plugins/logger/ui.py b/apps/knx-gui/src/knx_gui/plugins/logger/ui.py index 8c7391fb..e9e24212 100644 --- a/apps/knx-gui/src/knx_gui/plugins/logger/ui.py +++ b/apps/knx-gui/src/knx_gui/plugins/logger/ui.py @@ -33,6 +33,29 @@ def render(self) -> None: self._render_toolbar() self._render_table() + def _filtered_records(self) -> list[LogRecord]: + records = self._get_records() + if not self._filter_text: + return records + fl = self._filter_text.lower() + return [ + r + for r in records + if fl in r.event.lower() + or fl in r.plugin.lower() + or fl in r.level.lower() + or any(fl in v.lower() for v in r.payload.values()) + ] + + def _record_as_text(self, record: LogRecord) -> str: + line = ( + f"{record.timestamp_str} {record.level.upper()} " + f"[{record.plugin}] {record.event}" + ) + if record.payload: + line += " " + " ".join(f"{k}={v}" for k, v in record.payload.items()) + return line + def _render_toolbar(self) -> None: records = self._get_records() imgui.text_disabled(str(len(records))) @@ -42,22 +65,19 @@ def _render_toolbar(self) -> None: "##logfilter", S.FILTER_PLACEHOLDER, self._filter_text ) imgui.same_line() + # Rows are plain text (not selectable), so offer an explicit copy-all-to-clipboard. + filtered = self._filtered_records() + if imgui.button(S.COPY_LOG) and filtered: + imgui.set_clipboard_text( + "\n".join(self._record_as_text(r) for r in filtered) + ) + imgui.same_line() if imgui.button(S.BTN_CLEAR): self._on_clear() self._last_count = 0 def _render_table(self) -> None: - records = self._get_records() - if self._filter_text: - fl = self._filter_text.lower() - records = [ - r - for r in records - if fl in r.event.lower() - or fl in r.plugin.lower() - or fl in r.level.lower() - or any(fl in v.lower() for v in r.payload.values()) - ] + records = self._filtered_records() avail = imgui.get_content_region_avail() flags = ( diff --git a/apps/knx-gui/src/knx_gui/plugins/monitor/__init__.py b/apps/knx-gui/src/knx_gui/plugins/monitor/__init__.py new file mode 100644 index 00000000..51ac724c --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/monitor/__init__.py @@ -0,0 +1,3 @@ +from knx_gui.plugins.monitor.plugin import MonitorPlugin + +__all__ = ["MonitorPlugin"] diff --git a/apps/knx-gui/src/knx_gui/plugins/monitor/plugin.py b/apps/knx-gui/src/knx_gui/plugins/monitor/plugin.py new file mode 100644 index 00000000..0a058086 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/monitor/plugin.py @@ -0,0 +1,39 @@ +"""Group monitor plugin: read/write KNX group-address values with DPT decoding.""" + +from knx_gui.plugins.base import Logger, PanelDefinition, PluginAPI +from knx_gui.plugins.monitor.service import MonitorService +from knx_gui.plugins.monitor.strings import S +from knx_gui.plugins.monitor.ui import MonitorPanel + + +class MonitorPlugin: + name = "monitor" + + def __init__(self, api: PluginAPI) -> None: + self._api = api + self._service = MonitorService(api.connection) + self._service.set_logger(Logger(api.log, "monitor")) + self._panel = MonitorPanel( + service=self._service, + get_group_addresses=lambda: api.project.group_addresses, + is_connected=lambda: api.connection.xknx is not None, + ) + api.connection.add_raw_cemi_listener(self._service.on_raw_cemi) + api.connection.add_connected_listener(self._service.clear) + + @property + def panels(self) -> list[PanelDefinition]: + return [ + PanelDefinition( + name="monitor", + label=S.PANEL_MONITOR, + dock="BottomSpace", + render=self._panel.render, + ) + ] + + def on_load(self) -> None: + pass + + def on_unload(self) -> None: + pass diff --git a/apps/knx-gui/src/knx_gui/plugins/monitor/service.py b/apps/knx-gui/src/knx_gui/plugins/monitor/service.py new file mode 100644 index 00000000..ce9a98a3 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/monitor/service.py @@ -0,0 +1,115 @@ +"""Group monitor service: track the latest group-address value seen on the bus, and send +GroupValueWrite / GroupValueRead. Decoding is left to the UI thread (it has the project DPT); this +service only stores the raw payload captured on the interface thread.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from xknx.cemi import CEMIFrame, CEMILData, CEMIMessageCode +from xknx.telegram import GroupAddress as XGroupAddress +from xknx.telegram import Telegram +from xknx.telegram.apci import GroupValueRead, GroupValueResponse, GroupValueWrite + +from knx_gui.dpt import transcoder_for + +if TYPE_CHECKING: + from knx_gui.net import TelegramSource + from knx_gui.plugins.base import Logger + from knx_gui.plugins.connection.service import ConnectionService + + +@dataclass +class LiveValue: + payload: Any # DPTArray | DPTBinary + timestamp: datetime + service: str # "Write" | "Response" + + +class MonitorService: + def __init__(self, connection: ConnectionService) -> None: + self._connection = connection + self._log: Logger + # address string -> latest value seen. Written on the interface thread, read on the UI + # thread; dict item assignment is atomic under the GIL, which is enough here. + self._values: dict[str, LiveValue] = {} + + def set_logger(self, log: Logger) -> None: + self._log = log + + # --- incoming (interface thread) -------------------------------------- + + def on_raw_cemi(self, raw_cemi: bytes, _source: TelegramSource) -> None: + try: + frame = CEMIFrame.from_knx(raw_cemi) + except Exception: + return + data = frame.data + if not isinstance(data, CEMILData): + return + payload = data.payload + if isinstance(payload, GroupValueWrite): + kind = "Write" + elif isinstance(payload, GroupValueResponse): + kind = "Response" + else: + return + self._values[str(data.dst_addr)] = LiveValue( + payload=payload.value, timestamp=datetime.now(), service=kind + ) + + def latest(self, address: str) -> LiveValue | None: + return self._values.get(address) + + def clear(self) -> None: + self._values = {} + + # --- outgoing (UI thread) --------------------------------------------- + + def send_write(self, address: str, dpt: str | None, text: str) -> bool: + """Encode ``text`` per the group address' DPT and send a GroupValueWrite. Returns success.""" + transcoder = transcoder_for(dpt) + if transcoder is None: + self._log.warning("cannot write: unknown DPT", address=address, dpt=dpt) + return False + try: + knx_value = transcoder.to_knx(_coerce(text)) + except Exception as e: # xknx raises ConversionError/ValueError on bad input + self._log.warning( + "cannot encode value", address=address, value=text, error=str(e) + ) + return False + self._send(address, GroupValueWrite(knx_value)) + return True + + def send_read(self, address: str) -> None: + self._send(address, GroupValueRead()) + + def _send(self, address: str, payload: GroupValueWrite | GroupValueRead) -> None: + telegram = Telegram(destination_address=XGroupAddress(address), payload=payload) + frame = CEMIFrame( + code=CEMIMessageCode.L_DATA_REQ, + data=CEMILData.init_from_telegram(telegram), + ) + self._connection.send_cemi(frame.to_knx()) + + +def _coerce(text: str) -> Any: + """Best-effort interpret a user-typed value: bool words, then int, then float, else the string.""" + t = text.strip() + low = t.lower() + if low in ("on", "true", "yes"): + return True + if low in ("off", "false", "no"): + return False + try: + return int(t) + except ValueError: + pass + try: + return float(t) + except ValueError: + pass + return t diff --git a/apps/knx-gui/src/knx_gui/plugins/monitor/strings.py b/apps/knx-gui/src/knx_gui/plugins/monitor/strings.py new file mode 100644 index 00000000..d483e855 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/monitor/strings.py @@ -0,0 +1,45 @@ +"""Group monitor plugin strings.""" + +from pathlib import Path + +from knx_gui.strings import create_translator + +_locale_dir = Path(__file__).parent / "locales" +_ = create_translator("monitor", _locale_dir) + + +class MonitorStrings: + @property + def PANEL_MONITOR(self) -> str: + return _("Group Monitor") + + @property + def MONITOR_NO_GAS(self) -> str: + return _("No group addresses (open or import a project)") + + @property + def MONITOR_DISCONNECTED(self) -> str: + return _("Not connected") + + @property + def MONITOR_VALUE_HINT(self) -> str: + return _("value (e.g. on / 21.5 / 50)") + + @property + def MONITOR_VALUE_LABEL(self) -> str: + return _("Value:") + + @property + def MONITOR_FILTER_HINT(self) -> str: + return _("Filter by address or name…") + + @property + def MONITOR_WRITE(self) -> str: + return _("Write") + + @property + def MONITOR_READ(self) -> str: + return _("Read") + + +S = MonitorStrings() diff --git a/apps/knx-gui/src/knx_gui/plugins/monitor/ui.py b/apps/knx-gui/src/knx_gui/plugins/monitor/ui.py new file mode 100644 index 00000000..6c66c706 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/monitor/ui.py @@ -0,0 +1,136 @@ +"""Group monitor panel: a table of the project's group addresses with their latest bus value, +plus a command bar to write/read the selected group address (values decoded via the GA's DPT).""" + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +from imgui_bundle import imgui + +from knx_gui.dpt import transcoder_for +from knx_gui.plugins.monitor.strings import S + +if TYPE_CHECKING: + from knx_gui.plugins.monitor.service import MonitorService + from knx_gui.plugins.project.service import _GroupAddress + + +class MonitorPanel: + def __init__( + self, + service: "MonitorService", + get_group_addresses: "Callable[[], list[_GroupAddress]]", + is_connected: Callable[[], bool], + ) -> None: + self._service = service + self._get_group_addresses = get_group_addresses + self._is_connected = is_connected + self._selected: str | None = None + self._write_value = "" + self._filter = "" + + def render(self) -> None: + gas = self._get_group_addresses() + if not gas: + imgui.text_disabled(S.MONITOR_NO_GAS) + return + + self._render_command_bar() + imgui.set_next_item_width(-1) + _, self._filter = imgui.input_text_with_hint( + "##mon_filter", S.MONITOR_FILTER_HINT, self._filter + ) + needle = self._filter.lower() + shown = [ + ga + for ga in gas + if not needle or needle in ga.address.lower() or needle in ga.name.lower() + ] + imgui.separator() + self._render_table(shown) + + def _render_command_bar(self) -> None: + connected = self._is_connected() + if not connected: + imgui.text_disabled(S.MONITOR_DISCONNECTED) + imgui.begin_disabled(not connected or self._selected is None) + # The selected group address (set by clicking a table row), then the value to send. + imgui.text(self._selected or "-") + imgui.same_line() + imgui.text_disabled(S.MONITOR_VALUE_LABEL) + imgui.same_line() + imgui.set_next_item_width(160.0) + submitted, self._write_value = imgui.input_text_with_hint( + "##mon_value", + S.MONITOR_VALUE_HINT, + self._write_value, + imgui.InputTextFlags_.enter_returns_true, + ) + imgui.same_line() + write = imgui.button(S.MONITOR_WRITE) or submitted + imgui.same_line() + read = imgui.button(S.MONITOR_READ) + imgui.end_disabled() + + if self._selected is None or not connected: + return + dpt = self._dpt_for(self._selected) + if write: + self._service.send_write(self._selected, dpt, self._write_value) + if read: + self._service.send_read(self._selected) + + def _render_table(self, gas: "list[_GroupAddress]") -> None: + flags = ( + imgui.TableFlags_.borders_inner + | imgui.TableFlags_.sizing_stretch_prop + | imgui.TableFlags_.resizable + | imgui.TableFlags_.scroll_y + ) + if not imgui.begin_table("##monitor", 5, flags): + return + imgui.table_setup_column("Address", imgui.TableColumnFlags_.width_stretch, 0.16) + imgui.table_setup_column("Name", imgui.TableColumnFlags_.width_stretch, 0.34) + imgui.table_setup_column("DPT", imgui.TableColumnFlags_.width_stretch, 0.14) + imgui.table_setup_column("Value", imgui.TableColumnFlags_.width_stretch, 0.26) + imgui.table_setup_column("Time", imgui.TableColumnFlags_.width_stretch, 0.1) + imgui.table_headers_row() + + for ga in gas: + imgui.table_next_row() + imgui.table_set_column_index(0) + if imgui.selectable( + f"{ga.address}##mon{ga.id}", + self._selected == ga.address, + imgui.SelectableFlags_.span_all_columns, + )[0]: + self._selected = ga.address + imgui.table_set_column_index(1) + imgui.text(ga.name) + imgui.table_set_column_index(2) + imgui.text_disabled(ga.datapoint_type or "") + latest = self._service.latest(ga.address) + imgui.table_set_column_index(3) + imgui.text(_decode(latest.payload, ga.datapoint_type) if latest else "") + imgui.table_set_column_index(4) + imgui.text_disabled(latest.timestamp.strftime("%H:%M:%S") if latest else "") + + imgui.end_table() + + def _dpt_for(self, address: str) -> str | None: + for ga in self._get_group_addresses(): + if ga.address == address: + return ga.datapoint_type + return None + + +def _decode(payload: Any, dpt: str | None) -> str: + transcoder = transcoder_for(dpt) + if transcoder is not None: + try: + return str(transcoder.from_knx(payload)) + except Exception: + pass + value = getattr(payload, "value", None) + if isinstance(value, tuple): + return " ".join(f"{b:02x}" for b in value) + return str(value) if value is not None else "?" diff --git a/apps/knx-gui/src/knx_gui/plugins/node_editor/strings.py b/apps/knx-gui/src/knx_gui/plugins/node_editor/strings.py index f10af6b9..82e813eb 100644 --- a/apps/knx-gui/src/knx_gui/plugins/node_editor/strings.py +++ b/apps/knx-gui/src/knx_gui/plugins/node_editor/strings.py @@ -89,5 +89,9 @@ def BTN_REMOVE_LINKS(self) -> str: def TOOLTIP_LOCKED(self) -> str: return _("{name} (locked)") + @property + def SEARCH_HINT(self) -> str: + return _("Search…") + S = NodeEditorStrings() diff --git a/apps/knx-gui/src/knx_gui/plugins/project/knxproj_manufacturer.py b/apps/knx-gui/src/knx_gui/plugins/project/knxproj_manufacturer.py new file mode 100644 index 00000000..922aef5b --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/project/knxproj_manufacturer.py @@ -0,0 +1,138 @@ +"""Collect the manufacturer archive members a ``.knxproj`` export needs to bundle. + +``xknx-project`` is catalog-free, so its :func:`~xknxmono.project.export_knxproj` writes only the +project structure. To make the archive self-contained (applications resolvable), the GUI — which has +the catalog — re-extracts each used manufacturer's ``M-XXXX/`` tree (Hardware/Catalog/application +program XMLs, baggages) and its ``M-XXXX.signature`` from the original ``.knxprod`` the device was +imported from, and merges those archives' ``knx_master.xml`` into one. The result is passed to +``export_knxproj(..., extra_files=..., master_xml=...)``. + +The original ``.knxprod`` files must still exist at the paths the catalog recorded; refs that can't +be resolved are reported in :attr:`ManufacturerBundle.skipped_refs` and simply left out. +""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET +import zipfile +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from knx_gui.plugins.catalog.service import CatalogService + + +@dataclass +class ManufacturerBundle: + """Result of :func:`collect_manufacturer_bundle`.""" + + extra_files: dict[str, bytes] = field(default_factory=dict) + master_xml: bytes | None = None + resolved_manufacturers: set[str] = field(default_factory=set) + skipped_refs: set[str] = field(default_factory=set) + + +def collect_manufacturer_bundle( + program_refs: Iterable[str], catalog: CatalogService +) -> ManufacturerBundle: + """Gather manufacturer archive members for the given hardware-program refs. + + Args: + program_refs: Distinct hardware-program (or product) refs used by the project's devices. + catalog: The GUI catalog service (resolves a ref to its source ``.knxprod`` and manufacturer). + + Returns: + A :class:`ManufacturerBundle` with the verbatim ``extra_files`` to add, a merged + ``knx_master.xml`` (or ``None`` if nothing resolved), and bookkeeping of what was resolved + or skipped. + """ + bundle = ManufacturerBundle() + + # Map each source .knxprod to the manufacturer ids we need from it. + needed: dict[str, set[str]] = {} + for ref in program_refs: + source = catalog.get_program_source(ref) + if source is None: + bundle.skipped_refs.add(ref) + continue + knxprod_path, manufacturer_id = source + needed.setdefault(knxprod_path, set()).add(manufacturer_id) + + masters: list[bytes] = [] + for knxprod_path, manufacturer_ids in needed.items(): + try: + with zipfile.ZipFile(knxprod_path) as zf: + names = zf.namelist() + if "knx_master.xml" in names: + masters.append(zf.read("knx_master.xml")) + for mid in manufacturer_ids: + prefix = f"{mid}/" + signature = f"{mid}.signature" + for name in names: + if name.endswith("/"): + continue # zip directory entry + member = name == signature or name.startswith(prefix) + # First archive wins on collision (same manufacturer, two sources). + if member and name not in bundle.extra_files: + bundle.extra_files[name] = zf.read(name) + bundle.resolved_manufacturers.add(mid) + except (OSError, zipfile.BadZipFile): + # Source archive gone or unreadable: skip every ref that pointed at it. + bundle.skipped_refs.update(manufacturer_ids) + + bundle.master_xml = _merge_masters(masters) + return bundle + + +def _localname(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def _find_manufacturers(root: ET.Element) -> ET.Element | None: + for el in root.iter(): + if _localname(el.tag) == "Manufacturers": + return el + return None + + +def _merge_masters(masters: list[bytes]) -> bytes | None: + """Merge several ``knx_master.xml`` blobs into one, unioning their ```` entries. + + The blob with the most manufacturers is the base (it carries the full MasterData ETS needs); + manufacturers present only in the others are appended to it. + """ + if not masters: + return None + + parsed = [ET.fromstring(blob) for blob in masters] + + def manufacturer_count(root: ET.Element) -> int: + container = _find_manufacturers(root) + return len(list(container)) if container is not None else 0 + + base_index = max(range(len(parsed)), key=lambda i: manufacturer_count(parsed[i])) + base = parsed[base_index] + base_container = _find_manufacturers(base) + if base_container is None: + return masters[base_index] + + known = {m.get("Id") for m in base_container} + for i, root in enumerate(parsed): + if i == base_index: + continue + container = _find_manufacturers(root) + if container is None: + continue + for manufacturer in container: + mid = manufacturer.get("Id") + if mid not in known: + base_container.append(manufacturer) + known.add(mid) + + # Keep the default namespace unprefixed so the merged master parses like the originals. + if base.tag.startswith("{"): + ET.register_namespace("", base.tag[1:].split("}", 1)[0]) + return b'\n' + ET.tostring( + base, encoding="unicode" + ).encode("utf-8") diff --git a/apps/knx-gui/src/knx_gui/plugins/project/locales/de/LC_MESSAGES/project.mo b/apps/knx-gui/src/knx_gui/plugins/project/locales/de/LC_MESSAGES/project.mo index f989e721250fe8408b07a57eb2a544dc03e12b3d..20a154ba2efa97810f1dd9855f9641a56dbe0c95 100644 GIT binary patch delta 1384 zcmY+@L1()Q9rQzR5U)LdFn5cQ;>2swJ{O^D$4x0w|jnElMWH*e-Y|C#;1 z=T2|!ac9eSjxs{+qyDhTxk5rqv zokV?CLLp~iiIZLIGu($K)G!JqMc`qyz3@1Zi$&g!xc-^LMq4<}HWIEBhw6}Msy z70B1Y-%#Uj;ZOJwYnt#JUmey!)IInI-@^Y<@3l?Fx-KLZw=?tyP!kq#H+~!}<1YG3 zsPBG9efK9S1An9Dy}O0{*Uyj#ay<;_;wI5_`%yjf`zs>-rCVdOrQE>^+qFJA{(+QL1nq0XXT9Pe0tDIQI_e) pj{0=n6YUS0UIJr-xI_Q| delta 1078 zcmY+^J7^S96oBEA?5?`&YMv$=UrCHvABm3$TSc&lML<)86lq1UiWZ86V6lj#Fhw9l zh4iK|SOp8QP!I!x0Sns%Y@&@Khy*Rf{~zxb9_G$BcXsC9GiT=RdgXCfeX~9PD2Ow( zgS3ZD5o1`+u_G$^h$8mjE*wY>V>|sCcH=ni#w*F|xR3sw}fOZ0Ci>&O;!?0Rqk9d89&@HIN&2V{#ab|&@}JDERzdVnR) z9>89#;V4?Fo5@*hqrZgvaRsf!8d|wc+>2k(M7EP{WbbuD=y+pz4yUnxmV-GC_TbBO z!8>G&4R(wXTdDsI9cUZ-v4vT61P|jB@>ASG@4JgW`F(WaCG`Gh$+tz`zZYz9!jk?# zOaBM0NRE>xRzZKSrv3=}sxF`_67U>v8NrVZF3tiUOntD+IE>Y>!>!|XSgN0sJ(0BbTUVHXZA Y4(69vCkqd9O_vrY3mvQT#aGRL0px~EZvX%Q diff --git a/apps/knx-gui/src/knx_gui/plugins/project/locales/de/LC_MESSAGES/project.po b/apps/knx-gui/src/knx_gui/plugins/project/locales/de/LC_MESSAGES/project.po index a0d4ea07..d1c6ae57 100644 --- a/apps/knx-gui/src/knx_gui/plugins/project/locales/de/LC_MESSAGES/project.po +++ b/apps/knx-gui/src/knx_gui/plugins/project/locales/de/LC_MESSAGES/project.po @@ -141,3 +141,6 @@ msgstr "Linie {number} entfernt" msgid "Line: {old} -> {new}" msgstr "Linie: {old} -> {new}" + +msgid "Cannot run: no KNX connection. Connect to a gateway (Connection menu) and make sure the device is reachable at its individual address." +msgstr "Nicht möglich: Keine KNX-Verbindung. Verbinde ein Gateway (Menü Connection) und stelle sicher, dass das Gerät unter seiner Individualadresse erreichbar ist." diff --git a/apps/knx-gui/src/knx_gui/plugins/project/plugin.py b/apps/knx-gui/src/knx_gui/plugins/project/plugin.py index 868b71dc..b3a0c9ce 100644 --- a/apps/knx-gui/src/knx_gui/plugins/project/plugin.py +++ b/apps/knx-gui/src/knx_gui/plugins/project/plugin.py @@ -1,14 +1,57 @@ +import functools from collections.abc import Callable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from knx_gui.plugins.base import Logger, PanelDefinition, PluginAPI from knx_gui.plugins.project.strings import S -from knx_gui.plugins.project.ui import ConfigurePanel, DevicesPanel, HistoryPanel +from knx_gui.plugins.project.ui import ( + ConfigurePanel, + DevicesPanel, + GroupAddressesPanel, + HistoryPanel, + ProjectInfoPanel, + SpacesPanel, +) from knx_gui.plugins.project.ui.devices import Area, Line from knx_gui.plugins.project.ui.memory_preview import MemoryPreviewWindow +from knx_gui.plugins.project.ui.preflight_result import PreflightResultWindow if TYPE_CHECKING: + from concurrent.futures import Future + from knx_gui.device import Device + from xknxmono.download.image import GroupCommunication + from xknxmono.download.scope import DownloadScope + + +def _parse_individual_address(text: str) -> int | None: + """Parse ``area.line.device`` into a raw 16-bit individual address.""" + parts = text.split(".") + if len(parts) != 3: + return None + try: + area, line, device = (int(p) for p in parts) + except ValueError: + return None + if not (0 <= area <= 0xF and 0 <= line <= 0xF and 0 <= device <= 0xFF): + return None + return (area << 12) | (line << 8) | device + + +def _parse_group_address(text: str) -> int | None: + """Parse a 3-level (``a/b/c``), 2-level (``a/b``) or free group address.""" + parts = text.split("/") + try: + nums = [int(p) for p in parts] + except ValueError: + return None + if len(nums) == 3: + return (nums[0] << 11) | (nums[1] << 8) | nums[2] + if len(nums) == 2: + return (nums[0] << 11) | nums[1] + if len(nums) == 1: + return nums[0] + return None class ProjectPlugin: @@ -26,6 +69,7 @@ def __init__( self._memory_preview = MemoryPreviewWindow( get_devices=lambda: api.project.devices ) + self._preflight_result = PreflightResultWindow() self._devices_panel = DevicesPanel( get_devices=lambda: api.project.devices, @@ -49,10 +93,35 @@ def __init__( on_individual_address_change=self._handle_individual_address_change, on_name_change=self._handle_name_change, set_flag=self._handle_flag_change, - on_program_device=api.connection.assign_individual_address_for_device, + get_links_for_com_object=self._links_for_com_object, + get_all_group_addresses=self._all_group_addresses, + on_link_com_object=self._link_com_object, + on_unlink_com_object=api.project.unlink_com_object_from_ga, + get_device_info=api.project.get_device_info, + on_program_device=self._program_device, + on_eval_device=self._eval_device, open_memory_preview=self._memory_preview.open, ) + self._group_addresses_panel = GroupAddressesPanel( + get_range_tree=api.project.get_group_range_tree, + get_assignments_for_ga=api.project.get_assignments_for_ga, + get_devices=lambda: api.project.devices, + on_create_ga=self._on_create_ga, + on_rename_ga=api.project.rename_group_address, + on_set_ga_dpt=api.project.set_group_address_dpt, + on_remove_ga=api.project.remove_group_address, + ) + + self._spaces_panel = SpacesPanel( + get_space_tree=api.project.get_space_tree, + on_select_device_id=self._select_device_by_id, + ) + + self._project_info_panel = ProjectInfoPanel( + get_project_info=api.project.get_project_metadata, + ) + self._history_panel = HistoryPanel( get_entries=self._get_history_entries, get_cursor=lambda: api.project.cursor, @@ -67,9 +136,21 @@ def __init__( render=self._devices_panel.render, ), PanelDefinition( - name="configure", - label=S.PANEL_CONFIGURE, - dock="RightSpace", + name="buildings", + label=S.PANEL_BUILDINGS, + dock="LeftSpace", + render=self._spaces_panel.render, + ), + PanelDefinition( + name="group_addresses", + label=S.PANEL_GROUP_ADDRESSES, + dock="LeftSpace", + render=self._group_addresses_panel.render, + ), + PanelDefinition( + name="editor", + label=S.PANEL_EDITOR, + dock="MainDockSpace", render=self._render_configure, ), PanelDefinition( @@ -78,6 +159,12 @@ def __init__( dock="RightSpace", render=self._history_panel.render, ), + PanelDefinition( + name="project_info", + label=S.PANEL_PROJECT_INFO, + dock="RightSpace", + render=self._project_info_panel.render, + ), ] def _get_areas(self) -> list[Area]: @@ -143,6 +230,36 @@ def _on_move_device( def _set_selected_device(self, device: "Device") -> None: self._api.project.selected_device = device + def _select_device_by_id(self, node_id: int) -> None: + device = self._api.project.find_device_by_node_id(node_id) + if device is not None: + self._api.project.selected_device = device + + def _on_create_ga(self, address: str, name: str) -> None: + self._api.project.create_group_address(address, name) + + def _links_for_com_object( + self, com_object_db_id: int + ) -> list[tuple[int, int, str, bool]]: + result: list[tuple[int, int, str, bool]] = [] + for link in self._api.project.get_links_for_com_object(com_object_db_id): + ga = self._api.project.get_group_address(link.group_address_id) + if ga is not None: + result.append( + (link.id, link.group_address_id, ga.address, link.is_sending) + ) + return result + + def _all_group_addresses(self) -> list[tuple[int, str]]: + return [(g.id, g.address) for g in self._api.project.group_addresses] + + def _link_com_object(self, com_object_db_id: int, group_address_id: int) -> None: + # The first group address a com-object links to becomes its sending address (like ETS). + existing = self._api.project.get_links_for_com_object(com_object_db_id) + self._api.project.link_com_object_to_ga( + com_object_db_id, group_address_id, is_sending=not existing + ) + def _handle_param_change( self, device: "Device", param_id: str, new_value: str ) -> None: @@ -169,6 +286,93 @@ def _handle_flag_change( ) -> None: self._api.project.set_flag(device, co_id, flag_name, new_value) + def _program_device(self, device: "Device", scope: "DownloadScope") -> None: + self._api.connection.program_device( + device, scope, self._group_communication_for(device) + ) + + def _eval_device(self, device: "Device", scope: "DownloadScope") -> None: + label = device.name or device.individual_address or "device" + # The test reads the live device, so it needs a bus connection; without one + # there is nothing to compare against. + if self._api.connection.xknx is None: + self._preflight_result.submit_error( + label, scope.name, S.PREFLIGHT_NO_CONNECTION + ) + return + # The test validates our image generation against the device's programmed state, which + # matches the project only as long as it is unedited (import produces a zero-event project). + # Any edit would make a diff ambiguous, so refuse the test once the project was changed. + if self._api.project.history(): + self._preflight_result.submit_error( + label, scope.name, S.PREFLIGHT_PROJECT_MODIFIED + ) + return + runtime = self._runtime_managed_addresses(device) + future = self._api.connection.evaluate_device( + device, scope, self._group_communication_for(device) + ) + if future is not None: + future.add_done_callback( + functools.partial(self._on_eval_done, label, scope.name, runtime) + ) + + @staticmethod + def _runtime_managed_addresses(device: "Device") -> set[int]: + """Best-effort set of device-managed (runtime) memory addresses; empty on error.""" + from knx_gui.programming import runtime_managed_addresses + + try: + return runtime_managed_addresses(device) + except Exception: + return set() + + def _on_eval_done( + self, label: str, scope_name: str, runtime: set[int], future: "Future[Any]" + ) -> None: + """Runs on the async loop thread; only hands the result to the (thread-safe) window.""" + if future.cancelled(): + return + exc = future.exception() + if exc is not None: + self._preflight_result.submit_error( + label, scope_name, f"{type(exc).__name__}: {exc}" + ) + return + self._preflight_result.submit_result( + label, scope_name, future.result(), runtime + ) + + def _group_communication_for(self, device: "Device") -> "GroupCommunication | None": + """Collect the device's group address links into a GroupCommunication.""" + from xknxmono.download import GroupCommunication + from xknxmono.download.project_data import GroupObjectLink + + device_address = _parse_individual_address(device.individual_address) + if device_address is None: + return None + links: list[GroupObjectLink] = [] + for com_object in device.com_objects: + if com_object.db_id is None: + continue + for assignment in self._api.project.get_links_for_com_object( + com_object.db_id + ): + ga = self._api.project.get_group_address(assignment.group_address_id) + if ga is None: + continue + address = _parse_group_address(ga.address) + if address is None: + continue + links.append( + GroupObjectLink( + com_object_ref_id=com_object.id, + group_address=address, + sending=assignment.is_sending, + ) + ) + return GroupCommunication(device_address=device_address, links=links) + def _get_history_entries(self): return self._api.project.history() @@ -201,6 +405,7 @@ def panels(self) -> list[PanelDefinition]: def render_overlays(self) -> None: self._memory_preview.render() + self._preflight_result.render() def on_load(self) -> None: pass diff --git a/apps/knx-gui/src/knx_gui/plugins/project/service.py b/apps/knx-gui/src/knx_gui/plugins/project/service.py index 4955433e..bd049090 100644 --- a/apps/knx-gui/src/knx_gui/plugins/project/service.py +++ b/apps/knx-gui/src/knx_gui/plugins/project/service.py @@ -13,16 +13,20 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +from knx_gui.concurrency import io_guarded from knx_gui.device import Device from knx_gui.plugins.project.ui.history import HistoryEntry from xknxmono.models.intermediate import ComObjectInstanceRef from xknxmono.models.intermediate.enable_t import Enable from xknxmono.product import Application from xknxmono.project import ProjectService as _ProjectService +from xknxmono.project import import_knxproj as _import_knxproj +from xknxmono.project.core.addressing import GroupAddressStyle, parse_ga if TYPE_CHECKING: from knx_gui.plugins.base import Logger from knx_gui.plugins.catalog.service import CatalogService + from xknxmono.project.core.service import DeviceInfo, GroupRangeInfo, SpaceInfo _INSTALLATION = 0 @@ -123,6 +127,22 @@ class _GroupAddress: id: int address: str name: str + datapoint_type: str | None = None + description: str = "" + comment: str = "" + data_secure: bool = False + + +@dataclass +class _ProjectInfo: + id: str + name: str + group_address_style: str + guid: str + created_by: str + last_modified: str + schema_version: str + tool_version: str @dataclass @@ -136,6 +156,8 @@ class _Assignment: class ProjectService: def __init__(self, catalog: "CatalogService") -> None: self._catalog = catalog + # Share the catalog's re-entrant lock: a background import holds it while writing both stores. + self._io_lock = catalog.io_lock self._svc = _ProjectService() self._pid: str | None = None self._path: Path | None = None @@ -185,12 +207,46 @@ def new(self, path: Path) -> None: self._log.info("project created", path=str(path)) def open(self, path: Path) -> None: - if self._pid is not None: - self.close() - self._pid = self._svc.open(path) - self._path = path - self._reset() - self._log.info("project opened", path=str(path), devices=len(self.devices)) + # Hold the shared lock for the whole open (incl. the device-view build) so per-frame UI reads + # on other threads bail to empty placeholders instead of racing it — lets a background open + # run behind a progress spinner. Re-entrant, so our own nested reads still work. + with self._io_lock: + if self._pid is not None: + self.close() + self._pid = self._svc.open(path) + self._path = path + self._reset() + self._log.info("project opened", path=str(path), devices=len(self.devices)) + + def import_knxproj( + self, source: Path, dest: Path, *, password: str | None = None + ) -> None: + """Import an ETS ``.knxproj`` into a new project at ``dest`` and open it. + + A ``.knxproj`` bundles the (unencrypted) manufacturer/application data for the products it + uses, so we also ingest it into the catalog — without it the device view cannot resolve the + applications and would skip every imported device.""" + if not dest.suffix: + dest = dest.with_suffix(".xknx") + # Hold the shared lock for the whole import so per-frame UI reads on other threads bail to + # empty placeholders (see knx_gui.concurrency) instead of racing these writes. This method is + # meant to run on a worker thread; the lock is re-entrant, so our own nested reads still work. + with self._io_lock: + try: + added = self._catalog.import_knxprod(source) + self._log.info("catalog updated from knxproj", added=len(added)) + except Exception as e: + # Best effort: catalog ingest is optional enrichment (it lets the device view resolve + # applications). Any failure here — including product-parser bugs on odd archives — + # must not block the project import; topology and group addresses still load. + self._log.warning( + "could not populate catalog from knxproj", + source=str(source), + error=f"{type(e).__name__}: {e}", + ) + _import_knxproj(source, dest, password=password) + self.open(dest) + self._log.info("project imported", source=str(source), path=str(dest)) def close(self) -> None: if self._pid is not None: @@ -271,6 +327,7 @@ def _build_device(self, row: Any) -> Device | None: return device @property + @io_guarded(list) def devices(self) -> list[Device]: if self._devices_cache is None or self._cache_version != self._version: devices: list[Device] = [] @@ -331,12 +388,14 @@ def _ensure_topology_cache(self) -> None: } self._cache_version = self._version + @io_guarded(list) def get_areas(self) -> list[_Area]: if self._pid is None: return [] self._ensure_topology_cache() return self._areas_cache or [] + @io_guarded(list) def get_lines(self, area_id: int) -> list[_Line]: if self._pid is None: return [] @@ -346,18 +405,28 @@ def get_lines(self, area_id: int) -> list[_Line]: # --- group address reads ---------------------------------------------- @property + @io_guarded(list) def group_addresses(self) -> list[_GroupAddress]: if self._pid is None: return [] if self._ga_cache is not None and self._cache_version == self._version: return self._ga_cache self._ga_cache = [ - _GroupAddress(id=g.id, address=g.text, name=g.name) + _GroupAddress( + id=g.id, + address=g.text, + name=g.name, + datapoint_type=g.datapoint_type, + description=g.description, + comment=g.comment, + data_secure=g.data_secure, + ) for g in self._svc.group_addresses(self._pid) ] self._cache_version = self._version return self._ga_cache + @io_guarded(lambda: None) def get_group_address(self, ga_id: int) -> _GroupAddress | None: if self._pid is None: return None @@ -365,8 +434,17 @@ def get_group_address(self, ga_id: int) -> _GroupAddress | None: g = self._svc.group_address(self._pid, ga_id) except KeyError: return None - return _GroupAddress(id=g.id, address=g.text, name=g.name) + return _GroupAddress( + id=g.id, + address=g.text, + name=g.name, + datapoint_type=g.datapoint_type, + description=g.description, + comment=g.comment, + data_secure=g.data_secure, + ) + @io_guarded(list) def get_assignments_for_ga(self, ga_id: int) -> list[_Assignment]: if self._pid is None: return [] @@ -380,6 +458,79 @@ def get_assignments_for_ga(self, ga_id: int) -> list[_Assignment]: for link in self._svc.group_address_links(self._pid, ga_id) ] + @io_guarded(list) + def get_links_for_com_object(self, com_object_db_id: int) -> list[_Assignment]: + """A device com-object's group-address links (the per-com-object direction, for the editor's + Group Objects view). ``com_object_db_id`` is ``ComObject.db_id``.""" + if self._pid is None: + return [] + return [ + _Assignment( + id=link.id, + com_object_id=link.com_object_id, + group_address_id=link.group_address_id, + is_sending=link.is_sending, + ) + for link in self._svc.com_object_links(self._pid, com_object_db_id) + ] + + @io_guarded(list) + def get_group_range_tree(self) -> list["GroupRangeInfo"]: + """The named group-address range tree (roots → children → GAs) for the GA view.""" + if self._pid is None: + return [] + return self._svc.group_ranges(self._pid, _INSTALLATION) + + @io_guarded(list) + def get_space_tree(self) -> list["SpaceInfo"]: + """The building/location tree (spaces → devices/functions) for the Buildings view.""" + if self._pid is None: + return [] + return self._svc.space_tree(self._pid, _INSTALLATION) + + @io_guarded(lambda: None) + def get_project_metadata(self) -> _ProjectInfo | None: + """Project-level metadata (name, author, tool/schema version, …) from the imported project.""" + if self._pid is None: + return None + p = self._svc.project(self._pid) + return _ProjectInfo( + id=p.id, + name=p.name, + group_address_style=p.group_address_style, + guid=p.guid, + created_by=p.created_by, + last_modified=p.last_modified, + schema_version=p.schema_version, + tool_version=p.tool_version, + ) + + @io_guarded(lambda: None) + def get_device_info(self, node_id: int) -> "DeviceInfo | None": + """Descriptive device metadata (manufacturer/order number/hardware/description) from the + imported project, independent of catalog resolution.""" + if self._pid is None: + return None + try: + return self._svc.device(self._pid, node_id) + except KeyError: + return None + + @io_guarded(set) + def program_refs(self) -> set[str]: + """Distinct hardware-program refs (fallback product refs) across all project devices. + + Used to collect the manufacturer archives a ``.knxproj`` export needs to bundle. + """ + if self._pid is None: + return set() + refs: set[str] = set() + for row in self._svc.devices(self._pid): + ref = row.hardware2program_ref_id or row.product_ref_id + if ref: + refs.add(ref) + return refs + # --- device edits ----------------------------------------------------- def add_device( @@ -498,13 +649,35 @@ def rename_line(self, line_id: int, old_name: str, new_name: str) -> None: def create_group_address( self, address: str | None = None, name: str = "" ) -> int | None: + """Create a group address. ``address`` is a style-formatted string (e.g. ``"1/2/3"``); when + omitted, the next free address is allocated. Returns ``None`` on an invalid address.""" if self._pid is None: return None - value = self._svc.next_free_group_address(self._pid, _INSTALLATION) + if address: + style = GroupAddressStyle(self._svc.project(self._pid).group_address_style) + try: + value = parse_ga(address, style) + except (ValueError, IndexError): + self._log.warning("invalid group address", address=address) + return None + else: + value = self._svc.next_free_group_address(self._pid, _INSTALLATION) ga_id = self._svc.create_group_address(self._pid, _INSTALLATION, value, name) self._bump() return ga_id + def rename_group_address(self, ga_id: int, name: str) -> None: + if self._pid is None: + return + self._svc.rename_group_address(self._pid, ga_id, name) + self._bump() + + def set_group_address_dpt(self, ga_id: int, dpt: str | None) -> None: + if self._pid is None: + return + self._svc.set_group_address_datapoint_type(self._pid, ga_id, dpt or None) + self._bump() + def remove_group_address( self, ga_id: int, address: str = "", name: str = "" ) -> None: @@ -554,13 +727,16 @@ def redo(self) -> bool: self._bump() return result + @io_guarded(lambda: False) def can_undo(self) -> bool: return self._pid is not None and self._svc.can_undo(self._pid) + @io_guarded(lambda: False) def can_redo(self) -> bool: return self._pid is not None and self._svc.can_redo(self._pid) @property + @io_guarded(lambda: 0) def cursor(self) -> int: return self._svc.cursor(self._pid) if self._pid is not None else 0 @@ -570,6 +746,7 @@ def jump_to(self, event_id: int) -> None: self._svc.jump_to(self._pid, event_id) self._bump() + @io_guarded(list) def history(self) -> list[HistoryEntry]: if self._pid is None: return [] diff --git a/apps/knx-gui/src/knx_gui/plugins/project/strings.py b/apps/knx-gui/src/knx_gui/plugins/project/strings.py index 2efc7ae9..915cbe66 100644 --- a/apps/knx-gui/src/knx_gui/plugins/project/strings.py +++ b/apps/knx-gui/src/knx_gui/plugins/project/strings.py @@ -21,6 +21,90 @@ def PANEL_CONFIGURE(self) -> str: def PANEL_HISTORY(self) -> str: return _("History") + @property + def PANEL_EDITOR(self) -> str: + return _("Editor") + + @property + def PANEL_GROUP_ADDRESSES(self) -> str: + return _("Group Addresses") + + @property + def PANEL_BUILDINGS(self) -> str: + return _("Buildings") + + @property + def PANEL_PROJECT_INFO(self) -> str: + return _("Project") + + @property + def SPACES_EMPTY(self) -> str: + return _("No buildings/rooms in this project") + + @property + def PROJECT_INFO_EMPTY(self) -> str: + return _("No project") + + @property + def PROJECT_INFO_NAME(self) -> str: + return _("Name") + + @property + def PROJECT_INFO_GA_STYLE(self) -> str: + return _("Group address style") + + @property + def PROJECT_INFO_CREATED_BY(self) -> str: + return _("Created by") + + @property + def PROJECT_INFO_TOOL_VERSION(self) -> str: + return _("Tool version") + + @property + def PROJECT_INFO_SCHEMA_VERSION(self) -> str: + return _("Schema version") + + @property + def PROJECT_INFO_LAST_MODIFIED(self) -> str: + return _("Last modified") + + @property + def PROJECT_INFO_GUID(self) -> str: + return _("GUID") + + @property + def CONFIGURE_ORDER_NUMBER(self) -> str: + return _("Order number") + + @property + def CONFIGURE_DESCRIPTION(self) -> str: + return _("Description") + + @property + def GA_DESCRIPTION(self) -> str: + return _("Description") + + @property + def GA_COMMENT(self) -> str: + return _("Comment") + + @property + def EDITOR_TAB_PARAMETERS(self) -> str: + return _("Parameters ({count})") + + @property + def EDITOR_TAB_GROUP_OBJECTS(self) -> str: + return _("Group Objects ({count})") + + @property + def GA_NO_PROJECT(self) -> str: + return _("No project") + + @property + def GA_ASSIGNED_OBJECTS(self) -> str: + return _("Assigned objects") + @property def CONFIGURE_NO_DEVICES(self) -> str: return _("No devices") @@ -37,6 +121,37 @@ def CONFIGURE_INDIVIDUAL_ADDRESS(self) -> str: def BTN_PROGRAM_DEVICE(self) -> str: return _("Program Device") + @property + def BTN_EVAL_DEVICE(self) -> str: + return _("Test Before Programming") + + @property + def PROGRAM_CONFIRM_TITLE(self) -> str: + return _("Program device?") + + @property + def PROGRAM_CONFIRM_TEXT(self) -> str: + return _( + "This writes to the device at {address} (scope: {scope}) and changes its " + "configuration. Continue?" + ) + + @property + def CONFIGURE_DOWNLOAD_SCOPE(self) -> str: + return _("Download") + + @property + def SCOPE_FULL(self) -> str: + return _("Full") + + @property + def SCOPE_PARAMETERS(self) -> str: + return _("Partial: Parameters") + + @property + def SCOPE_GROUP_COMMUNICATION(self) -> str: + return _("Partial: Group Communication") + @property def CONFIGURE_MANUFACTURER(self) -> str: return _("Manufacturer") @@ -73,6 +188,125 @@ def BTN_PREVIEW_MEMORY(self) -> str: def CONFIGURE_MEMORY_PREVIEW(self) -> str: return _("Memory Preview") + @property + def PREFLIGHT_RESULT_TITLE(self) -> str: + return _("Programming test result") + + @property + def PREFLIGHT_FAILED(self) -> str: + return _("Evaluation failed") + + @property + def PREFLIGHT_NO_CHANGES_MADE(self) -> str: + return _( + "Read-only test: checks that the download function generates the same image " + "as what is already programmed on the device. Nothing is written to the device." + ) + + @property + def PREFLIGHT_PROJECT_MODIFIED(self) -> str: + return _( + "Cannot run: the project was changed since import. This test compares the " + "generated image against the device's programmed state, which only matches " + "an unedited (freshly imported) project. Undo your changes or re-import to test." + ) + + @property + def PREFLIGHT_NO_CONNECTION(self) -> str: + return _( + "Cannot run: no KNX connection. Connect to a gateway (Connection menu) " + "and make sure the device is reachable at its individual address." + ) + + @property + def PREFLIGHT_MATCH(self) -> str: + return _("Match: the generated image is identical to what is on the device") + + @property + def PREFLIGHT_WOULD_CHANGE(self) -> str: + return _( + "Mismatch: {bytes} byte(s) differ from the device in {locations} location(s)" + ) + + @property + def PREFLIGHT_SUMMARY_COUNTS(self) -> str: + return _("{matched} matched, {changed} would change") + + @property + def PREFLIGHT_COL_LOCATION(self) -> str: + return _("Location") + + @property + def PREFLIGHT_COL_SIZE(self) -> str: + return _("Size") + + @property + def PREFLIGHT_COL_STATUS(self) -> str: + return _("Status") + + @property + def PREFLIGHT_COL_CHANGED(self) -> str: + return _("Changed") + + @property + def PREFLIGHT_STATUS_MATCH(self) -> str: + return _("match") + + @property + def PREFLIGHT_STATUS_CHANGE(self) -> str: + return _("would change") + + @property + def PREFLIGHT_STATUS_RUNTIME(self) -> str: + return _("device-managed") + + @property + def PREFLIGHT_RUNTIME_TOOLTIP(self) -> str: + return _( + "System byte the device sets itself at runtime (e.g. a download " + "detection byte reset by the application after a download). A " + "difference here is expected and does not indicate a bad download." + ) + + @property + def PREFLIGHT_RUNTIME_NOTE(self) -> str: + return _("plus {bytes} device-managed runtime byte(s) (expected, benign)") + + @property + def PREFLIGHT_EXPORT(self) -> str: + return _("Export Ist/Soll...") + + @property + def PREFLIGHT_EXPORT_PATH(self) -> str: + return _("Export path:") + + @property + def PREFLIGHT_MEM_LABEL(self) -> str: + return _("memory {address}") + + @property + def PREFLIGHT_PROP_LABEL(self) -> str: + return _("object {object} property {property}") + + @property + def DEVICE_FILTER_HINT(self) -> str: + return _("Filter devices (name, address)...") + + @property + def DEVICE_EMPTY_HINT(self) -> str: + return _( + "No devices. Open or import a project (File menu), " + "or right-click here to add an area." + ) + + @property + def GA_FILTER_HINT(self) -> str: + return _("Filter group addresses (address, name)...") + + @property + def SPACES_FILTER_HINT(self) -> str: + return _("Filter rooms, devices, functions...") + @property def DEVICE_AREA(self) -> str: return _("Area {area}") @@ -109,6 +343,10 @@ def CONTEXT_RENAME(self) -> str: def CONTEXT_DELETE(self) -> str: return _("Delete") + @property + def CONTEXT_COPY_ADDRESS(self) -> str: + return _("Copy address") + @property def POPUP_NEW_AREA(self) -> str: return _("New Area") @@ -129,6 +367,46 @@ def POPUP_NUMBER(self) -> str: def POPUP_NAME(self) -> str: return _("Name") + @property + def BTN_ADD(self) -> str: + return _("Add") + + @property + def BTN_OK(self) -> str: + return _("OK") + + @property + def BTN_SAVE(self) -> str: + return _("Save") + + @property + def COPY_LOG(self) -> str: + return _("Copy Log") + + @property + def BTN_CANCEL(self) -> str: + return _("Cancel") + + @property + def GA_NEW(self) -> str: + return _("New group address") + + @property + def GA_RENAME(self) -> str: + return _("Rename group address") + + @property + def GA_SET_DPT(self) -> str: + return _("Set datapoint type") + + @property + def GA_ADDRESS(self) -> str: + return _("Address") + + @property + def GA_DPT_HINT(self) -> str: + return _("Datapoint type (e.g. DPST-1-1); empty to clear") + @property def STATUS_PROJECT(self) -> str: return _("Project: {name}") diff --git a/apps/knx-gui/src/knx_gui/plugins/project/ui/__init__.py b/apps/knx-gui/src/knx_gui/plugins/project/ui/__init__.py index c763d7d6..5ff9fa4a 100644 --- a/apps/knx-gui/src/knx_gui/plugins/project/ui/__init__.py +++ b/apps/knx-gui/src/knx_gui/plugins/project/ui/__init__.py @@ -1,10 +1,16 @@ from knx_gui.plugins.project.ui.configure import ConfigurePanel from knx_gui.plugins.project.ui.devices import DevicesPanel +from knx_gui.plugins.project.ui.group_addresses import GroupAddressesPanel from knx_gui.plugins.project.ui.history import HistoryEntry, HistoryPanel +from knx_gui.plugins.project.ui.project_info import ProjectInfoPanel +from knx_gui.plugins.project.ui.spaces import SpacesPanel __all__ = [ "ConfigurePanel", "DevicesPanel", + "GroupAddressesPanel", "HistoryEntry", "HistoryPanel", + "ProjectInfoPanel", + "SpacesPanel", ] diff --git a/apps/knx-gui/src/knx_gui/plugins/project/ui/_filter.py b/apps/knx-gui/src/knx_gui/plugins/project/ui/_filter.py new file mode 100644 index 00000000..2256b53e --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/project/ui/_filter.py @@ -0,0 +1,23 @@ +"""Shared filter-box widget for the project trees (devices, group addresses, spaces). + +Renders a full-width hint input followed by a small clear button, and returns the +(possibly cleared) filter text. +""" + +from __future__ import annotations + +from imgui_bundle import imgui + + +def filter_box(widget_id: str, hint: str, text: str) -> str: + """Draw a filter input with a clear button; return the current text.""" + btn = imgui.get_frame_height() + avail = imgui.get_content_region_avail().x + imgui.set_next_item_width(max(avail - btn - 4.0, 1.0)) + _, text = imgui.input_text_with_hint(widget_id, hint, text) + imgui.same_line(0.0, 4.0) + imgui.begin_disabled(not text) + if imgui.button(f"x##{widget_id}_clear", imgui.ImVec2(btn, btn)): + text = "" + imgui.end_disabled() + return text diff --git a/apps/knx-gui/src/knx_gui/plugins/project/ui/configure.py b/apps/knx-gui/src/knx_gui/plugins/project/ui/configure.py index 9a37c3f3..7082cbc6 100644 --- a/apps/knx-gui/src/knx_gui/plugins/project/ui/configure.py +++ b/apps/knx-gui/src/knx_gui/plugins/project/ui/configure.py @@ -1,14 +1,23 @@ from collections.abc import Callable +from typing import TYPE_CHECKING from imgui_bundle import imgui from knx_gui.device import Device from knx_gui.plugins.project.strings import S +from xknxmono.download.scope import DownloadScope + +if TYPE_CHECKING: + from xknxmono.project.core.service import DeviceInfo from knx_gui.widgets import ( - ComFlagsTable, + GroupObjectsTable, count_parameters, render_ui_tree, ) +from knx_gui.widgets.group_objects_widgets import ( + GroupAddressCatalog, + GroupLinkResolver, +) class ConfigurePanel: @@ -21,7 +30,13 @@ def __init__( on_individual_address_change: Callable[[Device, str], None], on_name_change: Callable[[Device, str], None], set_flag: Callable[[Device, str, str, bool], None], - on_program_device: Callable[[Device], None] | None = None, + get_links_for_com_object: GroupLinkResolver, + get_all_group_addresses: GroupAddressCatalog, + on_link_com_object: Callable[[int, int], None], + on_unlink_com_object: Callable[[int], None], + get_device_info: "Callable[[int], DeviceInfo | None]", + on_program_device: Callable[[Device, DownloadScope], None] | None = None, + on_eval_device: Callable[[Device, DownloadScope], None] | None = None, open_memory_preview: Callable[[Device], None] | None = None, ) -> None: self._get_devices = get_devices @@ -31,11 +46,19 @@ def __init__( self._on_individual_address_change = on_individual_address_change self._on_name_change = on_name_change self._on_program_device = on_program_device + self._on_eval_device = on_eval_device self._open_memory_preview = open_memory_preview - self._com_flags_table = ComFlagsTable(set_flag) + self._get_links = get_links_for_com_object + self._get_all_gas = get_all_group_addresses + self._get_device_info = get_device_info + self._group_objects_table = GroupObjectsTable( + set_flag, on_link_com_object, on_unlink_com_object + ) self._name_buffer: str = "" self._address_buffer: str = "" self._buffer_device_id: int | None = None + self._download_scope: DownloadScope = DownloadScope.FULL + self._confirm_program_open: bool = False def render(self) -> None: devices = self._get_devices() @@ -96,81 +119,161 @@ def render(self) -> None: ): self._address_buffer = device.individual_address - if self._on_program_device is not None: + if self._on_program_device is not None or self._on_eval_device is not None: + self._render_scope_selector() + + # Workflow order: dry run (preview changes) → preview memory image → program. + if self._on_eval_device is not None: enabled = bool(device.individual_address) imgui.begin_disabled(not enabled) - if imgui.button(S.BTN_PROGRAM_DEVICE): - self._on_program_device(device) + if imgui.button(S.BTN_EVAL_DEVICE): + self._on_eval_device(device, self._download_scope) imgui.end_disabled() imgui.same_line() - if self._open_memory_preview is not None and imgui.button(S.BTN_PREVIEW_MEMORY): - self._open_memory_preview(device) + if self._open_memory_preview is not None: + if imgui.button(S.BTN_PREVIEW_MEMORY): + self._open_memory_preview(device) + imgui.same_line() + + if self._on_program_device is not None: + enabled = bool(device.individual_address) + imgui.begin_disabled(not enabled) + if imgui.button(S.BTN_PROGRAM_DEVICE): + # Programming writes to the device; confirm first (like ETS). + self._confirm_program_open = True + imgui.end_disabled() + if self._confirm_program_open: + imgui.open_popup(S.PROGRAM_CONFIRM_TITLE) + self._confirm_program_open = False + self._render_program_confirm(device) if imgui.collapsing_header( S.CONFIGURE_MANUFACTURER, imgui.TreeNodeFlags_.default_open ): + info = self._get_device_info(device.node_id) + manufacturer = ( + info.manufacturer_name if info and info.manufacturer_name else None + ) self._render_label_value( - S.CONFIGURE_MANUFACTURER, device.app.manufacturer_id + S.CONFIGURE_MANUFACTURER, manufacturer or device.app.manufacturer_id ) self._render_label_value(S.CONFIGURE_APPLICATION, device.app.id) + if info is not None: + if info.order_number: + self._render_label_value( + S.CONFIGURE_ORDER_NUMBER, info.order_number + ) + if info.hardware_name: + self._render_label_value(S.CONFIGURE_HARDWARE, info.hardware_name) + if info.description: + self._render_label_value(S.CONFIGURE_DESCRIPTION, info.description) - ui_nodes = device.get_ui() - param_count = count_parameters(ui_nodes) - if ui_nodes and imgui.collapsing_header( - S.CONFIGURE_PARAMETERS.format(count=param_count), - imgui.TreeNodeFlags_.default_open, - ): - render_ui_tree(device, ui_nodes, self._on_param_change) + if imgui.begin_tab_bar("##editor_tabs"): + ui_nodes = device.get_ui() + param_count = count_parameters(ui_nodes) + if imgui.begin_tab_item(S.EDITOR_TAB_PARAMETERS.format(count=param_count))[ + 0 + ]: + if ui_nodes: + render_ui_tree(device, ui_nodes, self._on_param_change) + else: + imgui.text_disabled(S.CONFIGURE_NO_DEVICES) + self._render_load_procedures(device) + imgui.end_tab_item() - visible_cos = device.get_visible_com_objects() - if imgui.collapsing_header( - S.CONFIGURE_COM_FLAGS.format(count=len(visible_cos)), - imgui.TreeNodeFlags_.default_open, - ): - self._com_flags_table.render(device, visible_cos) + visible_cos = device.get_visible_com_objects() + if imgui.begin_tab_item( + S.EDITOR_TAB_GROUP_OBJECTS.format(count=len(visible_cos)) + )[0]: + self._group_objects_table.render( + device, visible_cos, self._get_links, self._get_all_gas + ) + imgui.end_tab_item() + imgui.end_tab_bar() + + def _render_load_procedures(self, device: Device) -> None: lp = device.app.load_procedures procedures = getattr(lp, "procedures", None) - if procedures: - total_steps = sum(len(p.steps) for p in procedures) - if imgui.collapsing_header( - S.CONFIGURE_LOAD_PROCEDURES.format(count=total_steps) - ): - imgui.text_disabled(getattr(lp, "style", "")) - for i, proc in enumerate(procedures): - label = f"Procedure {i + 1} ({len(proc.steps)} steps)##lp{i}" - if imgui.tree_node(label): - _table_flags = ( - imgui.TableFlags_.borders_outer - | imgui.TableFlags_.borders_inner_v - | imgui.TableFlags_.sizing_stretch_prop - ) - if imgui.begin_table(f"##lpt{i}", 3, _table_flags): - imgui.table_setup_column( - "Kind", imgui.TableColumnFlags_.width_stretch, 0.3 - ) - imgui.table_setup_column( - "Applies To", - imgui.TableColumnFlags_.width_stretch, - 0.15, - ) - imgui.table_setup_column( - "Details", imgui.TableColumnFlags_.width_stretch, 0.55 - ) - imgui.table_headers_row() - for step in proc.steps: - imgui.table_next_row() - imgui.table_set_column_index(0) - imgui.text(step.kind) - imgui.table_set_column_index(1) - imgui.text_disabled(step.applies_to) - imgui.table_set_column_index(2) - imgui.text_disabled(step.details) - imgui.end_table() - imgui.tree_pop() + if not procedures: + return + total_steps = sum(len(p.steps) for p in procedures) + if not imgui.collapsing_header( + S.CONFIGURE_LOAD_PROCEDURES.format(count=total_steps) + ): + return + imgui.text_disabled(getattr(lp, "style", "")) + for i, proc in enumerate(procedures): + label = f"Procedure {i + 1} ({len(proc.steps)} steps)##lp{i}" + if imgui.tree_node(label): + _table_flags = ( + imgui.TableFlags_.borders_outer + | imgui.TableFlags_.borders_inner_v + | imgui.TableFlags_.sizing_stretch_prop + ) + if imgui.begin_table(f"##lpt{i}", 3, _table_flags): + imgui.table_setup_column( + "Kind", imgui.TableColumnFlags_.width_stretch, 0.3 + ) + imgui.table_setup_column( + "Applies To", imgui.TableColumnFlags_.width_stretch, 0.15 + ) + imgui.table_setup_column( + "Details", imgui.TableColumnFlags_.width_stretch, 0.55 + ) + imgui.table_headers_row() + for step in proc.steps: + imgui.table_next_row() + imgui.table_set_column_index(0) + imgui.text(step.kind) + imgui.table_set_column_index(1) + imgui.text_disabled(step.applies_to) + imgui.table_set_column_index(2) + imgui.text_disabled(step.details) + imgui.end_table() + imgui.tree_pop() def _render_label_value(self, label: str, value: str) -> None: imgui.text_disabled(label) imgui.same_line(120.0) imgui.text(value) + + def _render_program_confirm(self, device: Device) -> None: + if not imgui.begin_popup_modal( + S.PROGRAM_CONFIRM_TITLE, None, imgui.WindowFlags_.always_auto_resize + )[0]: + return + imgui.text_wrapped( + S.PROGRAM_CONFIRM_TEXT.format( + address=device.individual_address or "?", + scope=self._download_scope.name, + ) + ) + imgui.spacing() + btn_w = imgui.ImVec2(140, 0) + if imgui.button(S.BTN_PROGRAM_DEVICE, btn_w): + if self._on_program_device is not None: + self._on_program_device(device, self._download_scope) + imgui.close_current_popup() + imgui.same_line() + if imgui.button(S.BTN_CANCEL, btn_w): + imgui.close_current_popup() + imgui.end_popup() + + def _render_scope_selector(self) -> None: + """Select what a download/eval covers: full, parameters, or group comm.""" + order = [ + DownloadScope.FULL, + DownloadScope.PARAMETERS, + DownloadScope.GROUP_COMMUNICATION, + ] + labels = [S.SCOPE_FULL, S.SCOPE_PARAMETERS, S.SCOPE_GROUP_COMMUNICATION] + current = order.index(self._download_scope) + imgui.align_text_to_frame_padding() + imgui.text_disabled(S.CONFIGURE_DOWNLOAD_SCOPE) + imgui.same_line(120.0) + imgui.set_next_item_width(220.0) + changed, new_idx = imgui.combo("##download_scope", current, labels) + if changed: + self._download_scope = order[new_idx] diff --git a/apps/knx-gui/src/knx_gui/plugins/project/ui/devices.py b/apps/knx-gui/src/knx_gui/plugins/project/ui/devices.py index 57dee2c8..3a52ad53 100644 --- a/apps/knx-gui/src/knx_gui/plugins/project/ui/devices.py +++ b/apps/knx-gui/src/knx_gui/plugins/project/ui/devices.py @@ -5,6 +5,7 @@ from knx_gui.device import Device from knx_gui.plugins.project.strings import S +from knx_gui.plugins.project.ui._filter import filter_box @dataclass @@ -49,6 +50,7 @@ def __init__( self._on_remove_line = on_remove_line self._on_rename_line = on_rename_line self._dragging_device: Device | None = None + self._filter_text: str = "" self._popup_area_number: int = 0 self._popup_line_number: int = 0 @@ -64,6 +66,14 @@ def render(self) -> None: areas = self._get_areas() device_tree = self._build_device_tree(devices) + self._filter_text = filter_box( + "##device_filter", S.DEVICE_FILTER_HINT, self._filter_text + ) + flt = self._filter_text.strip().lower() + + if not areas and not devices: + imgui.text_disabled(S.DEVICE_EMPTY_HINT) + if imgui.begin_popup_context_window("##devices_context"): if imgui.menu_item(S.CONTEXT_ADD_AREA, "", False)[0]: self._popup_area_number = self._next_area_number(areas) @@ -93,54 +103,95 @@ def render(self) -> None: for area in areas: lines = self._get_lines(area.id) + if flt and not any( + self._device_matches(device, flt) + for line in lines + for device in device_tree.get(area.number, {}).get(line.number, []) + ): + continue # while filtering, hide areas with no matching device area_label = self._format_area_label(area) area_flags = ( imgui.TreeNodeFlags_.default_open | imgui.TreeNodeFlags_.span_avail_width ) + if flt: + imgui.set_next_item_open(True, imgui.Cond_.always) if imgui.tree_node_ex(f"{area_label}##area_{area.id}", area_flags): self._render_area_context_menu(area, lines) for line in lines: + line_devices = device_tree.get(area.number, {}).get(line.number, []) + if flt: + line_devices = [ + d for d in line_devices if self._device_matches(d, flt) + ] + if not line_devices: + continue # hide lines with no match while filtering line_label = self._format_line_label(area, line) line_flags = ( imgui.TreeNodeFlags_.default_open | imgui.TreeNodeFlags_.span_avail_width ) + if flt: + imgui.set_next_item_open(True, imgui.Cond_.always) if imgui.tree_node_ex(f"{line_label}##line_{line.id}", line_flags): self._render_line_context_menu(line) self._render_line_drop_target(area, line) - line_devices = device_tree.get(area.number, {}).get( - line.number, [] - ) for device in line_devices: - imgui.tree_node_ex( - f"{device.name} ({device.individual_address})", - leaf_flags, - ) + imgui.tree_node_ex(self._device_label(device), leaf_flags) if imgui.is_item_clicked(): self._on_select_device(device) + self._render_device_context_menu(device) self._render_device_drag_source(device) imgui.tree_pop() imgui.tree_pop() unassigned = self._get_unassigned_devices(devices, areas) + if flt: + unassigned = [d for d in unassigned if self._device_matches(d, flt)] if unassigned: unassigned_flags = ( imgui.TreeNodeFlags_.default_open | imgui.TreeNodeFlags_.span_avail_width ) + if flt: + imgui.set_next_item_open(True, imgui.Cond_.always) if imgui.tree_node_ex( S.DEVICE_UNASSIGNED.format(count=len(unassigned)), unassigned_flags ): for device in unassigned: - imgui.tree_node_ex(device.name, leaf_flags) + imgui.tree_node_ex(self._device_label(device), leaf_flags) if imgui.is_item_clicked(): self._on_select_device(device) + self._render_device_context_menu(device) self._render_device_drag_source(device) imgui.tree_pop() + def _render_device_context_menu(self, device: Device) -> None: + if not device.individual_address: + return + if imgui.begin_popup_context_item(f"##dev_ctx_{device.node_id}"): + if imgui.menu_item(S.CONTEXT_COPY_ADDRESS, "", False)[0]: + imgui.set_clipboard_text(device.individual_address) + imgui.end_popup() + + def _device_matches(self, device: Device, flt: str) -> bool: + """Case-insensitive match of a device against the filter (name, address, app name).""" + if flt in (device.name or "").lower(): + return True + if flt in (device.individual_address or "").lower(): + return True + app_name = getattr(device.app, "name", "") or "" + return flt in app_name.lower() + + def _device_label(self, device: Device) -> str: + # Imported devices are often unnamed; fall back to the application/product name (like ETS). + primary = device.name or getattr(device.app, "name", "") or "?" + if device.individual_address: + return f"{device.individual_address} {primary}##dev{device.node_id}" + return f"{primary}##dev{device.node_id}" + def _format_area_label(self, area: Area) -> str: if area.name: return S.DEVICE_AREA_NAMED.format(name=area.name, area=area.number) diff --git a/apps/knx-gui/src/knx_gui/plugins/project/ui/group_addresses.py b/apps/knx-gui/src/knx_gui/plugins/project/ui/group_addresses.py new file mode 100644 index 00000000..f0124a54 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/project/ui/group_addresses.py @@ -0,0 +1,247 @@ +"""ETS-like Group Addresses view: the named range tree (main/middle/…) down to group addresses. + +Shows, for the selected group address, the device com-objects assigned to it, and lets the user +create / rename / delete group addresses and set their datapoint type (context menus + modals, +mirroring the Devices panel).""" + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from imgui_bundle import imgui + +from knx_gui.plugins.project.strings import S +from knx_gui.plugins.project.ui._filter import filter_box + +if TYPE_CHECKING: + from knx_gui.device import Device + from knx_gui.plugins.project.service import _Assignment + from xknxmono.project.core.service import GroupRangeInfo + + +class GroupAddressesPanel: + def __init__( + self, + get_range_tree: "Callable[[], list[GroupRangeInfo]]", + get_assignments_for_ga: "Callable[[int], list[_Assignment]]", + get_devices: "Callable[[], list[Device]]", + on_create_ga: Callable[[str, str], None], + on_rename_ga: Callable[[int, str], None], + on_set_ga_dpt: Callable[[int, str], None], + on_remove_ga: Callable[[int], None], + ) -> None: + self._get_range_tree = get_range_tree + self._get_assignments_for_ga = get_assignments_for_ga + self._get_devices = get_devices + self._on_create_ga = on_create_ga + self._on_rename_ga = on_rename_ga + self._on_set_ga_dpt = on_set_ga_dpt + self._on_remove_ga = on_remove_ga + self._filter_text: str = "" + self._selected_ga_id: int | None = None + self._selected_ga_text: str = "" + self._selected_ga_description: str = "" + self._selected_ga_comment: str = "" + # Modal state. + self._popup_ga_id: int = 0 + self._popup_address: str = "" + self._popup_name: str = "" + self._popup_dpt: str = "" + self._open_new_ga = False + self._open_rename_ga = False + self._open_set_dpt = False + + def render(self) -> None: + tree = self._get_range_tree() + + if imgui.begin_popup_context_window("##ga_context"): + if imgui.menu_item(S.GA_NEW, "", False)[0]: + self._popup_address = "" + self._popup_name = "" + self._open_new_ga = True + imgui.end_popup() + + if self._open_new_ga: + imgui.open_popup(S.GA_NEW) + self._open_new_ga = False + if self._open_rename_ga: + imgui.open_popup(S.GA_RENAME) + self._open_rename_ga = False + if self._open_set_dpt: + imgui.open_popup(S.GA_SET_DPT) + self._open_set_dpt = False + self._render_new_ga_popup() + self._render_rename_popup() + self._render_dpt_popup() + + if not tree: + imgui.text_disabled(S.GA_NO_PROJECT) + return + + self._filter_text = filter_box( + "##ga_filter", S.GA_FILTER_HINT, self._filter_text + ) + flt = self._filter_text.strip().lower() + + avail = imgui.get_content_region_avail() + tree_height = max(avail.y * 0.6, 0.0) + if imgui.begin_child("##ga_tree", imgui.ImVec2(0.0, tree_height)): + for node in tree: + self._render_range(node, flt) + imgui.end_child() + + imgui.separator() + self._render_assignments() + + def _render_range(self, node: "GroupRangeInfo", flt: str = "") -> None: + if flt and not self._range_has_match(node, flt): + return # while filtering, hide ranges with no matching group address + label = ( + f"{node.name}##gr{node.id}" if node.name else f"[{node.id}]##gr{node.id}" + ) + if flt: + imgui.set_next_item_open(True, imgui.Cond_.always) + if not imgui.tree_node_ex(label, imgui.TreeNodeFlags_.default_open): + return + for child in node.children: + self._render_range(child, flt) + for ga in node.group_addresses: + if flt and not self._ga_matches(ga, flt): + continue + selected = ga.id == self._selected_ga_id + ga_label = f"{ga.text} {ga.name}##ga{ga.id}" + if imgui.selectable(ga_label, selected)[0]: + self._selected_ga_id = ga.id + self._selected_ga_text = f"{ga.text} {ga.name}" + self._selected_ga_description = ga.description + self._selected_ga_comment = ga.comment + self._render_ga_context_menu(ga) + imgui.tree_pop() + + @staticmethod + def _ga_matches(ga: object, flt: str) -> bool: + text = getattr(ga, "text", "") or "" + name = getattr(ga, "name", "") or "" + return flt in text.lower() or flt in name.lower() + + def _range_has_match(self, node: "GroupRangeInfo", flt: str) -> bool: + if any(self._ga_matches(ga, flt) for ga in node.group_addresses): + return True + return any(self._range_has_match(child, flt) for child in node.children) + + def _render_ga_context_menu(self, ga: object) -> None: + # ga is a core GroupAddressInfo (id, text, name, datapoint_type, …). + if not imgui.begin_popup_context_item(f"##ga_ctx_{ga.id}"): # type: ignore[attr-defined] + return + if imgui.menu_item(S.CONTEXT_RENAME, "", False)[0]: + self._popup_ga_id = ga.id # type: ignore[attr-defined] + self._popup_name = ga.name # type: ignore[attr-defined] + self._open_rename_ga = True + if imgui.menu_item(S.GA_SET_DPT, "", False)[0]: + self._popup_ga_id = ga.id # type: ignore[attr-defined] + self._popup_dpt = ga.datapoint_type or "" # type: ignore[attr-defined] + self._open_set_dpt = True + if imgui.menu_item(S.CONTEXT_COPY_ADDRESS, "", False)[0]: + imgui.set_clipboard_text(getattr(ga, "text", "") or "") + imgui.separator() + if imgui.menu_item(S.CONTEXT_DELETE, "", False)[0]: + self._on_remove_ga(ga.id) # type: ignore[attr-defined] + if self._selected_ga_id == ga.id: # type: ignore[attr-defined] + self._selected_ga_id = None + imgui.end_popup() + + def _render_new_ga_popup(self) -> None: + if not imgui.begin_popup_modal( + S.GA_NEW, None, imgui.WindowFlags_.always_auto_resize + )[0]: + return + imgui.text_disabled(S.GA_ADDRESS) + imgui.set_next_item_width(220.0) + _, self._popup_address = imgui.input_text_with_hint( + "##ga_addr", "1/2/3", self._popup_address + ) + imgui.text_disabled(S.POPUP_NAME) + imgui.set_next_item_width(220.0) + _, self._popup_name = imgui.input_text("##ga_new_name", self._popup_name) + btn_w = imgui.ImVec2(120, 0) + if imgui.button(S.BTN_OK, btn_w) and self._popup_address.strip(): + self._on_create_ga(self._popup_address.strip(), self._popup_name) + imgui.close_current_popup() + imgui.same_line() + if imgui.button(S.BTN_CANCEL, btn_w): + imgui.close_current_popup() + imgui.end_popup() + + def _render_rename_popup(self) -> None: + if not imgui.begin_popup_modal( + S.GA_RENAME, None, imgui.WindowFlags_.always_auto_resize + )[0]: + return + imgui.set_next_item_width(220.0) + _, self._popup_name = imgui.input_text("##ga_rename", self._popup_name) + btn_w = imgui.ImVec2(120, 0) + if imgui.button(S.BTN_OK, btn_w): + self._on_rename_ga(self._popup_ga_id, self._popup_name) + imgui.close_current_popup() + imgui.same_line() + if imgui.button(S.BTN_CANCEL, btn_w): + imgui.close_current_popup() + imgui.end_popup() + + def _render_dpt_popup(self) -> None: + if not imgui.begin_popup_modal( + S.GA_SET_DPT, None, imgui.WindowFlags_.always_auto_resize + )[0]: + return + imgui.text_disabled(S.GA_DPT_HINT) + imgui.set_next_item_width(220.0) + _, self._popup_dpt = imgui.input_text_with_hint( + "##ga_dpt", "DPST-1-1", self._popup_dpt + ) + btn_w = imgui.ImVec2(120, 0) + if imgui.button(S.BTN_OK, btn_w): + self._on_set_ga_dpt(self._popup_ga_id, self._popup_dpt.strip()) + imgui.close_current_popup() + imgui.same_line() + if imgui.button(S.BTN_CANCEL, btn_w): + imgui.close_current_popup() + imgui.end_popup() + + def _render_assignments(self) -> None: + if self._selected_ga_id is None: + imgui.text_disabled(S.GA_ASSIGNED_OBJECTS) + return + imgui.text_disabled(self._selected_ga_text) + if self._selected_ga_description: + imgui.text_wrapped(f"{S.GA_DESCRIPTION}: {self._selected_ga_description}") + if self._selected_ga_comment: + imgui.text_wrapped(f"{S.GA_COMMENT}: {self._selected_ga_comment}") + + names = self._com_object_names() + assignments = self._get_assignments_for_ga(self._selected_ga_id) + flags = imgui.TableFlags_.borders_inner | imgui.TableFlags_.sizing_stretch_prop + if not imgui.begin_table("##ga_assignments", 3, flags): + return + imgui.table_setup_column("Device", imgui.TableColumnFlags_.width_stretch, 0.5) + imgui.table_setup_column("Object", imgui.TableColumnFlags_.width_stretch, 0.4) + imgui.table_setup_column("S", imgui.TableColumnFlags_.width_fixed, 20.0) + imgui.table_headers_row() + for a in assignments: + device_name, co_name = names.get(a.com_object_id, ("?", "?")) + imgui.table_next_row() + imgui.table_set_column_index(0) + imgui.text(device_name) + imgui.table_set_column_index(1) + imgui.text_disabled(co_name) + imgui.table_set_column_index(2) + if a.is_sending: + imgui.text("→") + imgui.end_table() + + def _com_object_names(self) -> dict[int, tuple[str, str]]: + names: dict[int, tuple[str, str]] = {} + for device in self._get_devices(): + label = device.name or device.individual_address or "?" + for co in device.com_objects: + if co.db_id is not None: + names[co.db_id] = (label, co.name) + return names diff --git a/apps/knx-gui/src/knx_gui/plugins/project/ui/preflight_result.py b/apps/knx-gui/src/knx_gui/plugins/project/ui/preflight_result.py new file mode 100644 index 00000000..abdff63b --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/project/ui/preflight_result.py @@ -0,0 +1,285 @@ +"""A result window for a device pre-flight (dry run). + +Shows whether the device already matches the configuration and, per memory segment / property, +how many bytes would change. The pre-flight runs on a background thread; its result is handed in via +:meth:`submit_result` / :meth:`submit_error` (thread-safe) and picked up on the next UI frame. The +window can export the current (Ist) and planned (Soll) bytes of every location to a text file. +""" + +from __future__ import annotations + +import contextlib +import threading +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING + +from imgui_bundle import imgui + +from knx_gui.plugins.project.strings import S + +if TYPE_CHECKING: + from xknxmono.download.preflight import PreflightReport, SegmentDiff + +_GREEN = imgui.ImVec4(0.4, 0.85, 0.45, 1.0) +_ORANGE = imgui.ImVec4(0.95, 0.75, 0.2, 1.0) +_RED = imgui.ImVec4(0.9, 0.35, 0.35, 1.0) +_BLUE = imgui.ImVec4(0.45, 0.7, 0.95, 1.0) + + +class PreflightResultWindow: + def __init__(self) -> None: + self._lock = threading.Lock() + # Set from the worker thread, consumed on the UI thread. + self._pending: ( + tuple[str, str, PreflightReport | None, str | None, frozenset[int]] | None + ) = None + self._label = "" + self._scope = "" + self._report: PreflightReport | None = None + self._error: str | None = None + # Absolute addresses of device-managed (runtime) bytes, e.g. a download + # detection byte the firmware resets after a download; a difference there + # is benign, so those locations are annotated instead of flagged. + self._runtime: frozenset[int] = frozenset() + self._show = False + self._save_path_buf = "preflight.txt" + + # -- worker thread: no imgui here ------------------------------------- + def submit_result( + self, + label: str, + scope: str, + report: PreflightReport, + runtime_addresses: set[int] | frozenset[int] = frozenset(), + ) -> None: + with self._lock: + self._pending = (label, scope, report, None, frozenset(runtime_addresses)) + + def submit_error(self, label: str, scope: str, error: str) -> None: + with self._lock: + self._pending = (label, scope, None, error, frozenset()) + + # -- UI thread -------------------------------------------------------- + def render(self) -> None: + with self._lock: + if self._pending is not None: + ( + self._label, + self._scope, + self._report, + self._error, + self._runtime, + ) = self._pending + self._pending = None + self._show = True + if not self._show: + return + imgui.set_next_window_size(imgui.ImVec2(720, 520), imgui.Cond_.first_use_ever) + opened, p_open = imgui.begin(S.PREFLIGHT_RESULT_TITLE, self._show) + if p_open is not None: + self._show = p_open + if opened: + self._render_body() + imgui.end() + + def _render_body(self) -> None: + imgui.text_disabled(f"{self._label} scope={self._scope}") + imgui.same_line() + # Text is plain (not selectable), so offer explicit copy-to-clipboard. + if imgui.small_button(S.COPY_LOG): + imgui.set_clipboard_text(self._clipboard_text()) + imgui.separator() + # Reassure up front: a test never writes to the device. + imgui.text_disabled(S.PREFLIGHT_NO_CHANGES_MADE) + imgui.spacing() + + if self._error is not None: + imgui.push_style_color(imgui.Col_.text, _RED) + imgui.text_wrapped(f"{S.PREFLIGHT_FAILED}: {self._error}") + imgui.pop_style_color() + return + + report = self._report + if report is None: + return + + # Split real changes from device-managed (runtime) byte differences, which + # are benign (the firmware sets them after a download). + real_segments = [ + s + for s in report.changed_segments + if not self._segment_runtime_only(s) + ] + runtime_segments = [ + s for s in report.changed_segments if self._segment_runtime_only(s) + ] + real_locations = len(real_segments) + len(report.changed_properties) + real_bytes = sum(s.changed_bytes for s in real_segments) + sum( + p.changed_bytes for p in report.changed_properties + ) + if real_locations: + imgui.push_style_color(imgui.Col_.text, _ORANGE) + imgui.text( + S.PREFLIGHT_WOULD_CHANGE.format( + bytes=real_bytes, locations=real_locations + ) + ) + imgui.pop_style_color() + else: + imgui.push_style_color(imgui.Col_.text, _GREEN) + imgui.text(S.PREFLIGHT_MATCH) + imgui.pop_style_color() + if runtime_segments: + runtime_bytes = sum(s.changed_bytes for s in runtime_segments) + imgui.text_disabled(S.PREFLIGHT_RUNTIME_NOTE.format(bytes=runtime_bytes)) + + matched = sum(1 for s in report.segments if not s.changed) + sum( + 1 for p in report.properties if not p.changed + ) + imgui.text_disabled( + S.PREFLIGHT_SUMMARY_COUNTS.format(matched=matched, changed=real_locations) + ) + + if imgui.button(S.PREFLIGHT_EXPORT): + imgui.open_popup("##preflight_export") + self._render_export_modal(report) + imgui.separator() + self._render_table(report) + + def _render_table(self, report: PreflightReport) -> None: + flags = ( + imgui.TableFlags_.row_bg + | imgui.TableFlags_.borders_inner_h + | imgui.TableFlags_.scroll_y + ) + avail = imgui.get_content_region_avail() + if not imgui.begin_table( + "##preflight", 4, flags, imgui.ImVec2(avail.x, avail.y) + ): + return + imgui.table_setup_scroll_freeze(0, 1) + imgui.table_setup_column(S.PREFLIGHT_COL_LOCATION) + imgui.table_setup_column( + S.PREFLIGHT_COL_SIZE, imgui.TableColumnFlags_.width_fixed, 70 + ) + imgui.table_setup_column( + S.PREFLIGHT_COL_STATUS, imgui.TableColumnFlags_.width_fixed, 120 + ) + imgui.table_setup_column( + S.PREFLIGHT_COL_CHANGED, imgui.TableColumnFlags_.width_fixed, 90 + ) + imgui.table_headers_row() + + for segment in report.segments: + self._render_row( + S.PREFLIGHT_MEM_LABEL.format(address=f"{segment.address:#06x}"), + len(segment.planned), + segment.changed, + segment.changed_bytes, + runtime=self._segment_runtime_only(segment), + ) + for prop in report.properties: + self._render_row( + S.PREFLIGHT_PROP_LABEL.format( + object=prop.object_index, property=prop.property_id + ), + len(prop.planned), + prop.changed, + prop.changed_bytes, + ) + imgui.end_table() + + def _segment_runtime_only(self, segment: SegmentDiff) -> bool: + """Whether every changed byte of ``segment`` is a device-managed runtime byte.""" + if not segment.changed or not self._runtime: + return False + return all( + segment.address + r.start + i in self._runtime + for r in segment.changed_ranges + for i in range(r.length) + ) + + def _render_row( + self, + location: str, + size: int, + changed: bool, + changed_bytes: int, + runtime: bool = False, + ) -> None: + imgui.table_next_row() + imgui.table_set_column_index(0) + imgui.text(location) + imgui.table_set_column_index(1) + imgui.text_disabled(f"{size} B") + imgui.table_set_column_index(2) + if runtime: + imgui.push_style_color(imgui.Col_.text, _BLUE) + imgui.text(S.PREFLIGHT_STATUS_RUNTIME) + imgui.pop_style_color() + if imgui.is_item_hovered(): + imgui.set_tooltip(S.PREFLIGHT_RUNTIME_TOOLTIP) + else: + color = _ORANGE if changed else _GREEN + label = S.PREFLIGHT_STATUS_CHANGE if changed else S.PREFLIGHT_STATUS_MATCH + imgui.push_style_color(imgui.Col_.text, color) + imgui.text(label) + imgui.pop_style_color() + imgui.table_set_column_index(3) + imgui.text_disabled(str(changed_bytes) if changed else "-") + + def _render_export_modal(self, report: PreflightReport) -> None: + imgui.set_next_window_size(imgui.ImVec2(520, 0), imgui.Cond_.always) + if not imgui.begin_popup_modal( + "##preflight_export", + None, + imgui.WindowFlags_.no_title_bar | imgui.WindowFlags_.always_auto_resize, + )[0]: + return + imgui.text(S.PREFLIGHT_EXPORT_PATH) + imgui.set_next_item_width(-1) + _, self._save_path_buf = imgui.input_text("##pf_path", self._save_path_buf) + imgui.spacing() + btn_w = imgui.ImVec2(120, 0) + if imgui.button(S.BTN_SAVE, btn_w): + with contextlib.suppress(OSError): + Path(self._save_path_buf).write_text( + self._export_text(report), encoding="utf-8" + ) + imgui.close_current_popup() + imgui.same_line() + if imgui.button(S.BTN_CANCEL, btn_w): + imgui.close_current_popup() + imgui.end_popup() + + def _clipboard_text(self) -> str: + header = f"{self._label} scope={self._scope}" + if self._error is not None: + return f"{header}\n{S.PREFLIGHT_FAILED}: {self._error}" + if self._report is not None: + return f"{header}\n{self._export_text(self._report)}" + return header + + def _export_text(self, report: PreflightReport) -> str: + lines = [ + f"# Pre-flight {self._label} scope={self._scope} " + f"generated={datetime.now().isoformat(timespec='seconds')}", + report.summary(), + "", + ] + for segment in report.segments: + lines.append( + f"## memory {segment.address:#06x} ({len(segment.planned)}B) " + f"changed={segment.changed_bytes}" + ) + lines.append(f"ist : {segment.current.hex()}") + lines.append(f"soll: {segment.planned.hex()}") + for prop in report.properties: + lines.append( + f"## object {prop.object_index} property {prop.property_id} " + f"changed={prop.changed_bytes}" + ) + lines.append(f"ist : {prop.current.hex()}") + lines.append(f"soll: {prop.planned.hex()}") + return "\n".join(lines) + "\n" diff --git a/apps/knx-gui/src/knx_gui/plugins/project/ui/project_info.py b/apps/knx-gui/src/knx_gui/plugins/project/ui/project_info.py new file mode 100644 index 00000000..0700af1d --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/project/ui/project_info.py @@ -0,0 +1,43 @@ +"""Project information view: the ETS project metadata carried over on import.""" + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from imgui_bundle import imgui + +from knx_gui.plugins.project.strings import S + +if TYPE_CHECKING: + from knx_gui.plugins.project.service import _ProjectInfo + + +class ProjectInfoPanel: + def __init__(self, get_project_info: "Callable[[], _ProjectInfo | None]") -> None: + self._get_project_info = get_project_info + + def render(self) -> None: + info = self._get_project_info() + if info is None: + imgui.text_disabled(S.PROJECT_INFO_EMPTY) + return + rows = [ + (S.PROJECT_INFO_NAME, info.name), + (S.PROJECT_INFO_GA_STYLE, info.group_address_style), + (S.PROJECT_INFO_CREATED_BY, info.created_by), + (S.PROJECT_INFO_TOOL_VERSION, info.tool_version), + (S.PROJECT_INFO_SCHEMA_VERSION, info.schema_version), + (S.PROJECT_INFO_LAST_MODIFIED, info.last_modified), + (S.PROJECT_INFO_GUID, info.guid), + ] + flags = imgui.TableFlags_.borders_inner | imgui.TableFlags_.sizing_stretch_prop + if not imgui.begin_table("##project_info", 2, flags): + return + imgui.table_setup_column("", imgui.TableColumnFlags_.width_stretch, 0.35) + imgui.table_setup_column("", imgui.TableColumnFlags_.width_stretch, 0.65) + for label, value in rows: + imgui.table_next_row() + imgui.table_set_column_index(0) + imgui.text_disabled(label) + imgui.table_set_column_index(1) + imgui.text_wrapped(value or "-") + imgui.end_table() diff --git a/apps/knx-gui/src/knx_gui/plugins/project/ui/spaces.py b/apps/knx-gui/src/knx_gui/plugins/project/ui/spaces.py new file mode 100644 index 00000000..361a8c8c --- /dev/null +++ b/apps/knx-gui/src/knx_gui/plugins/project/ui/spaces.py @@ -0,0 +1,151 @@ +"""ETS-like Buildings view: the imported location tree (building → floor → room → …) with the +devices placed in each space and the functions assigned to it.""" + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from imgui_bundle import imgui + +from knx_gui.plugins.project.strings import S +from knx_gui.plugins.project.ui._filter import filter_box + +if TYPE_CHECKING: + from xknxmono.project.core.service import ( + FunctionInfo, + SpaceDeviceInfo, + SpaceInfo, + ) + + +class SpacesPanel: + def __init__( + self, + get_space_tree: "Callable[[], list[SpaceInfo]]", + on_select_device_id: Callable[[int], None], + ) -> None: + self._get_space_tree = get_space_tree + self._on_select_device_id = on_select_device_id + self._filter_text: str = "" + + def render(self) -> None: + tree = self._get_space_tree() + if not tree: + imgui.text_disabled(S.SPACES_EMPTY) + return + self._filter_text = filter_box( + "##spaces_filter", S.SPACES_FILTER_HINT, self._filter_text + ) + flt = self._filter_text.strip().lower() + for space in tree: + self._render_space(space, flt) + + def _render_space(self, space: "SpaceInfo", flt: str = "") -> None: + if flt and not self._space_has_match(space, flt): + return # while filtering, hide spaces with no match anywhere below + # A space that matches by its own name shows all its contents; otherwise + # only the matching devices/functions (and matching child spaces). + self_match = not flt or self._space_matches_self(space, flt) + label = space.name or space.space_type or "?" + if space.space_type: + label = f"{label} [{space.space_type}]" + if flt: + imgui.set_next_item_open(True, imgui.Cond_.always) + open_node = imgui.tree_node_ex( + f"{label}##sp{space.id}", imgui.TreeNodeFlags_.default_open + ) + if space.description and imgui.is_item_hovered(): + imgui.set_tooltip(space.description) + if not open_node: + return + if space.description: + imgui.text_disabled(space.description) + for child in space.children: + self._render_space(child, flt) + for device in space.devices: + if self_match or self._device_matches(device, flt): + self._render_device(device) + for function in space.functions: + if self_match or self._function_matches(function, flt): + self._render_function(function) + imgui.tree_pop() + + @staticmethod + def _space_matches_self(space: "SpaceInfo", flt: str) -> bool: + return ( + flt in (space.name or "").lower() or flt in (space.space_type or "").lower() + ) + + @staticmethod + def _device_matches(device: "SpaceDeviceInfo", flt: str) -> bool: + fields = ( + device.name, + device.individual_address, + device.product_name, + device.hardware_name, + device.manufacturer_name, + ) + return any(flt in (f or "").lower() for f in fields) + + @staticmethod + def _function_matches(function: "FunctionInfo", flt: str) -> bool: + fields = (function.usage_text, function.name, function.function_type) + return any(flt in (f or "").lower() for f in fields) + + def _space_has_match(self, space: "SpaceInfo", flt: str) -> bool: + if self._space_matches_self(space, flt): + return True + if any(self._device_matches(d, flt) for d in space.devices): + return True + if any(self._function_matches(fn, flt) for fn in space.functions): + return True + return any(self._space_has_match(child, flt) for child in space.children) + + def _render_device(self, device: "SpaceDeviceInfo") -> None: + # Fall back to the product/hardware name when the device is unnamed (like ETS). + ia = f"{device.individual_address} " if device.individual_address else "" + primary = ( + device.name + or device.product_name + or device.hardware_name + or device.description + or "?" + ) + detail = ( + device.description + if device.description and device.description != primary + else "" + ) + leaf = f"{ia}{primary}".strip() + if detail: + leaf = f"{leaf} — {detail}" + if imgui.selectable(f"{leaf}##spdev{device.id}", False)[0]: + self._on_select_device_id(device.id) + hovered = imgui.is_item_hovered() # capture before the context menu below + if device.individual_address and imgui.begin_popup_context_item( + f"##spdev_ctx_{device.id}" + ): + if imgui.menu_item(S.CONTEXT_COPY_ADDRESS, "", False)[0]: + imgui.set_clipboard_text(device.individual_address) + imgui.end_popup() + if hovered: + parts = [ + p + for p in ( + device.manufacturer_name, + device.product_name, + device.hardware_name, + device.description, + ) + if p + ] + if parts: + imgui.set_tooltip("\n".join(parts)) + + def _render_function(self, function: "FunctionInfo") -> None: + label = function.usage_text or function.name or function.function_type + if not imgui.tree_node_ex(f"ƒ {label}##fn{function.id}"): + return + for ref in function.group_addresses: + role = f" ({ref.role})" if ref.role else "" + imgui.bullet_text(f"{ref.text}{role}") + imgui.tree_pop() diff --git a/apps/knx-gui/src/knx_gui/programming.py b/apps/knx-gui/src/knx_gui/programming.py new file mode 100644 index 00000000..ccfacd7f --- /dev/null +++ b/apps/knx-gui/src/knx_gui/programming.py @@ -0,0 +1,113 @@ +"""Program a configured device onto the bus, or preview what a download would change. + +Bridges a GUI :class:`~knx_gui.device.Device` (which holds a live evaluator with the +current parameter state) to the ``xknxmono.download`` package: the download image is +built directly from the device's evaluator, so the bytes reflect exactly what the +editor shows. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +# Import from the concrete modules: the package also has a ``preflight`` submodule, +# so ``from xknxmono.download import preflight`` is ambiguous to type checkers. +from xknxmono.download.download import download, preflight +from xknxmono.download.image import build_image +from xknxmono.download.scope import DownloadScope + +if TYPE_CHECKING: + from collections.abc import Callable + + from xknx import XKNX + + from knx_gui.device import Device + from xknxmono.download.image import DownloadImage, GroupCommunication + from xknxmono.download.preflight import PreflightReport + + +class DeviceProgrammingError(RuntimeError): + """A device cannot be programmed as configured (no app, no address, ...).""" + + +def _image_for( + device: Device, group_communication: GroupCommunication | None = None +) -> DownloadImage: + ui = device.dynamic_ui + if ui is None: + raise DeviceProgrammingError("device has no dynamic application to program") + return build_image(device.app, ui=ui, group_communication=group_communication) + + +def _address(device: Device) -> str: + if not device.individual_address: + raise DeviceProgrammingError("device has no individual address") + return device.individual_address + + +def runtime_managed_addresses(device: Device) -> set[int]: + """Absolute memory addresses of system parameters the device sets at runtime. + + These are parameters with access "None" (not user configurable), e.g. a + download-detection byte the application firmware overwrites after a download. + A pre-flight difference at such an address is expected and benign, so the + result view can annotate it instead of flagging a real change. + """ + ui = device.dynamic_ui + if ui is None: + return set() + from xknxmono.product.parser_v2.application_indexer import ApplicationIndexer + + indexer = ApplicationIndexer(device.app.program) + base_addresses = ui.segment_base_addrs() + addresses: set[int] = set() + for segment_id, offset_map in ui.memory_param_map().items(): + base = base_addresses.get(segment_id) + if base is None: + continue + for offset, (parameter_id, _value) in offset_map.items(): + parameter = indexer.parameters.get(parameter_id) + access = getattr(parameter, "access", None) + if access is not None and access.name == "NONE": + addresses.add(base + offset) + return addresses + + +async def eval_device( + xknx: XKNX, + device: Device, + scope: DownloadScope = DownloadScope.FULL, + group_communication: GroupCommunication | None = None, +) -> PreflightReport: + """Dry run: report what programming ``device`` would change, writing nothing.""" + return await preflight( + xknx, + _address(device), + device.app, + image=_image_for(device, group_communication), + scope=scope, + ) + + +async def download_device( + xknx: XKNX, + device: Device, + scope: DownloadScope = DownloadScope.FULL, + group_communication: GroupCommunication | None = None, + progress: Callable[[int, int], None] | None = None, +) -> None: + """Program ``device``: build its image and run the load procedure on the bus. + + ``scope`` selects a full download or a partial one (parameters only, or group + communication only), mirroring the ETS download options. ``group_communication`` + supplies the address/association tables a full or group download writes. ``progress`` + (optional) is called ``progress(done, total)`` after each executed load control. + """ + await download( + xknx, + _address(device), + device.app, + image=_image_for(device, group_communication), + scope=scope, + progress=progress, + ) diff --git a/apps/knx-gui/src/knx_gui/settings.py b/apps/knx-gui/src/knx_gui/settings.py new file mode 100644 index 00000000..eb471a53 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/settings.py @@ -0,0 +1,45 @@ +"""Small dependency-free settings store: per-app JSON files in the platform config directory. + +Used for lightweight, non-project preferences (e.g. the last connection settings) that should +survive restarts. Not for project data — that lives in the ``.xknx`` document.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +_APP = "knx-gui" + + +def config_dir() -> Path: + """Platform-appropriate per-user config directory for this app.""" + if sys.platform == "darwin": + base = Path.home() / "Library" / "Application Support" + elif sys.platform == "win32": + base = Path(os.environ.get("APPDATA") or Path.home() / "AppData" / "Roaming") + else: + base = Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") + return base / _APP + + +def load_settings(name: str) -> dict[str, Any]: + """Load ``/.json`` as a dict, or an empty dict if missing/unreadable.""" + path = config_dir() / f"{name}.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def save_settings(name: str, data: dict[str, Any]) -> None: + """Write ``data`` to ``/.json`` (best effort; failures are ignored).""" + path = config_dir() / f"{name}.json" + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2), encoding="utf-8") + except OSError: + pass diff --git a/apps/knx-gui/src/knx_gui/strings/__init__.py b/apps/knx-gui/src/knx_gui/strings/__init__.py index fc660fd8..bac913b8 100644 --- a/apps/knx-gui/src/knx_gui/strings/__init__.py +++ b/apps/knx-gui/src/knx_gui/strings/__init__.py @@ -68,6 +68,34 @@ def BTN_STOP(self) -> str: def FILE_DIALOG_ALL_FILES(self) -> str: return _("All files") + @property + def STATUS_PROJECT(self) -> str: + return _("Project: {name} · {devices} devices · {gas} GAs") + + @property + def STATUS_NO_PROJECT(self) -> str: + return _("No project open") + + @property + def STATUS_PROGRAMMING(self) -> str: + return _("Programming {address}...") + + @property + def STATUS_TESTING(self) -> str: + return _("Testing {address}...") + + @property + def STATUS_PROGRAM_DONE(self) -> str: + return _("Programming complete") + + @property + def STATUS_PROGRAM_FAILED(self) -> str: + return _("Programming failed") + + @property + def STATUS_NO_CONNECTION(self) -> str: + return _("No KNX connection") + @property def SHORTCUT_UNDO(self) -> str: return "Ctrl+Z" @@ -90,6 +118,14 @@ def MENU_NEW_PROJECT(self) -> str: def MENU_OPEN_PROJECT(self) -> str: return _("Open Project") + @property + def MENU_EXPORT_KNXPROJ(self) -> str: + return _("Export .knxproj...") + + @property + def FILE_DIALOG_KNXPROJ_SAVE_TITLE(self) -> str: + return _("Export ETS project") + @property def MENU_LOAD_KNXPROD(self) -> str: return _("Load .knxprod...") @@ -118,6 +154,10 @@ def FILE_DIALOG_KNXPROD_TITLE(self) -> str: def FILE_DIALOG_KNXPROD_FILTER(self) -> str: return _("KNX product (*.knxprod)") + @property + def FILE_DIALOG_KNXPROJ_FILTER(self) -> str: + return _("ETS project (*.knxproj)") + @property def FILE_DIALOG_PROJECT_TITLE(self) -> str: return _("Open XKNX project") @@ -130,6 +170,46 @@ def FILE_DIALOG_PROJECT_SAVE_TITLE(self) -> str: def FILE_DIALOG_PROJECT_FILTER(self) -> str: return _("XKNX project (*.xknx)") + @property + def FILE_DIALOG_OPEN_FILTER(self) -> str: + return _("Projects (*.xknx, *.knxproj)") + + @property + def PROGRESS_TITLE(self) -> str: + return _("Working…") + + @property + def IMPORT_PROGRESS_TEXT(self) -> str: + return _("Importing project — this can take a while for large projects.") + + @property + def PROGRESS_LOAD_KNXPROD(self) -> str: + return _("Loading product catalog…") + + @property + def PROGRESS_OPEN_PROJECT(self) -> str: + return _("Opening project…") + + @property + def IMPORT_PASSWORD_TITLE(self) -> str: + return _("Project password") + + @property + def IMPORT_PASSWORD_PROMPT(self) -> str: + return _("This ETS project is password protected. Enter its password:") + + @property + def IMPORT_PASSWORD_WRONG(self) -> str: + return _("Wrong password, please try again.") + + @property + def BTN_OK(self) -> str: + return _("OK") + + @property + def BTN_CANCEL(self) -> str: + return _("Cancel") + class _CombinedStrings(BaseStrings, MenuStrings): pass diff --git a/apps/knx-gui/src/knx_gui/widgets/__init__.py b/apps/knx-gui/src/knx_gui/widgets/__init__.py index 7eb7d9d7..4269ff39 100644 --- a/apps/knx-gui/src/knx_gui/widgets/__init__.py +++ b/apps/knx-gui/src/knx_gui/widgets/__init__.py @@ -1,4 +1,5 @@ from knx_gui.widgets.com_flags_widgets import ComFlagsTable +from knx_gui.widgets.group_objects_widgets import GroupObjectsTable from knx_gui.widgets.hex_view import HexView from knx_gui.widgets.parameter_widgets import ( EnumPopup, @@ -12,6 +13,7 @@ "ComFlagsTable", "EnumPopup", "EnumPopupRequest", + "GroupObjectsTable", "HexView", "count_parameters", "render_param_widget", diff --git a/apps/knx-gui/src/knx_gui/widgets/group_objects_widgets.py b/apps/knx-gui/src/knx_gui/widgets/group_objects_widgets.py new file mode 100644 index 00000000..baa15637 --- /dev/null +++ b/apps/knx-gui/src/knx_gui/widgets/group_objects_widgets.py @@ -0,0 +1,144 @@ +"""ETS-like "Group Objects" table: each com-object with its assigned group addresses and flags. + +Complements :class:`~knx_gui.widgets.com_flags_widgets.ComFlagsTable` (which shows only flags) by +adding the group-address assignment column, including linking/unlinking addresses. +""" + +from collections.abc import Callable + +from imgui_bundle import imgui + +from knx_gui.device import FLAG_LABELS, ComObject, Device +from knx_gui.plugins.node_editor.strings import S + +# One com-object → group-address link: (assignment_id, group_address_id, text, is_sending). +GroupLink = tuple[int, int, str, bool] +# Resolver: com-object db id -> its links. +GroupLinkResolver = Callable[[int], list[GroupLink]] +# All assignable group addresses: (group_address_id, text). +GroupAddressCatalog = Callable[[], list[tuple[int, str]]] + + +class GroupObjectsTable: + def __init__( + self, + set_flag: Callable[[Device, str, str, bool], None], + on_link: Callable[[int, int], None], + on_unlink: Callable[[int], None], + ) -> None: + self._set_flag = set_flag + self._on_link = on_link + self._on_unlink = on_unlink + self._picker_filter = "" + + def render( + self, + device: Device, + com_objects: list[ComObject], + get_links: GroupLinkResolver, + get_all_group_addresses: GroupAddressCatalog, + ) -> None: + flags = ( + imgui.TableFlags_.borders_inner + | imgui.TableFlags_.sizing_stretch_prop + | imgui.TableFlags_.resizable + ) + n_cols = 4 + len(FLAG_LABELS) + if not imgui.begin_table(f"##group_objects_{device.node_id}", n_cols, flags): + return + + imgui.table_setup_column("#", imgui.TableColumnFlags_.width_fixed, 32.0) + imgui.table_setup_column("Name", imgui.TableColumnFlags_.width_stretch, 0.3) + imgui.table_setup_column("DPT", imgui.TableColumnFlags_.width_stretch, 0.12) + imgui.table_setup_column( + "Group Addresses", imgui.TableColumnFlags_.width_stretch, 0.58 + ) + for _attr, letter, _name in FLAG_LABELS: + imgui.table_setup_column(letter, imgui.TableColumnFlags_.width_fixed, 22.0) + imgui.table_headers_row() + + for com_obj in com_objects: + self._render_row(device, com_obj, get_links, get_all_group_addresses) + + imgui.end_table() + + def _render_row( + self, + device: Device, + com_object: ComObject, + get_links: GroupLinkResolver, + get_all_group_addresses: GroupAddressCatalog, + ) -> None: + row_id = f"{device.node_id}_{com_object.id}" + imgui.table_next_row() + + imgui.table_set_column_index(0) + imgui.text_disabled(str(com_object.number)) + + imgui.table_set_column_index(1) + imgui.text(com_object.name) + + imgui.table_set_column_index(2) + imgui.text_disabled(getattr(com_object.dpt, "name", "") or "") + + imgui.table_set_column_index(3) + db_id = com_object.db_id + links = get_links(db_id) if db_id is not None else [] + # Sending group address first (marked with an arrow); the rest are receive-only. + for assignment_id, _ga_id, text, is_sending in sorted( + links, key=lambda link: not link[3] + ): + if imgui.small_button(f"x##unlink{assignment_id}"): + self._on_unlink(assignment_id) + imgui.same_line() + imgui.text(f"→ {text}" if is_sending else text) + if db_id is not None: + self._render_add(db_id, links, get_all_group_addresses) + + for col, (attr, _letter, full_name) in enumerate(FLAG_LABELS, start=4): + imgui.table_set_column_index(col) + current = getattr(com_object.flags, attr) + is_locked = ( + getattr(com_object.flags, f"{attr}_locked", False) + if attr != "communication" + else False + ) + if is_locked: + imgui.begin_disabled() + changed, new_value = imgui.checkbox(f"##{row_id}_{attr}", current) + if changed and not is_locked: + self._set_flag(device, com_object.id, attr, new_value) + if is_locked: + imgui.end_disabled() + if imgui.is_item_hovered(imgui.HoveredFlags_.allow_when_disabled): + imgui.set_tooltip( + S.TOOLTIP_LOCKED.format(name=full_name) if is_locked else full_name + ) + + def _render_add( + self, + db_id: int, + links: list[GroupLink], + get_all_group_addresses: GroupAddressCatalog, + ) -> None: + popup_id = f"##addga{db_id}" + if imgui.small_button(f"+##add{db_id}"): + self._picker_filter = "" + imgui.open_popup(popup_id) + if not imgui.begin_popup(popup_id): + return + already = {ga_id for _aid, ga_id, _text, _sending in links} + imgui.set_next_item_width(240.0) + _, self._picker_filter = imgui.input_text_with_hint( + "##ga_filter", S.SEARCH_HINT, self._picker_filter + ) + needle = self._picker_filter.lower() + if imgui.begin_child("##ga_list", imgui.ImVec2(240.0, 260.0)): + for ga_id, text in get_all_group_addresses(): + if ga_id in already or (needle and needle not in text.lower()): + continue + if imgui.selectable(f"{text}##pick{db_id}_{ga_id}", False)[0]: + self._on_link(db_id, ga_id) + imgui.close_current_popup() + imgui.end_child() + imgui.end_popup() diff --git a/packages/catalog/src/xknxmono/catalog/core/hardware.py b/packages/catalog/src/xknxmono/catalog/core/hardware.py index 95640a80..7918a69e 100644 --- a/packages/catalog/src/xknxmono/catalog/core/hardware.py +++ b/packages/catalog/src/xknxmono/catalog/core/hardware.py @@ -203,3 +203,27 @@ def get_hardware_program( HardwareProgram.hardware_id == hardware_id, ) ).first() + + +def get_program_source(db: Session, program_id: str) -> tuple[str, str] | None: + """Return ``(knxprod_path, manufacturer_id)`` for a hardware program, or ``None``. + + Resolves a program by its id alone (as stored on a project device's + ``hardware2program_ref_id``) to the on-disk ``.knxprod`` it was imported from and its + manufacturer, so callers can re-extract the manufacturer XMLs from that archive. + + Args: + db: An active SQLAlchemy session. + program_id: The hardware program's primary-key identifier. + + Returns: + A ``(knxprod_path, manufacturer_id)`` tuple, or ``None`` if the program is unknown. + """ + row = db.execute( + select(HardwareProgram.knxprod_path, Hardware.manufacturer_id) + .join(Hardware, Hardware.id == HardwareProgram.hardware_id) + .where(HardwareProgram.id == program_id) + ).first() + if row is None: + return None + return row[0], row[1] diff --git a/packages/catalog/src/xknxmono/catalog/core/service.py b/packages/catalog/src/xknxmono/catalog/core/service.py index f5c89b1e..2d988855 100644 --- a/packages/catalog/src/xknxmono/catalog/core/service.py +++ b/packages/catalog/src/xknxmono/catalog/core/service.py @@ -30,6 +30,7 @@ HardwareFilters, get_hardware, get_hardware_program, + get_program_source, list_hardware, ) from xknxmono.catalog.core.manufacturers import get_manufacturer, list_manufacturers @@ -91,6 +92,11 @@ def get_application_xml(self, program_id: str) -> tuple[bytes, str] | None: with Session(self._engine) as db: return get_application_xml(db, program_id) + def get_program_source(self, program_id: str) -> tuple[str, str] | None: + """Return ``(knxprod_path, manufacturer_id)`` for a program id, or ``None``.""" + with Session(self._engine) as db: + return get_program_source(db, program_id) + def get_application_detail(self, program_id: str) -> Application | None: with Session(self._engine) as db: return get_application_detail(db, program_id) diff --git a/packages/download/CHANGELOG.md b/packages/download/CHANGELOG.md new file mode 100644 index 00000000..1b17dcd2 --- /dev/null +++ b/packages/download/CHANGELOG.md @@ -0,0 +1,33 @@ +# Changelog + +All notable changes to `xknx-download` will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Initial device download package. +- `download()` high level entry point driving a full download over a + point-to-point connection. +- `build_image()` assembling a `DownloadImage` from a parsed application program. +- `LoadProcedureRunner` interpreting Load Procedures into bus operations. +- `DeviceProgrammer` with chunked memory writes, property access, interface + object location and property based Load State Machine control. +- Load event encodings per KNX Standard 3/5/2 (including data relative + allocation, subtype 0x0B). +- Load Procedure resolution (`resolve_download_controls`) for product, default + and merged procedure styles, splicing application fragments into the mask + version's default procedure by merge id. +- Connection lifecycle following the procedure's Connect/Disconnect/Restart + controls (open per Connect, close per Disconnect, tear down + cooldown after + Restart, auto-connect before any bus control) via a `ConnectionManager`. +- Relative memory controls (`WriteRelMem`/`CompareRelMem`/`LoadImageRelMem`) with + table-reference base resolution; image-backed `WriteMem`/`WriteProp`; + element-aware property chunking; per-block memory verify. +- Partial downloads via `DownloadScope` (parameters / group communication). +- `program_individual_address` for commissioning a device's individual address. +- `project_data` adapters turning a configured device into parameter values and + group communication links. diff --git a/packages/download/README.md b/packages/download/README.md new file mode 100644 index 00000000..b6c82181 --- /dev/null +++ b/packages/download/README.md @@ -0,0 +1,147 @@ +# xknx-download + +**Program real KNX devices end-to-end, without ETS.** + +This package does the job ETS does when you press "Download": it takes a parsed +product database and an application's Load Procedure and actually commissions a +physical device over a live KNX bus - assembling the memory image, driving each +loadable part's Load State Machine, writing memory and properties, laying down the +group communication tables, and restarting the device. It is a vendor-independent +implementation derived from the KNX Standard v3.0.0. + +This is verified on real hardware, not just in theory: a full download run through +the GUI has programmed physical devices, and the System B group communication path +(tables plus Memory Control Block CRCs) has been confirmed byte-perfect against +devices programmed by ETS. It also programs a virgin device's individual address +and offers a read-only preflight to preview every change before writing. + +Under the hood it interprets the Load Procedure and executes it over a running +[`xknx`](https://github.com/XKNX/xknx) connection. It builds on the sibling +packages for the data side (`xknx-product` parses the `.knxprod` and encodes +parameter values into the memory image) and on `xknx` for the runtime side +(KNXnet/IP transport, application layer services, point-to-point management +connection). + +## Scope + +- Full download over a point-to-point connection: unload, load, write image, + load completed, restart, per loadable part. +- Property based Load State Machine control (writing load events to + `PID_LOAD_STATE_CONTROL`), following KNX Standard v3.0.0, 3/5/2. This is the + mechanism ETS uses for every mask, including the older BIM M112 masks + (MV-0700/0701/0705/5705) whose master data still lists a `StandardMemory` + Application Load Control: their product Load Procedures drive the state machine + through `PID_LOAD_STATE_CONTROL`, and the memory location is legacy and unused. +- Chunked memory writes with optional read-back verification, property writes, + interface object location by type. +- Optional device mask guard (`expected_descriptor`): the device descriptor is + read and checked before any write, refusing to program the wrong device. +- APDU length negotiation: `max_apdu_length` defaults to reading the device's + maximum (Device Object PID 56) for larger, fewer telegrams; pass an integer to + fix it. +- Master Reset via `LdCtrlMasterReset` (A_Restart with restart type 1, erase code + and channel; KNX Standard v3.0.0, 3/3/7 section 3.4.2.2). +- Function Property command and state read via `LdCtrlInvokeFunctionProp` / + `LdCtrlReadFunctionProp` (A_FunctionPropertyCommand / A_FunctionPropertyState_Read; + 3/3/7 section 3.4.7). + +Not covered yet: clearing a line coupler filter table (`LdCtrlClearLCFilterTable`, +coupler-only and not yet validated against a router), the load procedure control +flow directives `LdCtrlOnError` / `LdCtrlProcType`, KNX Data Secure, and USB +transport. Unsupported Load Controls are reported via `UnsupportedProcedureError` +rather than guessed (see [Implementation gaps](#implementation-gaps-and-diagnostics)). + +## Group communication tables + +A full or group-communication download also writes the three tables that link a +device's group objects to group addresses. Pass a `GroupCommunication` +(the device's own address plus its `GroupObjectLink`s) to `download`/`build_image`. +Two device models are handled, detected automatically from the application: + +- **memory-mapped** (masks MV-0701/MV-0705): the address, association and com + object tables sit at fixed segment addresses; a 1-octet count leads the address + and association tables, and the address table starts with the device's own + individual address. +- **System B** (mask MV-07B0): the tables live in relative memory addressed + through each object's table reference. The formats differ (2-octet counts, no + leading device address, a group object table covering every object number), and + the application program carries no controls for them, so the write controls are + synthesized (see `group_communication.py`). Table bytes are validated against + real hardware; the relative-segment allocation framing is modelled on the + application's own parameter segment. + +## Preview before writing + +`preflight(...)` performs the whole download read-only: it reads the device's +current bytes at every location a write would target and returns the diff without +changing anything (and runs the application-fingerprint compare as a gate). Run it +before any real download to confirm the change set. The System B group +communication path has been verified byte-perfect against real hardware (including +the Memory Control Block CRCs), but preflight still lets you review any device +before writing. + +The GUI builds a **Test Before Programming** action directly on top of `preflight`: +it reports per memory segment and property whether the image the toolkit would +generate matches what is already programmed on the device (so a programming run +cannot be made to write a wrongly generated image), and can export the current +and planned bytes of every location as a report. `download(...)` also accepts a +`progress(done, total)` callback so the GUI can show an ETS-like load progress. + +## Implementation gaps and diagnostics + +A Load Control the interpreter does not execute never fails silently. The runner +raises `UnsupportedProcedureError` with a message that names the KNX Standard +service the control maps to, its position in the procedure and the target, so a +bug report shows immediately what is missing. The read-only `preflight` logs the +same information instead of raising. The registry of known-but-unimplemented +controls (with their standard mapping) lives in `gaps.py`; controls that legitimately +have nothing to write are listed there too so the preview does not flag them. + +The runner and preflight also log their start (target, in-scope control count) and +every unsupported control through the `xknxmono.download.procedure` logger. + +## Usage + +```python +from xknx import XKNX +from xknx.io import ConnectionConfig, ConnectionType +from xknxmono.product import load +from xknxmono.download import download + +registry = load("device.knxprod") +application = next(iter(registry.applications.values())) + +xknx = XKNX( + connection_config=ConnectionConfig(connection_type=ConnectionType.TUNNELING) +) +async with xknx: + await download( + xknx, + "1.1.5", + application, + master=registry.master, # needed for default/merged procedure styles + parameter_values={"P-1_R-1": "1"}, + ) +``` + +Pass `master` (`registry.master`) so the Load Procedure can be resolved: for +`DefaultProcedure`/`MergedProcedure` applications the mask version's default +procedure is merged with the application's fragments; `ProductProcedure` +applications carry the full procedure and work without it. For a partial +download pass `scope=DownloadScope.PARAMETERS` or +`DownloadScope.GROUP_COMMUNICATION`. + +The device must already carry the target individual address; program a virgin +device's individual address first via `program_individual_address`. + +## Public API + +- `download(xknx, individual_address, application, *, master, device, image, group_communication, scope, parameter_values, max_apdu_length, expected_descriptor, progress)` +- `program_individual_address(xknx, individual_address, *, serial_number)` +- `preflight(...)` → `PreflightReport` (same arguments; read-only) +- `build_image(application, *, ui, device, parameter_values, group_communication)` → `DownloadImage` +- `GroupCommunication`, `GroupObjectLink`, `DownloadScope` +- `DeviceProgrammer`, `LoadProcedureRunner` +- `LoadState`, `LoadEvent` +- Errors: `DownloadError`, `LoadStateError`, `VerificationError`, + `UnsupportedProcedureError`, `ImageError` diff --git a/packages/download/pyproject.toml b/packages/download/pyproject.toml new file mode 100644 index 00000000..581ba5c4 --- /dev/null +++ b/packages/download/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "xknx-download" +version = "0.1.0" +description = "XKNX device download library" +requires-python = ">=3.12" +dependencies = [ + "xknx-models", + "xknx-product", + # Pinned: relies on P2PConnection.request(payload, expected) and the + # management API as published in the 3.x line. + "xknx>=3.20,<4", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/xknxmono"] diff --git a/packages/download/src/xknxmono/download/__init__.py b/packages/download/src/xknxmono/download/__init__.py new file mode 100644 index 00000000..9dd20cde --- /dev/null +++ b/packages/download/src/xknxmono/download/__init__.py @@ -0,0 +1,90 @@ +"""Download applications into KNX devices. + +The package interprets the Load Procedure of a parsed application program and +executes it over a running ``xknx`` connection: driving each loadable part's +Load State Machine and writing the assembled download image via memory and +property services. +""" + +from __future__ import annotations + +from .commissioning import program_individual_address +from .download import download, preflight +from .errors import ( + DownloadError, + ImageError, + LoadStateError, + UnsupportedProcedureError, + VerificationError, +) +from .image import ( + DownloadImage, + GroupCommunication, + MemorySegment, + PropertyValue, + build_image, +) +from .load_state import LoadEvent, LoadState +from .merge import resolve_download_controls +from .preflight import ( + ByteRange, + PreflightReport, + PropertyDiff, + SegmentDiff, +) +from .procedure import LoadProcedureRunner +from .programmer import ConnectionManager, DeviceProgrammer +from .project_data import ( + GroupObjectLink, + SeedDevice, + group_address_table, + group_communication_from_device, + module_instances_from_device, + parameter_instance_refs_from_device, + parameter_values_from_device, +) +from .scope import DownloadScope +from .tables import ( + Association, + build_association_table, + build_group_address_table, +) + +__version__ = "0.1.0" + +__all__ = [ + "Association", + "ByteRange", + "ConnectionManager", + "DeviceProgrammer", + "DownloadError", + "DownloadImage", + "DownloadScope", + "GroupCommunication", + "GroupObjectLink", + "ImageError", + "LoadEvent", + "LoadProcedureRunner", + "LoadState", + "LoadStateError", + "MemorySegment", + "PreflightReport", + "PropertyDiff", + "PropertyValue", + "SeedDevice", + "SegmentDiff", + "UnsupportedProcedureError", + "VerificationError", + "build_association_table", + "build_group_address_table", + "build_image", + "download", + "group_address_table", + "group_communication_from_device", + "module_instances_from_device", + "parameter_instance_refs_from_device", + "parameter_values_from_device", + "preflight", + "program_individual_address", + "resolve_download_controls", +] diff --git a/packages/download/src/xknxmono/download/commissioning.py b/packages/download/src/xknxmono/download/commissioning.py new file mode 100644 index 00000000..3c12412d --- /dev/null +++ b/packages/download/src/xknxmono/download/commissioning.py @@ -0,0 +1,46 @@ +"""Program a device's individual address before downloading an application. + +A virgin device (or one whose address is unknown) has to be given its individual +address first. Two ways are supported, both delegating to ``xknx``'s network +management procedures: + +- via programming mode: exactly one device on the bus must be in programming + mode; its address is written by broadcast. +- via serial number: the target device is addressed by its serial number, so no + programming mode is required and several devices may be on the bus. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from xknx.management.procedures.network.nm_individual_address_serial_number_write import ( + nm_individual_address_serial_number_write, +) +from xknx.management.procedures.network.nm_individual_address_write import ( + nm_individual_address_write, +) + +if TYPE_CHECKING: + from xknx import XKNX + from xknx.telegram.address import IndividualAddressableType + + +async def program_individual_address( + xknx: XKNX, + individual_address: IndividualAddressableType, + *, + serial_number: bytes | None = None, +) -> None: + """Program ``individual_address`` into a device. + + With ``serial_number`` the device is addressed by serial number; otherwise + the single device currently in programming mode is addressed. ``xknx`` has to + be started. + """ + if serial_number is not None: + await nm_individual_address_serial_number_write( + xknx, serial_number, individual_address + ) + return + await nm_individual_address_write(xknx, individual_address) diff --git a/packages/download/src/xknxmono/download/crc.py b/packages/download/src/xknxmono/download/crc.py new file mode 100644 index 00000000..c37285e2 --- /dev/null +++ b/packages/download/src/xknxmono/download/crc.py @@ -0,0 +1,34 @@ +"""Segment CRC used by the Memory Control Block (MCB) table. + +Per KNX Standard v3.0.0, Chapter 3/5/1 "Resources", section 4.2.27 +"PID_MCB_TABLE (PID = 27)", each loaded segment referenced by the Memory Control +Block Table is protected by a 16 bit CRC over the segment's data; the download +writes that CRC into the MCB entry so the device can verify the segment. + +That section specifies the CRC as CRC-16/CCITT with truncated polynomial +``0x1021``, input and output not reflected and no final xor, and gives the check +value ``0xE5CC`` for the string ``"123456789"``. That check value corresponds to +the initial value ``0x1D0F`` (the augmented CCITT variant) used here - the "FFFFh" +initial value quoted in the prose does not reproduce the specified check value. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +_POLYNOMIAL = 0x1021 +_INITIAL = 0x1D0F + + +def segment_crc(data: Iterable[int]) -> int: + """Return the 16 bit MCB CRC over ``data`` (an iterable of octets).""" + crc = _INITIAL + for octet in data: + crc ^= (octet & 0xFF) << 8 + for _ in range(8): + crc = ( + ((crc << 1) ^ _POLYNOMIAL) & 0xFFFF + if crc & 0x8000 + else (crc << 1) & 0xFFFF + ) + return crc diff --git a/packages/download/src/xknxmono/download/download.py b/packages/download/src/xknxmono/download/download.py new file mode 100644 index 00000000..7a9f93fc --- /dev/null +++ b/packages/download/src/xknxmono/download/download.py @@ -0,0 +1,194 @@ +"""High level entry point for downloading an application into a device. + +Assembles the download image and runs the device's Load Procedure over a +point-to-point connection, following KNX Standard v3.0.0, Chapter 3/5/3 +"Configuration Procedures": the complete download procedure (section 3.5.2) and +the partial download procedure (section 3.5.3). +""" + +from __future__ import annotations + +import contextlib +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING + +from xknx.exceptions import ManagementConnectionError +from xknx.telegram import IndividualAddress + +from .group_communication import synthesize_group_communication_controls +from .image import build_image +from .merge import resolve_download_controls +from .procedure import LoadProcedureRunner +from .programmer import MAX_NEGOTIATED_APDU_LENGTH +from .scope import DownloadScope + +if TYPE_CHECKING: + from xknx import XKNX + from xknx.telegram.address import IndividualAddressableType + + from xknxmono.product import Application, MasterData + + from .image import DownloadImage, GroupCommunication + from .preflight import PreflightReport + from .programmer import BusConnection + from .project_data import SeedDevice + + +class _XknxConnectionManager: + """Open/close point-to-point connections to one device via ``xknx``.""" + + def __init__(self, xknx: XKNX, address: IndividualAddress) -> None: + """Initialize for a target individual address.""" + self._xknx = xknx + self._address = address + self._connection: BusConnection | None = None + + async def open(self) -> BusConnection: + """Open a fresh connection to the device.""" + self._connection = await self._xknx.management.connect(self._address) + return self._connection + + async def close(self) -> None: + """Close the current connection, tolerating a peer that already dropped it.""" + if self._connection is None: + return + self._connection = None + with contextlib.suppress(ManagementConnectionError): + await self._xknx.management.disconnect(self._address) + + +def _apdu_settings(max_apdu_length: int | None) -> tuple[int, bool]: + """Resolve the APDU ceiling and whether to negotiate it from the device.""" + if max_apdu_length is None: + return MAX_NEGOTIATED_APDU_LENGTH, True + return max_apdu_length, False + + +async def download( + xknx: XKNX, + individual_address: IndividualAddressableType, + application: Application, + *, + master: MasterData | None = None, + device: SeedDevice | None = None, + image: DownloadImage | None = None, + group_communication: GroupCommunication | None = None, + scope: DownloadScope = DownloadScope.FULL, + parameter_values: Mapping[str, str] | None = None, + max_apdu_length: int | None = None, + expected_descriptor: int | None = None, + progress: Callable[[int, int], None] | None = None, +) -> None: + """Download ``application`` into the device at ``individual_address``. + + Assembles the download image (applying ``parameter_values`` on top of the + application defaults), resolves the Load Procedure (merging the application's + fragments into the mask version's default procedure from ``master`` for + default/merged procedure styles) and runs it. The transport connection is + opened and closed following the procedure's Connect/Disconnect/Restart + controls. ``scope`` selects a full or partial download. + + ``max_apdu_length`` defaults to ``None``, which negotiates the length from the + device (reading its maximum APDU length once, for larger and fewer telegrams); + pass a fixed integer to override. ``expected_descriptor`` (the device + descriptor type 0, e.g. ``0x07B0`` for a System B device) is checked against + the device before any write when given, guarding against programming the wrong + device. + + Pass a pre-built ``image`` to skip assembly (e.g. one an editor built from its + live evaluator); otherwise it is built from ``device``/``parameter_values``. + ``master`` (``registry.master``) is required for applications with a default + or merged procedure style. The device must already carry ``individual_address`` + (program the individual address first for a virgin device). ``xknx`` has to be + started. + """ + if image is None: + image = build_image( + application, + device=device, + parameter_values=parameter_values, + group_communication=group_communication, + ) + controls = [ + *resolve_download_controls( + application, master.raw if master is not None else None + ), + *synthesize_group_communication_controls(image), + ] + address = IndividualAddress(individual_address) + apdu_ceiling, negotiate_apdu = _apdu_settings(max_apdu_length) + + manager = _XknxConnectionManager(xknx, address) + runner = LoadProcedureRunner( + application, + image, + connection_manager=manager, + max_apdu_length=apdu_ceiling, + controls=controls, + scope=scope, + expected_descriptor=expected_descriptor, + negotiate_apdu=negotiate_apdu, + ) + try: + await runner.run(progress) + finally: + # Ensure the connection is closed even if the procedure omits a trailing + # Disconnect or fails partway through. + await manager.close() + + +async def preflight( + xknx: XKNX, + individual_address: IndividualAddressableType, + application: Application, + *, + master: MasterData | None = None, + device: SeedDevice | None = None, + image: DownloadImage | None = None, + group_communication: GroupCommunication | None = None, + scope: DownloadScope = DownloadScope.FULL, + parameter_values: Mapping[str, str] | None = None, + max_apdu_length: int | None = None, + expected_descriptor: int | None = None, +) -> PreflightReport: + """Report what :func:`download` would change on the device, changing nothing. + + Assembles the same download image and resolves the same Load Procedure as + :func:`download` (with the same arguments), then reads the device's current + bytes at every location a write would target and returns the diff. Nothing is + written and no load state is changed. Run this before a real download to + confirm the change set (and to catch a mismatched application via the + procedure's compare controls). Pass a pre-built ``image`` to skip assembly. + ``xknx`` has to be started. + """ + if image is None: + image = build_image( + application, + device=device, + parameter_values=parameter_values, + group_communication=group_communication, + ) + controls = [ + *resolve_download_controls( + application, master.raw if master is not None else None + ), + *synthesize_group_communication_controls(image), + ] + address = IndividualAddress(individual_address) + apdu_ceiling, negotiate_apdu = _apdu_settings(max_apdu_length) + + manager = _XknxConnectionManager(xknx, address) + runner = LoadProcedureRunner( + application, + image, + connection_manager=manager, + max_apdu_length=apdu_ceiling, + controls=controls, + scope=scope, + expected_descriptor=expected_descriptor, + negotiate_apdu=negotiate_apdu, + ) + try: + return await runner.preflight() + finally: + await manager.close() diff --git a/packages/download/src/xknxmono/download/errors.py b/packages/download/src/xknxmono/download/errors.py new file mode 100644 index 00000000..4fd38420 --- /dev/null +++ b/packages/download/src/xknxmono/download/errors.py @@ -0,0 +1,23 @@ +"""Exceptions raised while downloading data into a KNX device.""" + +from __future__ import annotations + + +class DownloadError(Exception): + """Base class for all download errors.""" + + +class LoadStateError(DownloadError): + """A Load State Machine did not reach the expected state.""" + + +class VerificationError(DownloadError): + """Data read back from the device does not match the data written.""" + + +class UnsupportedProcedureError(DownloadError): + """The Load Procedure contains a step that is not supported.""" + + +class ImageError(DownloadError): + """The download image could not be assembled from the application data.""" diff --git a/packages/download/src/xknxmono/download/gaps.py b/packages/download/src/xknxmono/download/gaps.py new file mode 100644 index 00000000..b1952597 --- /dev/null +++ b/packages/download/src/xknxmono/download/gaps.py @@ -0,0 +1,85 @@ +"""Registry of Load Controls this engine does not execute yet. + +The KNX Standard v3.0.0 (Chapter 2/3/1 "Load Controls" and the Application Layer +services in 3/3/7) plus the ETS load procedure define more Load Controls than +this engine currently implements. When the runner meets one it fails - or, in a +read-only preflight, logs - a message that names the KNX Standard service the +control maps to. That way a bug report shows exactly which piece of the +implementation is missing instead of an opaque class name. + +Keep :data:`KNOWN_GAPS` in sync with :mod:`xknxmono.download.procedure`: a control +handled in ``LoadProcedureRunner._execute`` must not appear here, and a control +that appears here must not be silently accepted anywhere. +""" + +from __future__ import annotations + +# Control name -> the KNX Standard v3.0.0 service / clause it maps to. These are +# defined by the standard and emitted by ETS load procedures but not yet executed +# by this engine; hitting one is a known implementation gap, not a data error. +KNOWN_GAPS: dict[str, str] = { + "LdCtrlClearLCFilterTable": ( + "clear the line coupler filter table - either the function-property " + "variant (UseFunctionProp) or the memory variant writing LcFilterMemory " + "with a router stop/start sequence (KNX Standard v3.0.0, 3/5/1 Resources). " + "This is coupler-only and needs a router to validate against" + ), + "LdCtrlOnError": ( + "load procedure error-branch directive " + "(KNX Standard v3.0.0, 2/3/1 Load Controls)" + ), + "LdCtrlProcType": ( + "load procedure type marker (KNX Standard v3.0.0, 2/3/1 Load Controls)" + ), +} + +# Controls that legitimately have nothing to write, so a read-only preflight can +# skip them without hiding a gap: state events, segment allocations, delays, +# restarts, read-backs and the client-side directives. +PREFLIGHT_NO_WRITE: frozenset[str] = frozenset( + { + "LdCtrlConnect", + "LdCtrlDisconnect", + "LdCtrlDelay", + "LdCtrlRestart", + "LdCtrlMasterReset", + "LdCtrlLoad", + "LdCtrlUnload", + "LdCtrlLoadCompleted", + "LdCtrlRelSegment", + "LdCtrlTaskSegment", + "LdCtrlTaskPtr", + "LdCtrlTaskCtrl", + "LdCtrlLoadImageMem", + "LdCtrlLoadImageProp", + "LdCtrlLoadImageRelMem", + "LdCtrlReadFunctionProp", + "LdCtrlInvokeFunctionProp", + "LdCtrlMaxLength", + "LdCtrlSetControlVariable", + "LdCtrlMapError", + "LdCtrlProgressText", + "LdCtrlClearCachedObjectTypes", + "LdCtrlDeclarePropDesc", + } +) + + +def gap_hint(control_name: str) -> str | None: + """The KNX Standard service a known-gap control maps to, else ``None``.""" + return KNOWN_GAPS.get(control_name) + + +def describe_missing(control_name: str) -> str: + """A bug-report-ready description of why ``control_name`` is not handled.""" + hint = KNOWN_GAPS.get(control_name) + if hint is not None: + return ( + f"load control {control_name!r} is a known but not-yet-implemented " + f"step in xknx-download; it maps to {hint}" + ) + return ( + f"load control {control_name!r} is not recognised by xknx-download and is " + f"not in its known-gap registry (gaps.py); it still needs to be mapped to a " + f"KNX Standard v3.0.0 service and implemented" + ) diff --git a/packages/download/src/xknxmono/download/group_communication.py b/packages/download/src/xknxmono/download/group_communication.py new file mode 100644 index 00000000..4353728c --- /dev/null +++ b/packages/download/src/xknxmono/download/group_communication.py @@ -0,0 +1,68 @@ +"""Synthesize the load controls that write the System B group communication tables. + +For the System B model the group communication tables (address, association, +group object) are not written by the application's own Load Procedure - they are +applied by a separate step. So when the download image carries these tables as +relative segments, this module produces the load controls that write them: +mirroring the application's own relative-segment pattern (two ``RelSegment`` +allocations followed by a ``WriteRelMem``), addressed by interface object type so +the partial-download scope filter treats them as group communication. + +The table *data* is validated byte-exact against real hardware; the allocation +framing here is modelled on the application's parameter segment (the only +relative segment a System B application program carries) rather than extracted +extracted from a reference implementation, so a full download's allocation must be confirmed by a +read-diff (preflight) before it is trusted. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from xknxmono.models.intermediate.ld_ctrl_rel_segment_t import LdCtrlRelSegment +from xknxmono.models.intermediate.ld_ctrl_write_rel_mem_t import LdCtrlWriteRelMem + +if TYPE_CHECKING: + from .image import DownloadImage + +# The order in which the three tables are written (address, association, group +# object), following the group communication table order in KNX Standard 3/5/1 Resources. +_TABLE_ORDER = (1, 2, 9) + + +def synthesize_group_communication_controls(image: DownloadImage) -> list[object]: + """Return the load controls that write the image's group communication tables. + + Emits nothing when the image carries no relative group communication segments + (e.g. a memory-mapped model, or a parameters-only image). For each table, in + address/association/group-object order, emits two ``RelSegment`` allocations + (mode 1 then mode 0, as the application does for its parameter segment) and a + ``WriteRelMem`` that writes the table data at relative offset 0. + """ + controls: list[object] = [] + for object_type in _TABLE_ORDER: + segment = image.relative_segment(object_type) + if segment is None: + continue + size = len(segment.data) + controls.append( + LdCtrlRelSegment( + obj_type=object_type, occurrence=0, size=size, mode=1, fill=0 + ) + ) + controls.append( + LdCtrlRelSegment( + obj_type=object_type, occurrence=0, size=size, mode=0, fill=0 + ) + ) + controls.append( + LdCtrlWriteRelMem( + obj_type=object_type, + occurrence=0, + offset=0, + size=size, + verify=False, + inline_data=None, + ) + ) + return controls diff --git a/packages/download/src/xknxmono/download/image.py b/packages/download/src/xknxmono/download/image.py new file mode 100644 index 00000000..c173b3b6 --- /dev/null +++ b/packages/download/src/xknxmono/download/image.py @@ -0,0 +1,520 @@ +"""The download image: the data a Load Procedure writes into a device. + +The image is assembled from the parsed application program. Parameter values are +encoded into their code segments (producing the memory image) and into interface +object properties. Group communication tables (address, association, group object) +are part of the segment data as well; project specific values can be supplied via +``parameter_values``. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING + +from xknxmono.product import Application + +from .errors import ImageError +from .project_data import ( + GroupObjectLink, + SeedDevice, + com_object_instance_refs_from_device, + module_instances_from_device, + parameter_instance_refs_from_device, +) +from .tables import ( + Association, + ComObjectDescriptor, + build_association_table, + build_com_object_table, + build_group_address_table, + com_object_flag_byte, + size_code, +) +from .tables_systemb import ( + build_association_table_b, + build_group_address_table_b, + build_group_object_table_b, + group_address_index_b, +) + +# Interface object types of the three group communication tables (KNX Standard +# v3.0.0, Chapter 3/5/1 "Resources": the Address Table Object, the Association +# Table Object, and the Group Object Table Object, Object Type 9, section 4.8). +_ADDRESS_TABLE_TYPE = 1 +_ASSOCIATION_TABLE_TYPE = 2 +_GROUP_OBJECT_TABLE_TYPE = 9 + +if TYPE_CHECKING: + from xknxmono.product.parser_v2.dynamic import DynamicUI + + +@dataclass(frozen=True, slots=True) +class GroupCommunication: + """The data needed to build a device's group communication tables.""" + + device_address: int + links: Sequence[GroupObjectLink] + + +@dataclass(frozen=True, slots=True) +class MemorySegment: + """A block of memory to be written at an absolute address. + + ``mask`` (when set) has one byte per data byte: ``0xFF`` for a byte to write, + ``0x00`` for a byte to leave untouched on the device. With no mask the whole + block is written. Masking lets a download write only the bytes it actually + encoded, never overwriting regions it did not produce. + """ + + address: int + data: bytes + mask: bytes | None = None + + @property + def end(self) -> int: + """First address past this segment.""" + return self.address + len(self.data) + + def masked_runs(self) -> list[tuple[int, bytes]]: + """Return ``(address, data)`` for each contiguous run this segment writes. + + With no mask the whole block is one run; otherwise each maximal run of + ``0xFF`` mask bytes becomes one write. + """ + if self.mask is None: + return [(self.address, self.data)] if self.data else [] + runs: list[tuple[int, bytes]] = [] + i, n = 0, len(self.data) + while i < n: + if self.mask[i]: + j = i + while j < n and self.mask[j]: + j += 1 + runs.append((self.address + i, bytes(self.data[i:j]))) + i = j + else: + i += 1 + return runs + + +@dataclass(frozen=True, slots=True) +class RelativeSegment: + """A block written relative to an interface object's table base. + + Used by the System B model, where the group communication tables live in + relative memory addressed through an object's table reference (resolved on + the device at run time), keyed here by that object's interface object type. + ``data``/``mask`` follow :class:`MemorySegment` starting at relative offset 0. + """ + + object_type: int + data: bytes + mask: bytes | None = None + + def masked_runs(self) -> list[tuple[int, bytes]]: + """Return ``(relative offset, data)`` for each contiguous written run.""" + return MemorySegment(0, self.data, self.mask).masked_runs() + + +@dataclass(frozen=True, slots=True) +class PropertyValue: + """A value to be written to an interface object property.""" + + object_index: int | None + property_id: int + occurrence: int + data: bytes + + +@dataclass(frozen=True, slots=True) +class DownloadImage: + """The complete set of data to be written into a device.""" + + segments: tuple[MemorySegment, ...] + properties: tuple[PropertyValue, ...] + relative_segments: tuple[RelativeSegment, ...] = () + # Interface object index -> the segment data whose MCB CRC covers it. + object_segments: Mapping[int, bytes] = MappingProxyType({}) + + def relative_segment(self, object_type: int) -> RelativeSegment | None: + """Return the relative segment for an interface object type, if any.""" + for segment in self.relative_segments: + if segment.object_type == object_type: + return segment + return None + + def read(self, address: int, size: int) -> bytes: + """Return ``size`` octets from ``address`` within a single segment. + + Load Procedure steps that reference the image by address (LoadImageMem) + read their data through here. The requested range has to lie completely + within one segment. + """ + for segment in self.segments: + if segment.address <= address and address + size <= segment.end: + start = address - segment.address + return segment.data[start : start + size] + raise ImageError( + f"no image data for address range {address:#06x}..{address + size:#06x}" + ) + + def read_optional(self, address: int, size: int) -> bytes | None: + """Like :meth:`read` but return ``None`` when the range is not in the image.""" + try: + return self.read(address, size) + except ImageError: + return None + + def masked_writes(self, address: int, size: int) -> list[tuple[int, bytes]] | None: + """Return the ``(address, data)`` runs to write within ``[address, size)``. + + Finds the single segment covering the range and returns its masked runs + clipped to the range (see :meth:`MemorySegment.masked_runs`). Returns + ``None`` when no segment covers the range (as :meth:`read_optional` does), + and an empty list when the segment covers it but writes nothing there. + """ + for segment in self.segments: + if segment.address <= address and address + size <= segment.end: + out: list[tuple[int, bytes]] = [] + for run_address, run_data in segment.masked_runs(): + lo = max(run_address, address) + hi = min(run_address + len(run_data), address + size) + if lo < hi: + out.append((lo, run_data[lo - run_address : hi - run_address])) + return out + return None + + +def build_image( + application: Application, + *, + ui: DynamicUI | None = None, + device: SeedDevice | None = None, + parameter_values: Mapping[str, str] | None = None, + group_communication: GroupCommunication | None = None, +) -> DownloadImage: + """Assemble the download image for ``application``. + + Provide exactly one parameter state source: ``ui`` (an already-configured + evaluator, e.g. one an editor mutates live), ``device`` (seed a fresh + evaluator from a configured project device - parameter values, module + instances and com object flags), or neither (the application defaults). + ``parameter_values`` maps parameter reference ids to values and overrides the + result before encoding (e.g. individual tweaks). ``group_communication`` adds + the address and association table segments (needed for a full or group + communication download); the load procedure writes them only in scope. + """ + if ui is None and device is not None: + from xknxmono.product.parser_v2.dynamic import DynamicUI + + ui = DynamicUI( + application.program, + parameter_instance_refs=parameter_instance_refs_from_device(device), + module_instances=module_instances_from_device(device), + com_object_instance_refs=com_object_instance_refs_from_device(device), + ) + elif ui is None: + ui = application.dynamic_ui() + if ui is None: + raise ImageError("application has no dynamic section to encode") + + if parameter_values: + for ref_id, value in parameter_values.items(): + ui.set_parameter_ref(ref_id, value) + + base_addresses = ui.segment_base_addrs() + encoded = ui.encode_to_memory_masked() + segments = [ + MemorySegment(address=base_addresses[segment_id], data=data, mask=mask) + for segment_id, (data, mask) in encoded.items() + if data and segment_id in base_addresses + ] + + relative_segments: list[RelativeSegment] = [] + if group_communication is not None: + static = application.program.static + memory_mapped = ( + static.address_table is not None + and static.address_table.code_segment is not None + ) + if memory_mapped: + segments.extend( + _group_communication_segments( + application, ui, base_addresses, encoded, group_communication + ) + ) + else: + relative_segments.extend( + _group_communication_relative_segments( + application, ui, group_communication + ) + ) + + properties = tuple( + PropertyValue( + object_index=key[0], + property_id=key[1], + occurrence=key[2], + data=data, + ) + for key, data in ui.encode_to_properties().items() + if data + ) + + # Map each relative segment (id "..._RS--...") to its data, so the MCB + # table CRC for that interface object can be computed at write time. + object_segments: dict[int, bytes] = {} + for segment_id, (data, _mask) in encoded.items(): + match = re.search(r"_RS-(\d+)-", segment_id) + if match and data: + object_segments[int(match.group(1))] = data + + return DownloadImage( + segments=tuple(segments), + properties=properties, + relative_segments=tuple(relative_segments), + object_segments=MappingProxyType(object_segments), + ) + + +def _group_communication_segments( + application: Application, + ui: DynamicUI, + base_addresses: Mapping[str, int], + encoded: Mapping[str, tuple[bytes, bytes]], + gc: GroupCommunication, +) -> list[MemorySegment]: + """Build the address, association and com object table segments from links. + + Group addresses come from the links; each association references a group + address by its address-table index and the com object by its number (read + from the application's com object table). The com object table overlays each + resolved com object's flags and size onto the segment seed. The table + locations come from the application's static table definitions. + """ + from xknxmono.product.parser_v2.application_indexer import ApplicationIndexer + + static = application.program.static + number_of: dict[str, int] = {} + indexer = ApplicationIndexer(application.program) + for ref_id, ref in indexer.com_object_refs.items(): + com_object = indexer.com_objects.get(ref.ref_id) + if com_object is not None: + number_of[ref_id] = com_object.number + + group_addresses = sorted({link.group_address for link in gc.links}) + index_of = {address: i + 1 for i, address in enumerate(group_addresses)} + linked_numbers = { + number_of[link.com_object_ref_id] + for link in gc.links + if link.com_object_ref_id in number_of + } + + result: list[MemorySegment] = [] + com_object_table = static.com_object_table + if com_object_table is not None and com_object_table.code_segment is not None: + seed = encoded.get(com_object_table.code_segment) + base = base_addresses.get(com_object_table.code_segment) + if seed is not None and base is not None: + data, mask = _build_com_object_table( + seed[0], ui, linked_numbers, set(number_of.values()) + ) + result.append(MemorySegment(base, data, mask=mask)) + address_table = static.address_table + if address_table is not None and address_table.code_segment is not None: + base = base_addresses.get(address_table.code_segment) + if base is not None: + offset = address_table.offset or 0 + data = build_group_address_table(gc.device_address, group_addresses) + # The download writes only the count and the group address + # entries; the two own-address octets after the count are written + # during individual-address programming, not by this download, so keep + # them out of the mask (they legitimately differ per realisation, e.g. + # a 0xffff placeholder). + mask = bytearray(b"\xff" * len(data)) + mask[1] = mask[2] = 0x00 + address = base + offset + result.append(MemorySegment(address, data, mask=bytes(mask))) + association_table = static.association_table + if association_table is not None and association_table.code_segment is not None: + base = base_addresses.get(association_table.code_segment) + if base is not None: + associations = [ + Association( + index_of[link.group_address], + number_of[link.com_object_ref_id], + sending=link.sending, + ) + for link in gc.links + if link.com_object_ref_id in number_of + and link.group_address in index_of + ] + data = build_association_table(associations) + address = base + (association_table.offset or 0) + result.append(MemorySegment(address, data, mask=b"\xff" * len(data))) + return result + + +def _group_communication_relative_segments( + application: Application, + ui: DynamicUI, + gc: GroupCommunication, +) -> list[RelativeSegment]: + """Build the System B group communication tables as relative segments. + + The address, association and group object tables live in relative memory + (System B), keyed here by interface object type. Group addresses come from + the links; associations reference a group address by its address-table index + and a group object by its number; the group object table carries flags and + size for every linked object and leaves the rest empty. + """ + from xknxmono.product.parser_v2.application_indexer import ApplicationIndexer + + indexer = ApplicationIndexer(application.program) + number_of: dict[str, int] = {} + for ref_id, ref in indexer.com_object_refs.items(): + com_object = indexer.com_objects.get(ref.ref_id) + if com_object is not None: + number_of[ref_id] = com_object.number + + group_addresses = sorted({link.group_address for link in gc.links}) + index_of = group_address_index_b(group_addresses) + + associations = [ + Association( + index_of[link.group_address] - 1, + number_of[link.com_object_ref_id], + sending=link.sending, + ) + for link in gc.links + if link.com_object_ref_id in number_of and link.group_address in index_of + ] + linked_numbers = {association.group_object_number for association in associations} + + descriptors = _resolved_group_object_descriptors(ui, linked_numbers) + highest_number = max((c.number for c in indexer.com_objects.values()), default=0) + + address_data = build_group_address_table_b(group_addresses) + association_data = build_association_table_b(associations) + group_object_data = build_group_object_table_b(descriptors, highest_number) + + return [ + RelativeSegment(_ADDRESS_TABLE_TYPE, address_data, b"\xff" * len(address_data)), + RelativeSegment( + _ASSOCIATION_TABLE_TYPE, association_data, b"\xff" * len(association_data) + ), + RelativeSegment( + _GROUP_OBJECT_TABLE_TYPE, + group_object_data, + b"\xff" * len(group_object_data), + ), + ] + + +def _resolved_group_object_descriptors( + ui: DynamicUI, linked_numbers: set[int] +) -> dict[int, tuple[int, int]]: + """Map each linked object's number to its ``(flag byte, size code)``. + + Walks the configured UI so flags and size come from the resolved com object + reference (the values a download actually writes). Only linked objects get a + descriptor; the table formatter leaves every other slot empty. + """ + from xknxmono.product.parser_v2.ui import UiComObject, UiParameterBlock, UiTab + + descriptors: dict[int, tuple[int, int]] = {} + stack: list[object] = list(ui.ui()) + while stack: + node = stack.pop() + if isinstance(node, UiComObject): + if node.number not in linked_numbers: + continue + try: + size = size_code(node.object_size) + except ImageError: + continue + flags = com_object_flag_byte( + priority=node.priority or "Low", + communication=node.communication, + read=node.read, + write=node.write, + transmit=node.transmit, + update=node.update, + read_on_init=node.read_on_init, + ) + descriptors[node.number] = (flags, size) + elif isinstance(node, (UiTab, UiParameterBlock)): + stack.extend(node.children) + return descriptors + + +# Layout of a com object table record: 5 octet header, then 4 octet records; the +# flag byte is the first octet of each record (offset 5 + number*4). +_COM_OBJECT_HEADER = 5 +_COM_OBJECT_RECORD = 4 +_COMMUNICATION_BIT = 0x04 + + +def _build_com_object_table( + seed: bytes, + ui: DynamicUI, + linked_numbers: set[int], + object_numbers: set[int], +) -> tuple[bytes, bytes]: + """Overlay each com object's flags onto the table seed. + + Follows the Group Object Table definition (3/5/1 Resources, 4.18): a com object visible in the current + configuration gets its full flag byte and size recomputed (communication bit + set only when the object is linked); every other defined object keeps the + manufacturer's seed flags with just the communication bit toggled by link + state. This second, seed-preserving path is what a byte-exact download needs + for realisations whose seed already carries the real per-object flags. + ``object_numbers`` bounds the table to the objects the application defines, so + the pass never touches bytes past the table (some products share the segment + with parameter data). + """ + from xknxmono.product.parser_v2.ui import UiComObject, UiParameterBlock, UiTab + + descriptors: list[ComObjectDescriptor] = [] + stack: list[object] = list(ui.ui()) + while stack: + node = stack.pop() + if isinstance(node, UiComObject): + try: + size = size_code(node.object_size) + except ImageError: + continue + flags = com_object_flag_byte( + priority=node.priority or "Low", + communication=node.communication and node.number in linked_numbers, + read=node.read, + write=node.write, + transmit=node.transmit, + update=node.update, + read_on_init=node.read_on_init, + ) + descriptors.append( + ComObjectDescriptor(number=node.number, flags=flags, size=size) + ) + elif isinstance(node, (UiTab, UiParameterBlock)): + stack.extend(node.children) + + data, mask = build_com_object_table(seed, descriptors) + visible = {descriptor.number for descriptor in descriptors} + data, mask = bytearray(data), bytearray(mask) + for number in object_numbers: + if number in visible: + continue + offset = _COM_OBJECT_HEADER + number * _COM_OBJECT_RECORD + if offset >= len(data): + continue + flags = seed[offset] & ~_COMMUNICATION_BIT + if number in linked_numbers: + flags |= _COMMUNICATION_BIT + data[offset] = flags + mask[offset] = 0xFF + return bytes(data), bytes(mask) diff --git a/packages/download/src/xknxmono/download/load_state.py b/packages/download/src/xknxmono/download/load_state.py new file mode 100644 index 00000000..169744f3 --- /dev/null +++ b/packages/download/src/xknxmono/download/load_state.py @@ -0,0 +1,187 @@ +"""Load State Machine states and load events. + +A loadable part of a KNX device (address table, association table, group object +table, application program, ...) is guarded by a Load State Machine. The machine +is driven by writing *load events* to ``PID_LOAD_STATE_CONTROL`` (property id 5) +of the part's interface object and reading back the resulting *load state*. + +The state machine and its load events are defined in KNX Standard v3.0.0, +Chapter 3/5/1 "Resources", section 4.23 "Load State Machine" (the property based +Realisation Type 1, section 4.23.2) and section 4.2.5 "PID_LOAD_STATE_CONTROL +(PID = 5)"; the procedures that issue them are in Chapter 3/5/2 "Management +Procedures" (DM_LoadStateMachineWrite). Every load event is a 10 octet control +value; unused octets are zero. +""" + +from __future__ import annotations + +from enum import IntEnum + +# Property id of the load state control property (PDT_CONTROL) and the number of +# octets one control element occupies. +PID_LOAD_STATE_CONTROL = 5 +LOAD_STATE_CONTROL_SIZE = 10 + + +class LoadState(IntEnum): + """State reported when reading ``PID_LOAD_STATE_CONTROL``. + + ``UNLOADING`` and ``LOAD_COMPLETING`` are optional transient states a device + may report while an unload or load-complete is still in progress. + """ + + UNLOADED = 0 + LOADED = 1 + LOADING = 2 + ERROR = 3 + UNLOADING = 4 + LOAD_COMPLETING = 5 + + +class LoadEvent(IntEnum): + """First octet of a load event written to ``PID_LOAD_STATE_CONTROL``.""" + + START_LOADING = 1 + LOAD_COMPLETE = 2 + ADDITIONAL = 3 + UNLOAD = 4 + + +class SegmentType(IntEnum): + """Subtype of an ``ADDITIONAL`` load event (segment allocation).""" + + ABS_DATA = 0 + ABS_STACK = 1 + ABS_TASK = 2 + TASK_PTR = 3 + TASK_CTRL_1 = 4 + TASK_CTRL_2 = 5 + RELATIVE_ALLOCATION = 0x0A + DATA_RELATIVE_ALLOCATION = 0x0B + + +def _pad(data: bytes) -> bytes: + """Pad a control value to the fixed load state control element size.""" + if len(data) > LOAD_STATE_CONTROL_SIZE: + raise ValueError( + f"load event too long: {len(data)} > {LOAD_STATE_CONTROL_SIZE}" + ) + return data + bytes(LOAD_STATE_CONTROL_SIZE - len(data)) + + +def start_loading() -> bytes: + """Load event moving the machine to ``LOADING``.""" + return _pad(bytes([LoadEvent.START_LOADING])) + + +def load_complete() -> bytes: + """Load event moving the machine to ``LOADED``.""" + return _pad(bytes([LoadEvent.LOAD_COMPLETE])) + + +def unload() -> bytes: + """Load event moving the machine to ``UNLOADED``.""" + return _pad(bytes([LoadEvent.UNLOAD])) + + +def alloc_absolute_segment( + segment_type: SegmentType, + start_address: int, + length: int, + *, + access_attributes: int = 0, + memory_type: int = 0, + memory_attributes: int = 0, +) -> bytes: + """Absolute data or stack segment allocation. + + Access attributes: bits 0-3 write access level, bits 4-7 read access level. + Memory type: bits 0-2 (1 = zero page RAM, 2 = RAM, 3 = EEPROM). + Memory attributes: bit 7 enables checksum control. + """ + if segment_type not in (SegmentType.ABS_DATA, SegmentType.ABS_STACK): + raise ValueError("segment_type must be ABS_DATA or ABS_STACK") + return _pad( + bytes([LoadEvent.ADDITIONAL, segment_type]) + + start_address.to_bytes(2, "big") + + length.to_bytes(2, "big") + + bytes( + [access_attributes & 0xFF, memory_type & 0xFF, memory_attributes & 0xFF] + ) + ) + + +def alloc_task_segment( + start_address: int, + pei_type: int, + application_id: bytes, +) -> bytes: + """Absolute task segment allocation. + + ``application_id`` is the 5 octet application id: manufacturer id (2), + application software type (2) and version (1). + """ + if len(application_id) != 5: + raise ValueError("application_id must be 5 octets") + return _pad( + bytes([LoadEvent.ADDITIONAL, SegmentType.ABS_TASK]) + + start_address.to_bytes(2, "big") + + bytes([pei_type & 0xFF]) + + application_id + ) + + +def task_pointer(init_address: int, save_address: int, pei_handler: int) -> bytes: + """Task pointer load event.""" + return _pad( + bytes([LoadEvent.ADDITIONAL, SegmentType.TASK_PTR]) + + init_address.to_bytes(2, "big") + + save_address.to_bytes(2, "big") + + pei_handler.to_bytes(2, "big") + ) + + +def task_control_1(interface_object_address: int, interface_object_count: int) -> bytes: + """Task control 1 load event.""" + return _pad( + bytes([LoadEvent.ADDITIONAL, SegmentType.TASK_CTRL_1]) + + interface_object_address.to_bytes(2, "big") + + bytes([interface_object_count & 0xFF]) + ) + + +def task_control_2( + callback_address: int, + com_object_pointer: int, + com_object_segment_pointer_1: int, + com_object_segment_pointer_2: int, +) -> bytes: + """Task control 2 load event.""" + return _pad( + bytes([LoadEvent.ADDITIONAL, SegmentType.TASK_CTRL_2]) + + callback_address.to_bytes(2, "big") + + com_object_pointer.to_bytes(2, "big") + + com_object_segment_pointer_1.to_bytes(2, "big") + + com_object_segment_pointer_2.to_bytes(2, "big") + ) + + +def relative_allocation(number_of_octets: int) -> bytes: + """Relative allocation load event (subtype 0x0A, 2 octet count).""" + return _pad( + bytes([LoadEvent.ADDITIONAL, SegmentType.RELATIVE_ALLOCATION]) + + number_of_octets.to_bytes(2, "big") + ) + + +def data_relative_allocation(size: int, *, mode: int = 0, fill: int = 0) -> bytes: + """Data relative allocation load event (subtype 0x0B, System B). + + Layout: ``03 0B ``. ``mode`` bit 0 + set fills the allocated memory with ``fill``; other bits are reserved. + """ + return _pad( + bytes([LoadEvent.ADDITIONAL, SegmentType.DATA_RELATIVE_ALLOCATION]) + + size.to_bytes(4, "big") + + bytes([mode & 0xFF, fill & 0xFF]) + ) diff --git a/packages/download/src/xknxmono/download/merge.py b/packages/download/src/xknxmono/download/merge.py new file mode 100644 index 00000000..704dc0f0 --- /dev/null +++ b/packages/download/src/xknxmono/download/merge.py @@ -0,0 +1,116 @@ +"""Resolve the effective Load Procedure to execute for a download. + +An application declares one of three load procedure styles: + +- ``ProductProcedure``: the application ships the complete procedure; use it as is. +- ``DefaultProcedure``: the application ships no procedure; use the mask version's + default procedure from the master data. +- ``MergedProcedure``: the application ships fragments identified by a merge id; + splice them into the mask version's default procedure at the matching + ``LdCtrlMerge`` placeholders. + +The default procedures live per mask version in the master data +(``MaskVersion`` -> configuration data -> ``Procedures``), keyed by procedure +type (``Load`` for a download, ``Unload`` for removal). +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from xknxmono.models.intermediate.ld_ctrl_merge_t import LdCtrlMerge +from xknxmono.models.intermediate.load_procedure_style_t import LoadProcedureStyle +from xknxmono.models.intermediate.procedure_type_t import ProcedureType + +from .errors import UnsupportedProcedureError + +if TYPE_CHECKING: + from xknxmono.models.intermediate.load_procedure_t import LoadProcedure + from xknxmono.models.intermediate.load_procedures_t import LoadProcedures + from xknxmono.models.intermediate.master_data_t import MasterData + from xknxmono.product import Application + + +def resolve_download_controls( + application: Application, + master_data: MasterData | None = None, + *, + procedure_type: ProcedureType = ProcedureType.LOAD, +) -> list[object]: + """Return the flat, ordered list of Load Controls to run for a download. + + ``master_data`` is required for default and merged procedures; without it (or + without a matching default) the application's own procedure is used. + """ + style = application.load_procedure_style + load_procedures = application.load_procedures + + if style == LoadProcedureStyle.PRODUCT_PROCEDURE: + return _flatten(load_procedures) + + default = _default_procedure( + master_data, application.program.mask_version, procedure_type + ) + if default is None: + # No default available: fall back to the application's own procedure. + return _flatten(load_procedures) + + return _splice(default.choice, _fragments_by_merge_id(load_procedures)) + + +def _flatten(load_procedures: LoadProcedures | None) -> list[object]: + """Concatenate the controls of every application load procedure.""" + if load_procedures is None: + raise UnsupportedProcedureError("application has no load procedure") + controls: list[object] = [] + for procedure in load_procedures.load_procedure: + controls.extend(procedure.choice) + return controls + + +def _fragments_by_merge_id( + load_procedures: LoadProcedures | None, +) -> dict[int, list[object]]: + """Group application procedure fragments by their merge id.""" + fragments: dict[int, list[object]] = {} + if load_procedures is None: + return fragments + for procedure in load_procedures.load_procedure: + if procedure.merge_id is None: + continue + fragments.setdefault(procedure.merge_id, []).extend(procedure.choice) + return fragments + + +def _splice( + controls: Sequence[object], fragments: dict[int, list[object]] +) -> list[object]: + """Replace each merge placeholder with the fragment of matching merge id.""" + result: list[object] = [] + for control in controls: + if isinstance(control, LdCtrlMerge): + result.extend(fragments.get(control.merge_id, [])) + else: + result.append(control) + return result + + +def _default_procedure( + master_data: MasterData | None, + mask_version_id: str, + procedure_type: ProcedureType, +) -> LoadProcedure | None: + """Find the mask version's default procedure of the given type.""" + if master_data is None or master_data.mask_versions is None: + return None + for mask_version in master_data.mask_versions.mask_version: + if mask_version.id != mask_version_id: + continue + for configuration in mask_version.hawk_configuration_data: + if configuration.procedures is None: + continue + for procedure in configuration.procedures.procedure: + if procedure.procedure_type == procedure_type: + return procedure + return None diff --git a/packages/download/src/xknxmono/download/preflight.py b/packages/download/src/xknxmono/download/preflight.py new file mode 100644 index 00000000..ac09bcb4 --- /dev/null +++ b/packages/download/src/xknxmono/download/preflight.py @@ -0,0 +1,148 @@ +"""Read-only pre-flight check: what a download would change on the device. + +Before writing anything, walk the resolved Load Procedure and, for every control +that would write, read the device's current bytes and compare them against the +data the download would write. Nothing is written and no load state is changed - +only reads (memory, properties, table references, object lookup) happen, plus the +procedure's own compare controls, which act as a fingerprint gate. + +The resulting :class:`PreflightReport` lists, per memory segment and per property, +how many bytes would change (and the changed ranges), so an operator can confirm a +download does what they expect before committing it to hardware. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ByteRange: + """A contiguous run of changed bytes within a diff, by offset from its start.""" + + start: int + length: int + + +def _changed_ranges(current: bytes, planned: bytes) -> tuple[ByteRange, ...]: + """Return the contiguous runs where ``planned`` differs from ``current``. + + The device's length is authoritative: beyond it, a planned octet counts as a + change only when it is non-zero. Trailing zero padding (a property carried at + its maximum element size while the device stores a shorter value) is therefore + not reported as a change. Memory diffs read the same length and are unaffected. + """ + ranges: list[ByteRange] = [] + run_start: int | None = None + for offset in range(len(planned)): + if offset < len(current): + differs = current[offset] != planned[offset] + else: + differs = planned[offset] != 0 + if differs and run_start is None: + run_start = offset + elif not differs and run_start is not None: + ranges.append(ByteRange(run_start, offset - run_start)) + run_start = None + if run_start is not None: + ranges.append(ByteRange(run_start, len(planned) - run_start)) + return tuple(ranges) + + +@dataclass(frozen=True, slots=True) +class SegmentDiff: + """The current versus planned bytes for one memory write.""" + + address: int + current: bytes + planned: bytes + + @property + def changed_ranges(self) -> tuple[ByteRange, ...]: + """Contiguous runs (offset from ``address``) that would change.""" + return _changed_ranges(self.current, self.planned) + + @property + def changed_bytes(self) -> int: + """Number of bytes that would change.""" + return sum(r.length for r in self.changed_ranges) + + @property + def changed(self) -> bool: + """Whether this write would change anything.""" + return self.changed_bytes > 0 + + +@dataclass(frozen=True, slots=True) +class PropertyDiff: + """The current versus planned bytes for one property write.""" + + object_index: int + property_id: int + current: bytes + planned: bytes + + @property + def changed_ranges(self) -> tuple[ByteRange, ...]: + """Contiguous runs (element offset) that would change.""" + return _changed_ranges(self.current, self.planned) + + @property + def changed_bytes(self) -> int: + """Number of bytes that would change.""" + return sum(r.length for r in self.changed_ranges) + + @property + def changed(self) -> bool: + """Whether this write would change anything.""" + return self.changed_bytes > 0 + + +@dataclass(frozen=True, slots=True) +class PreflightReport: + """The full set of changes a download would make, without making them.""" + + segments: tuple[SegmentDiff, ...] + properties: tuple[PropertyDiff, ...] + + @property + def changed_segments(self) -> tuple[SegmentDiff, ...]: + """The memory writes that would actually change bytes.""" + return tuple(s for s in self.segments if s.changed) + + @property + def changed_properties(self) -> tuple[PropertyDiff, ...]: + """The property writes that would actually change bytes.""" + return tuple(p for p in self.properties if p.changed) + + @property + def total_changed_bytes(self) -> int: + """Total number of bytes that would change across all writes.""" + return sum(s.changed_bytes for s in self.segments) + sum( + p.changed_bytes for p in self.properties + ) + + @property + def has_changes(self) -> bool: + """Whether the download would change anything on the device.""" + return self.total_changed_bytes > 0 + + def summary(self) -> str: + """Return a human readable, multi-line summary of the pending changes.""" + lines = [ + f"Pre-flight: {self.total_changed_bytes} byte(s) would change " + f"({len(self.changed_segments)} memory segment(s), " + f"{len(self.changed_properties)} property write(s))." + ] + for segment in self.segments: + state = f"{segment.changed_bytes}/{len(segment.planned)} changed" + marker = " " if segment.changed else " (no change)" + lines.append(f" memory {segment.address:#06x}: {state}{marker}") + for prop in self.properties: + state = f"{prop.changed_bytes}/{len(prop.planned)} changed" + marker = " " if prop.changed else " (no change)" + lines.append( + f" object {prop.object_index} property {prop.property_id}: " + f"{state}{marker}" + ) + return "\n".join(lines) diff --git a/packages/download/src/xknxmono/download/procedure.py b/packages/download/src/xknxmono/download/procedure.py new file mode 100644 index 00000000..f0737600 --- /dev/null +++ b/packages/download/src/xknxmono/download/procedure.py @@ -0,0 +1,928 @@ +"""Interpret a Load Procedure and execute it over a device connection. + +A Load Procedure is an ordered list of Load Controls parsed from the application +program (Load Controls are described in KNX Standard v3.0.0, Volume 2 Cookbook, +02_03_01 "Load Controls"). :class:`LoadProcedureRunner` walks that list and turns +each control into the corresponding bus operation, reading bulk data from the +:class:`DownloadImage` where a control refers to it. The download procedures +themselves - complete download, partial download and unload - follow Chapter 3/5/3 +"Configuration Procedures", sections 3.5.2, 3.5.3 and 3.5.4. + +Load State Machine control follows the property based Realisation Type 1 (writing +load events to ``PID_LOAD_STATE_CONTROL`` of the addressed interface object, see +Chapter 3/5/1 "Resources", section 4.23); the memory mapped variant used by very +old (BCU) device models is out of scope. + +Connection handling follows the point-to-point management connection lifecycle: the transport connection is +opened on a Connect control and closed on Disconnect; a Restart tears it down +(after a cooldown the next control reconnects); and any bus control encountered +without an open connection opens one first (auto-connect). This lifecycle is only +active when the runner is given a :class:`ConnectionManager`; with a fixed +programmer the connection is assumed to stay open (Connect/Disconnect are no-ops). +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING + +from xknxmono.models.intermediate.ld_ctrl_abs_segment_t import LdCtrlAbsSegment +from xknxmono.models.intermediate.ld_ctrl_compare_mem_t import LdCtrlCompareMem +from xknxmono.models.intermediate.ld_ctrl_compare_prop_t import LdCtrlCompareProp +from xknxmono.models.intermediate.ld_ctrl_compare_rel_mem_t import LdCtrlCompareRelMem +from xknxmono.models.intermediate.ld_ctrl_connect_t import LdCtrlConnect +from xknxmono.models.intermediate.ld_ctrl_delay_t import LdCtrlDelay +from xknxmono.models.intermediate.ld_ctrl_disconnect_t import LdCtrlDisconnect +from xknxmono.models.intermediate.ld_ctrl_invoke_function_prop_t import ( + LdCtrlInvokeFunctionProp, +) +from xknxmono.models.intermediate.ld_ctrl_load_completed_t import LdCtrlLoadCompleted +from xknxmono.models.intermediate.ld_ctrl_load_image_mem_t import LdCtrlLoadImageMem +from xknxmono.models.intermediate.ld_ctrl_load_image_prop_t import LdCtrlLoadImageProp +from xknxmono.models.intermediate.ld_ctrl_load_image_rel_mem_t import ( + LdCtrlLoadImageRelMem, +) +from xknxmono.models.intermediate.ld_ctrl_load_t import LdCtrlLoad +from xknxmono.models.intermediate.ld_ctrl_master_reset_t import LdCtrlMasterReset +from xknxmono.models.intermediate.ld_ctrl_mem_addr_space_t import LdCtrlMemAddrSpace +from xknxmono.models.intermediate.ld_ctrl_read_function_prop_t import ( + LdCtrlReadFunctionProp, +) +from xknxmono.models.intermediate.ld_ctrl_rel_segment_t import LdCtrlRelSegment +from xknxmono.models.intermediate.ld_ctrl_restart_t import LdCtrlRestart +from xknxmono.models.intermediate.ld_ctrl_task_ctrl1_t import LdCtrlTaskCtrl1 +from xknxmono.models.intermediate.ld_ctrl_task_ctrl2_t import LdCtrlTaskCtrl2 +from xknxmono.models.intermediate.ld_ctrl_task_ptr_t import LdCtrlTaskPtr +from xknxmono.models.intermediate.ld_ctrl_task_segment_t import LdCtrlTaskSegment +from xknxmono.models.intermediate.ld_ctrl_unload_t import LdCtrlUnload +from xknxmono.models.intermediate.ld_ctrl_write_mem_t import LdCtrlWriteMem +from xknxmono.models.intermediate.ld_ctrl_write_prop_t import LdCtrlWriteProp +from xknxmono.models.intermediate.ld_ctrl_write_rel_mem_t import LdCtrlWriteRelMem + +from . import gaps, load_state +from .crc import segment_crc +from .errors import ( + DownloadError, + ImageError, + UnsupportedProcedureError, + VerificationError, +) +from .merge import resolve_download_controls +from .preflight import PreflightReport, PropertyDiff, SegmentDiff +from .programmer import DEFAULT_MAX_APDU_LENGTH, DeviceProgrammer +from .scope import DownloadScope, control_in_scope + +if TYPE_CHECKING: + from xknxmono.product import Application + + from .image import DownloadImage + from .programmer import ConnectionManager + +logger = logging.getLogger(__name__) + +# Default cooldown (seconds) to wait after a Restart before reconnecting. +DEFAULT_RESTART_COOLDOWN = 3.0 + +# Memory Control Block table property and its per-segment entry layout (KNX +# Standard v3.0.0, Chapter 3/5/1 "Resources", section 4.2.27 "PID_MCB_TABLE"): +# an 8 octet entry per segment, CRC-protected when bit 0 of octet 4 is clear, with +# the segment CRC in octets 6..7 (big-endian). +_PID_MCB_TABLE = 27 +_MCB_ENTRY_SIZE = 8 +_MCB_CRC_PROTECTED_OCTET = 4 +_MCB_CRC_OFFSET = 6 + + +def _mcb_table_with_crc(data: bytes, segment: bytes) -> bytes: + """Patch the segment CRC into each CRC-protected Memory Control Block entry. + + ``data`` is the MCB table value (8 octet entries, plus any trailing padding); + ``segment`` is the loaded segment the CRC protects. Only entries whose + CRC-protected flag is set get their CRC octets filled. + """ + out = bytearray(data) + crc = segment_crc(segment) + for start in range(0, len(out) - _MCB_ENTRY_SIZE + 1, _MCB_ENTRY_SIZE): + if out[start + _MCB_CRC_PROTECTED_OCTET] & 1: + continue + out[start + _MCB_CRC_OFFSET] = (crc >> 8) & 0xFF + out[start + _MCB_CRC_OFFSET + 1] = crc & 0xFF + return bytes(out) + + +# Load Controls handled entirely on the client side, without any bus effect. +# LdCtrlDeclarePropDesc only declares a property's description (type, element +# count, access) to the client object model so later property writes know its +# layout; it sends no telegram (A_PropertyValue_Write already carries count and +# data here), so it is a no-op for this engine. +_CLIENT_SIDE = ( + "LdCtrlMaxLength", + "LdCtrlSetControlVariable", + "LdCtrlMapError", + "LdCtrlProgressText", + "LdCtrlClearCachedObjectTypes", + "LdCtrlDeclarePropDesc", +) + + +def _application_id(application: Application) -> bytes: + """Assemble the 5 octet application id (manufacturer, type, version).""" + manufacturer = application.manufacturer_id.split("-")[-1] + manufacturer_id = int(manufacturer, 16) + program = application.program + return ( + manufacturer_id.to_bytes(2, "big") + + program.application_number.to_bytes(2, "big") + + bytes([program.application_version & 0xFF]) + ) + + +class LoadProcedureRunner: + """Execute an application's Load Procedure over a device connection.""" + + def __init__( + self, + application: Application, + image: DownloadImage, + programmer: DeviceProgrammer | None = None, + *, + connection_manager: ConnectionManager | None = None, + max_apdu_length: int = DEFAULT_MAX_APDU_LENGTH, + restart_cooldown: float = DEFAULT_RESTART_COOLDOWN, + controls: Sequence[object] | None = None, + scope: DownloadScope = DownloadScope.FULL, + expected_descriptor: int | None = None, + negotiate_apdu: bool = False, + ) -> None: + """Initialize the runner. + + Provide either a fixed ``programmer`` (the connection stays open for the + whole run; Connect/Disconnect are no-ops) or a ``connection_manager`` + (the runner opens/closes the connection per Connect/Disconnect and after + a Restart, and auto-connects before any bus control). ``controls`` is the + resolved Load Control list; when omitted the application's own procedure + is flattened. ``scope`` selects a full or partial download. + + With a ``connection_manager``, when ``expected_descriptor`` is given the + device's mask version (device descriptor type 0) is read once on the first + connection and must match, guarding against programming the wrong device; + when ``negotiate_apdu`` is set the device's maximum APDU length is read + once and used for the rest of the run (chunked writes then use larger + telegrams). Both are skipped for a fixed ``programmer``. + """ + if programmer is None and connection_manager is None: + raise DownloadError("provide a programmer or a connection manager") + self.application = application + self.image = image + self.scope = scope + self.restarted = False + self._programmer = programmer + self._manager = connection_manager + self._max_apdu_length = ( + programmer.max_apdu_length if programmer is not None else max_apdu_length + ) + self._restart_cooldown = restart_cooldown + self._restart_at: float | None = None + # A one-shot cooldown that overrides the default for the next reconnect + # (used when a Master Reset reports a longer device process time). + self._pending_cooldown: float | None = None + self._expected_descriptor = expected_descriptor + self._negotiate_apdu = negotiate_apdu + self._descriptor_checked = False + self._negotiated_apdu: int | None = None + self._controls = ( + list(controls) + if controls is not None + else resolve_download_controls(application) + ) + # Position of the control currently being executed, for diagnostics. + self._position: tuple[int, int] | None = None + + async def run(self, progress: Callable[[int, int], None] | None = None) -> None: + """Execute the Load Procedure, honouring the selected download scope. + + ``progress`` (optional) is called ``progress(done, total)`` after each executed + control, where ``total`` is the number of in-scope controls, so a UI can show + download progress. + """ + in_scope = [c for c in self._controls if self._in_scope(c)] + total = len(in_scope) + logger.info( + "download run start: %s, %d of %d load controls in scope", + self._target(), + total, + len(self._controls), + ) + for done, control in enumerate(in_scope, start=1): + self._position = (done, total) + await self._execute(control) + if progress is not None: + progress(done, total) + self._position = None + + async def preflight(self) -> PreflightReport: + """Report what the download would change, without changing anything. + + Walks the same scoped Load Controls as :meth:`run` but performs no write + and drives no Load State Machine: for each control that would write, the + device's current bytes are read and compared against the data the download + would write. Compare controls (the application fingerprint gate) still run + - they only read. The connection is opened read-only and closed again. + """ + segments: list[SegmentDiff] = [] + properties: list[PropertyDiff] = [] + in_scope = [c for c in self._controls if self._in_scope(c)] + logger.info( + "download preflight start: %s, %d of %d load controls in scope", + self._target(), + len(in_scope), + len(self._controls), + ) + try: + for done, control in enumerate(in_scope, start=1): + self._position = (done, len(in_scope)) + await self._preflight_control(control, segments, properties) + finally: + self._position = None + await self._close() + return PreflightReport(segments=tuple(segments), properties=tuple(properties)) + + def _in_scope(self, control: object) -> bool: + """Whether a control participates in the requested download scope.""" + return control_in_scope(control, self.scope) + + def _target(self) -> str: + """A short, log-friendly identity of the download target for diagnostics.""" + try: + app = _application_id(self.application).hex() + except (ValueError, AttributeError): + app = "unknown" + return f"app={app} scope={self.scope.name}" + + def _unsupported( + self, control: object, *, reason: str | None = None + ) -> UnsupportedProcedureError: + """Build (and log) a diagnostic error for a control this engine can't run. + + The message names the KNX Standard service the control maps to (via + :mod:`xknxmono.download.gaps`), the position in the procedure and the + target, so a bug report shows immediately what is still missing. + """ + name = type(control).__name__ + detail = reason if reason is not None else gaps.describe_missing(name) + where = "" + if self._position is not None: + where = f" at in-scope load control {self._position[0]}/{self._position[1]}" + message = ( + f"{detail}{where} [{self._target()}]. Please file a bug report with this " + f"message, the device order number and the mask version so the missing " + f"step can be implemented." + ) + logger.warning("unsupported load control: %s", message) + return UnsupportedProcedureError(message) + + async def _bus(self) -> DeviceProgrammer: + """Return the connected programmer, opening a connection if needed.""" + if self._programmer is not None: + return self._programmer + if self._manager is None: + raise DownloadError("no connection available") + await self._await_restart_cooldown() + connection = await self._manager.open() + self._programmer = DeviceProgrammer( + connection, max_apdu_length=self._max_apdu_length + ) + await self._prepare_device(self._programmer) + return self._programmer + + async def _prepare_device(self, programmer: DeviceProgrammer) -> None: + """Guard the device mask and negotiate the APDU length, once per run. + + Runs on the first opened connection: reads the device descriptor to + confirm the mask matches ``expected_descriptor`` (guarding against + programming the wrong device) and reads the device's maximum APDU length + to use larger telegrams. The negotiated length is remembered and reapplied + after a reconnect, so it is read only once. + """ + if self._expected_descriptor is not None and not self._descriptor_checked: + actual = await programmer.read_device_descriptor() + if actual != self._expected_descriptor: + raise VerificationError( + f"device mask mismatch: expected descriptor " + f"{self._expected_descriptor:#06x}, device reports {actual:#06x}. " + f"Refusing to program - check the individual address points at " + f"the intended device." + ) + self._descriptor_checked = True + logger.info("device mask confirmed: descriptor %#06x", actual) + if self._negotiate_apdu: + if self._negotiated_apdu is None: + device_max = await programmer.read_max_apdu_length() + self._negotiated_apdu = max( + DEFAULT_MAX_APDU_LENGTH, + min(device_max, self._max_apdu_length), + ) + logger.info( + "negotiated APDU length: %d (device reports %d)", + self._negotiated_apdu, + device_max, + ) + programmer.max_apdu_length = self._negotiated_apdu + + async def _close(self) -> None: + """Close the current connection when the runner manages the lifecycle.""" + if self._manager is not None and self._programmer is not None: + await self._manager.close() + self._programmer = None + + async def _await_restart_cooldown(self) -> None: + """Wait out the restart cooldown before reconnecting, if one is pending.""" + if self._restart_at is None: + return + cooldown = ( + self._pending_cooldown + if self._pending_cooldown is not None + else self._restart_cooldown + ) + elapsed = time.monotonic() - self._restart_at + remaining = cooldown - elapsed + if remaining > 0: + await asyncio.sleep(remaining) + self._restart_at = None + self._pending_cooldown = None + + async def _resolve_index(self, control: object) -> int: + """Resolve the interface object index a control addresses. + + A control identifies its object by explicit ``obj_idx``, by + ``obj_type`` + zero-based ``occurrence`` (resolved on the device), or by + ``lsm_idx`` which, for property based management, is the object index. + """ + obj_idx = getattr(control, "obj_idx", None) + if obj_idx is not None: + return obj_idx + obj_type = getattr(control, "obj_type", None) + if obj_type is not None: + occurrence = getattr(control, "occurrence", 0) + programmer = await self._bus() + return await programmer.locate_object(obj_type, occurrence) + lsm_idx = getattr(control, "lsm_idx", None) + if lsm_idx is not None: + return lsm_idx + raise self._unsupported( + control, + reason=( + f"load control {type(control).__name__!r} addresses no interface " + f"object (neither obj_idx, obj_type nor lsm_idx is set)" + ), + ) + + async def _execute(self, control: object) -> None: + """Dispatch a single Load Control to the matching bus operation.""" + if isinstance(control, LdCtrlConnect): + await self._bus() + return + if isinstance(control, LdCtrlDisconnect): + await self._close() + return + if isinstance(control, LdCtrlDelay): + await asyncio.sleep(control.milli_seconds / 1000) + return + if isinstance(control, LdCtrlRestart): + programmer = await self._bus() + await programmer.restart() + self.restarted = True + # A Restart tears down the connection device-side; drop it and + # arm the cooldown so the next control reconnects after a pause. + await self._close() + self._restart_at = time.monotonic() + return + if isinstance(control, LdCtrlMasterReset): + programmer = await self._bus() + process_time = await programmer.master_reset( + control.erase_code, control.channel_number + ) + self.restarted = True + # Like a Restart, the device drops the connection; additionally it + # reports how long it stays unreachable, so honour that as the + # one-shot reconnect cooldown when it exceeds the default. + await self._close() + self._restart_at = time.monotonic() + self._pending_cooldown = max(self._restart_cooldown, process_time / 1000) + return + if isinstance(control, LdCtrlUnload): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, load_state.unload(), load_state.LoadState.UNLOADED + ) + return + if isinstance(control, LdCtrlLoad): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, load_state.start_loading(), load_state.LoadState.LOADING + ) + return + if isinstance(control, LdCtrlLoadCompleted): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, load_state.load_complete(), load_state.LoadState.LOADED + ) + return + if isinstance(control, LdCtrlWriteMem): + await self._write_mem(control) + return + if isinstance(control, LdCtrlLoadImageMem): + # LoadImageMem reads device memory into the image (read-back / + # compare), it does not write. + _require_standard(control.address_space) + programmer = await self._bus() + await programmer.read_memory(control.address, control.size) + return + if isinstance(control, LdCtrlCompareMem): + _require_standard(control.address_space) + await self._compare_mem(control.address, control.inline_data) + return + if isinstance(control, LdCtrlWriteRelMem): + await self._write_rel_mem(control) + return + if isinstance(control, LdCtrlCompareRelMem): + index = await self._resolve_index(control) + programmer = await self._bus() + base = await programmer.read_table_reference(index) + await self._compare_mem(base + control.offset, control.inline_data) + return + if isinstance(control, LdCtrlLoadImageRelMem): + # Read-back into the image (see LoadImageMem); it does not write. + index = await self._resolve_index(control) + programmer = await self._bus() + base = await programmer.read_table_reference(index) + await programmer.read_memory(base + control.offset, control.size) + return + if isinstance(control, LdCtrlWriteProp): + await self._write_prop(control) + return + if isinstance(control, LdCtrlLoadImageProp): + # LoadImageProp reads an interface object property into the image + # (read-back / compare), it does not write. + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.read_property( + index, + control.prop_id, + count=control.count, + start_index=control.start_element, + ) + return + if isinstance(control, LdCtrlCompareProp): + await self._compare_prop(control) + return + if isinstance(control, LdCtrlInvokeFunctionProp): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.invoke_function_property( + index, control.prop_id, control.inline_data or b"" + ) + return + if isinstance(control, LdCtrlReadFunctionProp): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.read_function_property(index, control.prop_id) + return + if isinstance(control, LdCtrlAbsSegment): + await self._abs_segment(control) + return + if isinstance(control, LdCtrlRelSegment): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, + load_state.data_relative_allocation( + control.size, mode=control.mode, fill=control.fill + ), + load_state.LoadState.LOADING, + ) + return + if isinstance(control, LdCtrlTaskSegment): + await self._task_segment(control) + return + if isinstance(control, LdCtrlTaskPtr): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, + load_state.task_pointer( + control.init_ptr, control.save_ptr, control.serial_ptr + ), + load_state.LoadState.LOADING, + ) + return + if isinstance(control, LdCtrlTaskCtrl1): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, + load_state.task_control_1(control.address, control.count), + load_state.LoadState.LOADING, + ) + return + if isinstance(control, LdCtrlTaskCtrl2): + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, + load_state.task_control_2( + control.callback, control.address, control.seg0, control.seg1 + ), + load_state.LoadState.LOADING, + ) + return + if type(control).__name__ in _CLIENT_SIDE: + return + raise self._unsupported(control) + + async def _write_mem(self, control: LdCtrlWriteMem) -> None: + """Write a WriteMem control to memory. + + The data is the control's inline data, or - when none is given - the + download image slice at the control's address (an image backed write). + """ + _require_standard(control.address_space) + programmer = await self._bus() + if control.inline_data is not None: + await programmer.write_memory( + control.address, control.inline_data, verify=control.verify + ) + return + runs = self.image.masked_writes(control.address, control.size) + if runs is None: + raise ImageError( + f"no image data for address range {control.address:#06x}.." + f"{control.address + control.size:#06x}" + ) + for address, data in runs: + await programmer.write_memory(address, data, verify=control.verify) + + async def _write_rel_mem(self, control: LdCtrlWriteRelMem) -> None: + """Write a WriteRelMem control relative to the object's table base. + + The image mirrors a relative segment in its own relative address space + (the segment sits at its relative base, e.g. ``0``); only the device write + adds the table base read from the object at run time. So the image is + looked up at ``control.offset`` and each run is written at ``base + run``. + """ + index = await self._resolve_index(control) + programmer = await self._bus() + base = await programmer.read_table_reference(index) + if control.inline_data is not None: + await programmer.write_memory( + base + control.offset, control.inline_data, verify=control.verify + ) + return + runs = self._relative_runs(control) + if runs is None: + raise ImageError( + f"no image data for relative range {control.offset:#06x}.." + f"{control.offset + control.size:#06x}" + ) + for run_offset, data in runs: + await programmer.write_memory( + base + run_offset, data, verify=control.verify + ) + + def _relative_runs( + self, control: LdCtrlWriteRelMem + ) -> list[tuple[int, bytes]] | None: + """Return the relative ``(offset, data)`` runs a WriteRelMem writes. + + Prefers a relative segment keyed by the control's interface object type + (the System B group communication tables); otherwise falls back to the + flat image at ``control.offset`` (the parameter relative segment). + """ + object_type = getattr(control, "obj_type", None) + if object_type is not None: + segment = self.image.relative_segment(object_type) + if segment is not None: + return segment.masked_runs() + return self.image.masked_writes(control.offset, control.size) + + async def _compare_mem(self, address: int, expected: bytes) -> None: + """Read memory and compare it against expected data.""" + programmer = await self._bus() + read_back = await programmer.read_memory(address, len(expected)) + if read_back != expected: + raise VerificationError( + f"memory compare failed at {address:#06x}: " + f"expected {expected.hex()} read {read_back.hex()}" + ) + + def _property_write_data(self, index: int, control: LdCtrlWriteProp) -> bytes: + """The octets a WriteProp writes: inline data, else the image's property. + + For the Memory Control Block table (PID_MCB_TABLE) the per-segment CRC is + computed over the segment data and patched into each 8 octet entry, since + the application program carries only a zero CRC placeholder there. + """ + data = ( + control.inline_data + if control.inline_data is not None + else self._image_property_data(index, control.prop_id) + ) + if control.prop_id == _PID_MCB_TABLE: + segment = self.image.object_segments.get(index) + if segment is not None: + data = _mcb_table_with_crc(data, segment) + return data + + async def _write_prop(self, control: LdCtrlWriteProp) -> None: + """Write a WriteProp control to a property. + + The data is the control's inline data, or - when none is given - the + matching property data from the download image (an image backed write: + inline data if present, else the image's property data). + """ + index = await self._resolve_index(control) + data = self._property_write_data(index, control) + programmer = await self._bus() + await programmer.write_property( + index, + control.prop_id, + data, + count=control.count, + start_index=control.start_element, + ) + if control.verify: + read_back = await programmer.read_property( + index, + control.prop_id, + count=control.count, + start_index=control.start_element, + ) + if read_back != data: + raise VerificationError( + f"property verification failed for object {index} " + f"property {control.prop_id}" + ) + + def _image_property_data(self, object_index: int, property_id: int) -> bytes: + """Find the download image's property data for an object and property.""" + for prop in self.image.properties: + if prop.property_id != property_id: + continue + if prop.object_index in (object_index, None): + return prop.data + raise ImageError( + f"no image property data for object {object_index} property {property_id}" + ) + + async def _compare_prop(self, control: LdCtrlCompareProp) -> None: + """Read a property and compare it against expected data. + + The compare is mask driven when the control carries a ``mask`` (only the + marked bits of each octet have to match, e.g. the application-number bytes + of the application id while manufacturer and version are ignored). The + device's property length is authoritative: a procedure often carries the + property's maximum element size padded with trailing zeros while the device + reports only its actual length, so only the overlapping prefix is compared. + """ + index = await self._resolve_index(control) + programmer = await self._bus() + read_back = await programmer.read_property( + index, + control.prop_id, + count=control.count, + start_index=control.start_element, + ) + expected = control.inline_data + mask = control.mask + length = min(len(read_back), len(expected)) + if mask: + matches = all( + read_back[i] & mask[i] == expected[i] & mask[i] for i in range(length) + ) + else: + matches = read_back[:length] == expected[:length] + if not matches: + raise VerificationError( + f"property compare failed for object {index} property " + f"{control.prop_id}: expected {expected.hex()} " + f"read {read_back.hex()}" + ) + + async def _abs_segment(self, control: LdCtrlAbsSegment) -> None: + """Allocate an absolute segment, then write the image data for its range. + + A procedure may carry no explicit memory-write controls (property based + System B products): the download image is written into the segments the + procedure allocates. So after allocating the segment we write the image + slice that covers it - the segment address and size match an image + segment exactly. Segments the image does not cover are only allocated. + """ + segment_type = load_state.SegmentType(control.seg_type) + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, + load_state.alloc_absolute_segment( + segment_type, + control.address, + control.size, + access_attributes=control.access, + memory_type=control.mem_type, + memory_attributes=control.seg_flags, + ), + load_state.LoadState.LOADING, + ) + # Per the KNX Load Controls (KNX Standard v3.0.0, 2/3/1) an AbsSegment + # allocation is followed by a verified memory write of the segment data - + # but only the bytes the image actually produced (its mask). Bytes the + # encoder did not write stay at their current device value. + runs = self.image.masked_writes(control.address, control.size) + if runs: + for address, data in runs: + await programmer.write_memory(address, data, verify=True) + + async def _task_segment(self, control: LdCtrlTaskSegment) -> None: + """Allocate the task segment via a load event.""" + index = await self._resolve_index(control) + programmer = await self._bus() + await programmer.send_load_event( + index, + load_state.alloc_task_segment( + control.address, + self.application.program.pei_type, + _application_id(self.application), + ), + load_state.LoadState.LOADING, + ) + + async def _preflight_control( + self, + control: object, + segments: list[SegmentDiff], + properties: list[PropertyDiff], + ) -> None: + """Read-only preview of a single control (see :meth:`preflight`). + + Only controls with a write effect are previewed (by reading the current + device bytes and recording a diff); compare controls still run as a gate. + Load state events, allocations, restart, delays and read-back controls + have no write to preview and are ignored. + """ + if isinstance(control, LdCtrlConnect): + await self._bus() + return + if isinstance(control, LdCtrlDisconnect): + await self._close() + return + if isinstance(control, LdCtrlWriteMem): + _require_standard(control.address_space) + if control.inline_data is not None: + await self._diff_memory(control.address, control.inline_data, segments) + else: + await self._diff_masked(control.address, control.size, segments) + return + if isinstance(control, LdCtrlWriteRelMem): + index = await self._resolve_index(control) + programmer = await self._bus() + base = await programmer.read_table_reference(index) + if control.inline_data is not None: + await self._diff_memory( + base + control.offset, control.inline_data, segments + ) + else: + await self._diff_masked_rel(base, control, segments) + return + if isinstance(control, LdCtrlAbsSegment): + # An allocation whose range the image covers is an image backed write; + # preview only the bytes the image actually writes (its mask). + await self._diff_masked(control.address, control.size, segments) + return + if isinstance(control, LdCtrlWriteProp): + index = await self._resolve_index(control) + planned = self._property_write_data(index, control) + await self._diff_property( + index, + control.prop_id, + control.count, + control.start_element, + planned, + properties, + ) + return + if isinstance(control, LdCtrlCompareMem): + _require_standard(control.address_space) + await self._compare_mem(control.address, control.inline_data) + return + if isinstance(control, LdCtrlCompareRelMem): + index = await self._resolve_index(control) + programmer = await self._bus() + base = await programmer.read_table_reference(index) + await self._compare_mem(base + control.offset, control.inline_data) + return + if isinstance(control, LdCtrlCompareProp): + await self._compare_prop(control) + return + # Anything left is either a control with no write to preview (fine) or a + # step this engine does not implement. Do not fail the read-only preview, + # but log the gap so a bug report shows the preview was incomplete. + name = type(control).__name__ + if name not in gaps.PREFLIGHT_NO_WRITE: + logger.warning( + "preflight cannot preview %s [%s]: %s", + name, + self._target(), + gaps.describe_missing(name), + ) + + async def _diff_memory( + self, address: int, planned: bytes, segments: list[SegmentDiff] + ) -> None: + """Read current memory at ``address`` and record a diff against ``planned``.""" + programmer = await self._bus() + current = await programmer.read_memory(address, len(planned)) + segments.append( + SegmentDiff(address=address, current=current, planned=bytes(planned)) + ) + + async def _diff_masked( + self, address: int, size: int, segments: list[SegmentDiff] + ) -> None: + """Diff each masked write run the image would apply within ``[address, size)``.""" + runs = self.image.masked_writes(address, size) + if not runs: + return + programmer = await self._bus() + for run_address, planned in runs: + current = await programmer.read_memory(run_address, len(planned)) + segments.append( + SegmentDiff( + address=run_address, current=current, planned=bytes(planned) + ) + ) + + async def _diff_masked_rel( + self, base: int, control: LdCtrlWriteRelMem, segments: list[SegmentDiff] + ) -> None: + """Diff a relative segment: image at ``offset``, device at ``base + run``. + + Mirrors :meth:`_write_rel_mem` - the image holds the segment in its own + relative address space, and the device address adds the run-time table base. + """ + runs = self._relative_runs(control) + if not runs: + return + programmer = await self._bus() + for run_offset, planned in runs: + current = await programmer.read_memory(base + run_offset, len(planned)) + segments.append( + SegmentDiff( + address=base + run_offset, + current=current, + planned=bytes(planned), + ) + ) + + async def _diff_property( + self, + object_index: int, + property_id: int, + count: int, + start_element: int, + planned: bytes, + properties: list[PropertyDiff], + ) -> None: + """Read the current property value and record a diff against ``planned``.""" + programmer = await self._bus() + current = await programmer.read_property( + object_index, + property_id, + count=count, + start_index=start_element, + ) + properties.append( + PropertyDiff( + object_index=object_index, + property_id=property_id, + current=current, + planned=bytes(planned), + ) + ) + + +def _require_standard(address_space: LdCtrlMemAddrSpace) -> None: + """Reject memory operations outside the standard address space.""" + if address_space != LdCtrlMemAddrSpace.STANDARD: + message = ( + f"memory address space {address_space.value!r} is not implemented; only " + f"the standard address space is handled so far (KNX Standard v3.0.0, 3/5/1 " + f"Resources). Please file a bug report with the product data so this " + f"address space can be added." + ) + logger.warning("unsupported load control: %s", message) + raise UnsupportedProcedureError(message) diff --git a/packages/download/src/xknxmono/download/programmer.py b/packages/download/src/xknxmono/download/programmer.py new file mode 100644 index 00000000..4e55f9a6 --- /dev/null +++ b/packages/download/src/xknxmono/download/programmer.py @@ -0,0 +1,454 @@ +"""Low level device programming operations over a point-to-point connection. + +The :class:`DeviceProgrammer` turns the primitive application layer services +provided by ``xknx`` into the operations a Load Procedure needs: chunked memory +writes, property writes, and driving a Load State Machine with read-back +verification. Those services are defined in KNX Standard v3.0.0, Chapter 3/3/7 +"Application Layer": A_Memory_Read (section 3.5.3), A_Memory_Write (section 3.5.4), +A_PropertyValue_Read/A_PropertyValue_Write, and A_DeviceDescriptor_Read +(section 3.4.2.1). Memory reads/writes are split to the connection's maximum APDU +length. + +It talks to any object implementing :class:`BusConnection`; at runtime this is +``xknx.management.P2PConnection``, in tests it is a fake. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Protocol + +from xknx.telegram.apci import ( + DeviceDescriptorRead, + DeviceDescriptorResponse, + FunctionPropertyCommand, + FunctionPropertyStateRead, + FunctionPropertyStateResponse, + MemoryRead, + MemoryResponse, + MemoryWrite, + PropertyValueRead, + PropertyValueResponse, + PropertyValueWrite, + Restart, + RestartMasterReset, + RestartMasterResetResponse, +) + +from . import load_state +from .errors import DownloadError, LoadStateError, VerificationError + +if TYPE_CHECKING: + from xknx.telegram import Telegram + from xknx.telegram.apci import APCI + +# Octets consumed by TPCI/APCI/count/address in a standard A_Memory_Write ASDU. +_MEMORY_OVERHEAD = 3 +# A_Memory_Write encodes the byte count in 6 bits. +_MAX_MEMORY_CHUNK = 0x3F +# Octets consumed by APCI/object/property/count/index in A_PropertyValue_Write. +_PROPERTY_OVERHEAD = 5 +# A_PropertyValue_Write encodes the element count in 4 bits. +_MAX_PROPERTY_ELEMENTS = 0xF +# Default APDU length every KNX device has to support. +DEFAULT_MAX_APDU_LENGTH = 15 +# Upper bound used when negotiating the APDU length up from the default. +MAX_NEGOTIATED_APDU_LENGTH = 254 +# Device Object property carrying the device's maximum APDU length (2 octets). +PID_MAX_APDU_LENGTH = 56 +# Property id carrying an interface object's type (PID_OBJECT_TYPE). +PID_OBJECT_TYPE = 1 +# Property id carrying a loadable part's table base address (PID_TABLE_REFERENCE). +PID_TABLE_REFERENCE = 7 +# Highest interface object index scanned when locating an object by type. +_MAX_OBJECT_INDEX = 255 + + +class BusConnection(Protocol): + """Subset of ``xknx.management.P2PConnection`` used for programming.""" + + async def send_data(self, payload: APCI, wait_for_ack: bool = True) -> None: + """Send a payload and, by default, wait for the transport layer ACK.""" + ... + + async def request(self, payload: APCI, expected: type[APCI] | None) -> Telegram: + """Send a payload and wait for the device's response telegram.""" + ... + + +class ConnectionManager(Protocol): + """Opens and closes point-to-point connections for a Load Procedure. + + A Load Procedure opens the connection on a Connect control and closes it on + a Disconnect (and after a Restart). Implementations map ``open`` to a fresh + T_Connect and ``close`` to T_Disconnect. + """ + + async def open(self) -> BusConnection: + """Open a connection to the device and return it.""" + ... + + async def close(self) -> None: + """Close the currently open connection, if any.""" + ... + + +class DeviceProgrammer: + """Perform programming operations over an open point-to-point connection.""" + + def __init__( + self, + connection: BusConnection, + *, + max_apdu_length: int = DEFAULT_MAX_APDU_LENGTH, + ) -> None: + """Initialize with a connected bus connection and the negotiated APDU length.""" + self.connection = connection + self.max_apdu_length = max_apdu_length + self._object_index_cache: dict[tuple[int, int], int] = {} + + @property + def memory_chunk_size(self) -> int: + """Largest memory payload that fits into a single telegram (at least 1).""" + return max(1, min(self.max_apdu_length - _MEMORY_OVERHEAD, _MAX_MEMORY_CHUNK)) + + async def read_device_descriptor(self) -> int: + """Read device descriptor type 0 (the mask version).""" + telegram = await self.connection.request( + DeviceDescriptorRead(descriptor=0), DeviceDescriptorResponse + ) + payload = telegram.payload + if not isinstance(payload, DeviceDescriptorResponse): + raise VerificationError("no device descriptor response received") + return payload.value + + async def read_max_apdu_length(self) -> int: + """Read the device's maximum APDU length from the Device Object. + + PID_MAX_APDU_LENGTH (property 56 of the Device Object, interface object + index 0) holds the largest APDU the device accepts, in octets. Falls back + to the mandatory default when the device does not expose it. + """ + data = await self.read_property(0, PID_MAX_APDU_LENGTH) + if not data: + return DEFAULT_MAX_APDU_LENGTH + return int.from_bytes(data, "big") + + async def read_memory(self, address: int, size: int) -> bytes: + """Read ``size`` octets starting at ``address``, chunked to the APDU length.""" + result = bytearray() + chunk = self.memory_chunk_size + offset = 0 + while offset < size: + count = min(chunk, size - offset) + telegram = await self.connection.request( + MemoryRead(address=address + offset, count=count), MemoryResponse + ) + payload = telegram.payload + if not isinstance(payload, MemoryResponse): + raise VerificationError( + f"no memory response for address {address + offset:#06x}" + ) + if len(payload.data) != count: + raise VerificationError( + f"short memory response at {address + offset:#06x}: " + f"asked {count} got {len(payload.data)}" + ) + result.extend(payload.data) + offset += count + return bytes(result) + + async def write_memory( + self, address: int, data: bytes, *, verify: bool = False + ) -> None: + """Write ``data`` starting at ``address``, chunked to the APDU length. + + With ``verify`` each block is read back and compared right after it is + written (per KNX 3/5/2), so a lost EEPROM write is caught immediately. + """ + chunk = self.memory_chunk_size + for offset in range(0, len(data), chunk): + block = data[offset : offset + chunk] + block_address = address + offset + await self.connection.send_data( + MemoryWrite(address=block_address, data=block) + ) + if verify: + read_back = await self.read_memory(block_address, len(block)) + if read_back != block: + raise VerificationError( + f"memory verification failed at {block_address:#06x}: " + f"wrote {block.hex()} read {read_back.hex()}" + ) + + async def read_property( + self, + object_index: int, + property_id: int, + *, + count: int = 1, + start_index: int = 1, + ) -> bytes: + """Read a property value from an interface object.""" + telegram = await self.connection.request( + PropertyValueRead( + object_index=object_index, + property_id=property_id, + count=count, + start_index=start_index, + ), + PropertyValueResponse, + ) + payload = telegram.payload + if not isinstance(payload, PropertyValueResponse): + raise VerificationError( + f"no property response for object {object_index} property {property_id}" + ) + return payload.data + + async def write_property( + self, + object_index: int, + property_id: int, + data: bytes, + *, + count: int = 1, + start_index: int = 1, + ) -> bytes: + """Write a property value and return the resulting value. + + A_PropertyValue_Write is confirmed by A_PropertyValue_Response carrying + the resulting value, so this waits for that response (via ``request``) + rather than only the transport ACK - otherwise the buffered response + would be mistaken for the answer to the next request. + + The element count is encoded in four bits and the data must fit the APDU, + so a value spanning more than 15 elements or one frame is written in + successive element ranges; the last response is returned. + """ + element_size = (len(data) // count) if count > 1 else len(data) + max_bytes = max(1, self.max_apdu_length - _PROPERTY_OVERHEAD) + if element_size and element_size <= max_bytes: + per_frame = max(1, min(_MAX_PROPERTY_ELEMENTS, max_bytes // element_size)) + else: + per_frame = _MAX_PROPERTY_ELEMENTS + + result = b"" + element = 0 + while element < count: + frame_count = min(per_frame, count - element) + frame_data = ( + data[element * element_size : (element + frame_count) * element_size] + if element_size + else data + ) + result = await self._write_property_frame( + object_index, + property_id, + frame_data, + frame_count, + start_index + element, + ) + element += frame_count + return result + + async def _write_property_frame( + self, + object_index: int, + property_id: int, + data: bytes, + count: int, + start_index: int, + ) -> bytes: + """Send a single A_PropertyValue_Write and return the resulting value.""" + telegram = await self.connection.request( + PropertyValueWrite( + object_index=object_index, + property_id=property_id, + count=count, + start_index=start_index, + data=data, + ), + PropertyValueResponse, + ) + payload = telegram.payload + if not isinstance(payload, PropertyValueResponse): + raise VerificationError( + f"no property write response for object {object_index} " + f"property {property_id}" + ) + return payload.data + + async def invoke_function_property( + self, object_index: int, property_id: int, data: bytes + ) -> bytes: + """Call a Function Property and return the resulting state. + + A_FunctionPropertyCommand invokes a use-case specific function on an + interface object and is confirmed by A_FunctionPropertyState_Response + carrying a return code and the resulting state (KNX Standard v3.0.0, + 3/3/7 section 3.4.7.1). A non-zero return code means the device rejected + the command. + """ + telegram = await self.connection.request( + FunctionPropertyCommand( + object_index=object_index, property_id=property_id, data=data + ), + FunctionPropertyStateResponse, + ) + return self._function_property_result(telegram, object_index, property_id) + + async def read_function_property( + self, object_index: int, property_id: int + ) -> bytes: + """Read a Function Property state and return it. + + A_FunctionPropertyState_Read reads the state of a function property and is + answered by A_FunctionPropertyState_Response (KNX Standard v3.0.0, 3/3/7 + section 3.4.7.2). + """ + telegram = await self.connection.request( + FunctionPropertyStateRead( + object_index=object_index, property_id=property_id + ), + FunctionPropertyStateResponse, + ) + return self._function_property_result(telegram, object_index, property_id) + + @staticmethod + def _function_property_result( + telegram: Telegram, object_index: int, property_id: int + ) -> bytes: + """Validate a Function Property response and return its state data.""" + payload = telegram.payload + if not isinstance(payload, FunctionPropertyStateResponse): + raise VerificationError( + f"no function property response for object {object_index} " + f"property {property_id}" + ) + if payload.return_code != 0: + raise LoadStateError( + f"function property {property_id} on object {object_index} " + f"returned error code {payload.return_code:#04x}" + ) + return payload.data + + async def read_table_reference(self, object_index: int) -> int: + """Read a loadable part's table base address (PID_TABLE_REFERENCE).""" + data = await self.read_property(object_index, PID_TABLE_REFERENCE) + if not data: + raise VerificationError(f"empty table reference for object {object_index}") + return int.from_bytes(data, "big") + + async def locate_object(self, object_type: int, occurrence: int = 0) -> int: + """Resolve the object index of the ``occurrence``-th object of a type. + + ``occurrence`` is zero-based, matching the product application program XML (``Occurrence="0"`` + is the first instance). Scans interface objects reading ``PID_OBJECT_TYPE`` + until the requested occurrence is found. Cached per programmer instance. + """ + cached = self._object_index_cache.get((object_type, occurrence)) + if cached is not None: + return cached + ordinal = 0 + for index in range(_MAX_OBJECT_INDEX + 1): + try: + data = await self.read_property(index, PID_OBJECT_TYPE) + except (VerificationError, DownloadError): + break + if len(data) < 2: + break + found_type = int.from_bytes(data[:2], "big") + if found_type == object_type: + if ordinal == occurrence: + self._object_index_cache[(object_type, occurrence)] = index + return index + ordinal += 1 + raise LoadStateError( + f"interface object type {object_type} occurrence {occurrence} not found" + ) + + async def read_load_state(self, object_index: int) -> load_state.LoadState: + """Read the current state of an object's Load State Machine.""" + data = await self.read_property(object_index, load_state.PID_LOAD_STATE_CONTROL) + if not data: + raise LoadStateError(f"empty load state for object {object_index}") + return _decode_load_state(data[0], object_index) + + async def send_load_event( + self, + object_index: int, + event: bytes, + expected: load_state.LoadState, + *, + retries: int = 30, + retry_delay: float = 1.0, + ) -> None: + """Write a load event and verify the machine reaches ``expected``. + + ``LOAD_COMPLETE`` triggers a checksum calculation that can take a while, + and a device may report a transient ``UNLOADING``/``LOAD_COMPLETING`` + state first, so the state is polled up to ``retries`` times. + """ + resulting = await self.write_property( + object_index, load_state.PID_LOAD_STATE_CONTROL, event + ) + state = ( + _decode_load_state(resulting[0], object_index) + if resulting + else await self.read_load_state(object_index) + ) + for _ in range(max(retries, 0)): + if state == expected: + return + if state == load_state.LoadState.ERROR: + raise LoadStateError( + f"object {object_index} entered ERROR state " + f"(expected {expected.name})" + ) + await asyncio.sleep(retry_delay) + state = await self.read_load_state(object_index) + if state != expected: + raise LoadStateError( + f"object {object_index} did not reach {expected.name}, " + f"last state {state.name}" + ) + + async def restart(self) -> None: + """Restart the device (also closes the transport connection).""" + await self.connection.send_data(Restart(), wait_for_ack=False) + + async def master_reset(self, erase_code: int, channel_number: int) -> int: + """Perform a Master Reset and return the device's process time in ms. + + A Master Reset is an A_Restart with restart_type = 1, carrying an erase + code and channel number (KNX Standard v3.0.0, 3/3/7 section 3.4.2.2; the + erase codes are defined with DM_Restart in 3/5/2). Unlike a Basic Restart + it is confirmed at the application layer by A_Restart_Master_Reset_Response + with an error code and the process time the device needs before it is + reachable again. A non-zero error code means the device refused the reset. + The device restarts afterwards, tearing down the connection. + """ + telegram = await self.connection.request( + RestartMasterReset(erase_code=erase_code, channel_number=channel_number), + RestartMasterResetResponse, + ) + payload = telegram.payload + if not isinstance(payload, RestartMasterResetResponse): + raise LoadStateError("no master reset response received") + if payload.error_code != 0: + raise LoadStateError( + f"device refused master reset (erase code {erase_code}, channel " + f"{channel_number}): error code {payload.error_code:#04x}" + ) + return payload.process_time + + +def _decode_load_state(value: int, object_index: int) -> load_state.LoadState: + """Map a raw load state octet to :class:`LoadState`, erroring on unknowns.""" + try: + return load_state.LoadState(value) + except ValueError as exc: + raise LoadStateError( + f"object {object_index} reported unknown load state {value:#04x}" + ) from exc diff --git a/packages/download/src/xknxmono/download/project_data.py b/packages/download/src/xknxmono/download/project_data.py new file mode 100644 index 00000000..56e4f273 --- /dev/null +++ b/packages/download/src/xknxmono/download/project_data.py @@ -0,0 +1,166 @@ +"""Adapt project state into download inputs. + +Turns a configured device (parameter values and group address assignments) into +the inputs a download needs: a parameter value map that feeds the download image, +and the structured group communication links a device transmits and receives on. + +The device is consumed structurally (see the protocols below), so this module has +no hard dependency on the project store; any object with the same shape works, +including ``xknxmono.project`` ORM devices. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Protocol + +from xknxmono.models.intermediate.com_object_instance_ref_t import ComObjectInstanceRef +from xknxmono.models.intermediate.enable_t import Enable +from xknxmono.models.intermediate.module_instance_t import ModuleInstance +from xknxmono.models.intermediate.parameter_instance_ref_t import ParameterInstanceRef + + +class _Parameter(Protocol): + ref_id: str + value: str + + +class _ModuleInstance(Protocol): + instance_id: str + ref_id: str + + +class _GroupAddress(Protocol): + address: int + + +class _ComObjectLink(Protocol): + is_sending: bool + + @property + def group_address(self) -> _GroupAddress: ... + + +class _ComObject(Protocol): + ref_id: str + + @property + def links(self) -> Sequence[_ComObjectLink]: ... + + +class _ComObjectConfig(Protocol): + ref_id: str + channel_id: str | None + read_flag: bool | None + write_flag: bool | None + communication_flag: bool | None + transmit_flag: bool | None + update_flag: bool | None + read_on_init_flag: bool | None + + +class _Device(Protocol): + @property + def parameters(self) -> Sequence[_Parameter]: ... + + @property + def com_objects(self) -> Sequence[_ComObject]: ... + + +class SeedDevice(Protocol): + @property + def parameters(self) -> Sequence[_Parameter]: ... + + @property + def module_instances(self) -> Sequence[_ModuleInstance]: ... + + @property + def com_objects(self) -> Sequence[_ComObjectConfig]: ... + + +def _enable(value: bool | None) -> Enable | None: + """Map a project boolean flag override to the model's Enable enum.""" + if value is None: + return None + return Enable.ENABLED if value else Enable.DISABLED + + +@dataclass(frozen=True, slots=True) +class GroupObjectLink: + """A link between a device com object and a group address.""" + + com_object_ref_id: str + group_address: int + sending: bool + + +def parameter_values_from_device(device: _Device) -> dict[str, str]: + """Collect the parameter reference id to value map of a configured device.""" + return {parameter.ref_id: parameter.value for parameter in device.parameters} + + +def group_communication_from_device(device: _Device) -> tuple[GroupObjectLink, ...]: + """Collect the group address links of a device's com objects.""" + return tuple( + GroupObjectLink( + com_object_ref_id=com_object.ref_id, + group_address=link.group_address.address, + sending=link.is_sending, + ) + for com_object in device.com_objects + for link in com_object.links + ) + + +def group_address_table(links: Sequence[GroupObjectLink]) -> tuple[int, ...]: + """Return the sorted, unique group addresses a device uses. + + This is the content of the group address table. Encoding it (and the + association table) into device memory is realisation-type specific and is + performed by the Load Procedure's table segments; this helper provides the + ordered address set those tables are built from. + """ + return tuple(sorted({link.group_address for link in links})) + + +def parameter_instance_refs_from_device( + device: SeedDevice, +) -> list[ParameterInstanceRef]: + """Build the application evaluator's parameter instance refs from a device.""" + return [ + ParameterInstanceRef(ref_id=parameter.ref_id, value=parameter.value) + for parameter in device.parameters + ] + + +def module_instances_from_device(device: SeedDevice) -> list[ModuleInstance]: + """Build the application evaluator's module instances from a device.""" + return [ + ModuleInstance(id=instance.instance_id, ref_id=instance.ref_id) + for instance in device.module_instances + ] + + +def com_object_instance_refs_from_device( + device: SeedDevice, +) -> list[ComObjectInstanceRef]: + """Build the evaluator's com object instance refs (flags, channel) from a device. + + Only configuration that influences the parameter memory image is carried + (flag overrides, channel); group address links belong to the group + communication tables, not the parameter part. + """ + return [ + ComObjectInstanceRef( + ref_id=com_object.ref_id, + channel_id=com_object.channel_id, + read_flag=_enable(com_object.read_flag), + write_flag=_enable(com_object.write_flag), + communication_flag=_enable(com_object.communication_flag), + transmit_flag=_enable(com_object.transmit_flag), + update_flag=_enable(com_object.update_flag), + read_on_init_flag=_enable(com_object.read_on_init_flag), + ) + for com_object in device.com_objects + ] diff --git a/packages/download/src/xknxmono/download/py.typed b/packages/download/src/xknxmono/download/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/packages/download/src/xknxmono/download/resources.py b/packages/download/src/xknxmono/download/resources.py new file mode 100644 index 00000000..21011ba6 --- /dev/null +++ b/packages/download/src/xknxmono/download/resources.py @@ -0,0 +1,156 @@ +"""Per-mask resource location map from the KNX master data. + +The KNX project schema (namespace ``http://knx.org/xml/project``) describes, per +device mask version, where every loadable resource lives: element +``MaskVersion`` -> ``HawkConfigurationData`` -> ``Resources`` -> ``Resource``, +each with a ``Location`` (an address space plus either a memory ``StartAddress`` +or an interface object ``InterfaceObjectRef`` + ``PropertyID``). This maps, per +mask, the Load State Machine control (``ApplicationLoadControl`` etc.), the table +base pointers (``GroupAddressTablePtr`` etc.), the Run State Machine control +(``ApplicationRunControl``) and more. + +Resolving these from the master data - rather than assuming fixed property ids or +addresses - lets the download follow whatever a given mask defines. Callers that +do not supply master data fall back to the conventional defaults used by the +memory-mapped and System B device models handled elsewhere in this package. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from functools import cache +from importlib.resources import files +from typing import TYPE_CHECKING + +from xknxmono.models.intermediate.resource_addr_space_t import ResourceAddrSpace +from xknxmono.models.intermediate.resource_location_t import ResourceLocation +from xknxmono.models.intermediate.resource_name_t import ResourceName + +if TYPE_CHECKING: + from collections.abc import Mapping + + from xknxmono.product import MasterData + +# Bundled per-mask resource table (see mask_resources.json / tools/gen_mask_resources.py). +_BUNDLED_TABLE = "mask_resources.json" + +# Address spaces that place a resource in interface object relative memory (the +# System B group communication tables and parameter segment). +_RELATIVE_SPACES = frozenset( + { + ResourceAddrSpace.RELATIVE_MEMORY, + ResourceAddrSpace.RELATIVE_MEMORY_BY_OBJECT_TYPE, + } +) + + +@dataclass(frozen=True, slots=True) +class MaskResources: + """The resolved resource locations of a single device mask version.""" + + _by_name: Mapping[ResourceName, ResourceLocation] + + def location(self, name: ResourceName) -> ResourceLocation | None: + """The raw location of a named resource, or ``None`` if the mask lacks it.""" + return self._by_name.get(name) + + def property_ref(self, name: ResourceName) -> tuple[int, int] | None: + """Return ``(interface object ref, property id)`` for a property resource.""" + location = self._by_name.get(name) + if ( + location is None + or location.address_space != ResourceAddrSpace.SYSTEM_PROPERTY + or location.interface_object_ref is None + or location.property_id is None + ): + return None + return location.interface_object_ref, location.property_id + + def memory_address(self, name: ResourceName) -> int | None: + """Return the start address for a memory-located resource, else ``None``.""" + location = self._by_name.get(name) + if location is None or location.address_space not in ( + ResourceAddrSpace.STANDARD_MEMORY, + ResourceAddrSpace.USER_MEMORY, + ): + return None + return location.start_address + + def is_relative(self, name: ResourceName) -> bool: + """Whether a resource lives in interface object relative memory (System B).""" + location = self._by_name.get(name) + return location is not None and location.address_space in _RELATIVE_SPACES + + +def mask_resources( + master: MasterData | None, mask_version_id: str +) -> MaskResources | None: + """Resolve the resource map for ``mask_version_id``. + + Prefers the loaded master data (the device's own product data); falls back to + the bundled per-mask table so any known mask can be downloaded even when the + loaded product data does not describe it. Returns ``None`` when neither has it. + """ + resolved = _from_master(master, mask_version_id) + return resolved if resolved is not None else bundled_mask_resources(mask_version_id) + + +def _from_master( + master: MasterData | None, mask_version_id: str +) -> MaskResources | None: + """Build the resource map for ``mask_version_id`` from loaded master data.""" + if master is None: + return None + raw = master.raw + if raw is None or raw.mask_versions is None: + return None + for mask_version in raw.mask_versions.mask_version: + if mask_version.id != mask_version_id: + continue + by_name: dict[ResourceName, ResourceLocation] = {} + for configuration in mask_version.hawk_configuration_data: + if configuration.resources is None: + continue + for resource in configuration.resources.resource: + if resource.location is not None: + by_name.setdefault(resource.name, resource.location) + if by_name: + return MaskResources(_by_name=by_name) + return None + + +@cache +def _bundled_table() -> Mapping[str, Mapping[str, list[object]]]: + """Load the bundled per-mask resource table (cached).""" + text = (files("xknxmono.download") / _BUNDLED_TABLE).read_text("utf-8") + return json.loads(text) + + +def bundled_mask_resources(mask_version_id: str) -> MaskResources | None: + """Build a resource map for ``mask_version_id`` from the bundled table.""" + entry = _bundled_table().get(mask_version_id) + if not entry: + return None + by_name: dict[ResourceName, ResourceLocation] = {} + for name, values in entry.items(): + space = values[0] + if not isinstance(space, str): + continue + try: + resource_name = ResourceName(name) + address_space = ResourceAddrSpace(space) + except ValueError: + continue + by_name[resource_name] = ResourceLocation( + address_space=address_space, + interface_object_ref=_as_int(values[1]), + property_id=_as_int(values[2]), + start_address=_as_int(values[3]), + ) + return MaskResources(_by_name=by_name) if by_name else None + + +def _as_int(value: object) -> int | None: + """Coerce a bundled-table field to ``int``; ``None`` for anything else.""" + return value if isinstance(value, int) else None diff --git a/packages/download/src/xknxmono/download/scope.py b/packages/download/src/xknxmono/download/scope.py new file mode 100644 index 00000000..17255f13 --- /dev/null +++ b/packages/download/src/xknxmono/download/scope.py @@ -0,0 +1,66 @@ +"""Download scope: full download versus a partial parameter/group download. + +A partial download runs only the Load Controls that target a given category of +loadable part (KNX Standard v3.0.0, Chapter 3/5/3 "Configuration Procedures", +section 3.5.3 "Load procedure for partial download"), classifying a control by +the interface object it addresses: + +- the Address Table, Association Table and Group Object Table objects hold the + group communication; +- the Application Program object holds the parameters; +- the Device Object and connection/restart controls are framing and always run. + +The applies_to marker (``LdCtrlProcType``) is uniform on many products, so the +object a control targets - not applies_to - is what distinguishes a partial +parameter download from a partial group communication download. +""" + +from __future__ import annotations + +from enum import Enum + +# Interface object types (and their conventional indices) that hold group +# communication: address table (1), association table (2), group object +# table (9), group object responder table (also object type 9). +_GROUP_COMMUNICATION_OBJECTS = frozenset({1, 2, 9}) +# The device object is framing (fingerprint compare) and always runs. +_DEVICE_OBJECT = 0 + + +class DownloadScope(Enum): + """Which part of a Load Procedure to execute.""" + + FULL = "full" + PARAMETERS = "par" + GROUP_COMMUNICATION = "grp" + + +def control_in_scope(control: object, scope: DownloadScope) -> bool: + """Return whether a control runs in ``scope``. + + Controls that do not target a loadable part (Connect/Disconnect/Restart/ + Delay and Device Object compares) are framing and always run. Targeted + controls run only when the object they address belongs to the requested + category; a full download runs everything. + """ + if scope is DownloadScope.FULL: + return True + target = _target_object(control) + if target is None or target == _DEVICE_OBJECT: + return True + is_group_communication = target in _GROUP_COMMUNICATION_OBJECTS + if scope is DownloadScope.GROUP_COMMUNICATION: + return is_group_communication + return not is_group_communication + + +def _target_object(control: object) -> int | None: + """The interface object type/index a control addresses, or None if framing.""" + obj_type = getattr(control, "obj_type", None) + if obj_type is not None: + return obj_type + lsm_idx = getattr(control, "lsm_idx", None) + if lsm_idx is not None: + return lsm_idx + obj_idx = getattr(control, "obj_idx", None) + return obj_idx diff --git a/packages/download/src/xknxmono/download/tables.py b/packages/download/src/xknxmono/download/tables.py new file mode 100644 index 00000000..da4cd531 --- /dev/null +++ b/packages/download/src/xknxmono/download/tables.py @@ -0,0 +1,212 @@ +"""Build the group communication tables a device links group addresses through. + +The formats follow KNX Standard v3.0.0, Chapter 3/5/1 "Resources": the Group +Address Table (section 4.16 "Group Address Table (GrAT)"), the Group Object +Association Table (section 4.17 "Group Object Association Table (GrOAT)") and the +Group Object Table (section 4.18). This module produces the Realisation Type +using a 1-octet count with the device's own individual address leading the group +address table: + +- the group address table lists the addresses the device sends and receives on, + led by the device's own individual address; +- the association table maps each configured link to a ``(group address, group + object)`` pair, referencing the group address by its index in the address table. + +These formats are validated byte-exact against real hardware; the 2-octet-count +System B realisation of the same tables is produced by :mod:`.tables_systemb`. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +from .errors import ImageError + + +@dataclass(frozen=True, slots=True) +class Association: + """One link: a group address (by address-table index) to a group object.""" + + group_address_index: int + group_object_number: int + sending: bool = True + + +def build_group_address_table( + device_address: int, group_addresses: Sequence[int] +) -> bytes: + """Return the group address table bytes. + + Layout: ``[count:1][device address:2][group address:2]*`` (big-endian). The + count includes the leading device-address entry. ``group_addresses`` is sorted + ascending and de-duplicated; the resulting order defines the 1-based index the + association table references (index 0 is the device address). + """ + addresses = sorted(set(group_addresses)) + count = 1 + len(addresses) + if count > 0xFF: + raise ImageError(f"group address table has too many entries: {count}") + out = bytearray() + out.append(count) + out += _u16(device_address) + for address in addresses: + out += _u16(address) + return bytes(out) + + +def group_address_index(group_addresses: Sequence[int]) -> dict[int, int]: + """Map each group address to its 1-based index in the address table.""" + return {address: i + 1 for i, address in enumerate(sorted(set(group_addresses)))} + + +def build_association_table(associations: Sequence[Association]) -> bytes: + """Return the association table bytes. + + Layout: ``[count:1]`` then one ``[group address index:1][group object + number:1]`` entry per association. Ordering (per the Group Object Association Table): + sending associations first, then by group object number, then by group + address index. + """ + ordered = sorted( + associations, + key=lambda a: ( + not a.sending, + a.group_object_number, + a.group_address_index, + ), + ) + if len(ordered) > 0xFF: + raise ImageError(f"association table has too many entries: {len(ordered)}") + out = bytearray() + out.append(len(ordered)) + for association in ordered: + _require_octet(association.group_address_index, "group address index") + _require_octet(association.group_object_number, "group object number") + out.append(association.group_address_index) + out.append(association.group_object_number) + return bytes(out) + + +# --- com object table ------------------------------------------------------- + +# ComObjectSize value -> KNX size code (Chapter 3/5/1 Resources, Group Object Table). +_SIZE_CODE: dict[str, int] = { + "1 Bit": 0, + "2 Bit": 1, + "3 Bit": 2, + "4 Bit": 3, + "5 Bit": 4, + "6 Bit": 5, + "7 Bit": 6, + "1 Byte": 7, + "2 Bytes": 8, + "3 Bytes": 9, + "4 Bytes": 10, + "6 Bytes": 11, + "8 Bytes": 12, + "10 Bytes": 13, + "14 Bytes": 14, + "5 Bytes": 15, + "7 Bytes": 16, + "9 Bytes": 17, + "11 Bytes": 18, + "12 Bytes": 19, + "13 Bytes": 20, +} + +# Communication flag byte bit layout (bits 0-1 priority, then enables). +_PRIORITY_BITS: dict[str, int] = {"System": 0, "High": 1, "Alert": 2, "Low": 3} + + +def com_object_flag_byte( + *, + priority: str, + communication: bool, + read: bool, + write: bool, + transmit: bool, + update: bool, + read_on_init: bool, +) -> int: + """Encode a com object's flags into the descriptor flag byte.""" + value = _PRIORITY_BITS.get(priority, 3) + if communication: + value |= 0x04 + if read: + value |= 0x08 + if write: + value |= 0x10 + if read_on_init: + value |= 0x20 + if transmit: + value |= 0x40 + if update: + value |= 0x80 + return value + + +def size_code(object_size: str) -> int: + """Return the KNX size code for a ``ComObjectSize`` value string.""" + try: + return _SIZE_CODE[object_size] + except KeyError as exc: + raise ImageError( + f"com object size {object_size!r} is not implemented; known sizes are " + f"{sorted(_SIZE_CODE)} (KNX Standard v3.0.0, 3/5/1 group object " + f"descriptor). Please file a bug report with the product data so this " + f"size can be added." + ) from exc + + +@dataclass(frozen=True, slots=True) +class ComObjectDescriptor: + """One com object's table entry: its number, flag byte and size code.""" + + number: int + flags: int + size: int + + +def build_com_object_table( + seed: bytes, + descriptors: Sequence[ComObjectDescriptor], + *, + header_size: int = 5, + record_size: int = 4, +) -> tuple[bytes, bytes]: + """Overlay com object descriptors onto the segment seed. + + Each record is ``[flag byte][size code][data pointer:2]`` at + ``header_size + number * record_size``. Only the flag and size bytes are + written (from ``descriptors``); the header and the manufacturer's data + pointers are kept from ``seed``. Returns ``(data, mask)`` where the mask marks + the two written bytes of each descriptor, so a download touches nothing else. + """ + data = bytearray(seed) + mask = bytearray(len(seed)) + for descriptor in descriptors: + offset = header_size + descriptor.number * record_size + if offset + 2 > len(data): + continue + _require_octet(descriptor.flags, "com object flags") + _require_octet(descriptor.size, "com object size code") + # Read-modify-write per the Group Object Table descriptor: the flag byte keeps + # bit 5 (value-read-on-init) from the seed, and the size byte keeps its + # top two bits (the size code occupies only the low six bits). + data[offset] = (data[offset] & 0x20) | descriptor.flags + data[offset + 1] = (data[offset + 1] & 0xC0) | (descriptor.size & 0x3F) + mask[offset] = 0xFF + mask[offset + 1] = 0xFF + return bytes(data), bytes(mask) + + +def _u16(value: int) -> bytes: + if not 0 <= value <= 0xFFFF: + raise ImageError(f"value {value} does not fit in two octets") + return value.to_bytes(2, "big") + + +def _require_octet(value: int, what: str) -> None: + if not 0 <= value <= 0xFF: + raise ImageError(f"{what} {value} does not fit in one octet") diff --git a/packages/download/src/xknxmono/download/tables_systemb.py b/packages/download/src/xknxmono/download/tables_systemb.py new file mode 100644 index 00000000..79ad3c98 --- /dev/null +++ b/packages/download/src/xknxmono/download/tables_systemb.py @@ -0,0 +1,124 @@ +"""Build the group communication tables for the System B device model. + +Same three tables as :mod:`.tables` (KNX Standard v3.0.0, Chapter 3/5/1 +"Resources", sections 4.16 "Group Address Table (GrAT)", 4.17 "Group Object +Association Table (GrOAT)" and 4.18 "Group Object Table"), but in the System B +realisation - the mask 57B0h configuration procedure (Chapter 3/5/3 +"Configuration Procedures", section 3.9.3) - which holds them in object relative +memory with a 2-octet count layout (validated byte-exact against real hardware): + +- the group address table is ``[count:2][group address:2]*`` (big-endian), the + count being the number of addresses. Unlike the memory-mapped variant it does + *not* lead with the device's own individual address; +- the association table is ``[count:2]`` followed by one entry per link, each + ``[group address index + 1 : 1][group object number : 1]`` (a wide variant uses + two octets per field); +- the group object table is ``[count:2 = highest object number]`` followed by one + ``[flags:1][size code:1]`` record per object number ``1..highest``. Only linked + objects carry their flags and size; every other slot stays ``00 00``. + +All three live in relative memory (addressed through the object's table +reference), not at fixed addresses. See :mod:`.tables` for the memory-mapped +model and the shared :class:`~.tables.Association` and flag/size helpers. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from .errors import ImageError +from .tables import Association + + +def build_group_address_table_b(group_addresses: Sequence[int]) -> bytes: + """Return the System B group address table bytes. + + Layout ``[count:2][group address:2]*`` (big-endian); ``group_addresses`` is + sorted ascending and de-duplicated. The count is the number of addresses (no + leading device-address entry). The resulting order defines the 1-based index + the association table references. + """ + addresses = sorted(set(group_addresses)) + if len(addresses) > 0xFFFF: + raise ImageError(f"group address table has too many entries: {len(addresses)}") + out = bytearray(_u16(len(addresses))) + for address in addresses: + out += _u16(address) + return bytes(out) + + +def group_address_index_b(group_addresses: Sequence[int]) -> dict[int, int]: + """Map each group address to its 1-based index in the System B address table.""" + return {address: i + 1 for i, address in enumerate(sorted(set(group_addresses)))} + + +def build_association_table_b( + associations: Sequence[Association], *, wide: bool = False +) -> bytes: + """Return the System B association table bytes. + + Layout ``[count:2]`` then one entry per association. A narrow entry is + ``[group address index + 1 : 1][group object number : 1]``; a wide entry uses + two big-endian octets per field. Ordering matches the memory-mapped model: + sending associations first, then by group object number, then by group + address index. + """ + ordered = sorted( + associations, + key=lambda a: (not a.sending, a.group_object_number, a.group_address_index), + ) + if len(ordered) > 0xFFFF: + raise ImageError(f"association table has too many entries: {len(ordered)}") + out = bytearray(_u16(len(ordered))) + for association in ordered: + reference = association.group_address_index + 1 + number = association.group_object_number + if wide: + out += _u16(reference) + out += _u16(number) + else: + _require_octet(reference, "group address index") + _require_octet(number, "group object number") + out.append(reference) + out.append(number) + return bytes(out) + + +def build_group_object_table_b( + descriptors: Mapping[int, tuple[int, int]], highest_number: int +) -> bytes: + """Return the System B group object table bytes. + + Layout ``[count:2 = highest_number]`` then one ``[flags:1][size code:1]`` + record for every object number ``1..highest_number``. ``descriptors`` maps a + linked object's number to its ``(flags, size code)``; numbers absent from it + (unlinked or undefined) stay ``00 00``; only linked objects carry flags, per + the Group Object Table definition (Chapter 3/5/1 Resources, section 4.18). + """ + if not 0 <= highest_number <= 0xFFFF: + raise ImageError( + f"group object count {highest_number} does not fit in two octets" + ) + out = bytearray(_u16(highest_number)) + for number in range(1, highest_number + 1): + descriptor = descriptors.get(number) + if descriptor is None: + out += b"\x00\x00" + continue + flags, size = descriptor + _require_octet(flags, "group object flags") + _require_octet(size, "group object size code") + out.append(flags) + out.append(size) + return bytes(out) + + +def _u16(value: int) -> bytes: + if not 0 <= value <= 0xFFFF: + raise ImageError(f"value {value} does not fit in two octets") + return value.to_bytes(2, "big") + + +def _require_octet(value: int, what: str) -> None: + if not 0 <= value <= 0xFF: + raise ImageError(f"{what} {value} does not fit in one octet") diff --git a/packages/download/tests/__init__.py b/packages/download/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/download/tests/conftest.py b/packages/download/tests/conftest.py new file mode 100644 index 00000000..b491a22e --- /dev/null +++ b/packages/download/tests/conftest.py @@ -0,0 +1,172 @@ +"""Shared test doubles for the download package.""" + +from __future__ import annotations + +from xknx.telegram import IndividualAddress, Telegram +from xknx.telegram.apci import ( + APCI, + DeviceDescriptorRead, + DeviceDescriptorResponse, + FunctionPropertyCommand, + FunctionPropertyStateRead, + FunctionPropertyStateResponse, + MemoryRead, + MemoryResponse, + MemoryWrite, + PropertyValueRead, + PropertyValueResponse, + PropertyValueWrite, + Restart, + RestartMasterReset, + RestartMasterResetResponse, +) + +from xknxmono.download.load_state import ( + PID_LOAD_STATE_CONTROL, + LoadEvent, + LoadState, +) +from xknxmono.download.programmer import PID_OBJECT_TYPE, PID_TABLE_REFERENCE + +_LOAD_STATE = IndividualAddress("1.1.1") + + +class FakeDevice: + """Simulate the bus behaviour of a KNX device being programmed. + + Records everything written and answers reads from an in-memory model: + a flat memory, interface object properties, per-object load states and a + mapping of object index to object type (for object location). + """ + + def __init__( + self, object_types: dict[int, int] | None = None, descriptor: int = 0x0705 + ) -> None: + """Initialize an empty device with an optional object type map.""" + self.memory: dict[int, int] = {} + self.properties: dict[tuple[int, int], bytes] = {} + self.load_states: dict[int, LoadState] = {} + self.table_references: dict[int, int] = {} + self.object_types = object_types or {} + self.descriptor = descriptor + self.sent: list[APCI] = [] + self.restarted = False + self.master_reset: tuple[int, int] | None = None + # Error code and process time (ms) reported in the Master Reset response. + self.master_reset_error_code = 0 + self.master_reset_process_time = 0 + # Function property command/read state, keyed by (object, property). + self.function_properties: dict[tuple[int, int], bytes] = {} + self.function_property_return_code = 0 + + async def send_data(self, payload: APCI, wait_for_ack: bool = True) -> None: + """Handle a payload the programmer does not expect an answer to.""" + self.sent.append(payload) + if isinstance(payload, MemoryWrite): + for index, byte in enumerate(payload.data): + self.memory[payload.address + index] = byte + elif isinstance(payload, PropertyValueWrite): + self._handle_property_write(payload) + elif isinstance(payload, Restart): + self.restarted = True + + async def request(self, payload: APCI, expected: type[APCI] | None) -> Telegram: + """Handle a payload the programmer waits for a response to.""" + self.sent.append(payload) + if isinstance(payload, MemoryRead): + data = bytes( + self.memory.get(payload.address + i, 0) for i in range(payload.count) + ) + return self._telegram(MemoryResponse(address=payload.address, data=data)) + if isinstance(payload, PropertyValueWrite): + # A_PropertyValue_Write is confirmed by a response carrying the + # resulting value; apply the write, then answer with a read. + self._handle_property_write(payload) + return self._telegram( + self._handle_property_read( + PropertyValueRead( + object_index=payload.object_index, + property_id=payload.property_id, + count=payload.count, + start_index=payload.start_index, + ) + ) + ) + if isinstance(payload, PropertyValueRead): + return self._telegram(self._handle_property_read(payload)) + if isinstance(payload, DeviceDescriptorRead): + return self._telegram( + DeviceDescriptorResponse(descriptor=0, value=self.descriptor) + ) + if isinstance(payload, FunctionPropertyCommand): + self.function_properties[(payload.object_index, payload.property_id)] = ( + payload.data + ) + return self._telegram(self._function_property_response(payload)) + if isinstance(payload, FunctionPropertyStateRead): + return self._telegram(self._function_property_response(payload)) + if isinstance(payload, RestartMasterReset): + self.master_reset = (payload.erase_code, payload.channel_number) + self.restarted = True + return self._telegram( + RestartMasterResetResponse( + error_code=self.master_reset_error_code, + process_time=self.master_reset_process_time, + ) + ) + raise AssertionError(f"unexpected request: {payload}") + + def _handle_property_write(self, payload: PropertyValueWrite) -> None: + """Apply a property write, interpreting load state control specially.""" + if payload.property_id == PID_LOAD_STATE_CONTROL and payload.data: + event = payload.data[0] + transitions = { + LoadEvent.START_LOADING: LoadState.LOADING, + LoadEvent.LOAD_COMPLETE: LoadState.LOADED, + LoadEvent.UNLOAD: LoadState.UNLOADED, + LoadEvent.ADDITIONAL: LoadState.LOADING, + } + self.load_states[payload.object_index] = transitions[LoadEvent(event)] + return + self.properties[(payload.object_index, payload.property_id)] = payload.data + + def _handle_property_read( + self, payload: PropertyValueRead + ) -> PropertyValueResponse: + """Answer a property read from the device model.""" + if payload.property_id == PID_LOAD_STATE_CONTROL: + state = self.load_states.get(payload.object_index, LoadState.UNLOADED) + data = bytes([state]) + elif payload.property_id == PID_OBJECT_TYPE: + object_type = self.object_types.get(payload.object_index) + # A missing object answers empty, as a real device does past its + # last interface object; locate_object stops scanning on that. + data = object_type.to_bytes(2, "big") if object_type is not None else b"" + elif payload.property_id == PID_TABLE_REFERENCE: + data = self.table_references.get(payload.object_index, 0).to_bytes(2, "big") + else: + data = self.properties.get((payload.object_index, payload.property_id), b"") + return PropertyValueResponse( + object_index=payload.object_index, + property_id=payload.property_id, + data=data, + ) + + def _function_property_response( + self, payload: FunctionPropertyCommand | FunctionPropertyStateRead + ) -> FunctionPropertyStateResponse: + """Answer a function property command/read from the device model.""" + data = self.function_properties.get( + (payload.object_index, payload.property_id), b"" + ) + return FunctionPropertyStateResponse( + object_index=payload.object_index, + property_id=payload.property_id, + return_code=self.function_property_return_code, + data=data, + ) + + @staticmethod + def _telegram(payload: APCI) -> Telegram: + """Wrap a response payload in an incoming telegram.""" + return Telegram(destination_address=_LOAD_STATE, payload=payload) diff --git a/packages/download/tests/test_commissioning.py b/packages/download/tests/test_commissioning.py new file mode 100644 index 00000000..8021c3b4 --- /dev/null +++ b/packages/download/tests/test_commissioning.py @@ -0,0 +1,46 @@ +"""Tests for individual address programming.""" + +from __future__ import annotations + +from typing import cast + +import pytest +from xknx import XKNX +from xknx.telegram.address import IndividualAddressableType + +import xknxmono.download.commissioning as commissioning + + +async def test_program_via_programming_mode(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, object]] = [] + + async def fake_write( + xknx: XKNX, individual_address: IndividualAddressableType + ) -> None: + calls.append(("prog", individual_address)) + + monkeypatch.setattr(commissioning, "nm_individual_address_write", fake_write) + + await commissioning.program_individual_address(cast("XKNX", object()), "1.1.5") + + assert calls == [("prog", "1.1.5")] + + +async def test_program_via_serial_number(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, bytes, object]] = [] + + async def fake_serial( + xknx: XKNX, serial: bytes, individual_address: IndividualAddressableType + ) -> None: + calls.append(("serial", serial, individual_address)) + + monkeypatch.setattr( + commissioning, "nm_individual_address_serial_number_write", fake_serial + ) + + serial = bytes.fromhex("00010203 0405".replace(" ", "")) + await commissioning.program_individual_address( + cast("XKNX", object()), "1.1.5", serial_number=serial + ) + + assert calls == [("serial", serial, "1.1.5")] diff --git a/packages/download/tests/test_crc.py b/packages/download/tests/test_crc.py new file mode 100644 index 00000000..6c682481 --- /dev/null +++ b/packages/download/tests/test_crc.py @@ -0,0 +1,34 @@ +"""Tests for the Memory Control Block segment CRC. + +The golden CRC is validated byte-exact against a real System B device (the MCB +table entry for its parameter segment reads 0x47c9). +""" + +from __future__ import annotations + +from xknxmono.download.crc import segment_crc +from xknxmono.download.procedure import _mcb_table_with_crc + + +def test_segment_crc_matches_hardware() -> None: + segment = bytes.fromhex("577a456e060300ff000000000000003c") + assert segment_crc(segment) == 0x47C9 + + +def test_segment_crc_empty() -> None: + # Initial value of the augmented CCITT variant. + assert segment_crc(b"") == 0x1D0F + + +def test_mcb_table_patches_crc_into_entry() -> None: + segment = bytes.fromhex("577a456e060300ff000000000000003c") + # 8 octet entry (CRC-protected: octet 4 bit 0 clear) plus two trailing octets + mcb = bytes.fromhex("00000010003200000000") + assert _mcb_table_with_crc(mcb, segment) == bytes.fromhex("00000010003247c90000") + + +def test_mcb_table_skips_unprotected_entry() -> None: + segment = bytes.fromhex("577a456e060300ff000000000000003c") + # octet 4 bit 0 set -> not CRC-protected, CRC octets left untouched + mcb = bytes.fromhex("0000001001320000") + assert _mcb_table_with_crc(mcb, segment) == mcb diff --git a/packages/download/tests/test_gaps.py b/packages/download/tests/test_gaps.py new file mode 100644 index 00000000..65dba82e --- /dev/null +++ b/packages/download/tests/test_gaps.py @@ -0,0 +1,77 @@ +"""Tests for the implementation-gap registry and its diagnostic messages.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import cast + +import pytest + +from xknxmono.download import gaps +from xknxmono.download.errors import UnsupportedProcedureError +from xknxmono.download.image import DownloadImage +from xknxmono.download.procedure import LoadProcedureRunner +from xknxmono.download.programmer import DeviceProgrammer + +from .conftest import FakeDevice + +if True: # keep import used for typing without a runtime dependency cycle + from xknxmono.product import Application + + +def test_known_gap_message_names_standard_service() -> None: + message = gaps.describe_missing("LdCtrlClearLCFilterTable") + assert "LdCtrlClearLCFilterTable" in message + assert "line coupler filter table" in message + assert "KNX Standard v3.0.0" in message + + +def test_unknown_control_message_flags_registry() -> None: + message = gaps.describe_missing("LdCtrlSomethingBrandNew") + assert "not recognised" in message + assert "gaps.py" in message + + +def test_known_gaps_are_never_silently_skipped_in_preflight() -> None: + # A control that is a known implementation gap must not also be in the + # no-write set, otherwise preflight would hide it instead of logging it. + assert gaps.KNOWN_GAPS.keys().isdisjoint(gaps.PREFLIGHT_NO_WRITE) + + +def test_registry_excludes_implemented_controls() -> None: + # Controls the runner executes must not be listed as gaps. + for implemented in ("LdCtrlWriteMem", "LdCtrlWriteProp", "LdCtrlLoad"): + assert implemented not in gaps.KNOWN_GAPS + + +def _application(*controls: object) -> Application: + fake = SimpleNamespace( + load_procedures=None, + manufacturer_id="M-0072", + program=SimpleNamespace( + pei_type=1, application_number=1, application_version=1 + ), + ) + return cast("Application", fake) + + +async def test_unsupported_control_error_is_diagnostic() -> None: + class LdCtrlBrandNew: # a control the runner does not handle + pass + + application = _application() + runner = LoadProcedureRunner( + application, + DownloadImage(segments=(), properties=()), + DeviceProgrammer(FakeDevice()), + controls=[LdCtrlBrandNew()], + ) + + with pytest.raises(UnsupportedProcedureError) as excinfo: + await runner.run() + + message = str(excinfo.value) + assert "LdCtrlBrandNew" in message + assert "in-scope load control 1/1" in message + assert "bug report" in message + assert "app=" in message diff --git a/packages/download/tests/test_group_communication.py b/packages/download/tests/test_group_communication.py new file mode 100644 index 00000000..882ba51c --- /dev/null +++ b/packages/download/tests/test_group_communication.py @@ -0,0 +1,59 @@ +"""Tests for the synthesized System B group communication load procedure.""" + +from __future__ import annotations + +from xknxmono.download.group_communication import ( + synthesize_group_communication_controls, +) +from xknxmono.download.image import DownloadImage, RelativeSegment +from xknxmono.models.intermediate.ld_ctrl_rel_segment_t import LdCtrlRelSegment +from xknxmono.models.intermediate.ld_ctrl_write_rel_mem_t import LdCtrlWriteRelMem + + +def _image(*relative: RelativeSegment) -> DownloadImage: + return DownloadImage(segments=(), properties=(), relative_segments=relative) + + +def test_no_relative_segments_yields_no_controls() -> None: + assert synthesize_group_communication_controls(_image()) == [] + + +def test_controls_ordered_address_association_group_object() -> None: + image = _image( + RelativeSegment(9, b"\x00\xc8", b"\xff\xff"), + RelativeSegment(1, b"\x00\x02", b"\xff\xff"), + RelativeSegment(2, b"\x00\x01", b"\xff\xff"), + ) + controls = synthesize_group_communication_controls(image) + # three controls per table (two allocations then a write), in table order + object_types = [ + c.obj_type + for c in controls + if isinstance(c, (LdCtrlRelSegment, LdCtrlWriteRelMem)) + ] + assert object_types == [1, 1, 1, 2, 2, 2, 9, 9, 9] + + +def test_write_control_carries_full_table_size_at_offset_zero() -> None: + image = _image(RelativeSegment(1, b"\x00\x02\x0b\x08", b"\xff\xff\xff\xff")) + writes = [ + c + for c in synthesize_group_communication_controls(image) + if isinstance(c, LdCtrlWriteRelMem) + ] + assert len(writes) == 1 + assert writes[0].obj_type == 1 + assert writes[0].offset == 0 + assert writes[0].size == 4 + assert writes[0].inline_data is None + + +def test_allocations_mirror_two_mode_pattern() -> None: + image = _image(RelativeSegment(1, b"\x00\x00", b"\xff\xff")) + allocations = [ + c + for c in synthesize_group_communication_controls(image) + if isinstance(c, LdCtrlRelSegment) + ] + assert [a.mode for a in allocations] == [1, 0] + assert all(a.size == 2 for a in allocations) diff --git a/packages/download/tests/test_image.py b/packages/download/tests/test_image.py new file mode 100644 index 00000000..a6cad752 --- /dev/null +++ b/packages/download/tests/test_image.py @@ -0,0 +1,82 @@ +"""Tests for download image assembly.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from xknxmono.download.errors import ImageError +from xknxmono.download.image import DownloadImage, MemorySegment, build_image +from xknxmono.product import load + +_FIXTURE = ( + Path(__file__).resolve().parents[2] + / "product" + / "tests" + / "fixtures" + / "gira_2gang_button_interface.knxprod" +) + + +def test_read_within_segment() -> None: + image = DownloadImage( + segments=(MemorySegment(address=0x100, data=bytes(range(16))),), + properties=(), + ) + assert image.read(0x104, 4) == bytes([4, 5, 6, 7]) + + +def test_read_outside_segment_raises() -> None: + image = DownloadImage( + segments=(MemorySegment(address=0x100, data=bytes(4)),), + properties=(), + ) + with pytest.raises(ImageError, match="no image data"): + image.read(0x100, 8) + + +def test_memory_segment_end() -> None: + assert MemorySegment(address=0x100, data=bytes(4)).end == 0x104 + + +def test_build_image_from_project_device() -> None: + from types import SimpleNamespace + from typing import cast + + from xknxmono.download.project_data import SeedDevice + + registry = load(_FIXTURE) + applications = [ + app for app in registry.applications.values() if app.dynamic_ui() is not None + ] + assert applications + application = applications[0] + + # A device with no configured parameters/modules must yield the same image + # as the plain default build (seeding with empty project data is a no-op). + device = cast( + "SeedDevice", + SimpleNamespace(parameters=[], module_instances=[], com_objects=[]), + ) + seeded = build_image(application, device=device) + default = build_image(application) + + assert isinstance(seeded, DownloadImage) + assert {s.address for s in seeded.segments} == {s.address for s in default.segments} + + +def test_build_image_from_fixture() -> None: + registry = load(_FIXTURE) + applications = [ + app for app in registry.applications.values() if app.dynamic_ui() is not None + ] + assert applications, "fixture should contain at least one application" + + image = build_image(applications[0]) + + assert isinstance(image, DownloadImage) + # every assembled segment carries data at a concrete address + for segment in image.segments: + assert segment.data + assert segment.address >= 0 diff --git a/packages/download/tests/test_load_state.py b/packages/download/tests/test_load_state.py new file mode 100644 index 00000000..2ff95a76 --- /dev/null +++ b/packages/download/tests/test_load_state.py @@ -0,0 +1,116 @@ +"""Tests for load event encodings. + +Byte layouts and the worked examples are taken from KNX Standard 3/5/2 +"Management Procedures", section 3.31.3 "DMP_LoadStateMachineWrite_RCo_IO", +where every load event written to PID_LOAD_STATE_CONTROL is a 10 octet value. +""" + +from __future__ import annotations + +import pytest + +from xknxmono.download import load_state as ls + + +def test_simple_events_are_ten_octets() -> None: + for event in (ls.start_loading(), ls.load_complete(), ls.unload()): + assert len(event) == ls.LOAD_STATE_CONTROL_SIZE + + +def test_start_loading() -> None: + # 3.31.3.2: data = 01 00 ... + assert ls.start_loading() == bytes([0x01]) + bytes(9) + + +def test_load_complete() -> None: + # 3.31.3.3: data = 02 00 ... + assert ls.load_complete() == bytes([0x02]) + bytes(9) + + +def test_unload() -> None: + # 3.31.3.1: data = 04 00 ... + assert ls.unload() == bytes([0x04]) + bytes(9) + + +def test_alloc_absolute_data_segment() -> None: + # AllocAbsDataSeg (segment type 0): 03 00 SSSS LLLL AA TT MM 00. + # Load Controls cookbook example: start 0x4000, end 0x41FE => length 0x01FF, + # read+write access (0xFF), EEPROM (0x03), checksum control enabled (0x80). + event = ls.alloc_absolute_segment( + ls.SegmentType.ABS_DATA, + start_address=0x4000, + length=0x01FF, + access_attributes=0xFF, + memory_type=0x03, + memory_attributes=0x80, + ) + assert event == bytes.fromhex("0300400001FFFF038000") + + +def test_alloc_absolute_stack_segment() -> None: + # AllocAbsStackSeg (segment type 1): same layout as data segment. + event = ls.alloc_absolute_segment( + ls.SegmentType.ABS_STACK, + start_address=0x0700, + length=0x0290, + access_attributes=0x00, + memory_type=0x02, + memory_attributes=0x00, + ) + assert event == bytes.fromhex("0301070002900002 0000".replace(" ", "")) + assert len(event) == ls.LOAD_STATE_CONTROL_SIZE + + +def test_alloc_task_segment() -> None: + # AllocAbsTaskSeg (segment type 2): 03 02 SSSS PP MMMM TTTT VV. + # Cookbook example: start 0x43FC, PEI type 0x01, manufacturer 0x0072, + # device type 0x0001, version 0x01. + event = ls.alloc_task_segment( + start_address=0x43FC, + pei_type=0x01, + application_id=bytes.fromhex("0072000101"), + ) + assert event == bytes.fromhex("030243FC010072000101") + assert len(event) == ls.LOAD_STATE_CONTROL_SIZE + + +def test_task_pointer_pads_reserved() -> None: + # TaskPtr (segment type 3): 03 03 IIII SSSS PPPP 0000. + event = ls.task_pointer(0x1234, 0x5678, 0x9ABC) + assert event == bytes.fromhex("0303123456789ABC0000") + + +def test_task_control_1() -> None: + # TaskCtrl1 (segment type 4): 03 04 AAAA NN 00... + event = ls.task_control_1(interface_object_address=0x1000, interface_object_count=3) + assert event == bytes.fromhex("0304100003 0000000000".replace(" ", "")) + + +def test_task_control_2_is_full_length() -> None: + # TaskCtrl2 (segment type 5): 03 05 CCCC OOOO 1111 2222. + event = ls.task_control_2(0x1111, 0x2222, 0x3333, 0x4444) + assert event == bytes.fromhex("03051111222233334444") + assert len(event) == ls.LOAD_STATE_CONTROL_SIZE + + +def test_relative_allocation() -> None: + # Relative allocation (subtype 0Ah): 03 0A 00... + assert ls.relative_allocation(0x0100) == bytes.fromhex("030A0100000000000000") + + +def test_data_relative_allocation() -> None: + # Data relative allocation (subtype 0Bh, System B): + # 03 0B . + event = ls.data_relative_allocation(0x00001E15, mode=0x01, fill=0x00) + assert event == bytes.fromhex("030B00001E1501000000") + assert len(event) == ls.LOAD_STATE_CONTROL_SIZE + + +def test_task_segment_rejects_wrong_application_id_length() -> None: + with pytest.raises(ValueError, match="5 octets"): + ls.alloc_task_segment(0x0000, 0x00, b"\x00") + + +def test_absolute_segment_rejects_task_type() -> None: + with pytest.raises(ValueError, match="ABS_DATA or ABS_STACK"): + ls.alloc_absolute_segment(ls.SegmentType.ABS_TASK, 0, 0) diff --git a/packages/download/tests/test_merge.py b/packages/download/tests/test_merge.py new file mode 100644 index 00000000..a9f7f36a --- /dev/null +++ b/packages/download/tests/test_merge.py @@ -0,0 +1,102 @@ +"""Tests for Load Procedure resolution (default / product / merged styles).""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import cast + +from xknxmono.download.merge import resolve_download_controls +from xknxmono.models.intermediate.ld_ctrl_connect_t import LdCtrlConnect +from xknxmono.models.intermediate.ld_ctrl_merge_t import LdCtrlMerge +from xknxmono.models.intermediate.ld_ctrl_restart_t import LdCtrlRestart +from xknxmono.models.intermediate.ld_ctrl_write_mem_t import LdCtrlWriteMem +from xknxmono.models.intermediate.load_procedure_style_t import LoadProcedureStyle +from xknxmono.models.intermediate.load_procedures_t import LoadProcedures +from xknxmono.models.intermediate.load_procedures_t_load_procedure import ( + LoadProceduresLoadProcedure, +) +from xknxmono.models.intermediate.master_data_t import MasterData +from xknxmono.models.intermediate.procedure_type_t import ProcedureType + +_MASK = "MV-0705" + + +def _application(style: LoadProcedureStyle, *fragments: object) -> object: + return SimpleNamespace( + load_procedure_style=style, + load_procedures=LoadProcedures(load_procedure=list(fragments)), # type: ignore[arg-type] + program=SimpleNamespace(mask_version=_MASK), + ) + + +def _master_with_default(*controls: object) -> MasterData: + default = SimpleNamespace(procedure_type=ProcedureType.LOAD, choice=list(controls)) + mask = SimpleNamespace( + id=_MASK, + hawk_configuration_data=[ + SimpleNamespace(procedures=SimpleNamespace(procedure=[default])) + ], + ) + return cast( + "MasterData", + SimpleNamespace(mask_versions=SimpleNamespace(mask_version=[mask])), + ) + + +def _fragment(merge_id: int, *controls: object) -> LoadProceduresLoadProcedure: + return LoadProceduresLoadProcedure(merge_id=merge_id, choice=list(controls)) # type: ignore[arg-type] + + +def test_product_procedure_uses_application_controls() -> None: + application = _application( + LoadProcedureStyle.PRODUCT_PROCEDURE, + _fragment(0, LdCtrlConnect(), LdCtrlRestart()), + ) + controls = resolve_download_controls(cast("object", application)) # type: ignore[arg-type] + assert [type(c).__name__ for c in controls] == ["LdCtrlConnect", "LdCtrlRestart"] + + +def test_merged_procedure_splices_fragments_into_default() -> None: + master = _master_with_default( + LdCtrlConnect(), + LdCtrlMerge(merge_id=2), + LdCtrlMerge(merge_id=4), + LdCtrlRestart(), + ) + write = LdCtrlWriteMem(address=0x10, size=1, verify=False, inline_data=b"\x01") + application = _application( + LoadProcedureStyle.MERGED_PROCEDURE, + _fragment(2, LdCtrlConnect()), + _fragment(4, write), + ) + + controls = resolve_download_controls(cast("object", application), master) # type: ignore[arg-type] + + # Connect, , , Restart + assert [type(c).__name__ for c in controls] == [ + "LdCtrlConnect", + "LdCtrlConnect", + "LdCtrlWriteMem", + "LdCtrlRestart", + ] + + +def test_default_procedure_drops_unmatched_merge_placeholders() -> None: + master = _master_with_default( + LdCtrlConnect(), LdCtrlMerge(merge_id=9), LdCtrlRestart() + ) + application = _application(LoadProcedureStyle.DEFAULT_PROCEDURE) + + controls = resolve_download_controls(cast("object", application), master) # type: ignore[arg-type] + + # no fragment for merge id 9 -> placeholder dropped + assert [type(c).__name__ for c in controls] == ["LdCtrlConnect", "LdCtrlRestart"] + + +def test_merged_without_master_falls_back_to_application() -> None: + application = _application( + LoadProcedureStyle.MERGED_PROCEDURE, + _fragment(0, LdCtrlConnect()), + ) + controls = resolve_download_controls(cast("object", application)) # type: ignore[arg-type] + assert [type(c).__name__ for c in controls] == ["LdCtrlConnect"] diff --git a/packages/download/tests/test_preflight.py b/packages/download/tests/test_preflight.py new file mode 100644 index 00000000..32624d34 --- /dev/null +++ b/packages/download/tests/test_preflight.py @@ -0,0 +1,257 @@ +"""Tests for the read-only pre-flight (dry-run) check.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import cast + +import pytest +from xknx.telegram.apci import MemoryRead, MemoryWrite, PropertyValueWrite + +from xknxmono.download.errors import VerificationError +from xknxmono.download.image import DownloadImage, MemorySegment +from xknxmono.download.preflight import ( + ByteRange, + PreflightReport, + PropertyDiff, + SegmentDiff, +) +from xknxmono.download.procedure import LoadProcedureRunner +from xknxmono.download.programmer import DeviceProgrammer +from xknxmono.models.intermediate.ld_ctrl_abs_segment_t import LdCtrlAbsSegment +from xknxmono.models.intermediate.ld_ctrl_compare_prop_t import LdCtrlCompareProp +from xknxmono.models.intermediate.ld_ctrl_write_mem_t import LdCtrlWriteMem +from xknxmono.models.intermediate.ld_ctrl_write_prop_t import LdCtrlWriteProp +from xknxmono.models.intermediate.load_procedure_style_t import LoadProcedureStyle +from xknxmono.models.intermediate.load_procedures_t import LoadProcedures +from xknxmono.models.intermediate.load_procedures_t_load_procedure import ( + LoadProceduresLoadProcedure, +) + +from .conftest import FakeDevice + +if True: # keep import used for typing without a runtime dependency cycle + from xknxmono.product import Application + +_ADDRESS_TABLE_TYPE = 1 + + +def _application(*controls: object) -> Application: + """Wrap Load Controls into a minimal application stand-in.""" + procedure = LoadProceduresLoadProcedure(choice=list(controls)) # type: ignore[arg-type] + load_procedures = LoadProcedures(load_procedure=[procedure]) + fake = SimpleNamespace( + load_procedures=load_procedures, + load_procedure_style=LoadProcedureStyle.PRODUCT_PROCEDURE, + manufacturer_id="M-0072", + program=SimpleNamespace( + pei_type=1, + application_number=1, + application_version=1, + mask_version="MV-0705", + ), + ) + return cast("Application", fake) + + +def _runner( + application: Application, image: DownloadImage +) -> tuple[LoadProcedureRunner, FakeDevice]: + device = FakeDevice(object_types={0: 0x0000, 1: _ADDRESS_TABLE_TYPE}) + programmer = DeviceProgrammer(device) + return LoadProcedureRunner(application, image, programmer), device + + +# --- report dataclass units ------------------------------------------------ + + +def test_changed_ranges_finds_contiguous_runs() -> None: + diff = SegmentDiff( + address=0x100, + current=b"\x00\x00\x00\x00\x00", + planned=b"\x00\xff\xff\x00\xff", + ) + assert diff.changed_ranges == (ByteRange(1, 2), ByteRange(4, 1)) + assert diff.changed_bytes == 3 + assert diff.changed + + +def test_report_totals_and_summary() -> None: + report = PreflightReport( + segments=( + SegmentDiff(address=0x10, current=b"\x00\x00", planned=b"\x01\x02"), + SegmentDiff(address=0x20, current=b"\xaa", planned=b"\xaa"), + ), + properties=( + PropertyDiff( + object_index=3, property_id=5, current=b"\x00", planned=b"\x09" + ), + ), + ) + assert report.total_changed_bytes == 3 + assert report.has_changes + assert len(report.changed_segments) == 1 + assert len(report.changed_properties) == 1 + assert "3 byte(s) would change" in report.summary() + + +# --- runner preflight behaviour --------------------------------------------- + + +async def test_preflight_reports_memory_change_without_writing() -> None: + image = DownloadImage( + segments=(MemorySegment(address=0x4000, data=b"\x01\x02\x03\x04"),), + properties=(), + ) + application = _application( + LdCtrlWriteMem(address=0x4000, size=4, verify=False, inline_data=None) + ) + runner, device = _runner(application, image) + # device currently holds different bytes at two positions + device.memory.update({0x4000: 0x01, 0x4001: 0xFF, 0x4002: 0x03, 0x4003: 0xFF}) + + report = await runner.preflight() + + assert not any(isinstance(p, MemoryWrite) for p in device.sent) # nothing written + assert any(isinstance(p, MemoryRead) for p in device.sent) # only read + (segment,) = report.segments + assert segment.address == 0x4000 + assert segment.current == b"\x01\xff\x03\xff" + assert segment.planned == b"\x01\x02\x03\x04" + assert segment.changed_ranges == (ByteRange(1, 1), ByteRange(3, 1)) + assert report.total_changed_bytes == 2 + + +async def test_preflight_reports_no_change_when_matching() -> None: + image = DownloadImage( + segments=(MemorySegment(address=0x4000, data=b"\xde\xad"),), + properties=(), + ) + application = _application( + LdCtrlWriteMem(address=0x4000, size=2, verify=False, inline_data=None) + ) + runner, device = _runner(application, image) + device.memory.update({0x4000: 0xDE, 0x4001: 0xAD}) + + report = await runner.preflight() + + assert not report.has_changes + assert report.segments[0].changed is False + + +async def test_preflight_previews_abs_segment_without_allocating() -> None: + image = DownloadImage( + segments=(MemorySegment(address=0x4400, data=b"\x11\x22\x33"),), + properties=(), + ) + application = _application( + LdCtrlAbsSegment( + obj_type=_ADDRESS_TABLE_TYPE, + occurrence=0, + seg_type=0, + address=0x4400, + size=3, + access=0xFF, + mem_type=3, + seg_flags=0x80, + ) + ) + runner, device = _runner(application, image) + + report = await runner.preflight() + + # no allocation (no load state changed) and no memory written + assert device.load_states == {} + assert not any(isinstance(p, MemoryWrite) for p in device.sent) + (segment,) = report.segments + assert segment.address == 0x4400 + assert segment.planned == b"\x11\x22\x33" + assert segment.changed_bytes == 3 # device is all zeros + + +async def test_preflight_previews_property_without_writing() -> None: + application = _application( + LdCtrlWriteProp( + obj_idx=5, + prop_id=0x33, + start_element=1, + count=1, + verify=False, + inline_data=b"\xaa\xbb", + ) + ) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + device.properties[(5, 0x33)] = b"\xaa\x00" + + report = await runner.preflight() + + assert not any(isinstance(p, PropertyValueWrite) for p in device.sent) + (prop,) = report.properties + assert prop.object_index == 5 + assert prop.property_id == 0x33 + assert prop.current == b"\xaa\x00" + assert prop.planned == b"\xaa\xbb" + assert prop.changed_ranges == (ByteRange(1, 1),) + + +async def test_preflight_compare_prop_gate_raises_on_mismatch() -> None: + device = FakeDevice() + device.properties[(0, 78)] = b"\x00\x00\x00\x00\x09\x99" + application = _application( + LdCtrlCompareProp( + obj_idx=0, + prop_id=78, + start_element=1, + count=1, + inline_data=b"\x00\x00\x00\x00\x02\x27", + ) + ) + runner = LoadProcedureRunner( + application, DownloadImage(segments=(), properties=()), DeviceProgrammer(device) + ) + with pytest.raises(VerificationError, match="compare failed"): + await runner.preflight() + + +async def test_preflight_ignores_inline_property_source() -> None: + # inline data is used directly as the planned value (no image lookup needed) + application = _application( + LdCtrlWriteProp( + obj_idx=5, + prop_id=0x33, + start_element=1, + count=1, + verify=False, + inline_data=b"\x01", + ) + ) + runner, _device = _runner(application, DownloadImage(segments=(), properties=())) + + report = await runner.preflight() + + assert report.properties[0].planned == b"\x01" + assert report.properties[0].current == b"" # unset property reads empty + + +def test_property_trailing_zero_padding_is_not_a_change() -> None: + # Device stores 8 octets; the planned value is the same 8 octets padded with + # two trailing zeros (property carried at its maximum element size). The pad + # must not count as a change. + current = bytes.fromhex("000040000000a611") + planned = bytes.fromhex("000040000000a6110000") + diff = PropertyDiff( + object_index=3, property_id=204, current=current, planned=planned + ) + assert diff.changed_bytes == 0 + assert not diff.changed + + +def test_property_nonzero_extension_still_counts() -> None: + # A non-zero octet beyond the device length is a real change, still reported. + diff = PropertyDiff( + object_index=3, + property_id=204, + current=bytes.fromhex("0000"), + planned=bytes.fromhex("000001"), + ) + assert diff.changed_bytes == 1 diff --git a/packages/download/tests/test_procedure.py b/packages/download/tests/test_procedure.py new file mode 100644 index 00000000..f756b26d --- /dev/null +++ b/packages/download/tests/test_procedure.py @@ -0,0 +1,709 @@ +"""Tests for the Load Procedure interpreter.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import cast + +import pytest +from xknx.telegram.apci import ( + FunctionPropertyCommand, + FunctionPropertyStateRead, + MemoryRead, + PropertyValueWrite, +) + +from xknxmono.download.errors import DownloadError +from xknxmono.download.image import ( + DownloadImage, + MemorySegment, + PropertyValue, + RelativeSegment, +) +from xknxmono.download.load_state import PID_LOAD_STATE_CONTROL, LoadState +from xknxmono.download.procedure import LoadProcedureRunner +from xknxmono.download.programmer import DeviceProgrammer +from xknxmono.download.scope import DownloadScope +from xknxmono.models.intermediate.ld_ctrl_abs_segment_t import LdCtrlAbsSegment +from xknxmono.models.intermediate.ld_ctrl_connect_t import LdCtrlConnect +from xknxmono.models.intermediate.ld_ctrl_declare_prop_desc_t import ( + LdCtrlDeclarePropDesc, +) +from xknxmono.models.intermediate.ld_ctrl_disconnect_t import LdCtrlDisconnect +from xknxmono.models.intermediate.ld_ctrl_invoke_function_prop_t import ( + LdCtrlInvokeFunctionProp, +) +from xknxmono.models.intermediate.ld_ctrl_load_completed_t import LdCtrlLoadCompleted +from xknxmono.models.intermediate.ld_ctrl_load_image_mem_t import LdCtrlLoadImageMem +from xknxmono.models.intermediate.ld_ctrl_load_t import LdCtrlLoad +from xknxmono.models.intermediate.ld_ctrl_master_reset_t import LdCtrlMasterReset +from xknxmono.models.intermediate.ld_ctrl_read_function_prop_t import ( + LdCtrlReadFunctionProp, +) +from xknxmono.models.intermediate.ld_ctrl_restart_t import LdCtrlRestart +from xknxmono.models.intermediate.ld_ctrl_unload_t import LdCtrlUnload +from xknxmono.models.intermediate.ld_ctrl_write_mem_t import LdCtrlWriteMem +from xknxmono.models.intermediate.ld_ctrl_write_prop_t import LdCtrlWriteProp +from xknxmono.models.intermediate.ld_ctrl_write_rel_mem_t import LdCtrlWriteRelMem +from xknxmono.models.intermediate.load_procedure_style_t import LoadProcedureStyle +from xknxmono.models.intermediate.load_procedures_t import LoadProcedures +from xknxmono.models.intermediate.load_procedures_t_load_procedure import ( + LoadProceduresLoadProcedure, +) + +from .conftest import FakeDevice + +if True: # keep import used for typing without a runtime dependency cycle + from xknxmono.product import Application + +# Interface object type of the address table object used in the fixtures below. +_ADDRESS_TABLE_TYPE = 1 + + +def _application(*controls: object) -> Application: + """Wrap Load Controls into a minimal application stand-in.""" + procedure = LoadProceduresLoadProcedure(choice=list(controls)) # type: ignore[arg-type] + load_procedures = LoadProcedures(load_procedure=[procedure]) + fake = SimpleNamespace( + load_procedures=load_procedures, + load_procedure_style=LoadProcedureStyle.PRODUCT_PROCEDURE, + manufacturer_id="M-0072", + program=SimpleNamespace( + pei_type=1, + application_number=1, + application_version=1, + mask_version="MV-0705", + ), + ) + return cast("Application", fake) + + +def _runner( + application: Application, image: DownloadImage +) -> tuple[LoadProcedureRunner, FakeDevice]: + device = FakeDevice(object_types={0: 0x0000, 1: _ADDRESS_TABLE_TYPE}) + programmer = DeviceProgrammer(device) + return LoadProcedureRunner(application, image, programmer), device + + +async def test_full_property_based_sequence() -> None: + image = DownloadImage( + segments=(MemorySegment(address=0x4000, data=bytes(range(8))),), + properties=(), + ) + application = _application( + LdCtrlConnect(), + LdCtrlUnload(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0), + LdCtrlLoad(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0), + LdCtrlWriteMem( + address=0x4000, size=8, verify=False, inline_data=bytes(range(8)) + ), + LdCtrlLoadCompleted(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0), + LdCtrlRestart(), + LdCtrlDisconnect(), + ) + runner, device = _runner(application, image) + + await runner.run() + + # the data was written to memory + assert bytes(device.memory[0x4000 + i] for i in range(8)) == bytes(range(8)) + # the load state machine ended up loaded and the device restarted + assert device.load_states[1] == LoadState.LOADED + assert device.restarted + assert runner.restarted + + +async def test_load_image_mem_reads_without_writing() -> None: + application = _application(LdCtrlLoadImageMem(address=0x4000, size=4)) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + # LoadImageMem reads device memory into the image; it must not write. + assert 0x4000 not in device.memory + assert any(isinstance(p, MemoryRead) and p.address == 0x4000 for p in device.sent) + + +async def test_master_reset_sends_restart_master_reset() -> None: + application = _application( + LdCtrlConnect(), + LdCtrlMasterReset(erase_code=4, channel_number=0), + LdCtrlDisconnect(), + ) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + # The Master Reset went out with its erase code / channel and marked a restart. + assert device.master_reset == (4, 0) + assert device.restarted + assert runner.restarted + + +async def test_master_reset_error_code_raises() -> None: + application = _application(LdCtrlMasterReset(erase_code=4, channel_number=0)) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + device.master_reset_error_code = 0x02 + + with pytest.raises(DownloadError, match="refused master reset"): + await runner.run() + + +async def test_invoke_function_property_sends_command() -> None: + application = _application( + LdCtrlInvokeFunctionProp(obj_idx=1, prop_id=52, inline_data=bytes([0x01, 0x02])) + ) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + assert device.function_properties[(1, 52)] == bytes([0x01, 0x02]) + assert any( + isinstance(p, FunctionPropertyCommand) + and p.object_index == 1 + and p.property_id == 52 + for p in device.sent + ) + + +async def test_invoke_function_property_error_code_raises() -> None: + application = _application( + LdCtrlInvokeFunctionProp(obj_idx=1, prop_id=52, inline_data=b"\x01") + ) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + device.function_property_return_code = 0x03 + + with pytest.raises(DownloadError, match="returned error code"): + await runner.run() + + +async def test_read_function_property_reads_without_writing() -> None: + application = _application(LdCtrlReadFunctionProp(obj_idx=1, prop_id=52)) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + assert (1, 52) not in device.function_properties + assert any( + isinstance(p, FunctionPropertyStateRead) and p.property_id == 52 + for p in device.sent + ) + + +async def test_declare_prop_desc_is_client_side_noop() -> None: + # DeclarePropDesc only declares a property description to the client object + # model; it must not send any telegram. + application = _application( + LdCtrlDeclarePropDesc( + obj_idx=1, + prop_id=52, + prop_type=1, + max_elements=1, + read_access=0, + write_access=0, + writable=True, + ) + ) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + assert device.sent == [] + + +async def test_write_mem_inline_data() -> None: + application = _application( + LdCtrlWriteMem(address=0x100, size=3, verify=False, inline_data=b"\x01\x02\x03") + ) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + assert bytes(device.memory[0x100 + i] for i in range(3)) == b"\x01\x02\x03" + + +async def test_write_mem_image_backed() -> None: + image = DownloadImage( + segments=(MemorySegment(address=0x500, data=b"\xde\xad\xbe\xef"),), + properties=(), + ) + application = _application( + LdCtrlWriteMem(address=0x500, size=4, verify=False, inline_data=None) + ) + runner, device = _runner(application, image) + + await runner.run() + + assert bytes(device.memory[0x500 + i] for i in range(4)) == b"\xde\xad\xbe\xef" + + +async def test_write_rel_mem_uses_table_reference() -> None: + application = _application( + LdCtrlWriteRelMem( + obj_type=_ADDRESS_TABLE_TYPE, + occurrence=0, + offset=4, + size=2, + verify=False, + inline_data=b"\x11\x22", + ) + ) + device = FakeDevice(object_types={0: 0x0000, 1: _ADDRESS_TABLE_TYPE}) + device.table_references[1] = 0x4000 + runner = LoadProcedureRunner( + application, DownloadImage(segments=(), properties=()), DeviceProgrammer(device) + ) + + await runner.run() + + # written at table base (0x4000) + offset (4) + assert bytes(device.memory[0x4004 + i] for i in range(2)) == b"\x11\x22" + + +async def test_write_rel_mem_image_backed_uses_relative_lookup() -> None: + # The image mirrors a relative segment in its own relative address space + # (base 0x0); the device write adds the table base read at run time. Only the + # masked bytes are written, at base + their relative offset. + image = DownloadImage( + segments=( + MemorySegment( + address=0x0, + data=b"\x00\x00\xab\xcd", + mask=b"\x00\x00\xff\xff", + ), + ), + properties=(), + ) + application = _application( + LdCtrlWriteRelMem( + obj_type=_ADDRESS_TABLE_TYPE, + occurrence=0, + offset=0, + size=4, + verify=False, + inline_data=None, + ) + ) + device = FakeDevice(object_types={0: 0x0000, 1: _ADDRESS_TABLE_TYPE}) + device.table_references[1] = 0x3804 + runner = LoadProcedureRunner(application, image, DeviceProgrammer(device)) + + await runner.run() + + # relative offsets 2..3 written at table base 0x3804 + 2 + assert bytes(device.memory[0x3806 + i] for i in range(2)) == b"\xab\xcd" + assert 0x3804 not in device.memory and 0x3805 not in device.memory + + +async def test_write_rel_mem_relative_segment_by_object_type() -> None: + # A System B group communication table: the image holds it as a relative + # segment keyed by object type, and the write lands at the table base. + image = DownloadImage( + segments=(), + properties=(), + relative_segments=( + RelativeSegment( + object_type=_ADDRESS_TABLE_TYPE, + data=b"\x00\x02\x0b\x08", + mask=b"\xff\xff\xff\xff", + ), + ), + ) + application = _application( + LdCtrlWriteRelMem( + obj_type=_ADDRESS_TABLE_TYPE, + occurrence=0, + offset=0, + size=4, + verify=False, + inline_data=None, + ) + ) + device = FakeDevice(object_types={0: 0x0000, 1: _ADDRESS_TABLE_TYPE}) + device.table_references[1] = 0x3400 + runner = LoadProcedureRunner(application, image, DeviceProgrammer(device)) + + await runner.run() + + assert bytes(device.memory[0x3400 + i] for i in range(4)) == b"\x00\x02\x0b\x08" + + +async def test_write_prop_inline_data() -> None: + application = _application( + LdCtrlWriteProp( + obj_idx=5, + prop_id=0x33, + start_element=1, + count=1, + verify=False, + inline_data=b"\xaa\xbb", + ) + ) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + writes = [p for p in device.sent if isinstance(p, PropertyValueWrite)] + assert writes and writes[0].object_index == 5 + assert device.properties[(5, 0x33)] == b"\xaa\xbb" + + +async def test_write_prop_image_backed() -> None: + image = DownloadImage( + segments=(), + properties=( + PropertyValue( + object_index=5, property_id=0x33, occurrence=0, data=b"\xca\xfe" + ), + ), + ) + application = _application( + LdCtrlWriteProp( + obj_idx=5, + prop_id=0x33, + start_element=1, + count=1, + verify=False, + inline_data=None, + ) + ) + runner, device = _runner(application, image) + + await runner.run() + + assert device.properties[(5, 0x33)] == b"\xca\xfe" + + +async def test_compare_prop_tolerates_trailing_padding() -> None: + from xknxmono.models.intermediate.ld_ctrl_compare_prop_t import LdCtrlCompareProp + + device = FakeDevice() + device.properties[(0, 78)] = ( + b"\x00\x00\x00\x00\x02\x27" # 6 bytes, as a real device reports + ) + application = _application( + LdCtrlCompareProp( + obj_idx=0, + prop_id=78, + start_element=1, + count=1, + inline_data=b"\x00\x00\x00\x00\x02\x27\x00\x00\x00\x00", # padded to 10 + ) + ) + runner = LoadProcedureRunner( + application, DownloadImage(segments=(), properties=()), DeviceProgrammer(device) + ) + await runner.run() # must not raise: the 6 significant bytes match + + +async def test_compare_prop_detects_real_mismatch() -> None: + from xknxmono.download.errors import VerificationError + from xknxmono.models.intermediate.ld_ctrl_compare_prop_t import LdCtrlCompareProp + + device = FakeDevice() + device.properties[(0, 78)] = b"\x00\x00\x00\x00\x09\x99" + application = _application( + LdCtrlCompareProp( + obj_idx=0, + prop_id=78, + start_element=1, + count=1, + inline_data=b"\x00\x00\x00\x00\x02\x27", + ) + ) + runner = LoadProcedureRunner( + application, DownloadImage(segments=(), properties=()), DeviceProgrammer(device) + ) + with pytest.raises(VerificationError, match="compare failed"): + await runner.run() + + +async def test_compare_prop_mask_ignores_unmarked_octets() -> None: + from xknxmono.models.intermediate.ld_ctrl_compare_prop_t import LdCtrlCompareProp + + device = FakeDevice() + # application id: manufacturer 0x0002, app number 0xa062, version 0x14 + device.properties[(0, 13)] = b"\x00\x02\xa0\x62\x14" + application = _application( + LdCtrlCompareProp( + obj_idx=0, + prop_id=13, + start_element=1, + count=1, + inline_data=b"\x00\x00\xa0\x62\x00", + mask=b"\x00\x00\xff\xff\x00", # only the app-number octets matter + ) + ) + runner = LoadProcedureRunner( + application, DownloadImage(segments=(), properties=()), DeviceProgrammer(device) + ) + await runner.run() # must not raise: manufacturer and version are masked out + + +async def test_compare_prop_mask_still_detects_marked_mismatch() -> None: + from xknxmono.download.errors import VerificationError + from xknxmono.models.intermediate.ld_ctrl_compare_prop_t import LdCtrlCompareProp + + device = FakeDevice() + device.properties[(0, 13)] = b"\x00\x02\xa0\x63\x14" # app number differs + application = _application( + LdCtrlCompareProp( + obj_idx=0, + prop_id=13, + start_element=1, + count=1, + inline_data=b"\x00\x00\xa0\x62\x00", + mask=b"\x00\x00\xff\xff\x00", + ) + ) + runner = LoadProcedureRunner( + application, DownloadImage(segments=(), properties=()), DeviceProgrammer(device) + ) + with pytest.raises(VerificationError, match="compare failed"): + await runner.run() + + +async def test_abs_segment_writes_image_data_for_its_range() -> None: + image = DownloadImage( + segments=(MemorySegment(address=0x4000, data=b"\x01\x02\x03\x04"),), + properties=(), + ) + application = _application( + LdCtrlAbsSegment( + obj_type=_ADDRESS_TABLE_TYPE, + occurrence=0, + seg_type=0, + address=0x4000, + size=4, + access=0xFF, + mem_type=3, + seg_flags=0x80, + ) + ) + runner, device = _runner(application, image) + + await runner.run() + + # segment allocated (LSM loading) and its image data written to memory + assert device.load_states[1] == LoadState.LOADING + assert bytes(device.memory[0x4000 + i] for i in range(4)) == b"\x01\x02\x03\x04" + + +async def test_abs_segment_without_image_only_allocates() -> None: + application = _application( + LdCtrlAbsSegment( + obj_type=_ADDRESS_TABLE_TYPE, + occurrence=0, + seg_type=0, + address=0x9000, + size=4, + access=0xFF, + mem_type=3, + seg_flags=0x80, + ) + ) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + assert device.load_states[1] == LoadState.LOADING + assert 0x9000 not in device.memory # nothing written when image lacks the range + + +async def test_load_event_written_to_load_state_control() -> None: + application = _application(LdCtrlLoad(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0)) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + load_events = [ + p + for p in device.sent + if isinstance(p, PropertyValueWrite) and p.property_id == PID_LOAD_STATE_CONTROL + ] + assert load_events and load_events[0].data[0] == 0x01 # start loading + + +async def test_lsm_idx_addresses_object_directly() -> None: + # For property based management LsmIdx is the interface object index. + application = _application(LdCtrlLoad(lsm_idx=1, occurrence=0)) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + assert device.load_states[1] == LoadState.LOADING + + +async def test_connect_and_disconnect_are_noops() -> None: + application = _application(LdCtrlConnect(), LdCtrlDisconnect()) + runner, device = _runner(application, DownloadImage(segments=(), properties=())) + + await runner.run() + + assert device.sent == [] + + +_ASSOCIATION_TABLE_TYPE = 2 +_APPLICATION_TYPE = 3 + + +def _scoped_application() -> Application: + # group communication parts (address=1, association=2) and the application + # part (3); a partial download runs only the matching parts. + return _application( + LdCtrlLoad(obj_type=1, occurrence=0), + LdCtrlLoad(obj_type=_ASSOCIATION_TABLE_TYPE, occurrence=0), + LdCtrlLoad(obj_type=_APPLICATION_TYPE, occurrence=0), + ) + + +def _scoped_device() -> FakeDevice: + return FakeDevice(object_types={0: 0x0000, 1: 1, 2: 2, 3: 3}) + + +async def test_partial_parameters_scope_runs_only_application_part() -> None: + device = _scoped_device() + runner = LoadProcedureRunner( + _scoped_application(), + DownloadImage(segments=(), properties=()), + DeviceProgrammer(device), + scope=DownloadScope.PARAMETERS, + ) + await runner.run() + assert set(device.load_states) == {3} # only the application part loaded + + +async def test_partial_group_scope_runs_only_table_parts() -> None: + device = _scoped_device() + runner = LoadProcedureRunner( + _scoped_application(), + DownloadImage(segments=(), properties=()), + DeviceProgrammer(device), + scope=DownloadScope.GROUP_COMMUNICATION, + ) + await runner.run() + assert set(device.load_states) == {1, 2} # address + association tables + + +async def test_full_scope_runs_all_parts() -> None: + device = _scoped_device() + runner = LoadProcedureRunner( + _scoped_application(), + DownloadImage(segments=(), properties=()), + DeviceProgrammer(device), + scope=DownloadScope.FULL, + ) + await runner.run() + assert set(device.load_states) == {1, 2, 3} + + +class _FakeManager: + """Connection manager handing out one fake device, counting open/close.""" + + def __init__(self, device: FakeDevice) -> None: + self.device = device + self.opens = 0 + self.closes = 0 + + async def open(self) -> FakeDevice: + self.opens += 1 + return self.device + + async def close(self) -> None: + self.closes += 1 + + +def _managed_runner( + *controls: object, **kwargs: object +) -> tuple[LoadProcedureRunner, _FakeManager]: + device = FakeDevice(object_types={0: 0x0000, 1: _ADDRESS_TABLE_TYPE}) + manager = _FakeManager(device) + runner = LoadProcedureRunner( + _application(*controls), + DownloadImage(segments=(), properties=()), + connection_manager=manager, + restart_cooldown=0, + **kwargs, # type: ignore[arg-type] + ) + return runner, manager + + +async def test_connect_opens_and_disconnect_closes() -> None: + runner, manager = _managed_runner( + LdCtrlConnect(), + LdCtrlLoad(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0), + LdCtrlDisconnect(), + ) + await runner.run() + assert manager.opens == 1 + assert manager.closes == 1 + assert manager.device.load_states[1] == LoadState.LOADING + + +async def test_bus_control_auto_connects() -> None: + runner, manager = _managed_runner( + LdCtrlLoad(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0) + ) + await runner.run() + assert manager.opens == 1 # opened automatically, no explicit Connect + + +async def test_restart_tears_down_and_next_control_reconnects() -> None: + runner, manager = _managed_runner( + LdCtrlConnect(), + LdCtrlRestart(), + LdCtrlLoad(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0), + ) + await runner.run() + assert runner.restarted + assert manager.device.restarted + # opened for Connect, torn down by Restart, reopened for Load + assert manager.opens == 2 + assert manager.closes >= 1 + + +async def test_runner_requires_programmer_or_manager() -> None: + with pytest.raises(DownloadError, match="programmer or a connection manager"): + LoadProcedureRunner(_application(), DownloadImage(segments=(), properties=())) + + +async def test_expected_descriptor_mismatch_refuses_to_program() -> None: + # Device reports mask 0x0705 (FakeDevice default); expecting System B 0x07B0. + runner, manager = _managed_runner( + LdCtrlLoad(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0), + expected_descriptor=0x07B0, + ) + with pytest.raises(DownloadError, match="device mask mismatch"): + await runner.run() + # It refused before driving the load state machine. + assert manager.device.load_states == {} + + +async def test_expected_descriptor_match_proceeds() -> None: + runner, manager = _managed_runner( + LdCtrlLoad(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0), + expected_descriptor=0x0705, + ) + await runner.run() + assert manager.device.load_states[1] == LoadState.LOADING + + +async def test_negotiate_apdu_uses_device_maximum() -> None: + runner, manager = _managed_runner( + LdCtrlLoad(obj_type=_ADDRESS_TABLE_TYPE, occurrence=0), + max_apdu_length=254, + negotiate_apdu=True, + ) + # Device Object PID_MAX_APDU_LENGTH (56) reports 40 octets. + manager.device.properties[(0, 56)] = (40).to_bytes(2, "big") + + await runner.run() + + # The device's maximum APDU length was read (negotiated) before programming. + assert any( + type(p).__name__ == "PropertyValueRead" + and getattr(p, "object_index", None) == 0 + and getattr(p, "property_id", None) == 56 + for p in manager.device.sent + ) diff --git a/packages/download/tests/test_programmer.py b/packages/download/tests/test_programmer.py new file mode 100644 index 00000000..2a761bcf --- /dev/null +++ b/packages/download/tests/test_programmer.py @@ -0,0 +1,154 @@ +"""Tests for the low level device programmer.""" + +from __future__ import annotations + +import pytest +from xknx.telegram.apci import MemoryRead, MemoryWrite, PropertyValueWrite + +from xknxmono.download.errors import LoadStateError, VerificationError +from xknxmono.download.load_state import LoadState, start_loading +from xknxmono.download.programmer import PID_OBJECT_TYPE, DeviceProgrammer + +from .conftest import FakeDevice + + +def test_memory_chunk_size_respects_apdu() -> None: + assert DeviceProgrammer(FakeDevice(), max_apdu_length=15).memory_chunk_size == 12 + assert DeviceProgrammer(FakeDevice(), max_apdu_length=55).memory_chunk_size == 52 + + +async def test_write_memory_is_chunked() -> None: + device = FakeDevice() + programmer = DeviceProgrammer(device, max_apdu_length=15) + data = bytes(range(30)) + + await programmer.write_memory(0x4000, data) + + writes = [p for p in device.sent if isinstance(p, MemoryWrite)] + assert [len(w.data) for w in writes] == [12, 12, 6] + assert [w.address for w in writes] == [0x4000, 0x400C, 0x4018] + read_back = bytes(device.memory[0x4000 + i] for i in range(30)) + assert read_back == data + + +async def test_read_memory_reassembles_chunks() -> None: + device = FakeDevice() + device.memory.update({0x100 + i: i for i in range(20)}) + programmer = DeviceProgrammer(device, max_apdu_length=15) + + assert await programmer.read_memory(0x100, 20) == bytes(range(20)) + + +async def test_write_memory_verify_success() -> None: + device = FakeDevice() + programmer = DeviceProgrammer(device) + await programmer.write_memory(0x10, b"\x01\x02\x03", verify=True) + + +async def test_write_memory_verify_detects_mismatch() -> None: + class DroppingDevice(FakeDevice): + async def send_data(self, payload: object, wait_for_ack: bool = True) -> None: + if isinstance(payload, MemoryWrite): + return # silently drop the write + await super().send_data(payload, wait_for_ack) # type: ignore[arg-type] + + programmer = DeviceProgrammer(DroppingDevice()) + with pytest.raises(VerificationError, match="verification failed"): + await programmer.write_memory(0x10, b"\x01\x02\x03", verify=True) + + +async def test_property_round_trip() -> None: + device = FakeDevice() + programmer = DeviceProgrammer(device) + await programmer.write_property(5, 0x33, b"\xaa\xbb") + assert await programmer.read_property(5, 0x33) == b"\xaa\xbb" + + +async def test_write_property_chunks_large_value() -> None: + device = FakeDevice() + programmer = DeviceProgrammer(device, max_apdu_length=15) + # 20 one-byte elements, 10 data octets fit a frame -> two frames of 10 + await programmer.write_property(5, 0x10, bytes(20), count=20) + writes = [p for p in device.sent if isinstance(p, PropertyValueWrite)] + assert [w.count for w in writes] == [10, 10] + assert [w.start_index for w in writes] == [1, 11] + + +async def test_write_memory_verify_reads_back_each_block() -> None: + device = FakeDevice() + programmer = DeviceProgrammer(device, max_apdu_length=15) # 12 byte chunks + await programmer.write_memory(0x10, bytes(20), verify=True) + reads = [p for p in device.sent if isinstance(p, MemoryRead)] + assert len(reads) == 2 # one read-back per written block + + +async def test_read_table_reference() -> None: + device = FakeDevice() + device.table_references[2] = 0x1234 + assert await DeviceProgrammer(device).read_table_reference(2) == 0x1234 + + +def test_memory_chunk_size_never_below_one() -> None: + assert DeviceProgrammer(FakeDevice(), max_apdu_length=2).memory_chunk_size == 1 + + +async def test_locate_object_by_type_and_occurrence() -> None: + device = FakeDevice(object_types={0: 0, 1: 0x0100, 2: 0x0100}) + programmer = DeviceProgrammer(device) + + # occurrence is zero-based: 0 == first instance, 1 == second + assert await programmer.locate_object(0x0100, occurrence=0) == 1 + assert await programmer.locate_object(0x0100, occurrence=1) == 2 + # second lookup is served from the cache without further reads + reads_before = len(device.sent) + assert await programmer.locate_object(0x0100, occurrence=0) == 1 + assert len(device.sent) == reads_before + + +async def test_locate_object_reads_object_type_property() -> None: + device = FakeDevice(object_types={0: 0x0000, 1: 0x0007}) + programmer = DeviceProgrammer(device) + await programmer.locate_object(0x0007) + assert any(getattr(p, "property_id", None) == PID_OBJECT_TYPE for p in device.sent) + + +async def test_locate_object_missing_raises() -> None: + device = FakeDevice(object_types={0: 0x0000}) + programmer = DeviceProgrammer(device) + with pytest.raises(LoadStateError, match="not found"): + await programmer.locate_object(0x9999) + + +async def test_send_load_event_reaches_expected_state() -> None: + device = FakeDevice() + programmer = DeviceProgrammer(device) + await programmer.send_load_event(3, start_loading(), LoadState.LOADING) + assert device.load_states[3] == LoadState.LOADING + + +async def test_send_load_event_error_state_raises() -> None: + class ErrorDevice(FakeDevice): + def _handle_property_write(self, payload: object) -> None: + self.load_states[payload.object_index] = LoadState.ERROR # type: ignore[attr-defined] + + programmer = DeviceProgrammer(ErrorDevice()) + with pytest.raises(LoadStateError, match="ERROR"): + await programmer.send_load_event(3, start_loading(), LoadState.LOADING) + + +async def test_send_load_event_timeout_raises() -> None: + class StuckDevice(FakeDevice): + def _handle_property_write(self, payload: object) -> None: + return # never changes state, stays UNLOADED + + programmer = DeviceProgrammer(StuckDevice()) + with pytest.raises(LoadStateError, match="did not reach"): + await programmer.send_load_event( + 3, start_loading(), LoadState.LOADING, retries=2, retry_delay=0 + ) + + +async def test_restart() -> None: + device = FakeDevice() + await DeviceProgrammer(device).restart() + assert device.restarted diff --git a/packages/download/tests/test_resources.py b/packages/download/tests/test_resources.py new file mode 100644 index 00000000..8dc77419 --- /dev/null +++ b/packages/download/tests/test_resources.py @@ -0,0 +1,78 @@ +"""Tests for the per-mask resource location resolver.""" + +from __future__ import annotations + +from xknxmono.download.resources import MaskResources, mask_resources +from xknxmono.models.intermediate.resource_addr_space_t import ResourceAddrSpace +from xknxmono.models.intermediate.resource_location_t import ResourceLocation +from xknxmono.models.intermediate.resource_name_t import ResourceName + + +def _prop(obj: int, pid: int) -> ResourceLocation: + return ResourceLocation( + address_space=ResourceAddrSpace.SYSTEM_PROPERTY, + interface_object_ref=obj, + property_id=pid, + ) + + +def test_property_ref_resolves_object_and_pid() -> None: + resources = MaskResources({ResourceName.APPLICATION_LOAD_CONTROL: _prop(4, 5)}) + assert resources.property_ref(ResourceName.APPLICATION_LOAD_CONTROL) == (4, 5) + assert resources.memory_address(ResourceName.APPLICATION_LOAD_CONTROL) is None + + +def test_memory_address_resolves_start_address() -> None: + resources = MaskResources( + { + ResourceName.GROUP_OBJECT_TABLE_PTR: ResourceLocation( + address_space=ResourceAddrSpace.STANDARD_MEMORY, start_address=0x112 + ) + } + ) + assert resources.memory_address(ResourceName.GROUP_OBJECT_TABLE_PTR) == 0x112 + assert not resources.is_relative(ResourceName.GROUP_OBJECT_TABLE_PTR) + + +def test_is_relative_for_object_relative_memory() -> None: + resources = MaskResources( + { + ResourceName.GROUP_ADDRESS_TABLE_PTR: ResourceLocation( + address_space=ResourceAddrSpace.RELATIVE_MEMORY_BY_OBJECT_TYPE + ) + } + ) + assert resources.is_relative(ResourceName.GROUP_ADDRESS_TABLE_PTR) + assert resources.memory_address(ResourceName.GROUP_ADDRESS_TABLE_PTR) is None + + +def test_missing_resource_returns_none() -> None: + resources = MaskResources({}) + assert resources.location(ResourceName.APPLICATION_RUN_CONTROL) is None + assert resources.property_ref(ResourceName.APPLICATION_RUN_CONTROL) is None + assert resources.memory_address(ResourceName.APPLICATION_RUN_CONTROL) is None + assert not resources.is_relative(ResourceName.APPLICATION_RUN_CONTROL) + + +def test_mask_resources_unknown_mask_is_none() -> None: + assert mask_resources(None, "MV-FFFF-unknown") is None + + +def test_bundled_table_covers_known_masks() -> None: + from xknxmono.download.resources import bundled_mask_resources + + system_b = bundled_mask_resources("MV-07B0") + assert system_b is not None + assert system_b.property_ref(ResourceName.APPLICATION_LOAD_CONTROL) == (4, 5) + + memory_mapped = bundled_mask_resources("MV-0705") + assert memory_mapped is not None + assert memory_mapped.memory_address(ResourceName.APPLICATION_LOAD_CONTROL) == 260 + + assert bundled_mask_resources("MV-9999-unknown") is None + + +def test_mask_resources_falls_back_to_bundled_without_master() -> None: + resolved = mask_resources(None, "MV-07B0") + assert resolved is not None + assert resolved.property_ref(ResourceName.APPLICATION_RUN_CONTROL) == (4, 6) diff --git a/packages/download/tests/test_scope.py b/packages/download/tests/test_scope.py new file mode 100644 index 00000000..0a433e97 --- /dev/null +++ b/packages/download/tests/test_scope.py @@ -0,0 +1,42 @@ +"""Tests for download scope filtering (by target loadable part).""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from xknxmono.download.scope import DownloadScope, control_in_scope + + +def _ctrl(**attrs: int) -> object: + return SimpleNamespace(**attrs) + + +@pytest.mark.parametrize( + ("control", "scope", "expected"), + [ + # framing controls (no target object) always run + (_ctrl(), DownloadScope.PARAMETERS, True), + (_ctrl(), DownloadScope.GROUP_COMMUNICATION, True), + # device object (0) is framing (fingerprint compare) -> always + (_ctrl(obj_idx=0), DownloadScope.PARAMETERS, True), + (_ctrl(obj_idx=0), DownloadScope.GROUP_COMMUNICATION, True), + # group communication objects: address(1), association(2), group object(9) + (_ctrl(obj_type=1), DownloadScope.GROUP_COMMUNICATION, True), + (_ctrl(obj_type=2), DownloadScope.GROUP_COMMUNICATION, True), + (_ctrl(lsm_idx=9), DownloadScope.GROUP_COMMUNICATION, True), + (_ctrl(obj_type=1), DownloadScope.PARAMETERS, False), + # application program object (3) holds parameters + (_ctrl(obj_type=3), DownloadScope.PARAMETERS, True), + (_ctrl(lsm_idx=3), DownloadScope.PARAMETERS, True), + (_ctrl(obj_type=3), DownloadScope.GROUP_COMMUNICATION, False), + # full download runs everything + (_ctrl(obj_type=1), DownloadScope.FULL, True), + (_ctrl(obj_type=3), DownloadScope.FULL, True), + ], +) +def test_control_in_scope( + control: object, scope: DownloadScope, expected: bool +) -> None: + assert control_in_scope(control, scope) is expected diff --git a/packages/download/tests/test_tables.py b/packages/download/tests/test_tables.py new file mode 100644 index 00000000..5fd454aa --- /dev/null +++ b/packages/download/tests/test_tables.py @@ -0,0 +1,103 @@ +"""Group communication table formatting, checked byte-exact against real hardware. + +The expected bytes were read from a Berker BE-GT2Tx.01 (1.1.74) after ETS +programmed it, so these assert the formatters reproduce ETS output exactly. +""" + +from __future__ import annotations + +import pytest + +from xknxmono.download.errors import ImageError +from xknxmono.download.tables import ( + Association, + build_association_table, + build_group_address_table, + group_address_index, +) + +# Device 1.1.74 = 0x114a; its 18 group addresses (sorted ascending). +_DEVICE_74 = 0x114A +_GAS_74 = [ + 0x0B00, + 0x0B02, + 0x0B03, + 0x0B2B, + 0x102C, + 0x1030, + 0x1031, + 0x1033, + 0x1035, + 0x1800, + 0x1801, + 0x1802, + 0x1803, + 0x181B, + 0x181C, + 0x181D, + 0x181E, + 0x2828, +] + +# (group address index into the address table, com object number) +_ASSOC_74 = [ + Association(12, 0), + Association(13, 1), + Association(17, 2), + Association(16, 3), + Association(5, 10), + Association(7, 11), + Association(9, 13), + Association(6, 21), + Association(8, 23), + Association(10, 30), + Association(11, 31), + Association(15, 32), + Association(14, 33), + Association(3, 106), + Association(18, 107), + Association(1, 112), + Association(2, 114), + Association(4, 122), +] + + +def test_group_address_table_matches_device() -> None: + expected = bytes.fromhex( + "13" # count = 1 + 18 + "114a" # device individual address 1.1.74 + "0b00 0b02 0b03 0b2b 102c 1030 1031 1033 1035" + "1800 1801 1802 1803 181b 181c 181d 181e 2828".replace(" ", "") + ) + assert build_group_address_table(_DEVICE_74, _GAS_74) == expected + + +def test_group_address_table_sorts_and_dedups() -> None: + table = build_group_address_table(0x1101, [0x0B02, 0x0B00, 0x0B02]) + assert table == bytes.fromhex("0311010b00 0b02".replace(" ", "")) + + +def test_group_address_index_is_one_based() -> None: + index = group_address_index([0x0B02, 0x0B00]) + assert index == {0x0B00: 1, 0x0B02: 2} + + +def test_association_table_matches_device() -> None: + expected = bytes.fromhex( + "12" # count = 18 + "0c00 0d01 1102 1003 050a 070b 090d 0615 0817" + "0a1e 0b1f 0f20 0e21 036a 126b 0170 0272 047a".replace(" ", "") + ) + assert build_association_table(_ASSOC_74) == expected + + +def test_association_table_orders_by_group_object_number() -> None: + table = build_association_table( + [Association(5, 20), Association(3, 7), Association(9, 12)] + ) + assert table == bytes.fromhex("030307 090c 0514".replace(" ", "")) + + +def test_group_address_table_rejects_overflow() -> None: + with pytest.raises(ImageError): + build_group_address_table(0x1101, list(range(300))) diff --git a/packages/download/tests/test_tables_systemb.py b/packages/download/tests/test_tables_systemb.py new file mode 100644 index 00000000..100bfc19 --- /dev/null +++ b/packages/download/tests/test_tables_systemb.py @@ -0,0 +1,99 @@ +"""Tests for the System B group communication table formatter. + +The golden vectors marked "device" are the exact bytes read from a real System B +device (M-015B A-0200, individual address 1.1.41) over the bus. +""" + +from __future__ import annotations + +import pytest + +from xknxmono.download.errors import ImageError +from xknxmono.download.tables import Association +from xknxmono.download.tables_systemb import ( + build_association_table_b, + build_group_address_table_b, + build_group_object_table_b, + group_address_index_b, +) + + +def test_address_table_has_two_octet_count_and_no_device_address() -> None: + data = build_group_address_table_b([0x0B0A, 0x0B08, 0x0B09]) + # count = 3 (big-endian, no leading device address), then sorted addresses + assert data == bytes.fromhex("0003 0b08 0b09 0b0a".replace(" ", "")) + + +def test_address_table_matches_real_device() -> None: + # 1.1.41: 43 consecutive addresses 0x0b08..0x0b32 plus 0x1c07. + addresses = [*range(0x0B08, 0x0B33), 0x1C07] + expected = "002c" + "".join(f"{a:04x}" for a in sorted(addresses)) + assert build_group_address_table_b(addresses).hex() == expected + + +def test_address_index_is_one_based_over_sorted_addresses() -> None: + assert group_address_index_b([0x0B0A, 0x0B08, 0x0B09]) == { + 0x0B08: 1, + 0x0B09: 2, + 0x0B0A: 3, + } + + +def test_association_table_narrow_entries() -> None: + associations = [ + Association(group_address_index=0, group_object_number=1, sending=True), + Association(group_address_index=1, group_object_number=2, sending=True), + ] + data = build_association_table_b(associations) + # count = 2, then [gaindex+1][object number] per entry + assert data == bytes.fromhex("0002 0101 0202".replace(" ", "")) + + +def test_association_table_wide_entries() -> None: + associations = [ + Association(group_address_index=0, group_object_number=1, sending=True) + ] + data = build_association_table_b(associations, wide=True) + assert data == bytes.fromhex("0001 0001 0001".replace(" ", "")) + + +def test_association_ordering_sending_first_then_object_then_index() -> None: + associations = [ + Association(group_address_index=5, group_object_number=2, sending=False), + Association(group_address_index=1, group_object_number=2, sending=True), + Association(group_address_index=0, group_object_number=1, sending=True), + ] + data = build_association_table_b(associations) + # sending (obj 1 idx 0), sending (obj 2 idx 1), then receiving (obj 2 idx 5); + # each entry is [group address index + 1][object number] + assert data == bytes.fromhex("0003 0101 0202 0602".replace(" ", "")) + + +def test_group_object_table_linked_records_and_gaps() -> None: + # Objects 1, 2 and 10 linked; everything up to 10 else stays 00 00. + descriptors = {1: (0x4F, 0x07), 2: (0xF7, 0x07), 10: (0x4F, 0x00)} + data = build_group_object_table_b(descriptors, highest_number=10) + expected = ( + "000a" # count = highest number = 10 + "4f07" # object 1 + "f707" # object 2 + + "0000" * 7 # objects 3..9 + + "4f00" # object 10 + ) + assert data.hex() == expected + + +def test_group_object_table_matches_real_device_prefix() -> None: + # 1.1.41 device prefix: obj 1 = 0x4f07, obj 2 = 0xf707, obj 3..9 gaps. + descriptors = {1: (0x4F, 0x07), 2: (0xF7, 0x07)} + data = build_group_object_table_b(descriptors, highest_number=200) + assert data[:2] == bytes.fromhex("00c8") # count = 200 + assert data[2:6] == bytes.fromhex("4f07f707") + assert data[6:20] == b"\x00" * 14 # objects 3..9 unlinked + + +def test_octet_overflow_is_rejected() -> None: + with pytest.raises(ImageError): + build_association_table_b( + [Association(group_address_index=300, group_object_number=1)] + ) diff --git a/packages/models/src/xknxmono/models/schemas/.gitignore b/packages/models/src/xknxmono/models/schemas/.gitignore index d6b7ef32..90b3d15f 100644 --- a/packages/models/src/xknxmono/models/schemas/.gitignore +++ b/packages/models/src/xknxmono/models/schemas/.gitignore @@ -1,2 +1,5 @@ +# Track the bundled KNX schema XSDs; ignore anything else dropped here. * !.gitignore +!README.md +!*.xsd diff --git a/packages/models/src/xknxmono/models/schemas/README.md b/packages/models/src/xknxmono/models/schemas/README.md new file mode 100644 index 00000000..3cebc979 --- /dev/null +++ b/packages/models/src/xknxmono/models/schemas/README.md @@ -0,0 +1,20 @@ +# KNX schema files + +The dataclass bindings under `../files/` are generated by xsdata from the KNX +project/product interchange schema (XML namespace `http://knx.org/xml/project/`). +That schema is the public KNX interchange format shipped with every `.knxprod` +and product database; `HawkConfigurationData` and the other elements the bindings +use are defined there. + +Bundled here: + +- `knx_project_v23.xsd` - schema version 23 (namespace `.../project/23`). + +To regenerate the bindings (see the repository README), place the schema XSD for +each version you need (10-14, 20-23) in this directory and run: + + cd packages/models/src && uvx --from "xsdata[cli]" xsdata generate \ + --config ../.xsdata.xml xknxmono/models/schemas/ + +Only the version XSDs present here are (re)generated, so keep the full set here if +you regenerate; committing only a subset and regenerating would drop the others. diff --git a/packages/models/src/xknxmono/models/schemas/knx_project_v23.xsd b/packages/models/src/xknxmono/models/schemas/knx_project_v23.xsd new file mode 100644 index 00000000..8b9a92b7 --- /dev/null +++ b/packages/models/src/xknxmono/models/schemas/knx_project_v23.xsd @@ -0,0 +1,5997 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant set + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + registration-relevant set + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + + + + + + + + + + registration-relevant set + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + + + + + registration-relevant set + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + registration-relevant set + + + + + + + + + + registration-relevant set + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + registration-relevant set + + + + + + + + + + + + + + + + + + + + + + registration-relevant list + This is a list to ensure deterministic behaviour in case of multiple active parameter refs + + + + + + + + + + + registration-relevant set + + + + + + + + + + + registration-relevant set + + + + + + + + + + + registration-relevant set + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + registration-relevant set + This is a list to ensure deterministic behaviour in case of multiple active communication object refs + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant set + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant set + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + + + + registration-relevant set + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + registration-relevant set + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + + + registration-relevant list + + + + + + + + + + + + + registration-relevant list + + + + + + + + + + + + + + + registration-relevant set + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + registration-relevant set + + + + + + + + + + + + + + + + + + registration-relevant set + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant set + + + + + + + + + + + + + + + + + + + + + registration-relevant list + This is a list to ensure deterministic behaviour in case of multiple active parameter refs + + + + + + + + + + + registration-relevant set + + + + + + + + + + + registration-relevant set + + + + + + + + + + + registration-relevant set + + + + + + + registration-relevant + + + + + + + + + + + + + + + registration-relevant set + This is a list to ensure deterministic behaviour in case of multiple active communication object refs + + + + + + + + + registration-relevant set + + + + + + + + + + + + + registration-relevant list + + + + + + + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + + + registration-relevant set + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant list + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + registration-relevant set + + + + + + + registration-relevant + + + + + + + + + + + + registration-relevant list + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant list + + + + + + + registration-relevant list + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + registration-relevant set + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + + + + + + registration-relevant set + + + + + + + + registration-relevant list + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + registration-relevant list + + + + + + + registration-relevant list + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant set + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + registration-relevant set + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + registration-relevant set + + + + + + + + + + + registration-relevant set + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + registration-relevant set + + + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + registration-relevant list + + + + + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + registration-relevant list + + + + + + + registration-relevant list + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant list + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + registration-relevant list + + + + + + + registration-relevant list + + + + + + + + + + + + + + + + + registration-relevant + + + + + + + + + registration-relevant list + + + + + + + registration-relevant list + + + + + + + + + + + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + + + + + registration-relevant + + + + + + registration-relevant + + + + + registration-relevant + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + registration-relevant list + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + registration-relevant + + + + + registration-relevant + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + registration-relevant + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/product/src/xknxmono/product/parser_v2/dynamic.py b/packages/product/src/xknxmono/product/parser_v2/dynamic.py index 58224810..bc14af21 100644 --- a/packages/product/src/xknxmono/product/parser_v2/dynamic.py +++ b/packages/product/src/xknxmono/product/parser_v2/dynamic.py @@ -36,6 +36,7 @@ build_property_param_map, collect_writes, encode_to_memory, + encode_to_memory_masked, encode_to_properties, resolve_param_values, ) @@ -82,6 +83,7 @@ "build_property_param_map", "collect_writes", "encode_to_memory", + "encode_to_memory_masked", "encode_to_properties", "resolve_param_values", ] @@ -269,6 +271,16 @@ def encode_to_memory(self) -> dict[str, bytes]: self._state, ) + def encode_to_memory_masked(self) -> dict[str, tuple[bytes, bytes]]: + """Encode into ``{segment_id: (data, mask)}``; mask marks written bytes.""" + self.ui() # ensure state is current + return encode_to_memory_masked( + self._app, + self._idx, + resolve_param_values(self._idx, self._state), + self._state, + ) + def memory_param_map(self) -> dict[str, dict[int, tuple[str, str]]]: """Return {seg_id: {byte_offset: (param_id, value)}} for hex viewer hover lookups.""" self.ui() diff --git a/packages/product/src/xknxmono/product/parser_v2/encode.py b/packages/product/src/xknxmono/product/parser_v2/encode.py index e17d20fa..653a28ae 100644 --- a/packages/product/src/xknxmono/product/parser_v2/encode.py +++ b/packages/product/src/xknxmono/product/parser_v2/encode.py @@ -31,21 +31,39 @@ ModuleDefStaticParametersUnionProperty, ) from xknxmono.models.intermediate.module_t_numeric_arg import ModuleNumericArg +from xknxmono.models.intermediate.parameter_type_t_type_color import ( + ParameterTypeTypeColor, +) +from xknxmono.models.intermediate.parameter_type_t_type_color_space import ( + ParameterTypeTypeColorSpace, +) +from xknxmono.models.intermediate.parameter_type_t_type_date import ( + ParameterTypeTypeDate, +) from xknxmono.models.intermediate.parameter_type_t_type_float import ( ParameterTypeTypeFloat, ) from xknxmono.models.intermediate.parameter_type_t_type_float_encoding import ( ParameterTypeTypeFloatEncoding, ) +from xknxmono.models.intermediate.parameter_type_t_type_ipaddress import ( + ParameterTypeTypeIpaddress, +) from xknxmono.models.intermediate.parameter_type_t_type_number import ( ParameterTypeTypeNumber, ) +from xknxmono.models.intermediate.parameter_type_t_type_raw_data import ( + ParameterTypeTypeRawData, +) from xknxmono.models.intermediate.parameter_type_t_type_restriction import ( ParameterTypeTypeRestriction, ) from xknxmono.models.intermediate.parameter_type_t_type_text import ( ParameterTypeTypeText, ) +from xknxmono.models.intermediate.parameter_type_t_type_time import ( + ParameterTypeTypeTime, +) from xknxmono.models.intermediate.property_parameter_t import PropertyParameter from xknxmono.models.intermediate.property_union_t import PropertyUnion from xknxmono.models.intermediate.union_parameter_t import UnionParameter @@ -85,6 +103,16 @@ def __init__(self) -> None: PropertyKey = tuple[int | None, int, int] # (object_index, property_id, occurrence) +def _program_little_endian(app: ApplicationProgram) -> bool: + """Whether the application program encodes parameter values little-endian. + + Read from the static options' ``ParameterByteOrder`` (default big-endian). + """ + options = getattr(app.static, "options", None) + order = getattr(options, "parameter_byte_order", None) + return order is not None and order.value == "LittleEndian" + + def _write_bits( buf: bytearray, offset: int, bit_offset: int, size_in_bit: int, value: int ) -> None: @@ -99,62 +127,264 @@ def _write_bits( buf[pos // 8] &= ~bit_mask -def _encode_value(str_value: str, size_in_bit: int, tc: object) -> int | None: - if isinstance(tc, (ParameterTypeTypeNumber, ParameterTypeTypeRestriction)): - try: - v = int(str_value) - except (ValueError, TypeError): - return None - return v & ((1 << size_in_bit) - 1) +def _encode_value( + str_value: str, size_in_bit: int, tc: object, *, little_endian: bool = False +) -> int | None: + """Encode a parameter value string into the integer that ``_write_bits`` packs. + + Values are packed MSB-first. The default byte order is big-endian; when the + application program selects little-endian (``ParameterByteOrder`` in its static + options), the byte order of a multi-octet numeric value is reversed, matching + the reference engine (which reverses the octets of integer and float values for + a little-endian program). Ports the reference engine's per-type value encoders. + """ + if isinstance(tc, (ParameterTypeTypeNumber, ParameterTypeTypeTime)): + # A time value is stored as a plain integer in the type's unit. + return _apply_byte_order( + _encode_number(str_value, size_in_bit), size_in_bit, little_endian + ) + + if isinstance(tc, ParameterTypeTypeRestriction): + return _encode_restriction(str_value, size_in_bit, tc, little_endian) if isinstance(tc, ParameterTypeTypeFloat): - try: - f = float(str_value) - except (ValueError, TypeError): - return None - if tc.encoding == ParameterTypeTypeFloatEncoding.DPT_9: - mantissa = round(f * 100) - exp = 0 - while mantissa < -2048 or mantissa > 2047: - mantissa >>= 1 - exp += 1 - if exp > 15: - return None - sign = 1 if mantissa < 0 else 0 - return (sign << 15) | (exp << 11) | (mantissa & 0x7FF) - if tc.encoding == ParameterTypeTypeFloatEncoding.IEEE_754_SINGLE: - return struct.unpack(">I", struct.pack(">f", f))[0] - if tc.encoding == ParameterTypeTypeFloatEncoding.IEEE_754_DOUBLE: - return struct.unpack(">Q", struct.pack(">d", f))[0] - return None + return _apply_byte_order( + _encode_float(str_value, size_in_bit, tc), size_in_bit, little_endian + ) if isinstance(tc, ParameterTypeTypeText): - encoded = str_value.encode("latin-1", errors="replace") - n_bytes = size_in_bit // 8 - padded = encoded[:n_bytes].ljust(n_bytes, b"\x00") - result = 0 - for b in padded: - result = (result << 8) | b - return result + return _encode_text(str_value, size_in_bit) + + if isinstance(tc, ParameterTypeTypeDate): + return _encode_date(str_value, tc) + + if isinstance(tc, ParameterTypeTypeIpaddress): + return _encode_ipaddress(str_value) + + if isinstance(tc, ParameterTypeTypeColor): + return _encode_color(str_value, size_in_bit, tc) + + if isinstance(tc, ParameterTypeTypeRawData): + return _encode_raw_data(str_value, size_in_bit, little_endian) + + return None + +def _apply_byte_order( + value: int | None, size_in_bit: int, little_endian: bool +) -> int | None: + """Reverse the octet order of a byte-aligned multi-octet numeric ``value``. + + Only applies for a little-endian program and a whole-octet field (at least two + octets); single-octet and sub-octet fields are unaffected. + """ + if value is None or not little_endian or size_in_bit < 16 or size_in_bit % 8 != 0: + return value + n = size_in_bit // 8 + return int.from_bytes(value.to_bytes(n, "big")[::-1], "big") + + +def _encode_restriction( + str_value: str, + size_in_bit: int, + tc: ParameterTypeTypeRestriction, + little_endian: bool, +) -> int | None: + """Encode an enumeration value of a restricted parameter type. + + An enumeration entry may carry an explicit ``BinaryValue`` (the exact octets to + store, e.g. a priority ordering); that is written verbatim, big-endian, and is + not subject to the program byte order. Entries without a binary value fall back + to encoding the enumeration value as an integer (honouring the byte order). + """ + for enumeration in tc.enumeration: + if str(enumeration.value) == str_value: + if enumeration.binary_value: + return int.from_bytes(enumeration.binary_value, "big") + break + return _apply_byte_order( + _encode_number(str_value, size_in_bit), size_in_bit, little_endian + ) + + +def _encode_number(str_value: str, size_in_bit: int) -> int | None: + """Signed/unsigned integer, masked to ``size_in_bit`` (two's complement).""" + try: + v = int(str_value, 0) if str_value[:2].lower() == "0x" else int(str_value) + except (ValueError, TypeError): + return None + return v & ((1 << size_in_bit) - 1) + + +def _encode_float( + str_value: str, size_in_bit: int, tc: ParameterTypeTypeFloat +) -> int | None: + try: + f = float(str_value) + except (ValueError, TypeError): + return None + if tc.encoding == ParameterTypeTypeFloatEncoding.DPT_9: + # KNX DPT 9 (2 octet float): value = 0.01 * m * 2^exp, m = 11 bit signed + # two's complement, sign in bit 15, exponent in bits 14..11. + mantissa = round(f * 100) + exp = 0 + while mantissa < -2048 or mantissa > 2047: + mantissa >>= 1 + exp += 1 + if exp > 15: + return None + sign = 1 if mantissa < 0 else 0 + return (sign << 15) | (exp << 11) | (mantissa & 0x7FF) + if tc.encoding == ParameterTypeTypeFloatEncoding.IEEE_754_SINGLE: + return struct.unpack(">I", struct.pack(">f", f))[0] + if tc.encoding == ParameterTypeTypeFloatEncoding.IEEE_754_DOUBLE: + return struct.unpack(">Q", struct.pack(">d", f))[0] return None +def _encode_text(str_value: str, size_in_bit: int) -> int: + """Code-page (Latin-1) text, truncated and zero-padded to the field width. + + Zero padding also provides the null terminator implicitly (the reference + engine relies on the zero-initialised buffer for termination). + """ + encoded = str_value.encode("latin-1", errors="replace") + n_bytes = size_in_bit // 8 + padded = encoded[:n_bytes].ljust(n_bytes, b"\x00") + return int.from_bytes(padded, "big") + + +def _encode_date(str_value: str, tc: ParameterTypeTypeDate) -> int | None: + """KNX date: 3 octets day, month, year mod 100 (value ``YYYY-MM-DD``). + + When the type does not display the year, the year octet stays zero (the + reference engine writes only day and month for that formatting). + """ + parts = str_value.split("-") + if len(parts) != 3: + return None + try: + year, month, day = (int(p) for p in parts) + except ValueError: + return None + year_octet = 0 if tc.display_the_year is False else year % 100 + return (day << 16) | (month << 8) | year_octet + + +def _encode_ipaddress(str_value: str) -> int | None: + """IPv4 dotted-quad to 4 octets in network (big-endian) order.""" + parts = str_value.split(".") + if len(parts) != 4: + return None + try: + octets = [int(p) for p in parts] + except ValueError: + return None + if any(o < 0 or o > 255 for o in octets): + return None + return int.from_bytes(bytes(octets), "big") + + +def _encode_color( + str_value: str, size_in_bit: int, tc: ParameterTypeTypeColor +) -> int | None: + """Colour from a ``#RRGGBB``/``#RRGGBBWW`` hex string, per the type's space. + + - RGB: three octets ``[R, G, B]``. + - RGBW: four octets ``[R, G, B, W]``. + - HSV: three octets ``[H, S, V]`` converted from the RGB value (H scaled to a + single octet), matching the reference engine's RGB-to-HSV conversion. + """ + try: + raw = bytes.fromhex(str_value.lstrip("#")) + except ValueError: + return None + if tc.space == ParameterTypeTypeColorSpace.RGBW: + raw = raw[:4].ljust(4, b"\x00") + return int.from_bytes(raw, "big") + if len(raw) < 3: + return None + r, g, b = raw[0], raw[1], raw[2] + if tc.space == ParameterTypeTypeColorSpace.HSV: + h, s, v = _rgb_to_hsv(r, g, b) + return (h << 16) | (s << 8) | v + return (r << 16) | (g << 8) | b + + +def _rgb_to_hsv(r: int, g: int, b: int) -> tuple[int, int, int]: + """Convert an RGB triple to the KNX HSV octet triple (H scaled to 0..255).""" + low = min(r, g, b) + high = max(r, g, b) + if low == high: + hue = 0.0 + elif high == r: + hue = 60.0 * (g - b) / (high - low) + elif high == g: + hue = 60.0 * (2.0 + (b - r) / (high - low)) + else: + hue = 60.0 * (4.0 + (r - g) / (high - low)) + if hue < 0.0: + hue += 360.0 + elif hue > 360.0: + hue -= 360.0 + saturation = 0 if high == 0 else int(255.0 * (high - low) / high) + return int(255.0 * hue / 360.0), saturation, high + + +def _encode_raw_data( + str_value: str, size_in_bit: int, little_endian: bool = False +) -> int | None: + """Raw octets from a hex string, truncated/zero-padded to the field width. + + For a little-endian program the octets are preceded by the data length as a + four octet little-endian prefix (matching the reference engine), then the whole + is padded to the field width. + """ + try: + data = bytes.fromhex(str_value) + except ValueError: + return None + if little_endian: + data = len(data).to_bytes(4, "little") + data + n_bytes = size_in_bit // 8 + padded = data[:n_bytes].ljust(n_bytes, b"\x00") + return int.from_bytes(padded, "big") if padded else 0 + + def resolve_param_values(idx: ApplicationIndexer, state: GlobalState) -> dict[str, str]: - """Build {param_id: state_value} for parameters with an explicit user override in state. + """Build {referenced_id: effective_value} for the ACTIVE ParameterRefs. - Does not include static defaults — collect_writes reads those directly from the - parameter objects as it iterates the static model. + ETS encodes each memory/property cell from the ParameterRef that is active in + the resolved UI, taking its instance value, else its ref default, else the base + Parameter default (``state.get`` resolves that whole chain). Parameters whose + ref is not active are not encoded at all - their cells stay at the segment seed. + + Keys are the referenced ``Parameter``/``UnionParameter`` id (``pr.ref_id``), + matching ``item.id`` in :func:`_collect_param` and ``up.id`` in a union pick. + Module-qualified active refs are not in ``idx.parameter_refs``; the module + collection path resolves those from its own instance state instead. """ - state_values = dict(state.relative_param_values()) + # When several active refs target the same parameter cell (the evaluator can + # reach more than one in overlapping conditional branches), the reference + # engine keeps the first one in definition order. Iterate parameter_refs in + # their (XML/insertion) order and take the first active ref per parameter. + active = state.active_param_refs() overrides: dict[str, str] = {} - for pr_id, pr in idx.parameter_refs.items(): - state_val = state_values.get(pr_id) - if state_val is None: + for ref_id, pr in idx.parameter_refs.items(): + if ref_id not in active or pr.ref_id in overrides: + continue + value = state.get(ref_id) + if value is not None: + overrides[pr.ref_id] = value + # Explicitly configured union alternatives drive the union selection even if + # the evaluator did not mark their ref active; carry them too. + for ref_id, _ in state.relative_param_values(): + pr = idx.parameter_refs.get(ref_id) + if pr is None or not isinstance(idx.parameters.get(pr.ref_id), UnionParameter): continue - param = idx.parameters.get(pr.ref_id) - if param is not None: - overrides[param.id] = state_val + value = state.get(ref_id) + if value is not None: + overrides[pr.ref_id] = value return overrides @@ -180,22 +410,26 @@ def _build_instance_overrides( return overrides -def _pick_union_param( +def _union_params_to_write( parameters: list[UnionParameter], overrides: dict[str, str], - location: str, -) -> tuple[UnionParameter, str] | None: + gated: bool = False, +) -> list[tuple[UnionParameter, str]]: + """Return the union alternatives to write, each with its value. + + Union alternatives can sit at different bit offsets within the shared cell, so + more than one may be active at once; each active alternative is written (their + bits accumulate), matching the reference engine. When nothing is active: in a + resolved UI (gated) the cell keeps its seed; a bare caller falls back to the + union's default alternative so ``collect_writes`` still yields a value. + """ active = [up for up in parameters if up.id in overrides] - assert len(active) <= 1, ( - f"union at {location} has {len(active)} active alternatives: " - + ", ".join(up.id for up in active) - ) if active: - return active[0], overrides[active[0].id] + return [(up, overrides[up.id]) for up in active] + if gated: + return [] default_up = next((up for up in parameters if up.default_union_parameter), None) - if default_up is not None: - return default_up, default_up.value - return None + return [(default_up, default_up.value)] if default_up is not None else [] def _collect_param( @@ -204,7 +438,12 @@ def _collect_param( overrides: dict[str, str], ms: ModuleState | None, out: Writes, + active_ids: set[str] | None = None, ) -> None: + # When an active-ref set is given (resolved UI), encode only active parameters; + # inactive cells are left at the segment seed, as the reference engine does. + if active_ids is not None and item.id not in active_ids: + return choice = item.choice value = overrides.get(item.id) or item.value # base_value on a module parameter shifts the encoded value by an arg-resolved offset. @@ -282,22 +521,18 @@ def _collect_union( overrides: dict[str, str], ms: ModuleState | None, out: Writes, + gated: bool = False, ) -> None: choice = item.choice if choice is None: return + selected = _union_params_to_write(item.parameter, overrides, gated) # Check subclasses before parents (module types extend their top-level counterparts) if isinstance(choice, ModuleDefStaticParametersUnionMemory): assert ms is not None base = _resolve_base(choice.base_offset, ms) if base is not None: - picked = _pick_union_param( - item.parameter, - overrides, - f"{choice.code_segment}+{base + choice.offset}", - ) - if picked is not None: - up, value = picked + for up, value in selected: out.mem.append( MemWrite( choice.code_segment, @@ -309,11 +544,7 @@ def _collect_union( ) ) elif isinstance(choice, MemoryUnion): - picked = _pick_union_param( - item.parameter, overrides, f"{choice.code_segment}+{choice.offset}" - ) - if picked is not None: - up, value = picked + for up, value in selected: out.mem.append( MemWrite( choice.code_segment, @@ -331,13 +562,7 @@ def _collect_union( boc = _resolve_base(choice.base_occurrence, ms) if bo is not None and bi is not None and boc is not None: obj_idx = (choice.object_index or 0) + bi if bi else choice.object_index - picked = _pick_union_param( - item.parameter, - overrides, - f"prop_id={choice.property_id}+{bo + choice.offset}", - ) - if picked is not None: - up, value = picked + for up, value in selected: out.prop.append( PropWrite( obj_idx, @@ -352,11 +577,7 @@ def _collect_union( ) else: assert isinstance(choice, PropertyUnion) - picked = _pick_union_param( - item.parameter, overrides, f"prop_id={choice.property_id}+{choice.offset}" - ) - if picked is not None: - up, value = picked + for up, value in selected: out.prop.append( PropWrite( choice.object_index, @@ -397,13 +618,22 @@ def collect_writes( """Collect all parameter writes into a Writes container (mem + prop), separated by destination type.""" out = Writes() s = app.static + # With a resolved state, gate top-level parameters to the active set (regular + # parameters present in overrides); inactive cells stay at the segment seed. + # Without a state (direct callers), no gating: every static parameter encodes + # as before. Unions keep their explicit-selection behaviour either way. + active_ids = ( + {k for k in overrides if not isinstance(idx.parameters.get(k), UnionParameter)} + if state is not None + else None + ) if s.parameters is not None: for item in s.parameters.choice: if isinstance(item, ApplicationProgramStaticParametersParameter): - _collect_param(item, overrides, None, out) + _collect_param(item, overrides, None, out, active_ids) else: assert isinstance(item, ApplicationProgramStaticParametersUnion) - _collect_union(item, overrides, None, out) + _collect_union(item, overrides, None, out, gated=active_ids is not None) if state is not None: for ms in state.module_children(): _collect_module_writes(ms, idx, out) @@ -422,6 +652,7 @@ def encode_to_memory( Bit layout: bit_offset=0 is the MSB of each byte; values stored big-endian. """ writes = collect_writes(app, idx, overrides, state) + little_endian = _program_little_endian(app) bufs: dict[str, bytearray] = { seg_id: bytearray(seg.data) if seg.data else bytearray(seg.size) for seg_id, seg in idx.code_segments.items() @@ -437,13 +668,61 @@ def encode_to_memory( size_in_bit = getattr(tc, "size_in_bit", None) if size_in_bit is None: continue - encoded = _encode_value(w.value, size_in_bit, tc) + encoded = _encode_value(w.value, size_in_bit, tc, little_endian=little_endian) if encoded is None: continue _write_bits(buf, w.offset, w.bit_offset, size_in_bit, encoded) return {seg_id: bytes(buf) for seg_id, buf in bufs.items()} +def encode_to_memory_masked( + app: ApplicationProgram, + idx: ApplicationIndexer, + overrides: dict[str, str], + state: GlobalState | None = None, +) -> dict[str, tuple[bytes, bytes]]: + """Like :func:`encode_to_memory` but also return a write mask per segment. + + Returns ``{segment_id: (data, mask)}`` where ``mask`` has one byte per data + byte: ``0xFF`` for bytes an encoded parameter actually wrote, ``0x00`` for + bytes left at the segment seed. A downloader writes only masked bytes, so it + never overwrites regions this encoder does not produce (e.g. the com object + table or RAM) - mirroring the reference engine, which loads only touched + bytes on a partial download. + """ + writes = collect_writes(app, idx, overrides, state) + little_endian = _program_little_endian(app) + bufs: dict[str, bytearray] = { + seg_id: bytearray(seg.data) if seg.data else bytearray(seg.size) + for seg_id, seg in idx.code_segments.items() + } + masks: dict[str, bytearray] = { + seg_id: bytearray(len(buf)) for seg_id, buf in bufs.items() + } + for w in writes.mem: + buf = bufs.get(w.seg_id) + if buf is None: + continue + pt = idx.parameter_types.get(w.parameter_type) + if pt is None: + continue + tc = pt.choice + size_in_bit = getattr(tc, "size_in_bit", None) + if size_in_bit is None: + continue + encoded = _encode_value(w.value, size_in_bit, tc, little_endian=little_endian) + if encoded is None: + continue + _write_bits(buf, w.offset, w.bit_offset, size_in_bit, encoded) + start_bit = w.offset * 8 + w.bit_offset + end_bit = start_bit + size_in_bit - 1 + mask = masks[w.seg_id] + for b in range(start_bit // 8, end_bit // 8 + 1): + if b < len(mask): + mask[b] = 0xFF + return {seg_id: (bytes(buf), bytes(masks[seg_id])) for seg_id, buf in bufs.items()} + + def build_memory_param_map( app: ApplicationProgram, idx: ApplicationIndexer, @@ -485,6 +764,7 @@ def encode_to_properties( Buffers are sized dynamically to fit all writes. """ writes = collect_writes(app, idx, overrides, state) + little_endian = _program_little_endian(app) bufs: dict[PropertyKey, bytearray] = {} for w in writes.prop: pt = idx.parameter_types.get(w.parameter_type) @@ -494,7 +774,7 @@ def encode_to_properties( size_in_bit = getattr(tc, "size_in_bit", None) if not size_in_bit: continue - encoded = _encode_value(w.value, size_in_bit, tc) + encoded = _encode_value(w.value, size_in_bit, tc, little_endian=little_endian) if encoded is None: continue key: PropertyKey = (w.object_index, w.property_id, w.occurrence) diff --git a/packages/product/src/xknxmono/product/parser_v2/nodes/com_object_ref_ref.py b/packages/product/src/xknxmono/product/parser_v2/nodes/com_object_ref_ref.py index f6cb54fe..ada6c161 100644 --- a/packages/product/src/xknxmono/product/parser_v2/nodes/com_object_ref_ref.py +++ b/packages/product/src/xknxmono/product/parser_v2/nodes/com_object_ref_ref.py @@ -46,6 +46,8 @@ class ComObjectRefRefNode(DynamicNode): "_local_ref_id", "_name_template", "_number", + "_object_size", + "_priority", "_read", "_read_locked", "_read_on_init", @@ -84,6 +86,11 @@ def __init__( cor.datapoint_type if cor else [], co.datapoint_type if co else [], ) + # Size and priority: the ref overrides the base, else the base value. + size = (cor.object_size if cor else None) or (co.object_size if co else None) + self._object_size: str = size.value if size is not None else "" + priority = (cor.priority if cor else None) or (co.priority if co else None) + self._priority: str = priority.value if priority is not None else "" self._communication = _flag( cor.communication_flag if cor else None, co.communication_flag if co else None, @@ -133,6 +140,8 @@ def eval(self, ctx: EvalContext) -> list[UiNode]: name=name, number=self._number + base, dpt_codes=self._dpt_codes, + object_size=self._object_size, + priority=self._priority, communication=_flag( ov.communication_flag if ov else None, Enable.ENABLED if self._communication else Enable.DISABLED, diff --git a/packages/product/src/xknxmono/product/parser_v2/ui/com_object.py b/packages/product/src/xknxmono/product/parser_v2/ui/com_object.py index 87f25dd1..1c9fe930 100644 --- a/packages/product/src/xknxmono/product/parser_v2/ui/com_object.py +++ b/packages/product/src/xknxmono/product/parser_v2/ui/com_object.py @@ -9,6 +9,10 @@ class UiComObject: name: str # resolved display name ({{0}} filled from text_parameter_ref_id) number: int dpt_codes: tuple[str, ...] # e.g. ("1.0", "1.1") + object_size: ( + str # resolved ComObjectSize value, e.g. "1 Bit" / "1 Byte" ("" if unset) + ) + priority: str # resolved ComObjectPriority value, e.g. "Low" ("" if unset) communication: bool read: bool write: bool diff --git a/packages/product/tests/parser_v2/test_encode_types.py b/packages/product/tests/parser_v2/test_encode_types.py new file mode 100644 index 00000000..1617ac01 --- /dev/null +++ b/packages/product/tests/parser_v2/test_encode_types.py @@ -0,0 +1,228 @@ +"""Unit tests for the per-type value encoders (_encode_value). + +Values are checked against the KNX Datapoint Type byte encodings; the integer the +encoder returns is packed big-endian (MSB first) by _write_bits. +""" + +from __future__ import annotations + +from xknxmono.models.intermediate.parameter_type_t_type_color import ( + ParameterTypeTypeColor, +) +from xknxmono.models.intermediate.parameter_type_t_type_color_space import ( + ParameterTypeTypeColorSpace, +) +from xknxmono.models.intermediate.parameter_type_t_type_date import ( + ParameterTypeTypeDate, +) +from xknxmono.models.intermediate.parameter_type_t_type_date_encoding import ( + ParameterTypeTypeDateEncoding, +) +from xknxmono.models.intermediate.parameter_type_t_type_float import ( + ParameterTypeTypeFloat, +) +from xknxmono.models.intermediate.parameter_type_t_type_float_encoding import ( + ParameterTypeTypeFloatEncoding, +) +from xknxmono.models.intermediate.parameter_type_t_type_ipaddress import ( + ParameterTypeTypeIpaddress, +) +from xknxmono.models.intermediate.parameter_type_t_type_ipaddress_address_type import ( + ParameterTypeTypeIpaddressAddressType, +) +from xknxmono.models.intermediate.parameter_type_t_type_number import ( + ParameterTypeTypeNumber, +) +from xknxmono.models.intermediate.parameter_type_t_type_number_type import ( + ParameterTypeTypeNumberType, +) +from xknxmono.models.intermediate.parameter_type_t_type_raw_data import ( + ParameterTypeTypeRawData, +) +from xknxmono.models.intermediate.parameter_type_t_type_restriction import ( + ParameterTypeTypeRestriction, +) +from xknxmono.models.intermediate.parameter_type_t_type_restriction_enumeration import ( + ParameterTypeTypeRestrictionEnumeration, +) +from xknxmono.models.intermediate.parameter_type_t_type_text import ( + ParameterTypeTypeText, +) +from xknxmono.models.intermediate.parameter_type_t_type_time import ( + ParameterTypeTypeTime, +) +from xknxmono.models.intermediate.parameter_type_t_type_time_unit import ( + ParameterTypeTypeTimeUnit, +) +from xknxmono.product.parser_v2.encode import _encode_value + + +def _number() -> ParameterTypeTypeNumber: + return ParameterTypeTypeNumber( + size_in_bit=8, + type_value=ParameterTypeTypeNumberType.UNSIGNED_INT, + min_inclusive=0, + max_inclusive=255, + ) + + +def _float(encoding: ParameterTypeTypeFloatEncoding) -> ParameterTypeTypeFloat: + return ParameterTypeTypeFloat(encoding=encoding, min_inclusive=0, max_inclusive=0) + + +def _text() -> ParameterTypeTypeText: + return ParameterTypeTypeText(size_in_bit=0) + + +def _date(display_the_year: bool = True) -> ParameterTypeTypeDate: + return ParameterTypeTypeDate( + encoding=ParameterTypeTypeDateEncoding.DPT_11, + display_the_year=display_the_year, + ) + + +def _ipaddress() -> ParameterTypeTypeIpaddress: + return ParameterTypeTypeIpaddress( + address_type=ParameterTypeTypeIpaddressAddressType.HOST_ADDRESS + ) + + +def _color(space: ParameterTypeTypeColorSpace) -> ParameterTypeTypeColor: + return ParameterTypeTypeColor(space=space) + + +def _raw_data() -> ParameterTypeTypeRawData: + return ParameterTypeTypeRawData(max_size=16) + + +def test_number_unsigned() -> None: + assert _encode_value("15", 8, _number()) == 0x0F + + +def test_number_negative_twos_complement() -> None: + assert _encode_value("-1", 8, _number()) == 0xFF + assert _encode_value("-2", 16, _number()) == 0xFFFE + + +def test_number_invalid_returns_none() -> None: + assert _encode_value("abc", 8, _number()) is None + + +def test_float_dpt9() -> None: + # 21.0 -> 0.01 * 1050 * 2^1; encoded 0x0C1A + tc = _float(ParameterTypeTypeFloatEncoding.DPT_9) + assert _encode_value("21.0", 16, tc) == 0x0C1A + assert _encode_value("0", 16, tc) == 0x0000 + + +def test_float_dpt9_negative() -> None: + tc = _float(ParameterTypeTypeFloatEncoding.DPT_9) + # -1.0 -> m = -100, exp 0, sign bit set, 11-bit two's complement of -100 + assert _encode_value("-1.0", 16, tc) == (0x8000 | (-100 & 0x7FF)) + + +def test_float_ieee_single() -> None: + tc = _float(ParameterTypeTypeFloatEncoding.IEEE_754_SINGLE) + assert _encode_value("1.0", 32, tc) == 0x3F800000 + + +def test_text_latin1_padded() -> None: + assert _encode_value("AB", 24, _text()) == 0x414200 + + +def test_text_truncated() -> None: + assert _encode_value("ABCD", 16, _text()) == 0x4142 + + +def test_date_dpt11() -> None: + # 2024-03-15 -> day 15, month 3, year%100 = 24 + assert _encode_value("2024-03-15", 24, _date()) == 0x0F0318 + + +def test_ipaddress_v4() -> None: + assert _encode_value("192.168.1.1", 32, _ipaddress()) == 0xC0A80101 + + +def test_ipaddress_invalid() -> None: + assert _encode_value("192.168.1", 32, _ipaddress()) is None + assert _encode_value("1.2.3.999", 32, _ipaddress()) is None + + +def test_color_rgb() -> None: + assert _encode_value("#FF8000", 24, _color(ParameterTypeTypeColorSpace.RGB)) == ( + 0xFF8000 + ) + + +def test_color_hsv() -> None: + # #FF8000 (255,128,0) -> H=30.12deg -> 21, S=255, V=255 + assert _encode_value("#FF8000", 24, _color(ParameterTypeTypeColorSpace.HSV)) == ( + 0x15FFFF + ) + + +def test_color_rgbw() -> None: + assert _encode_value( + "#FF800040", 32, _color(ParameterTypeTypeColorSpace.RGBW) + ) == 0xFF800040 + + +def test_raw_data_hex() -> None: + assert _encode_value("0a0b", 16, _raw_data()) == 0x0A0B + + +def test_raw_data_padded() -> None: + assert _encode_value("0a", 16, _raw_data()) == 0x0A00 + + +def test_raw_data_little_endian_length_prefix() -> None: + # little-endian program: 4 octet little-endian length prefix, then the data + assert _encode_value("0a0b", 48, _raw_data(), little_endian=True) == 0x020000000A0B + + +def test_number_little_endian_byte_swap() -> None: + # 500 = 0x01F4 big-endian; little-endian reverses the two octets + assert _encode_value("500", 16, _number()) == 0x01F4 + assert _encode_value("500", 16, _number(), little_endian=True) == 0xF401 + + +def test_date_without_year_zeroes_year_octet() -> None: + assert _encode_value("2024-03-17", 24, _date(display_the_year=False)) == ( + (17 << 16) | (3 << 8) + ) + + +def test_time_encodes_as_integer() -> None: + tc = ParameterTypeTypeTime( + size_in_bit=16, + unit=ParameterTypeTypeTimeUnit.SECONDS, + min_inclusive=0, + max_inclusive=65535, + ) + assert _encode_value("1000", 16, tc) == 1000 + assert _encode_value("1000", 16, tc, little_endian=True) == 0xE803 + + +def test_restriction_uses_enumeration_binary_value() -> None: + # An enumeration with an explicit binary value writes those octets verbatim. + tc = ParameterTypeTypeRestriction( + base="BinaryValue", + size_in_bit=24, + enumeration=[ + ParameterTypeTypeRestrictionEnumeration( + id="EN-0", value=0, binary_value=b"\x01\x00\x02" + ) + ], + ) + assert _encode_value("0", 24, tc) == 0x010002 + # even for a little-endian program the binary value is written as-is + assert _encode_value("0", 24, tc, little_endian=True) == 0x010002 + + +def test_restriction_without_binary_value_is_numeric() -> None: + tc = ParameterTypeTypeRestriction( + base="Value", + size_in_bit=8, + enumeration=[ParameterTypeTypeRestrictionEnumeration(id="EN-5", value=5)], + ) + assert _encode_value("5", 8, tc) == 5 diff --git a/packages/project/CHANGELOG.md b/packages/project/CHANGELOG.md index f733058f..3681e9e0 100644 --- a/packages/project/CHANGELOG.md +++ b/packages/project/CHANGELOG.md @@ -10,5 +10,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - KNX project state management on top of `xknx-models`. -- Load and save `.knxproj` project archives. +- `import_knxproj()` — import an ETS `.knxproj` archive into a new project (topology, group + addresses with DPTs, devices, and com-object links), parsed via `xknxproject`. - Typed read/write access to group addresses, topology, and device configuration. diff --git a/packages/project/pyproject.toml b/packages/project/pyproject.toml index 4014e3b4..07ccc214 100644 --- a/packages/project/pyproject.toml +++ b/packages/project/pyproject.toml @@ -6,6 +6,7 @@ requires-python = ">=3.12" dependencies = [ "sqlalchemy>=2.0.0", "xknx-models", + "xknxproject>=3.10", ] [build-system] diff --git a/packages/project/src/xknxmono/project/__init__.py b/packages/project/src/xknxmono/project/__init__.py index fe1c6700..a0a9000e 100644 --- a/packages/project/src/xknxmono/project/__init__.py +++ b/packages/project/src/xknxmono/project/__init__.py @@ -14,6 +14,6 @@ __version__ = "0.1.0" -from xknxmono.project.core import ProjectService +from xknxmono.project.core import ProjectService, export_knxproj, import_knxproj -__all__ = ["ProjectService"] +__all__ = ["ProjectService", "export_knxproj", "import_knxproj"] diff --git a/packages/project/src/xknxmono/project/core/__init__.py b/packages/project/src/xknxmono/project/core/__init__.py index 2c8fe77e..ff80250e 100644 --- a/packages/project/src/xknxmono/project/core/__init__.py +++ b/packages/project/src/xknxmono/project/core/__init__.py @@ -1,6 +1,8 @@ """Core project domain: a relational SQLite store edited through a command/event log.""" from xknxmono.project.core.event_store import EventStore +from xknxmono.project.core.knxproj_export import export_knxproj +from xknxmono.project.core.knxproj_import import import_knxproj from xknxmono.project.core.service import ProjectService -__all__ = ["EventStore", "ProjectService"] +__all__ = ["EventStore", "ProjectService", "export_knxproj", "import_knxproj"] diff --git a/packages/project/src/xknxmono/project/core/events.py b/packages/project/src/xknxmono/project/core/events.py index dd364211..ea01ebbd 100644 --- a/packages/project/src/xknxmono/project/core/events.py +++ b/packages/project/src/xknxmono/project/core/events.py @@ -603,6 +603,38 @@ def from_dict(cls, data: dict[str, Any]) -> SetGroupAddressDatapointType: return cls(**data) +@_register +@dataclass +class RenameGroupAddress(Event): + event_type: ClassVar[str] = "RenameGroupAddress" + + group_address_id: int + name: str + old_name: str | None = None + + def apply(self, session: Session) -> None: + ga = session.get(GroupAddress, self.group_address_id) + if ga is not None: + self.old_name = ga.name + ga.name = self.name + + def revert(self, session: Session) -> None: + ga = session.get(GroupAddress, self.group_address_id) + if ga is not None and self.old_name is not None: + ga.name = self.old_name + + def to_dict(self) -> dict[str, Any]: + return { + "group_address_id": self.group_address_id, + "name": self.name, + "old_name": self.old_name, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RenameGroupAddress: + return cls(**data) + + @_register @dataclass class SetComObjectSending(Event): diff --git a/packages/project/src/xknxmono/project/core/knxproj_export.py b/packages/project/src/xknxmono/project/core/knxproj_export.py new file mode 100644 index 00000000..a2eaaf17 --- /dev/null +++ b/packages/project/src/xknxmono/project/core/knxproj_export.py @@ -0,0 +1,332 @@ +"""Export a project SQLite document to a simple ETS ``.knxproj`` archive. + +This is a *simple* export: it writes the project structure (topology, devices with their +com-object → group-address links, the group-address tree, and the locations tree of spaces with +their device/function assignments) plus project metadata, in the ETS project XML shape +(schema ``project/20``) so it round-trips back through :func:`import_knxproj`. + +Manufacturer/application data (Hardware/Catalog/application program XMLs) is **not** read from a +catalog here — this package is catalog-free. Callers that have the catalog can pass those raw +archive members via ``extra_files`` (and a merged ``knx_master.xml`` via ``master_xml``) so the +resulting archive is self-contained and applications resolve. The **project-level** +``{pid}.signature`` is generated with a valid signature (see :mod:`knxproj_signing`); +manufacturer ``M-XXXX.signature`` files, when needed, come from the catalog via ``extra_files``. +""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET +import zipfile +from collections.abc import Mapping +from pathlib import Path + +from sqlalchemy.orm import Session + +from xknxmono.project.core.knxproj_signing import directory_signature +from xknxmono.project.db import make_engine, url_for +from xknxmono.project.models import ( + Device, + GroupRange, + Installation, + Project, + Space, +) + +_NS = "http://knx.org/xml/project/20" +_SCHEMA = "20" + + +def export_knxproj( + source: Path | str, + dest: Path | str, + *, + extra_files: Mapping[str, bytes] | None = None, + master_xml: bytes | None = None, +) -> None: + """Read the project at ``source`` (a ``.xknx``) and write a ``.knxproj`` archive to ``dest``. + + Args: + source: Path to the ``.xknx`` project document. + dest: Path to write the ``.knxproj`` archive to. + extra_files: Optional raw archive members to add verbatim (e.g. manufacturer ``M-XXXX/`` + trees and their ``M-XXXX.signature`` files, supplied by a caller that has the catalog). + Keys colliding with the export's own paths are ignored. + master_xml: Optional ``knx_master.xml`` bytes to write instead of the minimal generated one. + """ + engine = make_engine(url_for(Path(source))) + try: + with Session(engine) as session: + project = session.query(Project).first() + if project is None: + raise ValueError(f"{source} is not a project (no project row)") + installation = ( + session.query(Installation).order_by(Installation.index).first() + ) + _write_archive(Path(dest), project, installation, extra_files, master_xml) + finally: + engine.dispose() + + +def _el(parent: ET.Element, tag: str, **attrs: object) -> ET.Element: + child = ET.SubElement(parent, f"{{{_NS}}}{tag}") + for key, value in attrs.items(): + if value is not None: + child.set(key, str(value)) + return child + + +def _root() -> ET.Element: + return ET.Element(f"{{{_NS}}}KNX") + + +def _write_archive( + dest: Path, + project: Project, + installation: Installation | None, + extra_files: Mapping[str, bytes] | None, + master_xml: bytes | None, +) -> None: + pid = project.id + ga_link_id, di_id = _assign_ids(pid, installation) + + if master_xml is None: + master = _root() + _el(master, "MasterData", Version="1", Signature="") + master_xml = _serialize(master) + + proj_xml = _root() + p = _el(proj_xml, "Project", Id=pid) + _el( + p, + "ProjectInformation", + Name=project.name, + GroupAddressStyle=project.group_address_style, + Guid=project.guid or None, + LastModified=project.last_modified or None, + ) + + zero_xml = _build_project_xml(pid, project, installation, ga_link_id, di_id) + + own_paths = { + "knx_master.xml", + f"{pid}.signature", + f"{pid}/project.xml", + f"{pid}/0.xml", + } + project_xml = _serialize(proj_xml) + zero = _serialize(zero_xml) + # Sign the project folder so a strict import accepts it (see knxproj_signing). + # The reference tooling writes the .signature file as UTF-8 with a BOM. + project_signature = b"\xef\xbb\xbf" + directory_signature( + {"project.xml": project_xml, "0.xml": zero} + ) + dest.unlink(missing_ok=True) + with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("knx_master.xml", master_xml) + zf.writestr(f"{pid}.signature", project_signature) + zf.writestr(f"{pid}/project.xml", project_xml) + zf.writestr(f"{pid}/0.xml", zero) + for path, data in (extra_files or {}).items(): + if path not in own_paths: + zf.writestr(path, data) + + +def _assign_ids( + pid: str, installation: Installation | None +) -> tuple[dict[int, str], dict[int, str]]: + """Assign the stable element ids ETS uses: GA link tokens (``GA-n``) and device ids.""" + ga_link_id: dict[int, str] = {} + di_id: dict[int, str] = {} + if installation is None: + return ga_link_id, di_id + ga_seq = 0 + for group_range in installation.group_ranges: + for ga in group_range.group_addresses: + ga_seq += 1 + ga_link_id[ga.id] = f"GA-{ga_seq}" + di_seq = 0 + for area in installation.areas: + for line in area.lines: + for segment in line.segments: + for device in segment.devices: + di_seq += 1 + di_id[device.id] = f"{pid}-0_DI-{di_seq}" + return ga_link_id, di_id + + +def _build_project_xml( + pid: str, + project: Project, + installation: Installation | None, + ga_link_id: dict[int, str], + di_id: dict[int, str], +) -> ET.Element: + root = _root() + p = _el(root, "Project", Id=pid) + insts = _el(p, "Installations") + if installation is None: + return root + inst = _el(insts, "Installation", Name=installation.name) + + topo = _el(inst, "Topology") + l_seq = 0 + for a_seq, area in enumerate( + sorted(installation.areas, key=lambda x: x.address), start=1 + ): + area_el = _el( + topo, + "Area", + Id=f"{pid}-0_A-{a_seq}", + Address=area.address, + Name=area.name, + ) + for line in sorted(area.lines, key=lambda x: x.address): + l_seq += 1 + medium = line.segments[0].medium_type if line.segments else "MT-0" + line_el = _el( + area_el, + "Line", + Id=f"{pid}-0_L-{l_seq}", + Address=line.address, + Name=line.name, + MediumTypeRefId=medium, + ) + for segment in line.segments: + for device in segment.devices: + _build_device(line_el, device, di_id[device.id], ga_link_id) + + _build_locations(inst, installation, pid, di_id, ga_link_id) + + gas = _el(inst, "GroupAddresses") + ranges = _el(gas, "GroupRanges") + gr_seq = 0 + roots = [gr for gr in installation.group_ranges if gr.parent_id is None] + for group_range in sorted(roots, key=lambda x: x.range_start): + gr_seq = _build_range(ranges, group_range, pid, gr_seq, ga_link_id) + return root + + +def _build_locations( + inst_el: ET.Element, + installation: Installation, + pid: str, + di_id: dict[int, str], + ga_link_id: dict[int, str], +) -> None: + roots = [s for s in installation.spaces if s.parent_id is None] + if not roots: + return + locs = _el(inst_el, "Locations") + counters = {"sp": 0, "f": 0, "gar": 0} + for space in sorted(roots, key=lambda s: (s.order, s.id)): + _build_space(locs, space, pid, di_id, ga_link_id, counters) + + +def _build_space( + parent: ET.Element, + space: Space, + pid: str, + di_id: dict[int, str], + ga_link_id: dict[int, str], + counters: dict[str, int], +) -> None: + counters["sp"] += 1 + sp = _el( + parent, + "Space", + Type=space.space_type or "Room", + Id=f"{pid}-0_BP-{counters['sp']}", + Name=space.name, + Number=space.number or None, + ) + for device in space.devices: + if device.id in di_id: + _el(sp, "DeviceInstanceRef", RefId=di_id[device.id]) + for fn in sorted(space.functions, key=lambda f: (f.order, f.id)): + counters["f"] += 1 + fn_el = _el( + sp, + "Function", + Id=f"{pid}-0_F-{counters['f']}", + Name=fn.name, + Type=fn.function_type or None, + ) + for fga in fn.group_addresses: + if fga.group_address_id not in ga_link_id: + continue + counters["gar"] += 1 + _el( + fn_el, + "GroupAddressRef", + Id=f"{pid}-0_GAR-{counters['gar']}", + RefId=f"{pid}-0_{ga_link_id[fga.group_address_id]}", + Role=fga.role or None, + ) + for child in sorted(space.children, key=lambda s: (s.order, s.id)): + _build_space(sp, child, pid, di_id, ga_link_id, counters) + + +def _build_device( + line_el: ET.Element, + device: Device, + device_id: str, + ga_link_id: dict[int, str], +) -> None: + di = _el( + line_el, + "DeviceInstance", + Id=device_id, + Address=device.address, + Name=device.name, + ProductRefId=device.product_ref_id, + Hardware2ProgramRefId=device.hardware2program_ref_id, + ) + refs = _el(di, "ComObjectInstanceRefs") + for co in device.com_objects: + # ETS orders the sending link first; our ComObjectLink.is_sending marks it. + ordered = sorted(co.links, key=lambda link: not link.is_sending) + links = " ".join( + ga_link_id[link.group_address_id] + for link in ordered + if link.group_address_id in ga_link_id + ) + if not links: + continue # xknxproject only keeps linked com-object instance refs + _el(refs, "ComObjectInstanceRef", RefId=co.ref_id, Links=links) + + +def _build_range( + parent: ET.Element, + group_range: GroupRange, + pid: str, + seq: int, + ga_link_id: dict[int, str], +) -> int: + seq += 1 + gr_el = _el( + parent, + "GroupRange", + Id=f"{pid}-0_GR-{seq}", + RangeStart=group_range.range_start, + RangeEnd=group_range.range_end, + Name=group_range.name, + ) + for ga in sorted(group_range.group_addresses, key=lambda x: x.address): + _el( + gr_el, + "GroupAddress", + Id=f"{pid}-0_{ga_link_id[ga.id]}", + Address=ga.address, + Name=ga.name, + DatapointType=ga.datapoint_type, + ) + for child in sorted(group_range.children, key=lambda x: x.range_start): + seq = _build_range(gr_el, child, pid, seq, ga_link_id) + return seq + + +def _serialize(root: ET.Element) -> bytes: + ET.register_namespace("", _NS) + return b'\n' + ET.tostring( + root, encoding="unicode" + ).encode("utf-8") diff --git a/packages/project/src/xknxmono/project/core/knxproj_import.py b/packages/project/src/xknxmono/project/core/knxproj_import.py new file mode 100644 index 00000000..e21831db --- /dev/null +++ b/packages/project/src/xknxmono/project/core/knxproj_import.py @@ -0,0 +1,364 @@ +"""Import an ETS ``.knxproj`` archive into a fresh project SQLite document. + +Parsing is delegated to the ``xknxproject`` library. A ``.knxproj`` bundles its application +program XMLs, so ``xknxproject`` resolves every device's com-object/parameter/module refs and its +group-address links straight from the archive — the toolkit catalog is *not* needed. That keeps this +importer inside ``xknx-project`` (which depends only on ``xknx-models``) without pulling in the +catalog/product layers. + +The public ``parse()`` dict drops the ETS reference ids (``product_ref`` / ``hardware_program_ref``) +that a project :class:`~xknxmono.project.models.Device` requires, so we drive the internal +``XMLParser`` and read its populated ``devices`` / ``areas`` / ``group_ranges`` / +``group_addresses`` attributes instead. + +The whole project is written directly through the ORM in one commit (like +:func:`~xknxmono.project.core.skeleton.seed_new_project`): an import is initial state, not a sequence +of undoable edits, so it produces a valid zero-event project — far faster than one event per row for +a large installation, and it preserves the ETS group-range names. +""" + +from __future__ import annotations + +import zlib +from pathlib import Path +from uuid import uuid4 +from zipfile import BadZipFile + +from sqlalchemy.orm import Session +from xknxproject.exceptions import InvalidPasswordException, UnexpectedFileContent +from xknxproject.models.knxproject import DPTType +from xknxproject.models.models import ( + ComObjectInstanceRef, + DeviceInstance, + XMLGroupRange, + XMLSpace, +) +from xknxproject.xml import XMLParser +from xknxproject.zip.extractor import extract + +from xknxmono.project.core.addressing import GroupAddressStyle +from xknxmono.project.db import make_engine, url_for +from xknxmono.project.models import ( + Area, + ComObject, + ComObjectLink, + Device, + Function, + FunctionGroupAddress, + GroupAddress, + GroupRange, + Installation, + Line, + ModuleInstance, + Parameter, + Project, + Segment, + Space, +) + +_INSTALLATION_INDEX = 0 + + +def import_knxproj( + source: Path | str, + dest: Path | str, + *, + password: str | None = None, + language: str | None = None, + project_id: str | None = None, +) -> str: + """Parse ``source`` (a ``.knxproj``) and write it as a new project at ``dest``. + + ``password`` is the ETS project password (for encrypted ETS5/6 archives); ``language`` picks the + translation (e.g. ``"de-DE"``), falling back to the project default. Returns the project id. + + Raises :class:`~xknxproject.exceptions.InvalidPasswordException` when the archive is protected + and no/a wrong password was given (standard-zip encryption reports a wrong password only as a + decompression error, so any parse failure with a password set is treated as a wrong password), + and :class:`~xknxproject.exceptions.UnexpectedFileContent` when the file is not a readable + ``.knxproj`` archive. An existing ``dest`` is overwritten (an import is a fresh project). + """ + parser = _parse_checked(source, password, language) + pid = project_id or f"P-{uuid4().hex[:8].upper()}" + # An import always produces a new project; start from a clean file so re-importing over an + # existing target does not clash with its rows. Done only after a successful parse, so a failed + # import (e.g. wrong password) never destroys an existing project. + dest_path = Path(dest) + dest_path.unlink(missing_ok=True) + engine = make_engine(url_for(dest_path)) + try: + with Session(engine) as session: + _build(session, parser, pid) + session.commit() + finally: + engine.dispose() + return pid + + +def _parse_checked( + source: Path | str, password: str | None, language: str | None +) -> XMLParser: + """Parse, normalising the assorted low-level failure modes into a clear, typed error.""" + try: + return _parse(source, password, language) + except InvalidPasswordException: + raise # protected + no password (extractor reports this eagerly) + except (BadZipFile, zlib.error, RuntimeError) as e: + # Standard-zip (ETS4/5) encryption has only a one-byte password check: a wrong password may + # fail the header check (RuntimeError) or slip through and fail decompression later + # (zlib.error). With a password set, treat any such failure as a wrong password. + if password is not None: + raise InvalidPasswordException("Invalid password.") from e + raise UnexpectedFileContent(f"Not a readable .knxproj archive: {e}") from e + + +def _parse(source: Path | str, password: str | None, language: str | None) -> XMLParser: + with extract(Path(source), password) as contents: + parser = XMLParser(contents) + parser.parse( + language + ) # populates parser.* ; the returned dict drops the ref ids we need + return parser + + +def _build(session: Session, parser: XMLParser, pid: str) -> None: + info = parser.project_info + session.add( + Project( + id=pid, + name=info.name, + group_address_style=_style(info.group_address_style.value), + guid=info.guid, + created_by=info.created_by, + last_modified=info.last_modified or "", + schema_version=info.schema_version, + tool_version=info.tool_version, + ) + ) + + installation = Installation(index=_INSTALLATION_INDEX, name="") + session.add(installation) + + state = _ImportState() + _build_topology(installation, parser, state) + _build_group_addresses(installation, parser, state) + _build_links(state) + _build_spaces(installation, parser, state) + + +def _style(value: str) -> GroupAddressStyle: + try: + return GroupAddressStyle(value) + except ValueError: + return GroupAddressStyle.THREE_LEVEL + + +# --- topology + devices --------------------------------------------------- + + +def _build_topology( + installation: Installation, parser: XMLParser, state: _ImportState +) -> None: + for xarea in parser.areas: + area = Area(address=xarea.address, name=xarea.name) + installation.areas.append(area) + for xline in xarea.lines: + line = Line(address=xline.address, name=xline.name) + area.lines.append(line) + segment = Segment(number=0, medium_type=xline.medium_type) + line.segments.append(segment) + for xdevice in xline.devices: + segment.devices.append(_build_device(xdevice, state)) + + +def _build_device(xdevice: DeviceInstance, state: _ImportState) -> Device: + com_objects: list[ComObject] = [] + for coir in xdevice.com_object_instance_refs: + row = _build_com_object(coir) + com_objects.append(row) + state.register_com_object(coir, row) + device = Device( + address=xdevice.address, + name=xdevice.name, + product_ref_id=xdevice.product_ref, + hardware2program_ref_id=xdevice.hardware_program_ref, + description=xdevice.description, + order_number=xdevice.order_number, + hardware_name=xdevice.hardware_name, + product_name=xdevice.product_name, + manufacturer_name=xdevice.manufacturer_name, + com_objects=com_objects, + parameters=[ + Parameter(ref_id=ref, value=p.value or "") + for ref, p in xdevice.parameter_instance_refs.items() + ], + module_instances=[ + ModuleInstance(instance_id=mi.identifier, ref_id=mi.module_def_id) + for mi in xdevice.module_instances + ], + ) + state.register_device(xdevice.individual_address, device) + return device + + +def _build_com_object(coir: ComObjectInstanceRef) -> ComObject: + return ComObject( + ref_id=coir.com_object_ref_id or coir.ref_id, + channel_id=coir.channel, + read_flag=coir.read_flag, + write_flag=coir.write_flag, + communication_flag=coir.communication_flag, + transmit_flag=coir.transmit_flag, + update_flag=coir.update_flag, + read_on_init_flag=coir.read_on_init_flag, + ) + + +# --- group ranges + addresses + links ------------------------------------- + + +def _build_group_addresses( + installation: Installation, parser: XMLParser, state: _ImportState +) -> None: + ranges: list[GroupRange] = [] + _build_ranges(installation, parser.group_ranges, None, ranges) + + for xga in parser.group_addresses: + group_range = _range_for(ranges, xga.raw_address) + if group_range is None: + continue # a group address outside every range would violate the schema; skip it + ga = GroupAddress( + address=xga.raw_address, + name=xga.name, + datapoint_type=_dpt(xga.dpt), + description=xga.description, + comment=xga.comment, + data_secure=bool(xga.data_secure_key), + ) + group_range.group_addresses.append(ga) + state.register_group_address(xga.identifier, ga) + + +def _build_ranges( + installation: Installation, + xranges: list[XMLGroupRange], + parent: GroupRange | None, + collected: list[GroupRange], +) -> None: + for xr in xranges: + gr = GroupRange( + range_start=xr.range_start, + range_end=xr.range_end, + name=xr.name, + parent=parent, + ) + installation.group_ranges.append(gr) + collected.append(gr) + _build_ranges(installation, xr.group_ranges, gr, collected) + + +def _range_for(ranges: list[GroupRange], address: int) -> GroupRange | None: + """The smallest (leaf) range that contains ``address``.""" + best: GroupRange | None = None + for gr in ranges: + if gr.range_start <= address <= gr.range_end and ( + best is None + or (gr.range_end - gr.range_start) < (best.range_end - best.range_start) + ): + best = gr + return best + + +def _build_links(state: _ImportState) -> None: + for coir, com_object in state.com_objects: + for i, link in enumerate(coir.links or []): + ga = state.group_addresses.get(link) + if ga is None: + continue # link to a group address not present in the project; drop it + com_object.links.append( + ComObjectLink(group_address=ga, is_sending=(i == 0)) + ) + + +# --- spaces (buildings/rooms) + functions --------------------------------- + + +def _build_spaces( + installation: Installation, parser: XMLParser, state: _ImportState +) -> None: + space_by_identifier: dict[str, Space] = {} + for order, xspace in enumerate(parser.spaces): + _build_space(installation, xspace, None, order, state, space_by_identifier) + + for order, func in enumerate(parser.functions): + space = space_by_identifier.get(func.space_id) + if space is None: + continue # function references a space that was not present + function = Function( + function_type=func.function_type, + name=func.name, + usage_text=func.usage_text, + order=order, + ) + for ref in func.group_addresses: + ga = state.group_addresses.get(ref.ref_id) + if ga is not None: + function.group_addresses.append( + FunctionGroupAddress(group_address=ga, role=ref.role) + ) + space.functions.append(function) + + +def _build_space( + installation: Installation, + xspace: XMLSpace, + parent: Space | None, + order: int, + state: _ImportState, + space_by_identifier: dict[str, Space], +) -> None: + space = Space( + space_type=xspace.space_type.value, + name=xspace.name, + number=xspace.number, + usage_text=xspace.usage_text, + description=xspace.description, + order=order, + parent=parent, + ) + installation.spaces.append(space) + space_by_identifier[xspace.identifier] = space + for ia in xspace.devices: + device = state.device_by_ia.get(ia) + if device is not None: + device.space = space + for child_order, child in enumerate(xspace.spaces): + _build_space( + installation, child, space, child_order, state, space_by_identifier + ) + + +def _dpt(dpt: DPTType | None) -> str | None: + if dpt is None: + return None + main = dpt["main"] + sub = dpt["sub"] + return f"DPST-{main}-{sub}" if sub is not None else f"DPT-{main}" + + +class _ImportState: + """Cross-references built while walking devices, used to wire links once addresses exist.""" + + def __init__(self) -> None: + self.com_objects: list[tuple[ComObjectInstanceRef, ComObject]] = [] + self.group_addresses: dict[str, GroupAddress] = {} + self.device_by_ia: dict[str, Device] = {} + + def register_com_object(self, coir: ComObjectInstanceRef, row: ComObject) -> None: + self.com_objects.append((coir, row)) + + def register_group_address(self, identifier: str, row: GroupAddress) -> None: + self.group_addresses[identifier] = row + + def register_device(self, individual_address: str, row: Device) -> None: + self.device_by_ia[individual_address] = row diff --git a/packages/project/src/xknxmono/project/core/knxproj_signing.py b/packages/project/src/xknxmono/project/core/knxproj_signing.py new file mode 100644 index 00000000..486389f6 --- /dev/null +++ b/packages/project/src/xknxmono/project/core/knxproj_signing.py @@ -0,0 +1,71 @@ +"""Sign a ``.knxproj`` directory the way the KNX tooling expects on import. + +Each folder in a ``.knxproj`` archive (the project folder ``P-XXXX`` and each +manufacturer folder ``M-XXXX``) is accompanied by a ``.signature`` file +at the archive root. The signature is an RSA-PKCS#1 v1.5 signature over a SHA-1 +digest of the folder's contents: + +1. for every file in the folder (recursively), compute ``base64(sha1(content))``; +2. build the string ``"relpath:hash,relpath:hash,..."`` with the entries sorted + by relative path (ordinal) and joined by commas, relative paths being taken + from the folder root; +3. the digest is ``sha1(utf-8(that string))``; +4. the signature is ``RSA-PKCS#1v1.5(sha1)`` of that digest, base64 encoded. + +The signing key is the fixed "converter" RSA key that ships identically with the +KNX tooling (it is not per-installation or secret), so a project folder can be +re-signed offline. This module reproduces the algorithm in pure Python - verified +to produce byte-identical signatures to the reference implementation. + +Relative paths use ``/`` as the separator here. Folders exported by this package +are flat (``project.xml``, ``0.xml``), so no separator appears in the digest. +""" + +from __future__ import annotations + +import base64 +import hashlib +from collections.abc import Mapping + +# The converter RSA key (1024 bit, public exponent 65537). Public knowledge - the +# same key ships with the KNX tooling; kept here so folders can be re-signed. +_MODULUS_B64 = ( + "zSjrmVmM+ULXdrFHiSZZo7PEHo/sXBIkjxHkqQbxEI2YE1SBq0dbEfqW3eDSdjLlpMy5Yx9hcMS" + "nrmVUWh3PgBBQmzMBZpr/yJRny8UzB1pqTPyisWyfg7+NiAd1Ize4r/bQxKE4BaJ2wqEDwH8ggg" + "2faxJ2/WReGVrrzJL2u00=" +) +_PRIVATE_EXPONENT_B64 = ( + "p1DgE8h8uCxTHHGoLaohIOjS4TnvQYdqWWP2YANRRnazt9ALkGw5UYhU0c8w1UTdFHICH1zQUu+" + "O8SOij3wQZKMGcw4GgsJH8jUtlbSkHCtJVOBe817tNcuVUC1qfSt59uCyR6jKV2pm2+Hy8MCcsZ" + "kRXqDRcdgcYsiTpIwKcuE=" +) + +_MODULUS = int.from_bytes(base64.b64decode(_MODULUS_B64), "big") +_PRIVATE_EXPONENT = int.from_bytes(base64.b64decode(_PRIVATE_EXPONENT_B64), "big") +_KEY_SIZE = (_MODULUS.bit_length() + 7) // 8 +# ASN.1 DigestInfo prefix for a SHA-1 hash (RFC 3447). +_SHA1_DIGEST_INFO_PREFIX = bytes.fromhex("3021300906052b0e03021a05000414") + + +def directory_digest(files: Mapping[str, bytes]) -> bytes: + """Return the SHA-1 folder digest for ``files`` (relative path -> content).""" + entries = { + path: base64.b64encode(hashlib.sha1(content).digest()).decode("ascii") + for path, content in files.items() + } + joined = ",".join(f"{path}:{h}" for path, h in sorted(entries.items())) + return hashlib.sha1(joined.encode("utf-8")).digest() + + +def directory_signature(files: Mapping[str, bytes]) -> bytes: + """Return the base64 ``.signature`` bytes for a folder's ``files``. + + ``files`` maps each file's path (relative to the folder, ``/`` separated) to + its content. Suitable directly as the body of the folder's ``.signature`` file. + """ + digest = directory_digest(files) + block = _SHA1_DIGEST_INFO_PREFIX + digest + padding = b"\xff" * (_KEY_SIZE - 3 - len(block)) + encoded = b"\x00\x01" + padding + b"\x00" + block + signature = pow(int.from_bytes(encoded, "big"), _PRIVATE_EXPONENT, _MODULUS) + return base64.b64encode(signature.to_bytes(_KEY_SIZE, "big")) diff --git a/packages/project/src/xknxmono/project/core/service.py b/packages/project/src/xknxmono/project/core/service.py index 52d30167..f5b50c34 100644 --- a/packages/project/src/xknxmono/project/core/service.py +++ b/packages/project/src/xknxmono/project/core/service.py @@ -40,6 +40,7 @@ RemoveLine, RemoveSegment, RenameArea, + RenameGroupAddress, RenameLine, SetComObjectFlag, SetComObjectSending, @@ -54,12 +55,14 @@ Area, ComObjectLink, Device, + Function, GroupAddress, GroupRange, Installation, Line, Project, Segment, + Space, ) @@ -80,6 +83,69 @@ class GroupAddressInfo: name: str datapoint_type: str | None links: list[int] + description: str + comment: str + data_secure: bool + + +@dataclass(frozen=True) +class GroupRangeInfo: + """A node in the group-address range tree, resolved for display (recursive).""" + + id: int + name: str + range_start: int + range_end: int + children: list[GroupRangeInfo] + group_addresses: list[GroupAddressInfo] + + +@dataclass(frozen=True) +class SpaceDeviceInfo: + """A device as referenced from a space (building tree leaf), with display metadata.""" + + id: int + name: str + individual_address: str | None + description: str + product_name: str + hardware_name: str + manufacturer_name: str + + +@dataclass(frozen=True) +class FunctionGroupAddressInfo: + """A group address referenced by a function, with its role.""" + + group_address_id: int + text: str + role: str + + +@dataclass(frozen=True) +class FunctionInfo: + """A function assigned to a space, resolved for display.""" + + id: int + name: str + function_type: str + usage_text: str + group_addresses: list[FunctionGroupAddressInfo] + + +@dataclass(frozen=True) +class SpaceInfo: + """A node in the building/location tree, resolved for display (recursive).""" + + id: int + name: str + space_type: str + number: str + usage_text: str + description: str + children: list[SpaceInfo] + devices: list[SpaceDeviceInfo] + functions: list[FunctionInfo] @dataclass(frozen=True) @@ -92,6 +158,11 @@ class DeviceInfo: individual_address: str | None product_ref_id: str hardware2program_ref_id: str | None + description: str + order_number: str + hardware_name: str + product_name: str + manufacturer_name: str @dataclass(frozen=True) @@ -327,6 +398,13 @@ def rename_area(self, project_id: str, area_id: int, name: str) -> None: def rename_line(self, project_id: str, line_id: int, name: str) -> None: self._state(project_id).store.append(RenameLine(line_id=line_id, name=name)) + def rename_group_address( + self, project_id: str, group_address_id: int, name: str + ) -> None: + self._state(project_id).store.append( + RenameGroupAddress(group_address_id=group_address_id, name=name) + ) + def set_device_name(self, project_id: str, device_id: int, name: str) -> None: self._state(project_id).store.append( SetDeviceName(device_id=device_id, name=name) @@ -398,6 +476,11 @@ def device(self, project_id: str, device_id: int) -> DeviceInfo: individual_address=self._compose_ia(device), product_ref_id=device.product_ref_id, hardware2program_ref_id=device.hardware2program_ref_id, + description=device.description, + order_number=device.order_number, + hardware_name=device.hardware_name, + product_name=device.product_name, + manufacturer_name=device.manufacturer_name, ) def com_object_links(self, project_id: str, com_object_id: int) -> list[LinkInfo]: @@ -428,6 +511,36 @@ def group_addresses(self, project_id: str) -> list[GroupAddressInfo]: rows = state.session.query(GroupAddress).order_by(GroupAddress.id).all() return [self._ga_info(row, style) for row in rows] + def group_ranges(self, project_id: str, installation: int) -> list[GroupRangeInfo]: + """The installation's group-address range tree (roots → children), resolved for display.""" + state = self._state(project_id) + style = self._style(state) + inst = self._installation(state, installation) + roots = ( + state.session.query(GroupRange) + .filter( + GroupRange.installation_id == inst.id, + GroupRange.parent_id.is_(None), + ) + .order_by(GroupRange.range_start) + .all() + ) + return [self._range_info(r, style) for r in roots] + + def space_tree(self, project_id: str, installation: int) -> list[SpaceInfo]: + """The installation's building/location tree (roots → children), with devices and functions + resolved for display.""" + state = self._state(project_id) + style = self._style(state) + inst = self._installation(state, installation) + roots = ( + state.session.query(Space) + .filter(Space.installation_id == inst.id, Space.parent_id.is_(None)) + .order_by(Space.order, Space.id) + .all() + ) + return [self._space_info(s, style) for s in roots] + def group_address(self, project_id: str, group_address_id: int) -> GroupAddressInfo: state = self._state(project_id) ga = state.session.get(GroupAddress, group_address_id) @@ -533,6 +646,75 @@ def _ga_info(self, ga: GroupAddress, style: GroupAddressStyle) -> GroupAddressIn name=ga.name, datapoint_type=ga.datapoint_type, links=[link.com_object_id for link in ga.links], + description=ga.description, + comment=ga.comment, + data_secure=ga.data_secure, + ) + + def _range_info( + self, group_range: GroupRange, style: GroupAddressStyle + ) -> GroupRangeInfo: + return GroupRangeInfo( + id=group_range.id, + name=group_range.name, + range_start=group_range.range_start, + range_end=group_range.range_end, + children=[ + self._range_info(child, style) + for child in sorted(group_range.children, key=lambda c: c.range_start) + ], + group_addresses=[ + self._ga_info(ga, style) + for ga in sorted(group_range.group_addresses, key=lambda g: g.address) + ], + ) + + def _space_info(self, space: Space, style: GroupAddressStyle) -> SpaceInfo: + return SpaceInfo( + id=space.id, + name=space.name, + space_type=space.space_type, + number=space.number, + usage_text=space.usage_text, + description=space.description, + children=[ + self._space_info(child, style) + for child in sorted(space.children, key=lambda c: (c.order, c.id)) + ], + devices=[ + SpaceDeviceInfo( + id=device.id, + name=device.name, + individual_address=self._compose_ia(device), + description=device.description, + product_name=device.product_name, + hardware_name=device.hardware_name, + manufacturer_name=device.manufacturer_name, + ) + for device in space.devices + ], + functions=[ + self._function_info(fn, style) + for fn in sorted(space.functions, key=lambda f: (f.order, f.id)) + ], + ) + + def _function_info( + self, function: Function, style: GroupAddressStyle + ) -> FunctionInfo: + return FunctionInfo( + id=function.id, + name=function.name, + function_type=function.function_type, + usage_text=function.usage_text, + group_addresses=[ + FunctionGroupAddressInfo( + group_address_id=link.group_address_id, + text=format_ga(link.group_address.address, style), + role=link.role, + ) + for link in function.group_addresses + ], ) def _register(self, project_id: str, engine: Engine, session: Session) -> None: diff --git a/packages/project/src/xknxmono/project/models.py b/packages/project/src/xknxmono/project/models.py index 03b8f9bd..d5779356 100644 --- a/packages/project/src/xknxmono/project/models.py +++ b/packages/project/src/xknxmono/project/models.py @@ -31,6 +31,12 @@ class Project(Base): group_address_style: Mapped[str] = mapped_column( String, nullable=False, default="ThreeLevel" ) + # Descriptive metadata carried over from the imported .knxproj (ETS project information). + guid: Mapped[str] = mapped_column(String, nullable=False, default="") + created_by: Mapped[str] = mapped_column(Text, nullable=False, default="") + last_modified: Mapped[str] = mapped_column(String, nullable=False, default="") + schema_version: Mapped[str] = mapped_column(String, nullable=False, default="") + tool_version: Mapped[str] = mapped_column(String, nullable=False, default="") class Installation(Base): @@ -48,6 +54,9 @@ class Installation(Base): group_ranges: Mapped[list["GroupRange"]] = relationship( back_populates="installation", cascade="all, delete-orphan" ) + spaces: Mapped[list["Space"]] = relationship( + back_populates="installation", cascade="all, delete-orphan" + ) class Area(Base): @@ -123,8 +132,17 @@ class Device(Base): name: Mapped[str] = mapped_column(Text, nullable=False, default="") product_ref_id: Mapped[str] = mapped_column(String, nullable=False) hardware2program_ref_id: Mapped[str | None] = mapped_column(String) + # The building/room (Space) the device is placed in, if any. + space_id: Mapped[int | None] = mapped_column(ForeignKey("spaces.id"), index=True) + # Descriptive metadata carried over from the imported .knxproj (for display without a catalog). + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + order_number: Mapped[str] = mapped_column(String, nullable=False, default="") + hardware_name: Mapped[str] = mapped_column(Text, nullable=False, default="") + product_name: Mapped[str] = mapped_column(Text, nullable=False, default="") + manufacturer_name: Mapped[str] = mapped_column(Text, nullable=False, default="") segment: Mapped["Segment"] = relationship(back_populates="devices") + space: Mapped["Space | None"] = relationship(back_populates="devices") module_instances: Mapped[list["ModuleInstance"]] = relationship( back_populates="device", cascade="all, delete-orphan" ) @@ -238,11 +256,18 @@ class GroupAddress(Base): address: Mapped[int] = mapped_column(Integer, nullable=False) name: Mapped[str] = mapped_column(Text, nullable=False, default="") datapoint_type: Mapped[str | None] = mapped_column(String) + # Descriptive metadata carried over from the imported .knxproj. + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + comment: Mapped[str] = mapped_column(Text, nullable=False, default="") + data_secure: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) group_range: Mapped["GroupRange"] = relationship(back_populates="group_addresses") links: Mapped[list["ComObjectLink"]] = relationship( back_populates="group_address", cascade="all, delete-orphan" ) + function_links: Mapped[list["FunctionGroupAddress"]] = relationship( + back_populates="group_address", cascade="all, delete-orphan" + ) class ComObjectLink(Base): @@ -265,6 +290,79 @@ class ComObjectLink(Base): group_address: Mapped["GroupAddress"] = relationship(back_populates="links") +class Space(Base): + """A node in the building/location tree (recursive: building → floor → room → …), imported + from the ETS project's locations. ``space_type`` is the ETS type string (e.g. ``Building``, + ``Floor``, ``Room``). ``order`` preserves the project's original sibling order.""" + + __tablename__ = "spaces" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + installation_id: Mapped[int] = mapped_column( + ForeignKey("installations.id"), nullable=False, index=True + ) + parent_id: Mapped[int | None] = mapped_column(ForeignKey("spaces.id"), index=True) + space_type: Mapped[str] = mapped_column(String, nullable=False, default="") + name: Mapped[str] = mapped_column(Text, nullable=False, default="") + number: Mapped[str] = mapped_column(String, nullable=False, default="") + usage_text: Mapped[str] = mapped_column(Text, nullable=False, default="") + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + order: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + installation: Mapped["Installation"] = relationship(back_populates="spaces") + parent: Mapped["Space | None"] = relationship( + back_populates="children", remote_side="Space.id" + ) + children: Mapped[list["Space"]] = relationship( + back_populates="parent", cascade="all, delete-orphan" + ) + devices: Mapped[list["Device"]] = relationship(back_populates="space") + functions: Mapped[list["Function"]] = relationship( + back_populates="space", cascade="all, delete-orphan" + ) + + +class Function(Base): + """A function assigned to a space (ETS ``Function``): a named grouping of group addresses by + role (e.g. a light's switch/status/dimming addresses).""" + + __tablename__ = "functions" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + space_id: Mapped[int] = mapped_column( + ForeignKey("spaces.id"), nullable=False, index=True + ) + function_type: Mapped[str] = mapped_column(String, nullable=False, default="") + name: Mapped[str] = mapped_column(Text, nullable=False, default="") + usage_text: Mapped[str] = mapped_column(Text, nullable=False, default="") + order: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + space: Mapped["Space"] = relationship(back_populates="functions") + group_addresses: Mapped[list["FunctionGroupAddress"]] = relationship( + back_populates="function", cascade="all, delete-orphan" + ) + + +class FunctionGroupAddress(Base): + """A group address referenced by a function, with its ``role`` within that function.""" + + __tablename__ = "function_group_addresses" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + function_id: Mapped[int] = mapped_column( + ForeignKey("functions.id"), nullable=False, index=True + ) + group_address_id: Mapped[int] = mapped_column( + ForeignKey("group_addresses.id"), nullable=False, index=True + ) + role: Mapped[str] = mapped_column(String, nullable=False, default="") + + function: Mapped["Function"] = relationship(back_populates="group_addresses") + group_address: Mapped["GroupAddress"] = relationship( + back_populates="function_links" + ) + + class Event(Base): """The undo/redo history. ``data`` is the serialized command payload (incl. before-values).""" diff --git a/packages/project/tests/test_knxproj_export.py b/packages/project/tests/test_knxproj_export.py new file mode 100644 index 00000000..eb152d0d --- /dev/null +++ b/packages/project/tests/test_knxproj_export.py @@ -0,0 +1,144 @@ +"""Round-trip test for the simple .knxproj export: build a project via the core API, export it to +a ``.knxproj``, then re-import it (real xknxproject parse) and check topology/GAs/links survive.""" + +import zipfile +from pathlib import Path + +from sqlalchemy.orm import Session + +from xknxmono.project import ProjectService, export_knxproj, import_knxproj +from xknxmono.project.db import make_engine, url_for +from xknxmono.project.models import ( + Device, + Function, + FunctionGroupAddress, + Installation, + Space, +) + + +def test_export_import_round_trip(tmp_path: Path) -> None: + src = tmp_path / "src.xknx" + svc = ProjectService() + pid = svc.create(src, "P-RT") + + area_id = svc.create_area(pid, 0, 1, "Area 1") + line_id = svc.create_line(pid, area_id, 1, "Line 1") + segment_id = next( + line.segments[0].id + for area in svc.topology(pid, 0).areas + if area.id == area_id + for line in area.lines + if line.id == line_id + ) + device_id = svc.add_device( + pid, + segment_id, + "M-1_H-1_P-1", + address=5, + name="Dev", + hardware2program_ref_id="M-1_H-1_HP-1", + com_objects=[("M-1_A-1_O-1_R-1", None)], + ) + ga_id = svc.create_group_address(pid, 0, 0x0801, "GA One") # 1/0/1 + svc.set_group_address_datapoint_type(pid, ga_id, "DPST-1-1") + co_id = next( + co.id for d in svc.devices(pid) if d.id == device_id for co in d.com_objects + ) + svc.link_com_object(pid, co_id, ga_id, sending=True) + svc.close(pid) + + # The core API has no space/function commands yet; seed a location tree via ORM so the + # export's Locations round-trip is covered too. + engine = make_engine(url_for(src)) + with Session(engine) as session: + inst = session.query(Installation).one() + room = Space( + installation_id=inst.id, + parent_id=None, + space_type="Room", + name="Room 1", + number="1", + order=0, + ) + session.add(room) + session.flush() + device = session.query(Device).filter(Device.id == device_id).one() + device.space_id = room.id + fn = Function(space_id=room.id, function_type="FT-1", name="Light", order=0) + session.add(fn) + session.flush() + session.add( + FunctionGroupAddress(function_id=fn.id, group_address_id=ga_id, role="role") + ) + session.commit() + engine.dispose() + + out = tmp_path / "out.knxproj" + export_knxproj(src, out) + assert out.exists() and out.stat().st_size > 0 + + round_path = tmp_path / "round.xknx" + rpid = import_knxproj(out, round_path) + rsvc = ProjectService() + rsvc.open(round_path) + + inst = rsvc.topology(rpid, 0) + addresses = {a.address for a in inst.areas} + assert 1 in addresses # our Area 1 survived (Area 0 backbone also present) + gas = {g.text: g for g in rsvc.group_addresses(rpid)} + assert "1/0/1" in gas + assert gas["1/0/1"].name == "GA One" + assert gas["1/0/1"].datapoint_type == "DPST-1-1" + # the device and its sending link survived + devices = rsvc.devices(rpid) + assert any(d.product_ref_id == "M-1_H-1_P-1" for d in devices) + links = rsvc.group_address_links(rpid, gas["1/0/1"].id) + assert len(links) == 1 + assert links[0].is_sending + + # the location tree (space + its device + function) survived + with Session(make_engine(url_for(round_path))) as session: + rooms = session.query(Space).filter(Space.name == "Room 1").all() + assert len(rooms) == 1 + assert session.query(Device).filter(Device.space_id == rooms[0].id).count() == 1 + funcs = session.query(Function).filter(Function.space_id == rooms[0].id).all() + assert len(funcs) == 1 + assert funcs[0].name == "Light" + assert ( + session.query(FunctionGroupAddress) + .filter(FunctionGroupAddress.function_id == funcs[0].id) + .count() + == 1 + ) + + +def test_export_bundles_extra_files_and_master(tmp_path: Path) -> None: + src = tmp_path / "src.xknx" + svc = ProjectService() + pid = svc.create(src, "P-MFR") + svc.close(pid) + + out = tmp_path / "out.knxproj" + master = b'\n' + hardware = b"" + export_knxproj( + src, + out, + extra_files={ + "M-9999/Hardware.xml": hardware, + "M-9999.signature": b"sig", + # colliding with an own path must be ignored, not overwrite our master + "knx_master.xml": b"IGNORED", + }, + master_xml=master, + ) + + with zipfile.ZipFile(out) as zf: + names = zf.namelist() + assert "M-9999/Hardware.xml" in names + assert "M-9999.signature" in names + assert zf.read("M-9999/Hardware.xml") == hardware + # the colliding "knx_master.xml" key is ignored: only our master, written once + assert names.count("knx_master.xml") == 1 + assert zf.read("knx_master.xml") == master diff --git a/packages/project/tests/test_knxproj_import.py b/packages/project/tests/test_knxproj_import.py new file mode 100644 index 00000000..8075c1d0 --- /dev/null +++ b/packages/project/tests/test_knxproj_import.py @@ -0,0 +1,393 @@ +"""Tests for the ``.knxproj`` importer. + +The mapping logic is exercised through the public ``import_knxproj`` with its parse step patched to +return a lightweight fake parser that duck-types the ``xknxproject`` attributes the importer reads — +no binary fixture, no dependency on xknxproject's own parsing (which is its responsibility, not +ours). A separate smoke test runs the real ``import_knxproj`` against a local ``.knxproj`` when one +is available, and is skipped otherwise. +""" + +import os +import zlib +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from zipfile import BadZipFile + +import pytest +from sqlalchemy.orm import Session +from xknxproject.exceptions import InvalidPasswordException, UnexpectedFileContent + +from xknxmono.project import ProjectService, import_knxproj +from xknxmono.project.core import knxproj_import +from xknxmono.project.core.addressing import GroupAddressStyle +from xknxmono.project.db import make_engine, url_for +from xknxmono.project.models import ComObjectLink, GroupRange + + +def _coir(**kw: Any) -> SimpleNamespace: + defaults: dict[str, Any] = { + "com_object_ref_id": None, + "ref_id": "O-1_R-1", + "channel": None, + "read_flag": None, + "write_flag": None, + "communication_flag": None, + "transmit_flag": None, + "update_flag": None, + "read_on_init_flag": None, + "links": None, + } + return SimpleNamespace(**{**defaults, **kw}) + + +def _fake_parser() -> SimpleNamespace: + device = SimpleNamespace( + address=5, + individual_address="1.1.5", + name="Dev", + product_ref="M-1_H-1_P-1", + hardware_program_ref="M-1_H-1_HP-1", + description="A device", + order_number="ORD-1", + hardware_name="HW One", + product_name="Prod One", + manufacturer_name="ACME", + com_object_instance_refs=[ + _coir( + com_object_ref_id="M-1_A-1_O-1_R-1", + channel="CH-1", + write_flag=True, + communication_flag=True, + transmit_flag=True, + links=["GA-1", "GA-2"], + ) + ], + parameter_instance_refs={"M-1_A-1_P-1": SimpleNamespace(value="3")}, + module_instances=[], + ) + line = SimpleNamespace( + address=1, name="Line 1", medium_type="MT-0", devices=[device] + ) + area = SimpleNamespace(address=1, name="Area 1", lines=[line]) + middle = SimpleNamespace( + range_start=1, range_end=255, name="Middle", group_ranges=[] + ) + main = SimpleNamespace( + range_start=1, range_end=2047, name="Main", group_ranges=[middle] + ) + gas = [ + SimpleNamespace( + raw_address=1, + name="GA One", + dpt={"main": 1, "sub": 1}, + identifier="GA-1", + description="GA desc", + comment="GA comment", + data_secure_key="k", + ), + SimpleNamespace( + raw_address=2, + name="GA Two", + dpt={"main": 5, "sub": None}, + identifier="GA-2", + description="", + comment="", + data_secure_key=None, + ), + ] + room = SimpleNamespace( + identifier="SP-ROOM", + name="Wohnzimmer", + space_type=SimpleNamespace(value="Room"), + number="1", + usage_text="Living", + description="", + spaces=[], + devices=["1.1.5"], + functions=["F-1"], + ) + building = SimpleNamespace( + identifier="SP-B", + name="Haus", + space_type=SimpleNamespace(value="Building"), + number="", + usage_text="", + description="", + spaces=[room], + devices=[], + functions=[], + ) + function = SimpleNamespace( + identifier="F-1", + function_type="FT-1", + name="Light", + usage_text="Lighting", + space_id="SP-ROOM", + group_addresses=[ + SimpleNamespace(ref_id="GA-1", address="0/0/1", role="Switch") + ], + ) + return SimpleNamespace( + project_info=SimpleNamespace( + name="Test", + group_address_style=SimpleNamespace(value="ThreeLevel"), + guid="GUID-1", + created_by="ETS6", + last_modified="2020-01-01", + schema_version="20", + tool_version="6.0.0", + ), + areas=[area], + group_ranges=[main], + group_addresses=gas, + spaces=[building], + functions=[function], + ) + + +def _import_fake( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[ProjectService, str, Path]: + # Replace the xknxproject-backed parse with our fake so the public entry point is exercised. + def _fake(*_a: object, **_k: object) -> SimpleNamespace: + return _fake_parser() + + monkeypatch.setattr(knxproj_import, "_parse", _fake) + dest = tmp_path / "imported.xknx" + pid = import_knxproj("unused.knxproj", dest, project_id="P-TEST") + svc = ProjectService() + svc.open(dest) + return svc, pid, dest + + +def test_project_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + project = svc.project(pid) + assert project.name == "Test" + assert project.group_address_style == GroupAddressStyle.THREE_LEVEL + + +def test_topology(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + inst = svc.topology(pid, 0) + assert [a.address for a in inst.areas] == [1] + line = inst.areas[0].lines[0] + assert line.address == 1 + assert line.segments[0].medium_type == "MT-0" + + +def test_device(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + devices = svc.devices(pid) + assert len(devices) == 1 + dev = devices[0] + assert dev.product_ref_id == "M-1_H-1_P-1" + assert dev.hardware2program_ref_id == "M-1_H-1_HP-1" + assert svc.individual_address(pid, dev.id) == "1.1.5" + assert [(p.ref_id, p.value) for p in dev.parameters] == [("M-1_A-1_P-1", "3")] + assert len(dev.com_objects) == 1 + co = dev.com_objects[0] + assert ( + co.ref_id == "M-1_A-1_O-1_R-1" + ) # app-prefixed form the GUI's DynamicUI expects + assert co.channel_id == "CH-1" + assert (co.write_flag, co.communication_flag, co.read_flag) == (True, True, None) + + +def test_group_addresses_and_dpt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + gas = {g.address: g for g in svc.group_addresses(pid)} + assert gas[1].name == "GA One" + assert gas[1].datapoint_type == "DPST-1-1" + assert gas[2].datapoint_type == "DPT-5" # no sub-type + + +def test_group_ranges_preserve_names( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, _, dest = _import_fake(tmp_path, monkeypatch) + engine = make_engine(url_for(dest)) + with Session(engine) as session: + ranges = session.query(GroupRange).order_by(GroupRange.range_start).all() + names = {(r.range_start, r.range_end): r.name for r in ranges} + assert names[(1, 2047)] == "Main" + assert names[(1, 255)] == "Middle" + # the group address at value 1 sits under the smallest (leaf) range that contains it + middle = next(r for r in ranges if (r.range_start, r.range_end) == (1, 255)) + assert {ga.address for ga in middle.group_addresses} == {1, 2} + engine.dispose() + + +def test_group_ranges_read(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + roots = svc.group_ranges(pid, 0) + assert len(roots) == 1 + main = roots[0] + assert main.name == "Main" + assert (main.range_start, main.range_end) == (1, 2047) + assert len(main.children) == 1 + middle = main.children[0] + assert middle.name == "Middle" + assert {ga.address for ga in middle.group_addresses} == {1, 2} + + +def test_project_metadata_extra( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + p = svc.project(pid) + assert p.created_by == "ETS6" + assert p.tool_version == "6.0.0" + assert p.guid == "GUID-1" + assert p.schema_version == "20" + + +def test_device_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + dev = svc.devices(pid)[0] + info = svc.device(pid, dev.id) + assert info.order_number == "ORD-1" + assert info.hardware_name == "HW One" + assert info.manufacturer_name == "ACME" + assert info.description == "A device" + + +def test_group_address_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + gas = {g.address: g for g in svc.group_addresses(pid)} + assert gas[1].description == "GA desc" + assert gas[1].comment == "GA comment" + assert gas[1].data_secure is True + assert gas[2].data_secure is False + + +def test_space_tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + svc, pid, _ = _import_fake(tmp_path, monkeypatch) + roots = svc.space_tree(pid, 0) + assert len(roots) == 1 + building = roots[0] + assert building.space_type == "Building" + assert building.name == "Haus" + assert len(building.children) == 1 + room = building.children[0] + assert room.space_type == "Room" + # the device placed in the room + assert [d.individual_address for d in room.devices] == ["1.1.5"] + # the function assigned to the room, with its GA and role + assert len(room.functions) == 1 + fn = room.functions[0] + assert fn.name == "Light" + assert [(g.text, g.role) for g in fn.group_addresses] == [("0/0/1", "Switch")] + + +def test_links_sending_flag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _, _, dest = _import_fake(tmp_path, monkeypatch) + engine = make_engine(url_for(dest)) + with Session(engine) as session: + links = session.query(ComObjectLink).all() + assert len(links) == 2 + by_addr = {link.group_address.address: link.is_sending for link in links} + assert by_addr == { + 1: True, + 2: False, + } # first link of the com-object is the sender + engine.dispose() + + +@pytest.mark.parametrize( + "raised", + [zlib.error("bad"), BadZipFile("bad"), RuntimeError("Bad password for file")], +) +def test_parse_failure_with_password_is_invalid_password( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, raised: Exception +) -> None: + def _boom(*_a: object, **_k: object) -> SimpleNamespace: + raise raised + + monkeypatch.setattr(knxproj_import, "_parse", _boom) + with pytest.raises(InvalidPasswordException): + import_knxproj("x.knxproj", tmp_path / "o.xknx", password="whatever") + + +def test_parse_failure_without_password_is_unexpected_content( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def _boom(*_a: object, **_k: object) -> SimpleNamespace: + raise zlib.error("garbage") + + monkeypatch.setattr(knxproj_import, "_parse", _boom) + with pytest.raises(UnexpectedFileContent): + import_knxproj("x.knxproj", tmp_path / "o.xknx") + + +def test_missing_password_propagates( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def _boom(*_a: object, **_k: object) -> SimpleNamespace: + raise InvalidPasswordException("Password required.") + + monkeypatch.setattr(knxproj_import, "_parse", _boom) + with pytest.raises(InvalidPasswordException): + import_knxproj("x.knxproj", tmp_path / "o.xknx") + + +def test_reimport_overwrites_existing_dest( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def _fake(*_a: object, **_k: object) -> SimpleNamespace: + return _fake_parser() + + monkeypatch.setattr(knxproj_import, "_parse", _fake) + dest = tmp_path / "reimport.xknx" + import_knxproj("x.knxproj", dest, project_id="P-1") + # A second import to the same file must overwrite, not clash on unique rows (installation index). + import_knxproj("x.knxproj", dest, project_id="P-2") + svc = ProjectService() + assert svc.open(dest) == "P-2" + + +def test_failed_parse_preserves_existing_dest( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def _fake(*_a: object, **_k: object) -> SimpleNamespace: + return _fake_parser() + + monkeypatch.setattr(knxproj_import, "_parse", _fake) + dest = tmp_path / "keep.xknx" + import_knxproj("x.knxproj", dest, project_id="P-KEEP") + before = dest.read_bytes() + + def _boom(*_a: object, **_k: object) -> SimpleNamespace: + raise zlib.error("garbage") + + monkeypatch.setattr(knxproj_import, "_parse", _boom) + with pytest.raises(UnexpectedFileContent): + import_knxproj("x.knxproj", dest) + assert ( + dest.read_bytes() == before + ) # the failed import must not touch the existing project + + +# Optional smoke test against a real project. Provide your own ``.knxproj`` via the +# ``XKNX_TEST_KNXPROJ`` environment variable (no sample is bundled). +_REAL_ENV = os.environ.get("XKNX_TEST_KNXPROJ") +_REAL = Path(_REAL_ENV) if _REAL_ENV else None + + +@pytest.mark.skipif( + _REAL is None or not _REAL.exists(), + reason="set XKNX_TEST_KNXPROJ to a .knxproj to run this test", +) +def test_import_real_file(tmp_path: Path) -> None: + assert _REAL is not None + dest = tmp_path / "real.xknx" + pid = import_knxproj(_REAL, dest) + svc = ProjectService() + svc.open(dest) + assert svc.devices(pid) + assert svc.group_addresses(pid) diff --git a/packages/project/tests/test_knxproj_signing.py b/packages/project/tests/test_knxproj_signing.py new file mode 100644 index 00000000..515d7c2d --- /dev/null +++ b/packages/project/tests/test_knxproj_signing.py @@ -0,0 +1,41 @@ +"""Tests for the .knxproj folder signing. + +The golden signature is produced by the reference signing implementation for a +folder containing ``project.xml`` = ``b"hello"`` and ``0.xml`` = ``b"world!!"``. +""" + +from __future__ import annotations + +import base64 +import hashlib + +from xknxmono.project.core.knxproj_signing import ( + directory_digest, + directory_signature, +) + +_GOLDEN_SIGNATURE = ( + b"WZlt/FbYQ8wulqmLNqwSgfJCjNt33R3AQ+CPAxJyopgmmWG+ghtJ2Nra1zWNSmoo7YLtGWKsvYL" + b"bT/EzYueNp+PwvKsKfVj2uGN9kY/ldYagcXlgm/gIQKxCOL7847nLFlvkir+MH1ycs1TlnMewTy" + b"8f7gyCuUJjOXCxwzvWIB0=" +) +_FILES = {"project.xml": b"hello", "0.xml": b"world!!"} + + +def test_directory_signature_matches_reference() -> None: + assert directory_signature(_FILES) == _GOLDEN_SIGNATURE + + +def test_directory_digest_is_colon_joined_sorted_sha1() -> None: + # "0.xml" sorts before "project.xml"; each value is base64(sha1(content)). + expected_string = ( + f"0.xml:{base64.b64encode(hashlib.sha1(b'world!!').digest()).decode()}," + f"project.xml:{base64.b64encode(hashlib.sha1(b'hello').digest()).decode()}" + ) + assert directory_digest(_FILES) == hashlib.sha1( + expected_string.encode("utf-8") + ).digest() + + +def test_signature_is_deterministic() -> None: + assert directory_signature(_FILES) == directory_signature(dict(_FILES)) diff --git a/packages/project/tests/test_project_core.py b/packages/project/tests/test_project_core.py index 3d406115..e0bccf15 100644 --- a/packages/project/tests/test_project_core.py +++ b/packages/project/tests/test_project_core.py @@ -490,6 +490,15 @@ def test_group_address_datapoint_type(tmp_path: Path): assert svc.group_address(pid, gid).datapoint_type is None +def test_rename_group_address(tmp_path: Path): + svc, pid = _new(tmp_path) + gid = svc.create_group_address(pid, 0, 1, "Switch") + svc.rename_group_address(pid, gid, "Living room light") + assert svc.group_address(pid, gid).name == "Living room light" + svc.undo(pid) + assert svc.group_address(pid, gid).name == "Switch" + + def test_device_info_carries_refs(tmp_path: Path): svc, pid = _new(tmp_path) seg = _backbone_segment(svc, pid) diff --git a/pyproject.toml b/pyproject.toml index 118779ce..9c93ea3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "xknx-project", "xknx-keyring", "xknx-catalog", + "xknx-download", ] [dependency-groups] @@ -35,3 +36,4 @@ xknx-project = { workspace = true } xknx-keyring = { workspace = true } knx-gui = { workspace = true } xknx-catalog = { workspace = true } +xknx-download = { workspace = true } diff --git a/tools/gen_mask_resources.py b/tools/gen_mask_resources.py new file mode 100644 index 00000000..b6600b89 --- /dev/null +++ b/tools/gen_mask_resources.py @@ -0,0 +1,58 @@ +"""Regenerate the bundled mask resource table from a KNX master data file. + +Usage: uv run python tools/gen_mask_resources.py +Extracts the download-relevant resource locations (Load State Machine control, +table pointers, load status, run control) per device mask version from the KNX +master data. Only these few resources are extracted - not the manufacturer data. +""" + +import json +import sys +from pathlib import Path + +from xknxmono.models.intermediate.resource_name_t import ResourceName as N +from xknxmono.product.master import parse_master_xml + +RELEVANT = { + N.APPLICATION_LOAD_CONTROL, + N.APPLICATION_LOAD_STATUS, + N.APPLICATION_RUN_CONTROL, + N.APPLICATION_DATA_PTR, + N.GROUP_ADDRESS_TABLE_LOAD_CONTROL, + N.GROUP_ADDRESS_TABLE_PTR, + N.GROUP_ADDRESS_TABLE_LOAD_STATUS, + N.GROUP_ASSOCIATION_TABLE_LOAD_CONTROL, + N.GROUP_ASSOCIATION_TABLE_PTR, + N.GROUP_ASSOCIATION_TABLE_LOAD_STATUS, + N.GROUP_OBJECT_TABLE_LOAD_CONTROL, + N.GROUP_OBJECT_TABLE_PTR, + N.GROUP_OBJECT_TABLE_LOAD_STATUS, +} + + +def main(master_path: str, out_path: str) -> None: + master = parse_master_xml(Path(master_path).read_bytes()) + table: dict = {} + for mv in master.raw.mask_versions.mask_version: + entry = {} + for cfg in mv.hawk_configuration_data: + if cfg.resources is None: + continue + for r in cfg.resources.resource: + if r.name in RELEVANT and r.location is not None: + loc = r.location + entry[r.name.value] = [ + loc.address_space.value, + loc.interface_object_ref, + loc.property_id, + loc.start_address, + ] + if entry: + table[mv.id] = dict(sorted(entry.items())) + out = {mask: table[mask] for mask in sorted(table)} + Path(out_path).write_text(json.dumps(out, indent=1, sort_keys=True) + "\n") + print(f"wrote {len(out)} masks to {out_path}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) diff --git a/uv.lock b/uv.lock index 7797d40c..3ccbf272 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,7 @@ resolution-markers = [ members = [ "knx-gui", "xknx-catalog", + "xknx-download", "xknx-keyring", "xknx-models", "xknx-product", @@ -683,6 +684,7 @@ dependencies = [ { name = "structlog" }, { name = "xknx" }, { name = "xknx-catalog" }, + { name = "xknx-download" }, { name = "xknx-models" }, { name = "xknx-product" }, { name = "xknx-project" }, @@ -706,6 +708,7 @@ requires-dist = [ { name = "structlog", specifier = ">=24.0.0" }, { name = "xknx", specifier = ">=3.18.0" }, { name = "xknx-catalog", editable = "packages/catalog" }, + { name = "xknx-download", editable = "packages/download" }, { name = "xknx-models", editable = "packages/models" }, { name = "xknx-product", editable = "packages/product" }, { name = "xknx-project", editable = "packages/project" }, @@ -844,6 +847,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] +[[package]] +name = "pycryptodomex" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/85/e24bf90972a30b0fcd16c73009add1d7d7cd9140c2498a68252028899e41/pycryptodomex-3.23.0.tar.gz", hash = "sha256:71909758f010c82bc99b0abf4ea12012c98962fbf0583c2164f8b84533c2e4da", size = 4922157, upload-time = "2025-05-17T17:23:41.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/00/10edb04777069a42490a38c137099d4b17ba6e36a4e6e28bdc7470e9e853/pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:7b37e08e3871efe2187bc1fd9320cc81d87caf19816c648f24443483005ff886", size = 2498764, upload-time = "2025-05-17T17:22:21.453Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3f/2872a9c2d3a27eac094f9ceaa5a8a483b774ae69018040ea3240d5b11154/pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:91979028227543010d7b2ba2471cf1d1e398b3f183cb105ac584df0c36dac28d", size = 1643012, upload-time = "2025-05-17T17:22:23.702Z" }, + { url = "https://files.pythonhosted.org/packages/70/af/774c2e2b4f6570fbf6a4972161adbb183aeeaa1863bde31e8706f123bf92/pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b8962204c47464d5c1c4038abeadd4514a133b28748bcd9fa5b6d62e3cec6fa", size = 2187643, upload-time = "2025-05-17T17:22:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/de/a3/71065b24cb889d537954cedc3ae5466af00a2cabcff8e29b73be047e9a19/pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a33986a0066860f7fcf7c7bd2bc804fa90e434183645595ae7b33d01f3c91ed8", size = 2273762, upload-time = "2025-05-17T17:22:28.313Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0b/ff6f43b7fbef4d302c8b981fe58467b8871902cdc3eb28896b52421422cc/pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7947ab8d589e3178da3d7cdeabe14f841b391e17046954f2fbcd941705762b5", size = 2313012, upload-time = "2025-05-17T17:22:30.57Z" }, + { url = "https://files.pythonhosted.org/packages/02/de/9d4772c0506ab6da10b41159493657105d3f8bb5c53615d19452afc6b315/pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c25e30a20e1b426e1f0fa00131c516f16e474204eee1139d1603e132acffc314", size = 2186856, upload-time = "2025-05-17T17:22:32.819Z" }, + { url = "https://files.pythonhosted.org/packages/28/ad/8b30efcd6341707a234e5eba5493700a17852ca1ac7a75daa7945fcf6427/pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:da4fa650cef02db88c2b98acc5434461e027dce0ae8c22dd5a69013eaf510006", size = 2347523, upload-time = "2025-05-17T17:22:35.386Z" }, + { url = "https://files.pythonhosted.org/packages/0f/02/16868e9f655b7670dbb0ac4f2844145cbc42251f916fc35c414ad2359849/pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:58b851b9effd0d072d4ca2e4542bf2a4abcf13c82a29fd2c93ce27ee2a2e9462", size = 2272825, upload-time = "2025-05-17T17:22:37.632Z" }, + { url = "https://files.pythonhosted.org/packages/ca/18/4ca89ac737230b52ac8ffaca42f9c6f1fd07c81a6cd821e91af79db60632/pycryptodomex-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:a9d446e844f08299236780f2efa9898c818fe7e02f17263866b8550c7d5fb328", size = 1772078, upload-time = "2025-05-17T17:22:40Z" }, + { url = "https://files.pythonhosted.org/packages/73/34/13e01c322db027682e00986873eca803f11c56ade9ba5bbf3225841ea2d4/pycryptodomex-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bc65bdd9fc8de7a35a74cab1c898cab391a4add33a8fe740bda00f5976ca4708", size = 1803656, upload-time = "2025-05-17T17:22:42.139Z" }, + { url = "https://files.pythonhosted.org/packages/54/68/9504c8796b1805d58f4425002bcca20f12880e6fa4dc2fc9a668705c7a08/pycryptodomex-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:c885da45e70139464f082018ac527fdaad26f1657a99ee13eecdce0f0ca24ab4", size = 1707172, upload-time = "2025-05-17T17:22:44.704Z" }, + { url = "https://files.pythonhosted.org/packages/dd/9c/1a8f35daa39784ed8adf93a694e7e5dc15c23c741bbda06e1d45f8979e9e/pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:06698f957fe1ab229a99ba2defeeae1c09af185baa909a31a5d1f9d42b1aaed6", size = 2499240, upload-time = "2025-05-17T17:22:46.953Z" }, + { url = "https://files.pythonhosted.org/packages/7a/62/f5221a191a97157d240cf6643747558759126c76ee92f29a3f4aee3197a5/pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2c2537863eccef2d41061e82a881dcabb04944c5c06c5aa7110b577cc487545", size = 1644042, upload-time = "2025-05-17T17:22:49.098Z" }, + { url = "https://files.pythonhosted.org/packages/8c/fd/5a054543c8988d4ed7b612721d7e78a4b9bf36bc3c5ad45ef45c22d0060e/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43c446e2ba8df8889e0e16f02211c25b4934898384c1ec1ec04d7889c0333587", size = 2186227, upload-time = "2025-05-17T17:22:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/c8/a9/8862616a85cf450d2822dbd4fff1fcaba90877907a6ff5bc2672cafe42f8/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f489c4765093fb60e2edafdf223397bc716491b2b69fe74367b70d6999257a5c", size = 2272578, upload-time = "2025-05-17T17:22:53.676Z" }, + { url = "https://files.pythonhosted.org/packages/46/9f/bda9c49a7c1842820de674ab36c79f4fbeeee03f8ff0e4f3546c3889076b/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdc69d0d3d989a1029df0eed67cc5e8e5d968f3724f4519bd03e0ec68df7543c", size = 2312166, upload-time = "2025-05-17T17:22:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/03/cc/870b9bf8ca92866ca0186534801cf8d20554ad2a76ca959538041b7a7cf4/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bbcb1dd0f646484939e142462d9e532482bc74475cecf9c4903d4e1cd21f003", size = 2185467, upload-time = "2025-05-17T17:22:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/96/e3/ce9348236d8e669fea5dd82a90e86be48b9c341210f44e25443162aba187/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:8a4fcd42ccb04c31268d1efeecfccfd1249612b4de6374205376b8f280321744", size = 2346104, upload-time = "2025-05-17T17:23:02.112Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e869bcee87beb89040263c416a8a50204f7f7a83ac11897646c9e71e0daf/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55ccbe27f049743a4caf4f4221b166560d3438d0b1e5ab929e07ae1702a4d6fd", size = 2271038, upload-time = "2025-05-17T17:23:04.872Z" }, + { url = "https://files.pythonhosted.org/packages/8d/67/09ee8500dd22614af5fbaa51a4aee6e342b5fa8aecf0a6cb9cbf52fa6d45/pycryptodomex-3.23.0-cp37-abi3-win32.whl", hash = "sha256:189afbc87f0b9f158386bf051f720e20fa6145975f1e76369303d0f31d1a8d7c", size = 1771969, upload-time = "2025-05-17T17:23:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/69/96/11f36f71a865dd6df03716d33bd07a67e9d20f6b8d39820470b766af323c/pycryptodomex-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:52e5ca58c3a0b0bd5e100a9fbc8015059b05cffc6c66ce9d98b4b45e023443b9", size = 1803124, upload-time = "2025-05-17T17:23:09.267Z" }, + { url = "https://files.pythonhosted.org/packages/f9/93/45c1cdcbeb182ccd2e144c693eaa097763b08b38cded279f0053ed53c553/pycryptodomex-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:02d87b80778c171445d67e23d1caef279bf4b25c3597050ccd2e13970b57fd51", size = 1707161, upload-time = "2025-05-17T17:23:11.414Z" }, +] + [[package]] name = "pydantic" version = "2.13.4" @@ -1095,6 +1128,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "pyzipper" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycryptodomex" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/5a/548039b202f85fcdfbadaddde2f4182c6dbf7730bbe005b24f903ae886ee/pyzipper-0.4.0.tar.gz", hash = "sha256:a4b96afcac04c5589d5abdc6158dd362166374e3cc6810aa441e65f8a17cb9e3", size = 36780, upload-time = "2026-05-14T04:06:28.903Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/9a/db7b20df854ea3a7f25ee9d7bc31ff3ecce085843bcf4c06bd9c1dcc34e0/pyzipper-0.4.0-py3-none-any.whl", hash = "sha256:aa7b8a0fe741d67aac36ead85f6e735af107b72f84e0775f2ed565fc0d3a2f02", size = 36643, upload-time = "2026-05-14T04:06:27.332Z" }, +] + [[package]] name = "rich" version = "15.0.0" @@ -1296,6 +1341,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/e1/b2df4bc09a1e51ff664c1e17018a4274b42e5e9352e4a478ea540512dc88/starlette-1.0.1-py3-none-any.whl", hash = "sha256:7c0e69b2ee1c848bd54669d908500117a3ee13de603a21427e5c6fc1adf98dcd", size = 72802, upload-time = "2026-05-21T21:58:56.551Z" }, ] +[[package]] +name = "striprtf" +version = "0.0.33" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/3b/c42830804cb2da515d0cb8aa200fb199ce57f7dcc344ff73db9dfe37cf3b/striprtf-0.0.33.tar.gz", hash = "sha256:c2d3d9ff3118df6dab558675f10a31ff8bb999ac1f8921f00c6dc9ea19961f18", size = 8010, upload-time = "2026-08-17T21:11:15.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/85/bee751fd2096accfc8b76186d49fbb2871163723e45f1e721e03c3f044ae/striprtf-0.0.33-py3-none-any.whl", hash = "sha256:f9637632a4414de05b1c399ee34d324dc336133ae45769992143a024e0f919ef", size = 8446, upload-time = "2026-08-17T21:11:14.696Z" }, +] + [[package]] name = "structlog" version = "25.5.0" @@ -1539,15 +1593,15 @@ wheels = [ [[package]] name = "xknx" -version = "3.18.0" +version = "3.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "ifaddr" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/21/bc/f80479543890e2b83a8d32c31af3899adcbe9143c2deccfdcf8d7dfb3af0/xknx-3.18.0.tar.gz", hash = "sha256:bbca50110fd32e28274bee998462c1e7e992260d7489b957b85af10f15e5991d", size = 190189, upload-time = "2026-08-02T14:11:20.328Z" } +sdist = { url = "https://files.pythonhosted.org/packages/af/c1/de7cbc042cd6f3d570ae5425af76a0f38534a76fa09ce6597fb2bdded611/xknx-3.20.0.tar.gz", hash = "sha256:8be97bcca6c7768f02d3bc18f6236c786e8707327fde55c015272753880cc0df", size = 200881, upload-time = "2026-08-16T05:40:41.198Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/db/044894d64f0726bcf35555110235ba1927c5d4cb9b707600f4483a3e20f0/xknx-3.18.0-py3-none-any.whl", hash = "sha256:98841b34ab28d2f688d6906f5d994b2f3c36b1807204d662e278f7fbd322b54c", size = 280395, upload-time = "2026-08-02T14:11:18.916Z" }, + { url = "https://files.pythonhosted.org/packages/b6/d9/891f86d8d53876cf1e8e3275c6ea71a22db67607a388bb112c84b4f7fdc0/xknx-3.20.0-py3-none-any.whl", hash = "sha256:064353a2d295d793a4198994d732fb4e5bba922d5e5a47d0f7e14ac3ac16dacc", size = 294407, upload-time = "2026-08-16T05:40:39.925Z" }, ] [[package]] @@ -1569,6 +1623,23 @@ requires-dist = [ { name = "xknx-product", editable = "packages/product" }, ] +[[package]] +name = "xknx-download" +version = "0.1.0" +source = { editable = "packages/download" } +dependencies = [ + { name = "xknx" }, + { name = "xknx-models" }, + { name = "xknx-product" }, +] + +[package.metadata] +requires-dist = [ + { name = "xknx", specifier = ">=3.20,<4" }, + { name = "xknx-models", editable = "packages/models" }, + { name = "xknx-product", editable = "packages/product" }, +] + [[package]] name = "xknx-keyring" version = "0.1.0" @@ -1630,12 +1701,27 @@ source = { editable = "packages/project" } dependencies = [ { name = "sqlalchemy" }, { name = "xknx-models" }, + { name = "xknxproject" }, ] [package.metadata] requires-dist = [ { name = "sqlalchemy", specifier = ">=2.0.0" }, { name = "xknx-models", editable = "packages/models" }, + { name = "xknxproject", specifier = ">=3.10" }, +] + +[[package]] +name = "xknxproject" +version = "3.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyzipper" }, + { name = "striprtf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/7e/5a5d23f62e55e270f84fcff1cb643182b83f161da6efb0f4e8823ecfae44/xknxproject-3.10.0.tar.gz", hash = "sha256:b8ae4a63538992a072e819c32a9ec75947f34ab57cf0adbb3b0e84155fabaede", size = 50662, upload-time = "2026-08-02T14:17:25.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/95/504bb859ef2105c58b0d284ca50b25f5e1f62e7478d4082ba241c3086df3/xknxproject-3.10.0-py3-none-any.whl", hash = "sha256:cf8f1c2d6c846cad74b31987e745b06a46b1bfa71ad7406df4bd5b19803f7714", size = 51283, upload-time = "2026-08-02T14:17:24.77Z" }, ] [[package]] @@ -1644,6 +1730,7 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "xknx-catalog" }, + { name = "xknx-download" }, { name = "xknx-keyring" }, { name = "xknx-models" }, { name = "xknx-product" }, @@ -1663,6 +1750,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "xknx-catalog", editable = "packages/catalog" }, + { name = "xknx-download", editable = "packages/download" }, { name = "xknx-keyring", editable = "packages/keyring" }, { name = "xknx-models", editable = "packages/models" }, { name = "xknx-product", editable = "packages/product" }, From 29014d25f46f2f29bd8ddcdd32541aed2d6dbf67 Mon Sep 17 00:00:00 2001 From: knx-ai Date: Mon, 31 Aug 2026 16:20:42 +0200 Subject: [PATCH 2/2] feat(download): KNX Data Secure (tool key) + keyring, and review hardening Add KNX Data Secure point-to-point programming with Tool Key access (data_secure, secure_session) and a keyring bridge (secure_keyring), wired into download()/preflight() via a security= parameter. The CCM construction is verified byte-for-byte against 3/3/7 Annex C (C.1.1-C.1.4). Harden the load-procedure engine per an external review, cross-checked against the KNX standard and the decompiled ETS/Falcon reference: - MCB CRC computed per segment sub-entry (gd.cs), not over the whole object - split memory at the 64 KiB boundary into A_Memory_ / A_UserMemory_ (eo.cs) - property write validation (element fits APDU, divisibility, positive count) - reject empty/mismatched property responses; zero table reference - master-reset process time is seconds (DPT 7.005), not milliseconds - pre-validate unsupported controls before touching the device Deliberate deviations from the spec (object-based scope, fingerprint-tolerant compare, no-ACK restart) are documented in the code with their rationale. --- packages/download/README.md | 29 +- .../src/xknxmono/download/__init__.py | 6 + .../src/xknxmono/download/data_secure.py | 610 ++++++++++++++++++ .../src/xknxmono/download/download.py | 53 +- .../download/src/xknxmono/download/merge.py | 12 +- .../src/xknxmono/download/procedure.py | 115 +++- .../src/xknxmono/download/programmer.py | 208 ++++-- .../download/src/xknxmono/download/scope.py | 14 + .../src/xknxmono/download/secure_keyring.py | 63 ++ .../src/xknxmono/download/secure_session.py | 125 ++++ packages/download/tests/conftest.py | 12 +- packages/download/tests/test_data_secure.py | 406 ++++++++++++ packages/download/tests/test_programmer.py | 14 + packages/download/tests/test_review_fixes.py | 141 ++++ .../download/tests/test_secure_keyring.py | 57 ++ .../download/tests/test_secure_session.py | 123 ++++ pyrightconfig.json | 6 +- 17 files changed, 1935 insertions(+), 59 deletions(-) create mode 100644 packages/download/src/xknxmono/download/data_secure.py create mode 100644 packages/download/src/xknxmono/download/secure_keyring.py create mode 100644 packages/download/src/xknxmono/download/secure_session.py create mode 100644 packages/download/tests/test_data_secure.py create mode 100644 packages/download/tests/test_review_fixes.py create mode 100644 packages/download/tests/test_secure_keyring.py create mode 100644 packages/download/tests/test_secure_session.py diff --git a/packages/download/README.md b/packages/download/README.md index b6c82181..98ce2489 100644 --- a/packages/download/README.md +++ b/packages/download/README.md @@ -47,9 +47,32 @@ connection). Not covered yet: clearing a line coupler filter table (`LdCtrlClearLCFilterTable`, coupler-only and not yet validated against a router), the load procedure control -flow directives `LdCtrlOnError` / `LdCtrlProcType`, KNX Data Secure, and USB -transport. Unsupported Load Controls are reported via `UnsupportedProcedureError` -rather than guessed (see [Implementation gaps](#implementation-gaps-and-diagnostics)). +flow directives `LdCtrlOnError` / `LdCtrlProcType`, and USB transport. Unsupported +Load Controls are reported via `UnsupportedProcedureError` rather than guessed (see +[Implementation gaps](#implementation-gaps-and-diagnostics)). + +## KNX Data Secure (Tool Key) + +For a secure device, pass a `DeviceSecurity(address, tool_key)` as the `security` +argument to `download`/`preflight`. The whole session is then KNX Data Secure +protected with the device's Tool Key (the mode ETS uses for programming), the same +way a plain download runs - the programmer is unaware of it. + +- The CCM construction (B0/Ctr0, the `A`/`P` split per S-AL service, the AES-CBC-MAC + and AES-CTR steps) follows KNX Standard v3.0.0, 3/3/7 section 5 and is verified + byte-for-byte against the worked examples in 3/3/7 Annex C (C.1.1-C.1.4). +- Before the first secured frame the session is synchronised with an S-A_Sync + exchange (`DM_SecureSync`, 3/5/2), learning the device's Sequence Numbers. +- Frames are secured on the CEMI path (`xknx.cemi_handler.data_secure`), after the + transport layer has assigned the connection-oriented sequence number that the B0 + block binds - the same hook xknx uses for group Data Secure. + +The Tool Key can be supplied directly (`DeviceSecurity(address, tool_key)`) or read +from a KNX keyring: `load_device_security(path, password, address)` loads and +decrypts a `.knxkeys` file and returns the `DeviceSecurity` for the device, and +`device_security_from_keyring(keyring, address)` does the same from an already +loaded keyring. Decryption uses xknx's keyring loader. Wiring this into the GUI is +not part of this package yet. ## Group communication tables diff --git a/packages/download/src/xknxmono/download/__init__.py b/packages/download/src/xknxmono/download/__init__.py index 9dd20cde..0fbe8f34 100644 --- a/packages/download/src/xknxmono/download/__init__.py +++ b/packages/download/src/xknxmono/download/__init__.py @@ -9,6 +9,7 @@ from __future__ import annotations from .commissioning import program_individual_address +from .data_secure import DeviceSecurity, SecureProgrammingError from .download import download, preflight from .errors import ( DownloadError, @@ -44,6 +45,7 @@ parameter_values_from_device, ) from .scope import DownloadScope +from .secure_keyring import device_security_from_keyring, load_device_security from .tables import ( Association, build_association_table, @@ -57,6 +59,7 @@ "ByteRange", "ConnectionManager", "DeviceProgrammer", + "DeviceSecurity", "DownloadError", "DownloadImage", "DownloadScope", @@ -71,6 +74,7 @@ "PreflightReport", "PropertyDiff", "PropertyValue", + "SecureProgrammingError", "SeedDevice", "SegmentDiff", "UnsupportedProcedureError", @@ -78,9 +82,11 @@ "build_association_table", "build_group_address_table", "build_image", + "device_security_from_keyring", "download", "group_address_table", "group_communication_from_device", + "load_device_security", "module_instances_from_device", "parameter_instance_refs_from_device", "parameter_values_from_device", diff --git a/packages/download/src/xknxmono/download/data_secure.py b/packages/download/src/xknxmono/download/data_secure.py new file mode 100644 index 00000000..5d30e8a0 --- /dev/null +++ b/packages/download/src/xknxmono/download/data_secure.py @@ -0,0 +1,610 @@ +"""KNX Data Secure for point-to-point programming (Tool Key access). + +Secures the management telegrams of a download run following KNX Standard +v3.0.0, 3/3/7 section 5 (the Secure Application Layer): S-A_Data in 5.3.1, +S-A_Sync in 5.3.2. Tool Key access (SCF ``Tool Access`` bit set) is the mode +ETS uses for device programming: the device accepts frames from any tool +address secured with the Tool Key, so no key table has to be provisioned. + +The CCM building blocks (B0, Ctr0, the AES-CBC-MAC and AES-CTR steps) are laid +out here per KNX 3/3/7 sections 5.1.3.2 and Annex A, and the AES steps +themselves are taken from xknx's ``security_primitives``. The construction is +verified byte-for-byte against the worked examples in 3/3/7 Annex C +(C.1.1-C.1.4). It is deliberately *not* delegated to xknx's +``SecureData.init_from_plain_apdu``: that path only handles secure group +communication and gets two things wrong for point-to-point programming - it +puts only the SCF into the additional data ``A`` (the S-A_Sync request needs +``SCF | KNX Serial Number``) and its B0 TPCI/APCI octet is only correct for a +zero TPCI, so it cannot encode the connection-oriented frames the download uses. + +Each B0/Ctr0 is built from a ``CEMILData`` derived from the telegram, so the +source/destination address, the address-type octet and the TPCI octet are +exactly what the frame carries on the wire. Both sides recompute the same +blocks from the frame's own fields. + +Because the S-AL state on the device advances with the Sequence Number of an +accepted frame, the session's Sequence Numbers are first synchronised with an +S-A_Sync request before the first S-A_Data frame - the step ETS performs +(DM_SecureSync, 3/5/2). +""" + +from __future__ import annotations + +import logging +import secrets +import time +from copy import copy +from datetime import datetime +from typing import Final, Protocol + +from xknx.cemi.cemi_frame import CEMILData +from xknx.exceptions import ConversionError, DataSecureError, UnsupportedAPCIService +from xknx.secure.data_secure_asdu import ( + SecureData, + SecurityAlgorithmIdentifier, + SecurityALService, + SecurityControlField, +) +from xknx.secure.security_primitives import ( + calculate_message_authentication_code_cbc, + decrypt_ctr, + encrypt_data_ctr, +) +from xknx.telegram import Telegram +from xknx.telegram.address import IndividualAddress +from xknx.telegram.apci import APCI, SecureAPDU + +from .errors import DownloadError + +logger = logging.getLogger(__name__) + +# Secure APCI is 0x3F1 (3/3/7 Figure 101). In B0 its low 8 bits are one octet +# and its top 2 bits sit in the low 2 bits of the TPCI octet. +_SECURE_APCI_HIGH_BITS: Final = 0x03 +_SECURE_APCI_LOW: Final = 0xF1 +# The MAC is the 32 most significant bits of the CBC result (3/3/7 5.1.3.4.2). +_MAC_LENGTH: Final = 4 +# Only the address type bit and the extended-frame nibble of the control field +# are authenticated in B0 (3/3/7 5.1.3.2, "AT = A000EEEEb"). +_B0_CONTROL_FIELD_MASK: Final = 0x8F +# Octets a Secure APDU adds around the plain APDU (3/3/7 5.1.3.5.1, Figure 104): +# 2 Secure APCI + 1 SCF + 6 SeqNr + 4 MAC. The plaintext APDU must be this much +# smaller than the device's wire APDU length so the secured frame still fits. +SECURE_APDU_OVERHEAD: Final = 13 + +# Security Control Fields of the session (3/3/7 5.1.3, Figure 106): Tool Access +# set, CCM with authentication and confidentiality. +DATA_SCF: Final = SecurityControlField( + tool_access=True, + algorithm=SecurityAlgorithmIdentifier.CCM_ENCRYPTION, + system_broadcast=False, + service=SecurityALService.S_A_DATA, +) +SYNC_REQ_SCF: Final = SecurityControlField( + tool_access=True, + algorithm=SecurityAlgorithmIdentifier.CCM_ENCRYPTION, + system_broadcast=False, + service=SecurityALService.S_A_SYNC_REQ, +) +SYNC_RES_SCF: Final = SecurityControlField( + tool_access=True, + algorithm=SecurityAlgorithmIdentifier.CCM_ENCRYPTION, + system_broadcast=False, + service=SecurityALService.S_A_SYNC_RES, +) + +# Upper bound for a Sequence Number Sending (48-bit counter). +_SEQUENCE_NUMBER_MAX: Final = 0xFFFFFFFFFFFF +# Base timestamp for the initial (request) sequence number: milliseconds since +# 2018-01-05, the same base xknx uses for Data Secure - it keeps the tool's +# Sequence Number Sending in the same range as the devices' counters. +_SEQUENCE_NUMBER_INIT_TIMESTAMP: Final = datetime.fromisoformat( + "2018-01-05T00:00:00+00:00" +).timestamp() + + +class SecureProgrammingError(DownloadError): + """The secure programming session could not be established or verified.""" + + +class CemiSecurer(Protocol): + """The ``xknx.cemi_handler.data_secure`` hook interface. + + Both xknx's group ``DataSecure`` and :class:`ToolKeyCemiSecure` implement it, + so a Tool-Key session can delegate the traffic it does not handle to a + securer that was already installed. + """ + + def outgoing_cemi(self, cemi_data: CEMILData) -> CEMILData: + """Secure or pass through an outgoing frame.""" + ... + + def received_cemi(self, cemi_data: CEMILData) -> CEMILData: + """Unwrap or pass through an incoming frame.""" + ... + + +class DeviceSecurity: + """Tool Key security material for programming one device. + + ``tool_key`` is the device's Tool Key (the FDSK for a device in factory + state), as 16 raw octets or 32 hex digits (the form the keyring stores it + in). + """ + + __slots__ = ("address", "tool_key") + + def __init__(self, address: IndividualAddress, tool_key: bytes | str) -> None: + """Initialize with the target address and the Tool Key.""" + self.address = address + self.tool_key = self._normalize_key(tool_key) + + @staticmethod + def _normalize_key(tool_key: bytes | str) -> bytes: + if isinstance(tool_key, str): + try: + tool_key = bytes.fromhex(tool_key) + except ValueError as exc: + raise SecureProgrammingError( + f"tool key hex string cannot be decoded: {exc}" + ) from exc + if len(tool_key) != 16: + raise SecureProgrammingError( + "tool key must be 16 octets (32 hex digits), " + f"got {len(tool_key)} octets" + ) + return bytes(tool_key) + + +class _BlockFields: + """The frame fields that go into a B0/Ctr0 block for one telegram. + + These are exactly the fields the receiver reconstructs from the frame it + receives, so both sides compute the same blocks (3/3/7 5.1.3.2). + """ + + __slots__ = ("address_type", "destination", "source", "tpci_octet") + + def __init__(self, cemi: CEMILData) -> None: + """Derive the block fields from a link-layer frame.""" + self.source = cemi.src_addr.to_knx() + self.destination = cemi.dst_addr.to_knx() + # AT = A000EEEEb: address type bit and extended-frame nibble only. + self.address_type = cemi.flags & _B0_CONTROL_FIELD_MASK + # The full TPCI octet as carried on the wire; its low 2 bits are 0 for a + # data frame and become the top 2 bits of the Secure APCI in B0. + self.tpci_octet = cemi.tpci.to_knx() + + @classmethod + def from_telegram( + cls, telegram: Telegram, *, source: IndividualAddress + ) -> _BlockFields: + """Derive the block fields from ``telegram`` as sent by ``source``.""" + return cls(CEMILData.init_from_telegram(telegram=telegram, src_addr=source)) + + def block_0(self, nonce: bytes, payload_length: int) -> bytes: + """Return B0 (3/3/7 Figure 100). ``nonce`` is SeqNr, or Random for S-A_Sync.res.""" + return ( + nonce + + self.source + + self.destination + + bytes( + ( + 0x00, + self.address_type, + self.tpci_octet | _SECURE_APCI_HIGH_BITS, + _SECURE_APCI_LOW, + 0x00, + payload_length, + ) + ) + ) + + def counter_0(self, nonce: bytes) -> bytes: + """Return Ctr0 (3/3/7 Figure 102). Octet 14 is 01h, [j] for Ctr0 is 00h.""" + return nonce + self.source + self.destination + b"\x00\x00\x00\x00\x01\x00" + + +def _encrypt( + key: bytes, + fields: _BlockFields, + nonce: bytes, + additional_data: bytes, + payload: bytes, +) -> tuple[bytes, bytes]: + """CCM-encrypt ``payload`` and return ``(cipher, mac)`` (3/3/7 Annex A).""" + mac_cbc = calculate_message_authentication_code_cbc( + key=key, + additional_data=additional_data, + payload=payload, + block_0=fields.block_0(nonce, len(payload)), + )[:_MAC_LENGTH] + return encrypt_data_ctr( + key=key, + counter_0=fields.counter_0(nonce), + mac_cbc=mac_cbc, + payload=payload, + ) + + +def _decrypt( + key: bytes, + fields: _BlockFields, + nonce: bytes, + additional_data: bytes, + cipher: bytes, + mac: bytes, +) -> bytes: + """CCM-decrypt ``cipher`` and verify the MAC, or raise (3/3/7 Annex A).""" + plain, mac_tr = decrypt_ctr( + key=key, counter_0=fields.counter_0(nonce), mac=mac, payload=cipher + ) + mac_cbc = calculate_message_authentication_code_cbc( + key=key, + additional_data=additional_data, + payload=plain, + block_0=fields.block_0(nonce, len(plain)), + )[:_MAC_LENGTH] + if mac_cbc != mac_tr: + raise SecureProgrammingError( + "MAC verification failed - wrong Tool Key or tampered frame" + ) + return plain + + +class SecureManagement: + """Tool-Key state and CCM operations for one programming session. + + Secures outgoing S-A_Data frames and unwraps incoming ones, and drives the + S-A_Sync exchange that establishes the Sequence Numbers before the first + S-A_Data frame: the response's SeqNrRemote becomes the last valid value for + the device and its SeqNrLocal the next Sequence Number Sending for the tool + (3/3/7 5.3.2, Figure 109/110). + + It works on ``CEMILData`` frames, not telegrams, because in the + connection-oriented mode ETS uses the B0 block includes the TPCI octet + (with the transport-layer sequence number), which is only final once the + transport layer has built the frame. :class:`ToolKeyCemiSecure` installs + this on the frame path where that is the case. + """ + + def __init__( + self, + *, + device: DeviceSecurity, + tool_address: IndividualAddress, + ) -> None: + """Initialize for the device's Tool Key. + + ``tool_address`` is the tool's own source individual address on the bus + (e.g. ``xknx.current_address``); it is bound into the secured data like + every other frame field, so it must be the address the frames actually + carry. + """ + self._device = device + self._tool_address = tool_address + self._next_sequence_number: int | None = None + self._last_valid_remote: int | None = None + self._sync_armed = False + self._sync_challenge: bytes | None = None + self._sync_sequence: int | None = None + # The exact secured S-A_Sync request, cached so a transport-layer + # retransmission of the trigger frame replays identical bytes instead of + # being turned into an S-A_Data frame (3/3/7 5.3.2, Note 38). + self._sync_request: SecureAPDU | None = None + + @property + def synchronized(self) -> bool: + """Whether the session's sequence numbers are synchronized.""" + return self._next_sequence_number is not None + + @property + def sync_armed(self) -> bool: + """Whether the next outgoing frame is to be turned into an S-A_Sync request.""" + return self._sync_armed + + # ------------------------------------------------------------------ sync + + def arm_sync(self) -> None: + """Prepare a fresh challenge for the next S-A_Sync request. + + Clears any previous session state, so a re-sync (e.g. after a device + restart) starts from a clean slate. The next outgoing frame to the + device is turned into the S-A_Sync request by + :meth:`build_sync_request_cemi`, and stays armed (replaying the same + request for retransmissions) until the response is decoded. + """ + self._next_sequence_number = None + self._last_valid_remote = None + self._sync_armed = True + self._sync_request = None + self._sync_challenge = secrets.token_bytes(6) + # The request's SeqNrLocal is the sequence number the tool *assumes* it + # will send next (3/3/7 5.3.2); the response corrects it. + self._sync_sequence = _initial_sequence_number() + + def build_sync_request_cemi(self, cemi: CEMILData) -> CEMILData: + """Replace ``cemi``'s payload with the S-A_Sync request (3/3/7 Figure 109). + + A = SCF | KNX Serial Number (0 for point-to-point), P = Challenge. The + KNX Serial Number travels in plain text on the wire between the + Sequence Number and the encrypted Challenge. The request is built once + and cached; a retransmission of the same trigger frame replays the + identical Secure APDU (a fresh MAC over a new SeqNr would be rejected by + the device), matching that an S-A_Sync request does not advance the + Sequence Number Sending (3/3/7 5.3.2, Note 38). + """ + if self._sync_challenge is None or self._sync_sequence is None: + raise SecureProgrammingError("S-A_Sync is not armed") + secured = copy(cemi) + if self._sync_request is not None: + secured.payload = self._sync_request + return secured + serial_number = b"\x00" * 6 + nonce = self._sync_sequence.to_bytes(6, "big") + cipher, mac = _encrypt( + key=self._device.tool_key, + fields=_BlockFields(secured), + nonce=nonce, + additional_data=SYNC_REQ_SCF.to_knx() + serial_number, + payload=self._sync_challenge, + ) + self._sync_request = SecureAPDU( + scf=SYNC_REQ_SCF, + secured_data=SecureData( + sequence_number_bytes=nonce, + secured_apdu=serial_number + cipher, + message_authentication_code=mac, + ), + ) + secured.payload = self._sync_request + return secured + + def decode_sync_response_cemi(self, cemi: CEMILData) -> bool: + """Consume an S-A_Sync response and store the synchronized state. + + Returns whether the frame was a valid S-A_Sync response for this + session. The response's nonce is the device's Random value, recovered + by XOR-ing the plain-text ``Challenge XOR Random`` field with our own + challenge (3/3/7 5.3.2). + """ + payload = cemi.payload + if not isinstance(payload, SecureAPDU): + return False + if not _is_tool_scheme(payload.scf, SecurityALService.S_A_SYNC_RES): + return False + if self._sync_challenge is None: + # Already synchronized (e.g. a duplicated response); ignore. + return self._next_sequence_number is not None + challenge_xor_random = payload.secured_data.sequence_number_bytes + random = bytes( + a ^ b + for a, b in zip(challenge_xor_random, self._sync_challenge, strict=True) + ) + try: + plain = _decrypt( + key=self._device.tool_key, + fields=_BlockFields(cemi), + nonce=random, + additional_data=payload.scf.to_knx(), + cipher=payload.secured_data.secured_apdu, + mac=payload.secured_data.message_authentication_code, + ) + except SecureProgrammingError: + logger.warning("S-A_Sync response failed MAC verification; ignoring") + return False + if len(plain) != 12: + return False + sequence_number_remote = int.from_bytes(plain[:6], "big") + sequence_number_local = int.from_bytes(plain[6:12], "big") + # Both are "next valid" Sequence Numbers and are never 0 (3/3/7 5.3.1; + # a device rejects them, and SeqNrRemote 0 would make last_valid_remote + # negative and admit a replayed frame with SeqNr 0). SeqNrLocal is what + # the device expects from the tool next (Note 43); SeqNrRemote is the + # device's own next Sequence Number Sending, so its last valid value is + # one less (Note 45). + if sequence_number_local == 0 or sequence_number_remote == 0: + raise SecureProgrammingError( + "S-A_Sync response returned sequence number 0, which is invalid" + ) + self._next_sequence_number = sequence_number_local + self._last_valid_remote = sequence_number_remote - 1 + self._sync_armed = False + self._sync_request = None + self._sync_challenge = None + self._sync_sequence = None + logger.info( + "data secure sync complete: tool sends from %#014x, " + "device answers from %#014x", + sequence_number_local, + sequence_number_remote, + ) + return True + + # ------------------------------------------------------------- wrapping + + def wrap_cemi(self, cemi: CEMILData) -> CEMILData: + """Return a copy of ``cemi`` with its APDU secured as an S-A_Data frame. + + Must be called after synchronization. Only the APDU is replaced by a + Secure APDU; the frame's addresses and TPCI drive the B0/Ctr0 blocks. + """ + if self._next_sequence_number is None: + raise SecureProgrammingError( + "data secure session not synchronized - run S-A_Sync first" + ) + if not isinstance(cemi.payload, APCI): + raise SecureProgrammingError( + "cannot secure a control frame (no APDU to protect)" + ) + sequence_number = self._next_sequence_number + if sequence_number > _SEQUENCE_NUMBER_MAX: + raise SecureProgrammingError( + "Sequence Number Sending exhausted (48-bit counter overflow); " + "re-synchronize the session" + ) + self._next_sequence_number += 1 + secured = copy(cemi) + nonce = sequence_number.to_bytes(6, "big") + cipher, mac = _encrypt( + key=self._device.tool_key, + fields=_BlockFields(secured), + nonce=nonce, + additional_data=DATA_SCF.to_knx(), + payload=bytes(cemi.payload.to_knx()), + ) + secured.payload = SecureAPDU( + scf=DATA_SCF, + secured_data=SecureData( + sequence_number_bytes=nonce, + secured_apdu=cipher, + message_authentication_code=mac, + ), + ) + return secured + + def unwrap_cemi(self, cemi: CEMILData) -> CEMILData: + """Return a copy of ``cemi`` with its S-A_Data APDU decrypted and verified. + + Raises :class:`SecureProgrammingError` if the frame is not a Tool Key + secured S-A_Data frame, the MAC does not verify, or the sequence number + is not higher than the last valid value for the device. + """ + if self._last_valid_remote is None: + raise SecureProgrammingError( + "data secure session not synchronized - cannot decrypt response" + ) + payload = cemi.payload + if not isinstance(payload, SecureAPDU): + raise SecureProgrammingError( + f"expected a secure APDU in the response, got {payload!r} - the " + "device is not using KNX Data Secure (Tool Key access)" + ) + if not _is_tool_scheme(payload.scf, SecurityALService.S_A_DATA): + raise SecureProgrammingError( + "response secured with an unexpected scheme: " + f"service {payload.scf.service!r}, " + f"algorithm {payload.scf.algorithm!r}, " + f"tool access {payload.scf.tool_access}" + ) + sequence_number = int.from_bytes( + payload.secured_data.sequence_number_bytes, "big" + ) + if sequence_number <= self._last_valid_remote: + raise SecureProgrammingError( + f"device sequence number too low: {sequence_number:#014x} " + f"(last valid {self._last_valid_remote:#014x})" + ) + plain = _decrypt( + key=self._device.tool_key, + fields=_BlockFields(cemi), + nonce=payload.secured_data.sequence_number_bytes, + additional_data=payload.scf.to_knx(), + cipher=payload.secured_data.secured_apdu, + mac=payload.secured_data.message_authentication_code, + ) + # The plain APDU starts with the 000000b prefix before the 10-bit APCI + # (3/3/7 Figure 103); its top 6 bits must be zero. APCI.from_knx masks + # them off, so check here to reject an authenticated but malformed APDU. + if not plain or plain[0] & 0xFC: + raise SecureProgrammingError( + "decrypted APDU has a non-zero reserved prefix" + ) + try: + decoded = APCI.from_knx(plain) + except (ConversionError, UnsupportedAPCIService) as exc: + raise SecureProgrammingError( + f"decrypted APDU could not be parsed: {exc}" + ) from exc + self._last_valid_remote = sequence_number + plain_cemi = copy(cemi) + plain_cemi.payload = decoded + return plain_cemi + + +class ToolKeyCemiSecure: + """CEMI-layer securer for one device, for ``xknx.cemi_handler.data_secure``. + + Installed for the duration of a secure programming session. It secures the + frames to the target device with the Tool Key and unwraps the device's + secured answers, and passes every other frame through unchanged. It mirrors + how xknx installs its own group Data Secure on the same hook, so the B0/Ctr0 + blocks see the final transport-layer sequence number of each frame. + """ + + def __init__( + self, + management: SecureManagement, + device: IndividualAddress, + previous: CemiSecurer | None = None, + ) -> None: + """Initialize for ``management``'s Tool Key session and ``device``. + + ``previous`` is the securer that was installed before this one (e.g. a + group Data Secure); traffic not addressed to ``device`` is delegated to + it so concurrent secure group communication keeps working. + """ + self._management = management + self._device = device + self._previous = previous + + def outgoing_cemi(self, cemi_data: CEMILData) -> CEMILData: + """Secure an outgoing frame to the device; delegate everything else.""" + if cemi_data.dst_addr != self._device: + return ( + self._previous.outgoing_cemi(cemi_data) + if self._previous is not None + else cemi_data + ) + payload = cemi_data.payload + # Control frames (T_Connect/T_Ack/T_Disconnect) carry no APDU, and an + # already-secure APDU must not be wrapped again. + if not isinstance(payload, APCI) or isinstance(payload, SecureAPDU): + return cemi_data + if self._management.sync_armed: + return self._management.build_sync_request_cemi(cemi_data) + return self._management.wrap_cemi(cemi_data) + + def received_cemi(self, cemi_data: CEMILData) -> CEMILData: + """Unwrap a secured frame from the device; delegate everything else. + + A frame that fails verification is reported as :class:`DataSecureError`, + the contract xknx's CEMI handler expects on this hook: it logs the + failure and drops the frame instead of letting it crash the receive + loop. The pending request then times out, surfacing the failure. + """ + if cemi_data.src_addr != self._device: + return ( + self._previous.received_cemi(cemi_data) + if self._previous is not None + else cemi_data + ) + payload = cemi_data.payload + if not isinstance(payload, SecureAPDU): + return cemi_data + try: + if payload.scf.service is SecurityALService.S_A_SYNC_RES: + # Consume the sync state; leave the frame as-is so the request + # that triggered the sync still gets a (secure) response telegram. + self._management.decode_sync_response_cemi(cemi_data) + return cemi_data + return self._management.unwrap_cemi(cemi_data) + except SecureProgrammingError as exc: + raise DataSecureError(str(exc)) from exc + + +def _is_tool_scheme(scf: SecurityControlField, service: SecurityALService) -> bool: + """Whether ``scf`` is Tool Key access with CCM for the given S-AL service.""" + return ( + scf.service is service + and scf.algorithm is SecurityAlgorithmIdentifier.CCM_ENCRYPTION + and scf.tool_access + ) + + +def _initial_sequence_number() -> int: + """Return the initial (request) sequence number for S-A_Sync.""" + return min( + int((time.time() - _SEQUENCE_NUMBER_INIT_TIMESTAMP) * 1000), + _SEQUENCE_NUMBER_MAX, + ) diff --git a/packages/download/src/xknxmono/download/download.py b/packages/download/src/xknxmono/download/download.py index 7a9f93fc..761fcae4 100644 --- a/packages/download/src/xknxmono/download/download.py +++ b/packages/download/src/xknxmono/download/download.py @@ -28,9 +28,10 @@ from xknxmono.product import Application, MasterData + from .data_secure import DeviceSecurity from .image import DownloadImage, GroupCommunication from .preflight import PreflightReport - from .programmer import BusConnection + from .programmer import BusConnection, ConnectionManager from .project_data import SeedDevice @@ -64,6 +65,32 @@ def _apdu_settings(max_apdu_length: int | None) -> tuple[int, bool]: return max_apdu_length, False +def _connection_manager( + xknx: XKNX, address: IndividualAddress, security: DeviceSecurity | None +) -> ConnectionManager: + """Return a plain or a Tool-Key secured connection manager for ``address``.""" + if security is None: + return _XknxConnectionManager(xknx, address) + from .data_secure import SecureProgrammingError + from .secure_session import SecureConnectionManager + + if security.address != address: + raise SecureProgrammingError( + f"security material is for {security.address}, not the download " + f"target {address}" + ) + return SecureConnectionManager(xknx, address, security) + + +def _apdu_overhead(security: DeviceSecurity | None) -> int: + """Wire APDU overhead a secure session adds around each plaintext APDU.""" + if security is None: + return 0 + from .data_secure import SECURE_APDU_OVERHEAD + + return SECURE_APDU_OVERHEAD + + async def download( xknx: XKNX, individual_address: IndividualAddressableType, @@ -77,6 +104,7 @@ async def download( parameter_values: Mapping[str, str] | None = None, max_apdu_length: int | None = None, expected_descriptor: int | None = None, + security: DeviceSecurity | None = None, progress: Callable[[int, int], None] | None = None, ) -> None: """Download ``application`` into the device at ``individual_address``. @@ -113,12 +141,20 @@ async def download( *resolve_download_controls( application, master.raw if master is not None else None ), + # System B products carry no Load Controls for the group communication + # tables, so they are synthesized and appended here. NOTE: they run after + # the application procedure; for a product whose procedure ends in a + # Restart the runner reconnects and writes the tables in a fresh session. + # ETS/Falcon instead bind the table images to the master procedure's own + # allocation/write controls (before its restart). This ordering has only + # been validated read-only via preflight, not on a real write+restart, so + # it should be verified against hardware before relying on it there. *synthesize_group_communication_controls(image), ] address = IndividualAddress(individual_address) apdu_ceiling, negotiate_apdu = _apdu_settings(max_apdu_length) - manager = _XknxConnectionManager(xknx, address) + manager = _connection_manager(xknx, address, security) runner = LoadProcedureRunner( application, image, @@ -128,6 +164,7 @@ async def download( scope=scope, expected_descriptor=expected_descriptor, negotiate_apdu=negotiate_apdu, + apdu_overhead=_apdu_overhead(security), ) try: await runner.run(progress) @@ -150,6 +187,7 @@ async def preflight( parameter_values: Mapping[str, str] | None = None, max_apdu_length: int | None = None, expected_descriptor: int | None = None, + security: DeviceSecurity | None = None, ) -> PreflightReport: """Report what :func:`download` would change on the device, changing nothing. @@ -172,12 +210,20 @@ async def preflight( *resolve_download_controls( application, master.raw if master is not None else None ), + # System B products carry no Load Controls for the group communication + # tables, so they are synthesized and appended here. NOTE: they run after + # the application procedure; for a product whose procedure ends in a + # Restart the runner reconnects and writes the tables in a fresh session. + # ETS/Falcon instead bind the table images to the master procedure's own + # allocation/write controls (before its restart). This ordering has only + # been validated read-only via preflight, not on a real write+restart, so + # it should be verified against hardware before relying on it there. *synthesize_group_communication_controls(image), ] address = IndividualAddress(individual_address) apdu_ceiling, negotiate_apdu = _apdu_settings(max_apdu_length) - manager = _XknxConnectionManager(xknx, address) + manager = _connection_manager(xknx, address, security) runner = LoadProcedureRunner( application, image, @@ -187,6 +233,7 @@ async def preflight( scope=scope, expected_descriptor=expected_descriptor, negotiate_apdu=negotiate_apdu, + apdu_overhead=_apdu_overhead(security), ) try: return await runner.preflight() diff --git a/packages/download/src/xknxmono/download/merge.py b/packages/download/src/xknxmono/download/merge.py index 704dc0f0..0f33f1c4 100644 --- a/packages/download/src/xknxmono/download/merge.py +++ b/packages/download/src/xknxmono/download/merge.py @@ -101,7 +101,17 @@ def _default_procedure( mask_version_id: str, procedure_type: ProcedureType, ) -> LoadProcedure | None: - """Find the mask version's default procedure of the given type.""" + """Find the mask version's default procedure of the given type. + + Returns the first procedure of ``procedure_type`` (Load), regardless of its + ProcedureSubType (ap1/all/grp/par/par,grp). ETS/Falcon pick a subtype-specific + procedure per requested scope; this engine instead resolves one Load procedure + and applies the download scope afterwards by the interface object each control + targets (see :mod:`xknxmono.download.scope`). That object-based filtering was + validated byte-perfect against real hardware (memory-mapped and System B), + whereas ProcedureSubType/AppliesTo selection was not, so it is deliberately + the single point that decides full vs partial here. + """ if master_data is None or master_data.mask_versions is None: return None for mask_version in master_data.mask_versions.mask_version: diff --git a/packages/download/src/xknxmono/download/procedure.py b/packages/download/src/xknxmono/download/procedure.py index f0737600..324f94cf 100644 --- a/packages/download/src/xknxmono/download/procedure.py +++ b/packages/download/src/xknxmono/download/procedure.py @@ -100,14 +100,36 @@ def _mcb_table_with_crc(data: bytes, segment: bytes) -> bytes: """Patch the segment CRC into each CRC-protected Memory Control Block entry. ``data`` is the MCB table value (8 octet entries, plus any trailing padding); - ``segment`` is the loaded segment the CRC protects. Only entries whose - CRC-protected flag is set get their CRC octets filled. + ``segment`` is the loaded object segment the CRCs protect. Each entry declares + its own sub-segment size in octets 0..3 and carries the CRC (octets 6..7) over + just that sub-segment; the sub-segments tile the object data in order, so a + device with several segments in one object gets one CRC per segment rather + than one CRC over everything (KNX 3/5/1 4.2.27; matches Hawk gd.cs SegmentCRCs, + which reads the size via gy.c at 8*n and advances the offset cumulatively). + The size is a 32-bit value stored as two 16-bit big-endian halves, low half + first (gy.c order). Only entries whose CRC-protected flag (bit 0 of octet 4) + is clear get their CRC octets filled. + + If the declared sizes do not tile the segment exactly, fall back to a single + CRC over the whole segment - the behaviour validated byte-perfect on real + single-segment devices (e.g. System B 1.1.41), so an unexpected size encoding + can never regress it. """ out = bytearray(data) - crc = segment_crc(segment) - for start in range(0, len(out) - _MCB_ENTRY_SIZE + 1, _MCB_ENTRY_SIZE): + starts = list(range(0, len(out) - _MCB_ENTRY_SIZE + 1, _MCB_ENTRY_SIZE)) + sizes = [ + int.from_bytes(out[s : s + 2], "big") + | (int.from_bytes(out[s + 2 : s + 4], "big") << 16) + for s in starts + ] + per_entry = bool(sizes) and sum(sizes) == len(segment) + offset = 0 + for start, size in zip(starts, sizes, strict=True): + sub_segment = segment[offset : offset + size] if per_entry else segment + offset += size if out[start + _MCB_CRC_PROTECTED_OCTET] & 1: continue + crc = segment_crc(sub_segment) out[start + _MCB_CRC_OFFSET] = (crc >> 8) & 0xFF out[start + _MCB_CRC_OFFSET + 1] = crc & 0xFF return bytes(out) @@ -127,6 +149,42 @@ def _mcb_table_with_crc(data: bytes, segment: bytes) -> bytes: "LdCtrlDeclarePropDesc", ) +# Load Controls this engine executes on the bus (the isinstance branches of +# :meth:`LoadProcedureRunner._execute`). Used to pre-validate a resolved, +# scoped procedure before touching the device, so an unsupported control is +# reported up front instead of after earlier controls have already unloaded or +# written the device (a real master procedure can contain e.g. +# ``LdCtrlClearLCFilterTable``, which is not implemented). +_SUPPORTED_CONTROLS = frozenset( + { + "LdCtrlConnect", + "LdCtrlDisconnect", + "LdCtrlDelay", + "LdCtrlRestart", + "LdCtrlMasterReset", + "LdCtrlUnload", + "LdCtrlLoad", + "LdCtrlLoadCompleted", + "LdCtrlWriteMem", + "LdCtrlLoadImageMem", + "LdCtrlCompareMem", + "LdCtrlWriteRelMem", + "LdCtrlCompareRelMem", + "LdCtrlLoadImageRelMem", + "LdCtrlWriteProp", + "LdCtrlLoadImageProp", + "LdCtrlCompareProp", + "LdCtrlInvokeFunctionProp", + "LdCtrlReadFunctionProp", + "LdCtrlAbsSegment", + "LdCtrlRelSegment", + "LdCtrlTaskSegment", + "LdCtrlTaskPtr", + "LdCtrlTaskCtrl1", + "LdCtrlTaskCtrl2", + } +) + def _application_id(application: Application) -> bytes: """Assemble the 5 octet application id (manufacturer, type, version).""" @@ -156,6 +214,7 @@ def __init__( scope: DownloadScope = DownloadScope.FULL, expected_descriptor: int | None = None, negotiate_apdu: bool = False, + apdu_overhead: int = 0, ) -> None: """Initialize the runner. @@ -184,6 +243,9 @@ def __init__( self._max_apdu_length = ( programmer.max_apdu_length if programmer is not None else max_apdu_length ) + self._apdu_overhead = ( + programmer.apdu_overhead if programmer is not None else apdu_overhead + ) self._restart_cooldown = restart_cooldown self._restart_at: float | None = None # A one-shot cooldown that overrides the default for the next reconnect @@ -209,6 +271,7 @@ async def run(self, progress: Callable[[int, int], None] | None = None) -> None: download progress. """ in_scope = [c for c in self._controls if self._in_scope(c)] + self._prevalidate(in_scope) total = len(in_scope) logger.info( "download run start: %s, %d of %d load controls in scope", @@ -235,6 +298,7 @@ async def preflight(self) -> PreflightReport: segments: list[SegmentDiff] = [] properties: list[PropertyDiff] = [] in_scope = [c for c in self._controls if self._in_scope(c)] + self._prevalidate(in_scope) logger.info( "download preflight start: %s, %d of %d load controls in scope", self._target(), @@ -254,6 +318,23 @@ def _in_scope(self, control: object) -> bool: """Whether a control participates in the requested download scope.""" return control_in_scope(control, self.scope) + def _prevalidate(self, in_scope: list[object]) -> None: + """Reject an unsupported control before any device state is changed. + + Scans the resolved, scoped controls up front so a procedure containing a + control this engine cannot execute fails before the connection is opened + or any Load State Machine is unloaded, rather than partway through + (leaving the device unloaded). Client-side no-ops are accepted. + """ + for position, control in enumerate(in_scope, start=1): + name = type(control).__name__ + if name in _SUPPORTED_CONTROLS or name in _CLIENT_SIDE: + continue + self._position = (position, len(in_scope)) + error = self._unsupported(control) + self._position = None + raise error + def _target(self) -> str: """A short, log-friendly identity of the download target for diagnostics.""" try: @@ -293,7 +374,9 @@ async def _bus(self) -> DeviceProgrammer: await self._await_restart_cooldown() connection = await self._manager.open() self._programmer = DeviceProgrammer( - connection, max_apdu_length=self._max_apdu_length + connection, + max_apdu_length=self._max_apdu_length, + apdu_overhead=self._apdu_overhead, ) await self._prepare_device(self._programmer) return self._programmer @@ -411,7 +494,9 @@ async def _execute(self, control: object) -> None: # one-shot reconnect cooldown when it exceeds the default. await self._close() self._restart_at = time.monotonic() - self._pending_cooldown = max(self._restart_cooldown, process_time / 1000) + # Process Time is a 2 octet unsigned value in *seconds* (DPT 7.005, + # KNX 3/5/2 3.7.1.2.2), so use it directly - not milliseconds. + self._pending_cooldown = max(self._restart_cooldown, process_time) return if isinstance(control, LdCtrlUnload): index = await self._resolve_index(control) @@ -670,7 +755,15 @@ async def _write_prop(self, control: LdCtrlWriteProp) -> None: ) def _image_property_data(self, object_index: int, property_id: int) -> bytes: - """Find the download image's property data for an object and property.""" + """Find the download image's property data for an object and property. + + Matches on the resolved device object index (or an object-agnostic image + property). Note: the image keys properties by object index, not by + (object type, occurrence), so a product with several instances of the + same object type that carry *different* per-instance property images is + not distinguished here - the first matching property wins. No such device + has been observed; a fix would resolve by (object type, occurrence). + """ for prop in self.image.properties: if prop.property_id != property_id: continue @@ -699,6 +792,14 @@ async def _compare_prop(self, control: LdCtrlCompareProp) -> None: start_index=control.start_element, ) expected = control.inline_data + # A device that returns no data (absent property / rejected read) must + # not pass the compare vacuously - the overlapping-prefix rule below + # would otherwise match against an empty prefix. + if not read_back: + raise VerificationError( + f"property compare for object {index} property {control.prop_id} " + "read no data from the device" + ) mask = control.mask length = min(len(read_back), len(expected)) if mask: diff --git a/packages/download/src/xknxmono/download/programmer.py b/packages/download/src/xknxmono/download/programmer.py index 4e55f9a6..ba31d67a 100644 --- a/packages/download/src/xknxmono/download/programmer.py +++ b/packages/download/src/xknxmono/download/programmer.py @@ -33,6 +33,9 @@ Restart, RestartMasterReset, RestartMasterResetResponse, + UserMemoryRead, + UserMemoryResponse, + UserMemoryWrite, ) from . import load_state @@ -62,6 +65,12 @@ PID_TABLE_REFERENCE = 7 # Highest interface object index scanned when locating an object by type. _MAX_OBJECT_INDEX = 255 +# A_Memory_Read/Write carry a 16-bit address, so they only reach the first 64 +# KiB; at and above this boundary A_UserMemory_Read/Write (3-byte address) are +# used. ETS splits a transfer straddling the boundary into a Standard part below +# and a UserMemory part at/above it (Hawk eo.cs, e.g. the 65536 split around +# lines 16704-16707 and 16647). See KNX 3/5/1 4.2 and 3/3/7 3.5. +_USER_MEMORY_BOUNDARY = 0x10000 class BusConnection(Protocol): @@ -101,16 +110,31 @@ def __init__( connection: BusConnection, *, max_apdu_length: int = DEFAULT_MAX_APDU_LENGTH, + apdu_overhead: int = 0, ) -> None: - """Initialize with a connected bus connection and the negotiated APDU length.""" + """Initialize with a connected bus connection and the negotiated APDU length. + + ``apdu_overhead`` is the number of octets a lower layer adds around each + APDU on the wire (13 for a KNX Data Secure session, 0 otherwise). It is + subtracted from ``max_apdu_length`` when sizing chunks so the fully + encoded frame still fits the device's APDU length. + """ self.connection = connection self.max_apdu_length = max_apdu_length + self.apdu_overhead = apdu_overhead self._object_index_cache: dict[tuple[int, int], int] = {} + @property + def _plain_apdu_length(self) -> int: + """Usable plaintext APDU length after the wire overhead (at least 1).""" + return max(1, self.max_apdu_length - self.apdu_overhead) + @property def memory_chunk_size(self) -> int: """Largest memory payload that fits into a single telegram (at least 1).""" - return max(1, min(self.max_apdu_length - _MEMORY_OVERHEAD, _MAX_MEMORY_CHUNK)) + return max( + 1, min(self._plain_apdu_length - _MEMORY_OVERHEAD, _MAX_MEMORY_CHUNK) + ) async def read_device_descriptor(self) -> int: """Read device descriptor type 0 (the mask version).""" @@ -134,30 +158,48 @@ async def read_max_apdu_length(self) -> int: return DEFAULT_MAX_APDU_LENGTH return int.from_bytes(data, "big") + def _block_count(self, address: int, remaining: int) -> int: + """Octets to transfer next: the APDU chunk, clamped to the 64 KiB boundary. + + A single A_Memory_/A_UserMemory_ transfer must not straddle the 64 KiB + boundary (the address space and APCI change there), so a block that would + cross it is cut at the boundary (Hawk eo.cs splits the same way). + """ + count = min(self.memory_chunk_size, remaining) + if address < _USER_MEMORY_BOUNDARY < address + count: + count = _USER_MEMORY_BOUNDARY - address + return count + async def read_memory(self, address: int, size: int) -> bytes: """Read ``size`` octets starting at ``address``, chunked to the APDU length.""" result = bytearray() - chunk = self.memory_chunk_size offset = 0 while offset < size: - count = min(chunk, size - offset) - telegram = await self.connection.request( - MemoryRead(address=address + offset, count=count), MemoryResponse - ) - payload = telegram.payload - if not isinstance(payload, MemoryResponse): - raise VerificationError( - f"no memory response for address {address + offset:#06x}" - ) - if len(payload.data) != count: - raise VerificationError( - f"short memory response at {address + offset:#06x}: " - f"asked {count} got {len(payload.data)}" - ) - result.extend(payload.data) + block_address = address + offset + count = self._block_count(block_address, size - offset) + result.extend(await self._read_block(block_address, count)) offset += count return bytes(result) + async def _read_block(self, address: int, count: int) -> bytes: + """Read one block via A_Memory_Read or A_UserMemory_Read by address.""" + if address >= _USER_MEMORY_BOUNDARY: + request: APCI = UserMemoryRead(address=address, count=count) + expected: type[APCI] = UserMemoryResponse + else: + request = MemoryRead(address=address, count=count) + expected = MemoryResponse + telegram = await self.connection.request(request, expected) + payload = telegram.payload + if not isinstance(payload, MemoryResponse | UserMemoryResponse): + raise VerificationError(f"no memory response for address {address:#06x}") + if len(payload.data) != count: + raise VerificationError( + f"short memory response at {address:#06x}: " + f"asked {count} got {len(payload.data)}" + ) + return payload.data + async def write_memory( self, address: int, data: bytes, *, verify: bool = False ) -> None: @@ -166,13 +208,12 @@ async def write_memory( With ``verify`` each block is read back and compared right after it is written (per KNX 3/5/2), so a lost EEPROM write is caught immediately. """ - chunk = self.memory_chunk_size - for offset in range(0, len(data), chunk): - block = data[offset : offset + chunk] + offset = 0 + while offset < len(data): block_address = address + offset - await self.connection.send_data( - MemoryWrite(address=block_address, data=block) - ) + count = self._block_count(block_address, len(data) - offset) + block = data[offset : offset + count] + await self._write_block(block_address, block) if verify: read_back = await self.read_memory(block_address, len(block)) if read_back != block: @@ -180,6 +221,16 @@ async def write_memory( f"memory verification failed at {block_address:#06x}: " f"wrote {block.hex()} read {read_back.hex()}" ) + offset += count + + async def _write_block(self, address: int, block: bytes) -> None: + """Write one block via A_Memory_Write or A_UserMemory_Write by address.""" + if address >= _USER_MEMORY_BOUNDARY: + await self.connection.send_data( + UserMemoryWrite(address=address, data=block) + ) + else: + await self.connection.send_data(MemoryWrite(address=address, data=block)) async def read_property( self, @@ -199,12 +250,10 @@ async def read_property( ), PropertyValueResponse, ) - payload = telegram.payload - if not isinstance(payload, PropertyValueResponse): - raise VerificationError( - f"no property response for object {object_index} property {property_id}" - ) - return payload.data + response = _validate_property_response( + telegram.payload, object_index, property_id, "read" + ) + return response.data async def write_property( self, @@ -226,9 +275,26 @@ async def write_property( so a value spanning more than 15 elements or one frame is written in successive element ranges; the last response is returned. """ + if count <= 0: + raise VerificationError( + f"property write for object {object_index} property {property_id} " + "has a non-positive element count (an unresolved wildcard count is " + "not supported)" + ) + if count > 1 and len(data) % count: + raise VerificationError( + f"property data ({len(data)} octets) is not divisible by the " + f"element count {count} for object {object_index} property {property_id}" + ) element_size = (len(data) // count) if count > 1 else len(data) - max_bytes = max(1, self.max_apdu_length - _PROPERTY_OVERHEAD) - if element_size and element_size <= max_bytes: + max_bytes = max(1, self._plain_apdu_length - _PROPERTY_OVERHEAD) + if element_size > max_bytes: + raise VerificationError( + f"a single property element ({element_size} octets) does not fit " + f"the APDU ({max_bytes} octets) for object {object_index} " + f"property {property_id}; element fragmentation is not implemented" + ) + if element_size: per_frame = max(1, min(_MAX_PROPERTY_ELEMENTS, max_bytes // element_size)) else: per_frame = _MAX_PROPERTY_ELEMENTS @@ -271,13 +337,10 @@ async def _write_property_frame( ), PropertyValueResponse, ) - payload = telegram.payload - if not isinstance(payload, PropertyValueResponse): - raise VerificationError( - f"no property write response for object {object_index} " - f"property {property_id}" - ) - return payload.data + response = _validate_property_response( + telegram.payload, object_index, property_id, "write", require_nonzero=True + ) + return response.data async def invoke_function_property( self, object_index: int, property_id: int, data: bytes @@ -334,11 +397,25 @@ def _function_property_result( return payload.data async def read_table_reference(self, object_index: int) -> int: - """Read a loadable part's table base address (PID_TABLE_REFERENCE).""" + """Read a loadable part's table base address (PID_TABLE_REFERENCE). + + The reference width depends on the realisation type - 2 octets for the + memory-mapped BCU/System 7 families, 4 octets (PDT_GENERIC_04, KNX 3/5/1 + 4.2.7) for System B - so the raw value is taken as-is rather than fixed + to one width. A zero reference means the segment has not been allocated + (or allocation failed) and must not be used as a write target (KNX 3/5/3 + 3.5.1.4). + """ data = await self.read_property(object_index, PID_TABLE_REFERENCE) if not data: raise VerificationError(f"empty table reference for object {object_index}") - return int.from_bytes(data, "big") + reference = int.from_bytes(data, "big") + if reference == 0: + raise LoadStateError( + f"object {object_index} reports a zero table reference " + "(segment not allocated)" + ) + return reference async def locate_object(self, object_type: int, occurrence: int = 0) -> int: """Resolve the object index of the ``occurrence``-th object of a type. @@ -415,11 +492,20 @@ async def send_load_event( ) async def restart(self) -> None: - """Restart the device (also closes the transport connection).""" + """Restart the device (also closes the transport connection). + + Sent without waiting for a transport ACK on purpose: a Basic Restart has + no application-layer response and the device tears the connection down + immediately, so it often restarts before (or instead of) ACKing. Waiting + for the ACK would usually time out and just delay the teardown the caller + already performs. The connection-oriented confirmation the standard + mentions (3/3/7 3.4.2.2) is therefore not relied upon here; a Master + Reset, which *is* application-layer confirmed, uses the response path. + """ await self.connection.send_data(Restart(), wait_for_ack=False) async def master_reset(self, erase_code: int, channel_number: int) -> int: - """Perform a Master Reset and return the device's process time in ms. + """Perform a Master Reset and return the device's process time in seconds. A Master Reset is an A_Restart with restart_type = 1, carrying an erase code and channel number (KNX Standard v3.0.0, 3/3/7 section 3.4.2.2; the @@ -444,6 +530,42 @@ async def master_reset(self, erase_code: int, channel_number: int) -> int: return payload.process_time +def _validate_property_response( + payload: APCI | None, + object_index: int, + property_id: int, + operation: str, + *, + require_nonzero: bool = False, +) -> PropertyValueResponse: + """Return the response, or raise if it is missing or mismatched. + + A response echoes the addressed object and property; a differing echo means a + stale/buffered telegram was mistaken for the answer. With ``require_nonzero`` + a count (nr_of_elem) of 0 is treated as the device rejecting the access (KNX + 3/3/7 3.4.4.1/3.4.4.2). Reads leave it off: a 0-element/empty response is how + a device reports an absent property, which callers such as object location and + APDU-length negotiation handle by falling back rather than failing. + """ + if not isinstance(payload, PropertyValueResponse): + raise VerificationError( + f"no property {operation} response for object {object_index} " + f"property {property_id}, got {payload!r}" + ) + if payload.object_index != object_index or payload.property_id != property_id: + raise VerificationError( + f"property {operation} response addresses object " + f"{payload.object_index} property {payload.property_id}, " + f"expected object {object_index} property {property_id}" + ) + if require_nonzero and payload.count == 0: + raise VerificationError( + f"device rejected property {operation}: object {object_index} " + f"property {property_id} returned 0 elements" + ) + return payload + + def _decode_load_state(value: int, object_index: int) -> load_state.LoadState: """Map a raw load state octet to :class:`LoadState`, erroring on unknowns.""" try: diff --git a/packages/download/src/xknxmono/download/scope.py b/packages/download/src/xknxmono/download/scope.py index 17255f13..ed029287 100644 --- a/packages/download/src/xknxmono/download/scope.py +++ b/packages/download/src/xknxmono/download/scope.py @@ -13,6 +13,20 @@ The applies_to marker (``LdCtrlProcType``) is uniform on many products, so the object a control targets - not applies_to - is what distinguishes a partial parameter download from a partial group communication download. + +This is a deliberate deviation from ETS/Falcon, which evaluate ``AppliesTo`` (and +select a subtype-specific procedure). Object-based scoping was validated +byte-perfect on real hardware (e.g. a partial parameter download on 1.1.74), +whereas an ``AppliesTo``-driven scope was observed to be wrong on that device, so +the object a control addresses is the authoritative signal here. + +The classification looks at the object *type* for controls that carry one +(``obj_type``, e.g. the synthesized group-communication table writes use types +1/2/9) and at the interface object *index* otherwise (``obj_idx``/``lsm_idx``). +For our control set these do not collide - group-communication controls always +carry ``obj_type`` and parameter controls carry the Application Program index - +but a hand-built procedure that group-addressed the System B group object at +*index* 3 (rather than type 9) would need an index-to-type map to classify. """ from __future__ import annotations diff --git a/packages/download/src/xknxmono/download/secure_keyring.py b/packages/download/src/xknxmono/download/secure_keyring.py new file mode 100644 index 00000000..a7a40b04 --- /dev/null +++ b/packages/download/src/xknxmono/download/secure_keyring.py @@ -0,0 +1,63 @@ +"""Build a :class:`DeviceSecurity` from a KNX keyring (``.knxkeys``). + +A keyring stores each secure device's Tool Key encrypted with the keyring +password (KNX Standard v3.0.0, 3/5/1 and the ETS keyring export). xknx already +parses and decrypts a keyring - :func:`xknx.secure.keyring.sync_load_keyring` +returns a :class:`~xknx.secure.keyring.Keyring` whose ``devices`` carry the +``decrypted_tool_key`` - so this module only bridges that to the Tool-Key +material a secure download needs, keyed by the device's individual address. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from xknx.secure.keyring import sync_load_keyring +from xknx.telegram.address import IndividualAddress + +from .data_secure import DeviceSecurity, SecureProgrammingError + +if TYPE_CHECKING: + import os + + from xknx.secure.keyring import Keyring + from xknx.telegram.address import IndividualAddressableType + + +def device_security_from_keyring( + keyring: Keyring, address: IndividualAddressableType +) -> DeviceSecurity: + """Return the Tool-Key security material for ``address`` from a loaded keyring. + + ``keyring`` must already be decrypted (as returned by + :func:`xknx.secure.keyring.sync_load_keyring`). Raises + :class:`SecureProgrammingError` if the keyring has no entry for the device + or the entry carries no Tool Key. + """ + individual_address = IndividualAddress(address) + for device in keyring.devices: + if device.individual_address != individual_address: + continue + if device.decrypted_tool_key is None: + raise SecureProgrammingError( + f"keyring entry for {individual_address} has no Tool Key" + ) + return DeviceSecurity(individual_address, device.decrypted_tool_key) + raise SecureProgrammingError(f"device {individual_address} not found in keyring") + + +def load_device_security( + path: str | os.PathLike[str], + password: str, + address: IndividualAddressableType, + *, + validate_signature: bool = True, +) -> DeviceSecurity: + """Load a ``.knxkeys`` file and return the Tool-Key material for ``address``. + + Convenience wrapper over :func:`xknx.secure.keyring.sync_load_keyring` and + :func:`device_security_from_keyring`. Reads and decrypts the file + synchronously; call it off the event loop in async contexts. + """ + keyring = sync_load_keyring(path, password, validate_signature=validate_signature) + return device_security_from_keyring(keyring, address) diff --git a/packages/download/src/xknxmono/download/secure_session.py b/packages/download/src/xknxmono/download/secure_session.py new file mode 100644 index 00000000..d4e2cac2 --- /dev/null +++ b/packages/download/src/xknxmono/download/secure_session.py @@ -0,0 +1,125 @@ +"""Open a Tool-Key secured point-to-point session for a download. + +Wraps a point-to-point connection so that all its management traffic is KNX +Data Secure protected with the device's Tool Key: it installs +:class:`ToolKeyCemiSecure` on ``xknx.cemi_handler.data_secure`` (the same hook +xknx uses for group Data Secure, where each frame's transport-layer sequence +number is already assigned), opens the connection and runs the S-A_Sync +exchange, then restores the previous hook when the connection closes. + +The :class:`DeviceProgrammer` sees a plain :class:`BusConnection`; securing and +unwrapping happen transparently on the CEMI path underneath it. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +from typing import TYPE_CHECKING, Final + +from xknx.exceptions import ManagementConnectionError, ManagementConnectionTimeout +from xknx.telegram.apci import DeviceDescriptorRead, SecureAPDU + +from .data_secure import ( + CemiSecurer, + DeviceSecurity, + SecureManagement, + SecureProgrammingError, + ToolKeyCemiSecure, +) + +if TYPE_CHECKING: + from xknx import XKNX + from xknx.telegram.address import IndividualAddress + + from .programmer import BusConnection + +logger = logging.getLogger(__name__) + +# The S-AL user starts a 6 s timeout for the S-A_Sync response, and the device +# will not answer two requests within 1 s of each other (3/3/7 5.3.2). +_SYNC_RETRY_DELAY: Final = 1.0 +_SYNC_ATTEMPTS: Final = 3 +# The armed hook replaces this outgoing APDU with the S-A_Sync request, so the +# device never sees it - it is only a vehicle to drive one connection-oriented +# request/response over the transport layer. +_SYNC_TRIGGER: Final = DeviceDescriptorRead(descriptor=0) + + +class SecureConnectionManager: + """Open/close a Tool-Key secured connection to one device via ``xknx``. + + Implements the same open/close contract as the plain connection manager, so + a Load Procedure runner can use it unchanged. + """ + + def __init__( + self, xknx: XKNX, address: IndividualAddress, security: DeviceSecurity + ) -> None: + """Initialize for a target address and its Tool Key security material.""" + self._xknx = xknx + self._address = address + self._management = SecureManagement( + device=security, tool_address=xknx.current_address + ) + self._connection: BusConnection | None = None + self._previous_secure: CemiSecurer | None = None + self._installed = False + + async def open(self) -> BusConnection: + """Install the securer, open the connection and run S-A_Sync.""" + self._install() + try: + self._connection = await self._xknx.management.connect(self._address) + await self._synchronize(self._connection) + except BaseException: + self._restore() + raise + return self._connection + + async def close(self) -> None: + """Close the connection and restore the previous CEMI securer.""" + try: + if self._connection is not None: + self._connection = None + with contextlib.suppress(ManagementConnectionError): + await self._xknx.management.disconnect(self._address) + finally: + self._restore() + + def _install(self) -> None: + self._previous_secure = self._xknx.cemi_handler.data_secure + hook = ToolKeyCemiSecure( + self._management, self._address, previous=self._previous_secure + ) + self._xknx.cemi_handler.data_secure = hook # type: ignore[assignment] + self._installed = True + + def _restore(self) -> None: + if self._installed: + self._xknx.cemi_handler.data_secure = self._previous_secure # type: ignore[assignment] + self._installed = False + + async def _synchronize(self, connection: BusConnection) -> None: + """Run the S-A_Sync exchange, retrying within the device's answer rules.""" + for attempt in range(_SYNC_ATTEMPTS): + self._management.arm_sync() + try: + await connection.request(_SYNC_TRIGGER, SecureAPDU) + except ManagementConnectionTimeout: + logger.warning( + "S-A_Sync response not received (attempt %d/%d)", + attempt + 1, + _SYNC_ATTEMPTS, + ) + await asyncio.sleep(_SYNC_RETRY_DELAY) + continue + if self._management.synchronized: + return + await asyncio.sleep(_SYNC_RETRY_DELAY) + raise SecureProgrammingError( + "no S-A_Sync response received - the device did not answer with the " + "Tool Key; check the device address and that the Tool Key is current " + "(a Master Reset replaces it with the FDSK)" + ) diff --git a/packages/download/tests/conftest.py b/packages/download/tests/conftest.py index b491a22e..39fc652e 100644 --- a/packages/download/tests/conftest.py +++ b/packages/download/tests/conftest.py @@ -19,6 +19,9 @@ Restart, RestartMasterReset, RestartMasterResetResponse, + UserMemoryRead, + UserMemoryResponse, + UserMemoryWrite, ) from xknxmono.download.load_state import ( @@ -62,7 +65,7 @@ def __init__( async def send_data(self, payload: APCI, wait_for_ack: bool = True) -> None: """Handle a payload the programmer does not expect an answer to.""" self.sent.append(payload) - if isinstance(payload, MemoryWrite): + if isinstance(payload, MemoryWrite | UserMemoryWrite): for index, byte in enumerate(payload.data): self.memory[payload.address + index] = byte elif isinstance(payload, PropertyValueWrite): @@ -78,6 +81,13 @@ async def request(self, payload: APCI, expected: type[APCI] | None) -> Telegram: self.memory.get(payload.address + i, 0) for i in range(payload.count) ) return self._telegram(MemoryResponse(address=payload.address, data=data)) + if isinstance(payload, UserMemoryRead): + data = bytes( + self.memory.get(payload.address + i, 0) for i in range(payload.count) + ) + return self._telegram( + UserMemoryResponse(address=payload.address, data=data) + ) if isinstance(payload, PropertyValueWrite): # A_PropertyValue_Write is confirmed by a response carrying the # resulting value; apply the write, then answer with a read. diff --git a/packages/download/tests/test_data_secure.py b/packages/download/tests/test_data_secure.py new file mode 100644 index 00000000..1ab4cfb6 --- /dev/null +++ b/packages/download/tests/test_data_secure.py @@ -0,0 +1,406 @@ +"""Tests for KNX Data Secure point-to-point programming (Tool Key access). + +The vector tests reproduce the worked examples in KNX Standard v3.0.0, 3/3/7 +Annex C (C.1.1-C.1.4) byte-for-byte, so the CCM construction is pinned to the +authoritative reference and cannot drift. +""" + +from __future__ import annotations + +import pytest +from xknx.cemi.cemi_frame import CEMILData +from xknx.secure.data_secure_asdu import SecureData +from xknx.telegram import Telegram +from xknx.telegram.address import IndividualAddress +from xknx.telegram.apci import MemoryRead, SecureAPDU +from xknx.telegram.tpci import TDataConnected + +from xknxmono.download.data_secure import ( + DATA_SCF, + SYNC_REQ_SCF, + SYNC_RES_SCF, + DeviceSecurity, + SecureManagement, + SecureProgrammingError, + ToolKeyCemiSecure, + _BlockFields, + _decrypt, + _encrypt, +) + +# 3/3/7 Annex C security parameters. +TOOL_KEY = bytes.fromhex("000102030405060708090a0b0c0d0e0f") +DEVICE = IndividualAddress("15.15.0") # FF00h +TOOL = IndividualAddress("15.15.103") # FF67h + + +def _cemi( + source: IndividualAddress, + destination: IndividualAddress, + *, + sequence_number: int = 0, + payload: object = None, +) -> CEMILData: + """Build a connection-oriented CEMILData, like the transport layer would.""" + return CEMILData.init_from_telegram( + telegram=Telegram( + source_address=source, + destination_address=destination, + tpci=TDataConnected(sequence_number=sequence_number), + payload=payload, # type: ignore[arg-type] + ), + src_addr=source, + ) + + +def test_scf_values_match_reference() -> None: + # 3/3/7 Annex C: SCF 90h (S-A_Data), 92h (S-A_Sync.req), 93h (S-A_Sync.res). + assert DATA_SCF.to_knx() == bytes.fromhex("90") + assert SYNC_REQ_SCF.to_knx() == bytes.fromhex("92") + assert SYNC_RES_SCF.to_knx() == bytes.fromhex("93") + + +def test_annex_c_1_1_s_a_data() -> None: + # C.1.1: SA=FF67 DA=FF00 SCF=90 SeqNr=4, connectionless (TPCI/APCI 03F1). + telegram = Telegram(source_address=TOOL, destination_address=DEVICE) + fields = _BlockFields.from_telegram(telegram, source=TOOL) + plain = bytes.fromhex("03D705351001202122232425262728292A2B2C2D2E2F") + cipher, mac = _encrypt( + key=TOOL_KEY, + fields=fields, + nonce=(4).to_bytes(6, "big"), + additional_data=DATA_SCF.to_knx(), + payload=plain, + ) + assert ( + cipher + mac + ).hex() == "6767242a2308ca76a11774214ee4cf5d94909f743d050d8fc168" + + +def test_annex_c_1_2_s_a_data_response() -> None: + # C.1.2: SA=FF00 DA=FF67 SCF=90 SeqNr=3, connectionless. + telegram = Telegram(source_address=DEVICE, destination_address=TOOL) + fields = _BlockFields.from_telegram(telegram, source=DEVICE) + plain = bytes.fromhex("03D605351001202122232425262728292A2B2C2D2E2F") + cipher, mac = _encrypt( + key=TOOL_KEY, + fields=fields, + nonce=(3).to_bytes(6, "big"), + additional_data=DATA_SCF.to_knx(), + payload=plain, + ) + assert ( + cipher + mac + ).hex() == "706f533105503557cb2b24f1dd341b60b7e017ecd6b06849a72b" + + +def test_annex_c_1_3_s_a_sync_req() -> None: + # C.1.3: SCF=92 SeqNrLocal=1 Serial=0 Challenge=3, connection-oriented + # (TPCI/APCI 43F1). A = SCF | KNX Serial Number, P = Challenge. + fields = _BlockFields(_cemi(TOOL, DEVICE, sequence_number=0)) + cipher, mac = _encrypt( + key=TOOL_KEY, + fields=fields, + nonce=(1).to_bytes(6, "big"), + additional_data=SYNC_REQ_SCF.to_knx() + b"\x00" * 6, + payload=(3).to_bytes(6, "big"), + ) + assert (cipher + mac).hex() == "c1cf4506f09bd79fab55" + + +def test_annex_c_1_4_s_a_sync_res() -> None: + # C.1.4: SCF=93 SeqNrRemote=3 SeqNrLocal=4 Random=AA*6, connection-oriented. + # Nonce is the Random value; A = SCF, P = SeqNrRemote | SeqNrLocal. + fields = _BlockFields(_cemi(DEVICE, TOOL, sequence_number=0)) + cipher, mac = _encrypt( + key=TOOL_KEY, + fields=fields, + nonce=bytes.fromhex("AAAAAAAAAAAA"), + additional_data=SYNC_RES_SCF.to_knx(), + payload=(3).to_bytes(6, "big") + (4).to_bytes(6, "big"), + ) + assert (cipher + mac).hex() == "9c023ad25e146470693e638d5b70cac4" + + +def test_device_security_normalizes_key() -> None: + assert DeviceSecurity(DEVICE, TOOL_KEY).tool_key == TOOL_KEY + assert DeviceSecurity(DEVICE, TOOL_KEY.hex()).tool_key == TOOL_KEY + with pytest.raises(SecureProgrammingError, match="16 octets"): + DeviceSecurity(DEVICE, b"\x00" * 15) + with pytest.raises(SecureProgrammingError, match="hex"): + DeviceSecurity(DEVICE, "zz") + + +def _device_sync_response( + sync_request: CEMILData, *, sending: int, tool_expected: int +) -> CEMILData: + """Emulate a device answering an S-A_Sync request with the same Tool Key. + + Verifies the request, then builds the S-A_Sync response frame the tool must + accept, so a passing round trip proves both directions are consistent. + """ + payload = sync_request.payload + assert isinstance(payload, SecureAPDU) + challenge = _decrypt( + key=TOOL_KEY, + fields=_BlockFields(sync_request), + nonce=payload.secured_data.sequence_number_bytes, + additional_data=payload.scf.to_knx() + b"\x00" * 6, + cipher=payload.secured_data.secured_apdu[6:], + mac=payload.secured_data.message_authentication_code, + ) + random = bytes.fromhex("AAAAAAAAAAAA") + response = _cemi(DEVICE, TOOL, sequence_number=0) + cipher, mac = _encrypt( + key=TOOL_KEY, + fields=_BlockFields(response), + nonce=random, + additional_data=SYNC_RES_SCF.to_knx(), + payload=sending.to_bytes(6, "big") + tool_expected.to_bytes(6, "big"), + ) + challenge_xor_random = bytes(a ^ b for a, b in zip(challenge, random, strict=True)) + response.payload = SecureAPDU( + scf=SYNC_RES_SCF, + secured_data=SecureData( + sequence_number_bytes=challenge_xor_random, + secured_apdu=cipher, + message_authentication_code=mac, + ), + ) + return response + + +def _device_secured_response(payload: MemoryRead, *, sequence_number: int) -> CEMILData: + """Emulate a device sending a secured S-A_Data response frame.""" + response = _cemi(DEVICE, TOOL, sequence_number=0) + cipher, mac = _encrypt( + key=TOOL_KEY, + fields=_BlockFields(response), + nonce=sequence_number.to_bytes(6, "big"), + additional_data=DATA_SCF.to_knx(), + payload=bytes(payload.to_knx()), + ) + response.payload = SecureAPDU( + scf=DATA_SCF, + secured_data=SecureData( + sequence_number_bytes=sequence_number.to_bytes(6, "big"), + secured_apdu=cipher, + message_authentication_code=mac, + ), + ) + return response + + +def test_sync_then_data_round_trip() -> None: + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + management.arm_sync() + + # A frame to the device becomes the S-A_Sync request; it stays armed (so a + # transport retransmission replays the same request) until the response. + request = management.build_sync_request_cemi(_cemi(TOOL, DEVICE, sequence_number=0)) + assert management.sync_armed + response = _device_sync_response(request, sending=0x100, tool_expected=0x200) + assert management.decode_sync_response_cemi(response) is True + assert management.synchronized + assert not management.sync_armed + + # The first secured data frame uses the SeqNr the device returned. + wrapped = management.wrap_cemi( + _cemi( + TOOL, DEVICE, sequence_number=1, payload=MemoryRead(address=0x10, count=1) + ) + ) + assert isinstance(wrapped.payload, SecureAPDU) + assert ( + int.from_bytes(wrapped.payload.secured_data.sequence_number_bytes, "big") + == 0x200 + ) + + # A device response secured with a higher SeqNr decrypts back to the APDU. + device_frame = _device_secured_response( + MemoryRead(address=0x10, count=1), sequence_number=0x100 + ) + unwrapped = management.unwrap_cemi(device_frame) + assert isinstance(unwrapped.payload, MemoryRead) + assert unwrapped.payload.address == 0x10 + + +def test_wrap_before_sync_raises() -> None: + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + with pytest.raises(SecureProgrammingError, match="not synchronized"): + management.wrap_cemi( + _cemi(TOOL, DEVICE, payload=MemoryRead(address=0x10, count=1)) + ) + + +def test_unwrap_rejects_replayed_sequence_number() -> None: + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + management.arm_sync() + request = management.build_sync_request_cemi(_cemi(TOOL, DEVICE, sequence_number=0)) + response = _device_sync_response(request, sending=0x100, tool_expected=0x200) + management.decode_sync_response_cemi(response) + + # last valid remote is sending-1 = 0xFF; a frame at 0xFF must be rejected. + device_frame = _device_secured_response( + MemoryRead(address=0x10, count=1), sequence_number=0xFF + ) + with pytest.raises(SecureProgrammingError, match="sequence number too low"): + management.unwrap_cemi(device_frame) + + +def test_hook_passes_through_foreign_and_control_frames() -> None: + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + hook = ToolKeyCemiSecure(management, DEVICE) + + # A frame to another device is untouched. + other = IndividualAddress("1.1.1") + to_other = _cemi(TOOL, other, payload=MemoryRead(address=0x10, count=1)) + assert hook.outgoing_cemi(to_other) is to_other + + # A control frame to the device (no APDU) is untouched. + control = _cemi(TOOL, DEVICE) + assert hook.outgoing_cemi(control) is control + + +def test_hook_wraps_and_unwraps_over_session() -> None: + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + hook = ToolKeyCemiSecure(management, DEVICE) + + # Sync via the hook: the armed session turns the outgoing frame into the req. + management.arm_sync() + request = hook.outgoing_cemi( + _cemi(TOOL, DEVICE, sequence_number=0, payload=MemoryRead(address=0, count=1)) + ) + assert isinstance(request.payload, SecureAPDU) + assert request.payload.scf.service is SYNC_REQ_SCF.service + response = _device_sync_response(request, sending=0x100, tool_expected=0x200) + hook.received_cemi(response) + assert management.synchronized + + # Now a plain outgoing frame is wrapped as S-A_Data. + wrapped = hook.outgoing_cemi( + _cemi( + TOOL, DEVICE, sequence_number=1, payload=MemoryRead(address=0x10, count=1) + ) + ) + assert isinstance(wrapped.payload, SecureAPDU) + assert wrapped.payload.scf.service is DATA_SCF.service + + # A secured device frame is unwrapped back to the plain APDU. + device_frame = _device_secured_response( + MemoryRead(address=0x10, count=1), sequence_number=0x100 + ) + unwrapped = hook.received_cemi(device_frame) + assert isinstance(unwrapped.payload, MemoryRead) + + +def test_sync_request_replays_identical_on_retransmission() -> None: + # A missing T_ACK makes xknx resend the same trigger frame; the hook must + # replay the identical Secure APDU, not build a fresh one or wrap as data. + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + management.arm_sync() + first = management.build_sync_request_cemi(_cemi(TOOL, DEVICE, sequence_number=0)) + second = management.build_sync_request_cemi(_cemi(TOOL, DEVICE, sequence_number=0)) + assert isinstance(first.payload, SecureAPDU) + assert isinstance(second.payload, SecureAPDU) + assert first.payload.secured_data.to_knx() == second.payload.secured_data.to_knx() + + +def test_decode_rejects_zero_remote_sequence_number() -> None: + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + management.arm_sync() + request = management.build_sync_request_cemi(_cemi(TOOL, DEVICE, sequence_number=0)) + # SeqNrRemote 0 would make last_valid_remote negative and admit a replay. + response = _device_sync_response(request, sending=0, tool_expected=0x200) + with pytest.raises(SecureProgrammingError, match="sequence number 0"): + management.decode_sync_response_cemi(response) + + +def test_unwrap_rejects_bad_apdu_prefix() -> None: + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + management.arm_sync() + request = management.build_sync_request_cemi(_cemi(TOOL, DEVICE, sequence_number=0)) + management.decode_sync_response_cemi( + _device_sync_response(request, sending=0x100, tool_expected=0x200) + ) + # Encrypt a plaintext whose first octet has non-zero reserved bits. + response = _cemi(DEVICE, TOOL, sequence_number=0) + cipher, mac = _encrypt( + key=TOOL_KEY, + fields=_BlockFields(response), + nonce=(0x100).to_bytes(6, "big"), + additional_data=DATA_SCF.to_knx(), + payload=b"\xfc\x00", + ) + response.payload = SecureAPDU( + scf=DATA_SCF, + secured_data=SecureData( + sequence_number_bytes=(0x100).to_bytes(6, "big"), + secured_apdu=cipher, + message_authentication_code=mac, + ), + ) + with pytest.raises(SecureProgrammingError, match="reserved prefix"): + management.unwrap_cemi(response) + + +def test_wrap_rejects_exhausted_sequence_number() -> None: + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + management.arm_sync() + request = management.build_sync_request_cemi(_cemi(TOOL, DEVICE, sequence_number=0)) + management.decode_sync_response_cemi( + _device_sync_response(request, sending=0x100, tool_expected=0x200) + ) + # Drive the 48-bit Sequence Number Sending past its maximum. + management._next_sequence_number = 0xFFFFFFFFFFFF + 1 + with pytest.raises(SecureProgrammingError, match="exhausted"): + management.wrap_cemi( + _cemi(TOOL, DEVICE, payload=MemoryRead(address=0x10, count=1)) + ) + + +def test_hook_delegates_foreign_traffic_to_previous_securer() -> None: + class _Recorder: + def __init__(self) -> None: + self.out: list[CEMILData] = [] + self.inc: list[CEMILData] = [] + + def outgoing_cemi(self, cemi_data: CEMILData) -> CEMILData: + self.out.append(cemi_data) + return cemi_data + + def received_cemi(self, cemi_data: CEMILData) -> CEMILData: + self.inc.append(cemi_data) + return cemi_data + + previous = _Recorder() + management = SecureManagement( + device=DeviceSecurity(DEVICE, TOOL_KEY), tool_address=TOOL + ) + hook = ToolKeyCemiSecure(management, DEVICE, previous=previous) + + other = IndividualAddress("1.1.1") + out = _cemi(TOOL, other, payload=MemoryRead(address=0x10, count=1)) + inc = _cemi(other, TOOL, payload=MemoryRead(address=0x10, count=1)) + hook.outgoing_cemi(out) + hook.received_cemi(inc) + assert previous.out == [out] + assert previous.inc == [inc] diff --git a/packages/download/tests/test_programmer.py b/packages/download/tests/test_programmer.py index 2a761bcf..db1e02a8 100644 --- a/packages/download/tests/test_programmer.py +++ b/packages/download/tests/test_programmer.py @@ -17,6 +17,20 @@ def test_memory_chunk_size_respects_apdu() -> None: assert DeviceProgrammer(FakeDevice(), max_apdu_length=55).memory_chunk_size == 52 +def test_memory_chunk_size_accounts_for_secure_overhead() -> None: + # A Data Secure session adds 13 octets, so the plaintext ceiling shrinks by + # 13 before the memory overhead is applied: 55 - 13 - 3 = 39. + programmer = DeviceProgrammer(FakeDevice(), max_apdu_length=55, apdu_overhead=13) + assert programmer.memory_chunk_size == 39 + # A tiny APDU still yields at least a 1-octet chunk. + assert ( + DeviceProgrammer( + FakeDevice(), max_apdu_length=15, apdu_overhead=13 + ).memory_chunk_size + == 1 + ) + + async def test_write_memory_is_chunked() -> None: device = FakeDevice() programmer = DeviceProgrammer(device, max_apdu_length=15) diff --git a/packages/download/tests/test_review_fixes.py b/packages/download/tests/test_review_fixes.py new file mode 100644 index 00000000..4dbcea8e --- /dev/null +++ b/packages/download/tests/test_review_fixes.py @@ -0,0 +1,141 @@ +"""Tests for the download hardening fixes from the code review. + +Each test pins a behaviour that was corrected against the KNX standard and the +decompiled ETS/Falcon reference (see the cited comments in the source). +""" + +from __future__ import annotations + +import pytest +from xknx.telegram.apci import UserMemoryRead, UserMemoryWrite + +from xknxmono.download.crc import segment_crc +from xknxmono.download.errors import ( + LoadStateError, + UnsupportedProcedureError, + VerificationError, +) +from xknxmono.download.procedure import LoadProcedureRunner, _mcb_table_with_crc +from xknxmono.download.programmer import DeviceProgrammer + +from .conftest import FakeDevice + + +def _mcb_entry(size: int, *, protected: bool) -> bytes: + # Octets: [0..3] size (low 16 bits first, each half big-endian, per gy.c), + # [4] flags (bit0 clear = CRC protected), [5] reserved, [6..7] CRC placeholder. + low = (size & 0xFFFF).to_bytes(2, "big") + high = (size >> 16).to_bytes(2, "big") + flags = 0x00 if protected else 0x01 + return low + high + bytes([flags, 0x00, 0x00, 0x00]) + + +def test_mcb_crc_single_entry_covers_whole_segment() -> None: + segment = bytes(range(16)) + table = _mcb_entry(16, protected=True) + out = _mcb_table_with_crc(table, segment) + crc = segment_crc(segment) + assert out[6:8] == bytes([(crc >> 8) & 0xFF, crc & 0xFF]) + + +def test_mcb_crc_per_entry_partitions_the_segment() -> None: + # Two entries: 4 protected octets, then 1 mutable octet (like the Gira + # fixture the review cited). The protected CRC must cover only the first 4. + segment = bytes(range(5)) + table = _mcb_entry(4, protected=True) + _mcb_entry(1, protected=False) + out = _mcb_table_with_crc(table, segment) + crc = segment_crc(segment[:4]) + assert out[6:8] == bytes([(crc >> 8) & 0xFF, crc & 0xFF]) + # The unprotected entry is left untouched (placeholder CRC 00 00). + assert out[14:16] == b"\x00\x00" + + +def test_mcb_crc_falls_back_when_sizes_do_not_tile() -> None: + # An entry whose declared size does not match the segment falls back to a + # single CRC over the whole segment (never regress the validated behaviour). + segment = bytes(range(16)) + table = _mcb_entry(99, protected=True) + out = _mcb_table_with_crc(table, segment) + crc = segment_crc(segment) + assert out[6:8] == bytes([(crc >> 8) & 0xFF, crc & 0xFF]) + + +async def test_write_memory_splits_at_64_kib_boundary() -> None: + device = FakeDevice() + programmer = DeviceProgrammer(device, max_apdu_length=55) + # A write straddling 0x10000 must split into a Memory part below and a + # UserMemory part at/above the boundary. + await programmer.write_memory(0xFFFE, bytes(range(8))) + memory_writes = [p for p in device.sent if type(p).__name__ == "MemoryWrite"] + user_writes = [p for p in device.sent if isinstance(p, UserMemoryWrite)] + assert memory_writes and user_writes + assert max(p.address for p in memory_writes) < 0x10000 + assert min(p.address for p in user_writes) == 0x10000 + # Bytes land contiguously regardless of the split. + assert bytes(device.memory[0xFFFE + i] for i in range(8)) == bytes(range(8)) + + +async def test_read_memory_above_boundary_uses_user_memory() -> None: + device = FakeDevice() + device.memory.update({0x10000 + i: i for i in range(4)}) + programmer = DeviceProgrammer(device, max_apdu_length=55) + assert await programmer.read_memory(0x10000, 4) == bytes(range(4)) + assert any(isinstance(p, UserMemoryRead) for p in device.sent) + + +async def test_write_property_rejects_zero_count() -> None: + programmer = DeviceProgrammer(FakeDevice(), max_apdu_length=55) + with pytest.raises(VerificationError, match="non-positive element count"): + await programmer.write_property(0, 5, b"\x01\x02", count=0) + + +async def test_write_property_rejects_indivisible_data() -> None: + programmer = DeviceProgrammer(FakeDevice(), max_apdu_length=55) + with pytest.raises(VerificationError, match="not divisible"): + await programmer.write_property(0, 5, b"\x01\x02\x03", count=2) + + +async def test_write_property_rejects_oversized_element() -> None: + programmer = DeviceProgrammer(FakeDevice(), max_apdu_length=15) + with pytest.raises(VerificationError, match="does not fit the APDU"): + await programmer.write_property(0, 5, bytes(64), count=1) + + +async def test_read_table_reference_rejects_zero() -> None: + device = FakeDevice() + device.table_references[2] = 0 + programmer = DeviceProgrammer(device, max_apdu_length=55) + with pytest.raises(LoadStateError, match="zero table reference"): + await programmer.read_table_reference(2) + + +class _UnknownControl: + """A load control this engine does not implement.""" + + +class _RecordingManager: + def __init__(self, device: FakeDevice) -> None: + self._device = device + self.opened = 0 + + async def open(self) -> FakeDevice: + self.opened += 1 + return self._device + + async def close(self) -> None: + pass + + +async def test_prevalidate_rejects_unsupported_control_before_connecting() -> None: + device = FakeDevice() + manager = _RecordingManager(device) + runner = LoadProcedureRunner( + object(), # type: ignore[arg-type] + object(), # type: ignore[arg-type] + connection_manager=manager, + controls=[_UnknownControl()], + ) + with pytest.raises(UnsupportedProcedureError): + await runner.run() + # The connection is never opened - the device is left untouched. + assert manager.opened == 0 diff --git a/packages/download/tests/test_secure_keyring.py b/packages/download/tests/test_secure_keyring.py new file mode 100644 index 00000000..25893998 --- /dev/null +++ b/packages/download/tests/test_secure_keyring.py @@ -0,0 +1,57 @@ +"""Tests for building DeviceSecurity from a KNX keyring.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from xknx.secure.keyring import Keyring, XMLDevice +from xknx.telegram.address import IndividualAddress + +from xknxmono.download.data_secure import SecureProgrammingError +from xknxmono.download.secure_keyring import ( + device_security_from_keyring, + load_device_security, +) + +# xknx's own keyring fixture (kept out of the repo, present locally). The device +# 1.0.0 in it decrypts to this Tool Key with the password "password". +_FIXTURE = ( + Path(__file__).resolve().parents[3] + / ".references/xknx/test/secure_tests/resources/testcase.knxkeys" +) +_FIXTURE_TOOL_KEY = bytes.fromhex("9bc4fc74043a332b80baa2c8fef72d9d") + + +def _keyring_with(address: str, tool_key: bytes | None) -> Keyring: + device = XMLDevice() + device.individual_address = IndividualAddress(address) + device.decrypted_tool_key = tool_key + keyring = Keyring() + keyring.devices = [device] + return keyring + + +def test_device_security_from_keyring_returns_tool_key() -> None: + keyring = _keyring_with("1.1.5", bytes(range(16))) + security = device_security_from_keyring(keyring, "1.1.5") + assert security.address == IndividualAddress("1.1.5") + assert security.tool_key == bytes(range(16)) + + +def test_device_security_from_keyring_missing_device() -> None: + keyring = _keyring_with("1.1.5", bytes(16)) + with pytest.raises(SecureProgrammingError, match="not found"): + device_security_from_keyring(keyring, "1.1.6") + + +def test_device_security_from_keyring_without_tool_key() -> None: + keyring = _keyring_with("1.1.5", None) + with pytest.raises(SecureProgrammingError, match="no Tool Key"): + device_security_from_keyring(keyring, "1.1.5") + + +@pytest.mark.skipif(not _FIXTURE.exists(), reason="keyring fixture not available") +def test_load_device_security_decrypts_real_keyring() -> None: + security = load_device_security(_FIXTURE, "password", "1.0.0") + assert security.tool_key == _FIXTURE_TOOL_KEY diff --git a/packages/download/tests/test_secure_session.py b/packages/download/tests/test_secure_session.py new file mode 100644 index 00000000..e3c75090 --- /dev/null +++ b/packages/download/tests/test_secure_session.py @@ -0,0 +1,123 @@ +"""Tests for the Tool-Key secured connection manager wiring. + +Drives :class:`SecureConnectionManager` with a fake ``xknx`` whose connection +emulates the device's S-A_Sync answer through the installed CEMI securer, so +the install/restore of the hook and the sync handshake are covered without a +real bus. +""" + +from __future__ import annotations + +import pytest +from xknx.telegram import Telegram +from xknx.telegram.address import IndividualAddress +from xknx.telegram.apci import APCI + +from xknxmono.download import secure_session +from xknxmono.download.data_secure import ( + DeviceSecurity, + SecureProgrammingError, + ToolKeyCemiSecure, +) +from xknxmono.download.download import _connection_manager +from xknxmono.download.secure_session import SecureConnectionManager + +from .test_data_secure import DEVICE, TOOL, TOOL_KEY, _cemi, _device_sync_response + + +class _FakeCemiHandler: + def __init__(self) -> None: + self.data_secure: object = None + + +class _FakeConnection: + """Emulates one connection-oriented request/response through the securer.""" + + def __init__(self, cemi_handler: _FakeCemiHandler) -> None: + self._cemi_handler = cemi_handler + self.disconnected = False + + async def request(self, payload: APCI, expected: type[APCI] | None) -> Telegram: + hook = self._cemi_handler.data_secure + assert hook is not None + secured = hook.outgoing_cemi( # type: ignore[attr-defined] + _cemi(TOOL, DEVICE, sequence_number=0, payload=payload) + ) + response = _device_sync_response(secured, sending=0x100, tool_expected=0x200) + hook.received_cemi(response) # type: ignore[attr-defined] + return response.telegram() + + async def send_data(self, payload: APCI, wait_for_ack: bool = True) -> None: + raise AssertionError("not used in this test") + + +class _FakeManagement: + def __init__(self, cemi_handler: _FakeCemiHandler) -> None: + self._cemi_handler = cemi_handler + self.connected: list[IndividualAddress] = [] + self.disconnected: list[IndividualAddress] = [] + + async def connect(self, address: IndividualAddress) -> _FakeConnection: + self.connected.append(address) + return _FakeConnection(self._cemi_handler) + + async def disconnect(self, address: IndividualAddress) -> None: + self.disconnected.append(address) + + +class _FakeXKNX: + def __init__(self) -> None: + self.cemi_handler = _FakeCemiHandler() + self.current_address = TOOL + self.management = _FakeManagement(self.cemi_handler) + + +async def test_open_installs_hook_syncs_and_close_restores() -> None: + xknx = _FakeXKNX() + sentinel = object() + xknx.cemi_handler.data_secure = sentinel # a pre-existing (e.g. group) securer + + manager = SecureConnectionManager(xknx, DEVICE, DeviceSecurity(DEVICE, TOOL_KEY)) # type: ignore[arg-type] + + await manager.open() + # The securer is installed and the session synchronized during open. + assert isinstance(xknx.cemi_handler.data_secure, ToolKeyCemiSecure) + assert xknx.management.connected == [DEVICE] + + await manager.close() + # The previous securer is restored and the connection is disconnected. + assert xknx.cemi_handler.data_secure is sentinel + assert xknx.management.disconnected == [DEVICE] + + +async def test_open_restores_hook_when_sync_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(secure_session, "_SYNC_RETRY_DELAY", 0.0) + xknx = _FakeXKNX() + sentinel = object() + xknx.cemi_handler.data_secure = sentinel + + # A connection that never delivers a valid sync response. + class _SilentConnection(_FakeConnection): + async def request(self, payload: APCI, expected: type[APCI] | None) -> Telegram: + return Telegram(source_address=DEVICE, destination_address=TOOL) + + async def connect(address: IndividualAddress) -> _SilentConnection: + return _SilentConnection(xknx.cemi_handler) + + xknx.management.connect = connect # type: ignore[assignment] + + manager = SecureConnectionManager(xknx, DEVICE, DeviceSecurity(DEVICE, TOOL_KEY)) # type: ignore[arg-type] + + with pytest.raises(SecureProgrammingError): + await manager.open() + # Even on failure the previous securer is restored. + assert xknx.cemi_handler.data_secure is sentinel + + +def test_connection_manager_rejects_mismatched_security_address() -> None: + xknx = _FakeXKNX() + security = DeviceSecurity(IndividualAddress("1.2.3"), TOOL_KEY) + with pytest.raises(SecureProgrammingError, match="not the download target"): + _connection_manager(xknx, DEVICE, security) # type: ignore[arg-type] diff --git a/pyrightconfig.json b/pyrightconfig.json index 05932111..8e1b0037 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,12 +1,16 @@ { "include": [ "packages/catalog/src", + "packages/download/src", "packages/keyring/src", "packages/models/src", "packages/product/src", "packages/project/src" ], - "exclude": ["**/files", "**/intermediate"], + "exclude": [ + "**/files", + "**/intermediate" + ], "pythonVersion": "3.12", "typeCheckingMode": "strict", "reportMissingTypeStubs": false