Support newer ESPHome versions with serial_proxy responses

.github/workflows/ci.yml

@@ -11,6 +11,9 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  ESPHOME_VERSION: "2026.9.0b3"
+
 jobs:
   prepare-base:
     name: Prepare base dependencies
@@ -188,7 +191,7 @@ jobs:
         with:
           path: tests/esphome/.esphome/build/serialx-host-daemon/.pioenvs/serialx-host-daemon/program
           key: >-
-            2-esphome-daemon-${{ hashFiles('tests/esphome/host_daemon.yaml', 'tests/esphome/external_components/**') }}
+            2-esphome-daemon-${{ env.ESPHOME_VERSION }}-${{ hashFiles('tests/esphome/host_daemon.yaml', 'tests/esphome/external_components/**') }}
       - name: Set up Python 3.13
         if: steps.cache-esphome.outputs.cache-hit != 'true'
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
@@ -202,7 +205,7 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
           uv venv esphome-venv
           . esphome-venv/bin/activate
-          uv pip install 'esphome>=2026.3.2'
+          uv pip install --prerelease=allow "esphome>=${ESPHOME_VERSION}"
           esphome compile tests/esphome/host_daemon.yaml
       - name: Cache ESPHome binary
         if: steps.cache-esphome.outputs.cache-hit != 'true'
@@ -245,7 +248,7 @@ jobs:
         with:
           path: tests/esphome/.esphome/build/serialx-host-daemon/.pioenvs/serialx-host-daemon/program
           key: >-
-            2-esphome-daemon-${{ hashFiles('tests/esphome/host_daemon.yaml', 'tests/esphome/external_components/**') }}
+            2-esphome-daemon-${{ env.ESPHOME_VERSION }}-${{ hashFiles('tests/esphome/host_daemon.yaml', 'tests/esphome/external_components/**') }}
       - name: Restore cached ser2net binary
         uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
         with:

pyproject.toml

@@ -26,7 +26,7 @@ repository = "https://github.com/puddly/serialx"
 documentation = "https://puddly.github.io/serialx/"
 
 [project.optional-dependencies]
-esphome = ["aioesphomeapi>=46.0.0; python_version >= '3.11'"]
+esphome = ["aioesphomeapi>=46.3.0; python_version >= '3.11'"]
 dev = [
   "uv>=0.11.14",
   "ruff>=0.14.6",
@@ -44,7 +44,7 @@ dev = [
   "types-psutil>=7.2.2.20260508",
   "types-pywin32>=311.0.0.20260508",
   "types-setuptools>=82.0.0.20260508",
-  "aioesphomeapi>=46.0.0 ; python_version >= '3.11'",
+  "aioesphomeapi>=46.3.0 ; python_version >= '3.11'",
 ]
 docs = [
   "sphinx>=7,<8.2.3; python_version < '3.11'",

serialx/platforms/serial_esphome.py

@@ -34,7 +34,7 @@
 import urllib.parse
 import warnings
 
-from aioesphomeapi.client import APIClient
+from aioesphomeapi.client import MIN_VERSION_PROXY_ACK, APIClient
 from aioesphomeapi.core import (  # type: ignore[attr-defined]
     APIConnectionError,
     PingRequest,
@@ -46,6 +46,8 @@
     DisconnectReason,
     SerialProxyDataReceived,
     SerialProxyParity,
+    SerialProxyRequestResponse,
+    SerialProxyStatus,
 )
 from typing_extensions import Buffer, Unpack
 
@@ -63,6 +65,7 @@
 )
 
 _T = TypeVar("_T")
+_S = TypeVar("_S", bound=SerialProxyRequestResponse | None)
 _P = ParamSpec("_P")
 
 LOGGER = logging.getLogger(__name__)
@@ -80,6 +83,25 @@
     StopBits.TWO: 2,
 }
 
+STATUS_TO_ERROR_MAP: dict[
+    SerialProxyStatus, None | Callable[[str], SerialException | OSError]
+] = {
+    SerialProxyStatus.OK: None,
+    SerialProxyStatus.ASSUMED_SUCCESS: None,
+    SerialProxyStatus.TIMEOUT: lambda msg: SerialException(
+        f"Operation timed out: {msg}"
+    ),
+    SerialProxyStatus.NOT_SUPPORTED: lambda msg: SerialException(
+        f"Operation is not supported: {msg}"
+    ),
+    SerialProxyStatus.PORT_IN_USE: lambda msg: OSError(
+        errno.EBUSY, f"Serial proxy port is already in use: {msg}"
+    ),
+    SerialProxyStatus.INVALID_ARGUMENT: lambda msg: SerialException(
+        f"Operation is invalid: {msg}"
+    ),
+}
+
 
 class InvalidSettingsError(SerialException):
     """Raised when the provided settings are invalid."""
@@ -205,7 +227,7 @@ def __init__(
         self._closed_unsub: Callable[[], None] | None = None
         self._instance_subscribed = False
 
-        self._last_line_state = LineStateFlag(0)
+        self._last_line_states = LineStateFlag(0)
 
     def _in_event_loop(self) -> bool:
         """Check if we are currently running in the event loop."""
@@ -242,6 +264,26 @@ async def _call_on_client_loop(self, coro: Coroutine[Any, Any, _T]) -> _T:
             asyncio.run_coroutine_threadsafe(coro, client_loop)
         )
 
+    async def _call_on_client_loop_validated(self, coro: Coroutine[Any, Any, _S]) -> _S:
+        """Await a serial proxy `coro` on the `APIClient`'s loop, bridging if needed."""
+        result = await self._call_on_client_loop(coro)
+
+        # Earlier ESPHome versions did not provide responses for many serial proxy
+        # requests. To work around this, we enqueue a request with a response
+        # immediately after and wait for _that_ to finish.
+        assert self._api is not None
+        version = self._api.api_version
+
+        if version is None or version < MIN_VERSION_PROXY_ACK:
+            await self._ping(timeout=self._connect_timeout)
+
+        if result is not None and result.status is not None:
+            error_factory = STATUS_TO_ERROR_MAP.get(result.status)
+            if error_factory is not None:
+                raise error_factory(result.error_message)
+
+        return result
+
     def _schedule_on_client_loop(
         self,
         fn: Callable[_P, Any],
@@ -476,13 +518,10 @@ async def _subscribe_instance(self) -> None:
         await self._resolve_instance_id()
         assert self._instance_id is not None
 
-        self._schedule_on_client_loop(
-            self._api.serial_proxy_subscribe, self._instance_id
+        await self._call_on_client_loop_validated(
+            self._api.serial_proxy_subscribe_await_response(self._instance_id)
         )
 
-        # Ping to ensure the daemon has processed the subscribe
-        await self._ping(timeout=self._connect_timeout)
-
         self._instance_subscribed = True
 
     def _unsubscribe_instance(self) -> None:
@@ -508,28 +547,24 @@ async def _async_configure_port(self) -> None:
         assert self._api is not None
         await self._resolve_instance_id()
         assert self._instance_id is not None
-        self._schedule_on_client_loop(
-            self._api.serial_proxy_configure,
-            instance=self._instance_id,
-            baudrate=self._baudrate,
-            flow_control=self._rtscts,
-            parity=PARITY_MAP[self._parity],
-            stop_bits=STOP_BITS_MAP[self._stopbits],
-            data_size=self._byte_size,
-        )
 
-        # Ping to ensure the daemon has processed the configure
-        await self._ping(timeout=self._connect_timeout)
+        await self._call_on_client_loop_validated(
+            self._api.serial_proxy_configure_await_response(
+                instance=self._instance_id,
+                baudrate=self._baudrate,
+                flow_control=self._rtscts,
+                parity=PARITY_MAP[self._parity],
+                stop_bits=STOP_BITS_MAP[self._stopbits],
+                data_size=self._byte_size,
+            )
+        )
 
         # Subscribe after configure has landed so we don't stream bytes
         # under stale UART settings. Idempotent on reconfigure.
         await self._subscribe_instance()
 
-    def _send_set_modem_pins(self, modem_pins: ModemPins) -> None:
-        """Send a signal to set modem control bits, without waiting for a response."""
-        assert self._api is not None
-        assert self._instance_id is not None
-        line_states = self._last_line_state
+    def _compute_line_states(self, modem_pins: ModemPins) -> LineStateFlag:
+        line_states = self._last_line_states
 
         if modem_pins.rts is PinState.HIGH:
             line_states |= LineStateFlag.RTS
@@ -541,13 +576,26 @@ def _send_set_modem_pins(self, modem_pins: ModemPins) -> None:
         elif modem_pins.dtr is PinState.LOW:
             line_states &= ~LineStateFlag.DTR
 
-        self._last_line_state = line_states
-        self._schedule_on_client_loop(
-            self._api.serial_proxy_set_modem_pins,
-            instance=self._instance_id,
-            line_states=line_states,
+        self._last_line_states = line_states
+
+        return line_states
+
+    @translate_esphome_errors
+    async def _async_set_modem_pins(self, modem_pins: ModemPins) -> None:
+        """Send a signal to set modem control bits, without waiting for a response."""
+        assert self._api is not None
+        assert self._instance_id is not None
+
+        line_states = self._compute_line_states(modem_pins)
+        await self._call_on_client_loop_validated(
+            self._api.serial_proxy_set_modem_pins_await_response(
+                instance=self._instance_id,
+                line_states=line_states,
+            )
         )
 
+        await self._async_get_modem_pins()
+
     def _set_modem_pins(self, modem_pins: ModemPins) -> None:
         """Set modem control bits."""
         assert self._loop is not None
@@ -560,17 +608,19 @@ def _set_modem_pins(self, modem_pins: ModemPins) -> None:
                 DeprecationWarning,
                 stacklevel=2,
             )
-            self._send_set_modem_pins(modem_pins)
-            return
 
-        self._call_on_loop(self._async_set_modem_pins(modem_pins))
+            assert self._api is not None
+            assert self._instance_id is not None
 
-    @translate_esphome_errors
-    async def _async_set_modem_pins(self, modem_pins: ModemPins) -> None:
-        assert self._api is not None
-        self._send_set_modem_pins(modem_pins)
+            line_states = self._compute_line_states(modem_pins)
+            self._schedule_on_client_loop(
+                self._api.serial_proxy_set_modem_pins,
+                instance=self._instance_id,
+                line_states=line_states,
+            )
+            return
 
-        await self._async_get_modem_pins()
+        self._call_on_loop(self._async_set_modem_pins(modem_pins))
 
     def _get_modem_pins(self) -> ModemPins:
         return self._call_on_loop(self._async_get_modem_pins())
@@ -582,7 +632,7 @@ async def _async_get_modem_pins(self) -> ModemPins:
         rsp = await self._call_on_client_loop(
             self._api.serial_proxy_get_modem_pins(instance=self._instance_id)
         )
-        self._last_line_state = LineStateFlag(rsp.line_states)
+        self._last_line_states = LineStateFlag(rsp.line_states)
 
         return ModemPins(
             dtr=PinState.convert(bool(rsp.line_states & LineStateFlag.DTR)),

tests/esphome/external_components/serialx_host_overrides/serialx_host_overrides.cpp

@@ -62,9 +62,15 @@ void SerialxHostOverridesComponent::setup() {
     } else {
       // Base64-encoded PSK
       auto decoded = base64_decode(noise_psk_value);
+#ifdef USE_NOISE
+      // NoiseContext stores the pointer instead of copying, so the PSK must outlive setup()
+      std::copy_n(decoded.begin(), std::min(decoded.size(), this->noise_psk_.size()), this->noise_psk_.begin());
+      api::global_api_server->set_noise_psk(this->noise_psk_.data());
+#else
       api::psk_t psk{};
       std::copy_n(decoded.begin(), std::min(decoded.size(), psk.size()), psk.begin());
       api::global_api_server->set_noise_psk(psk);
+#endif
       ESP_LOGI(TAG, "Overrode noise PSK from %s", this->noise_psk_env_.c_str());
     }
   }

tests/esphome/external_components/serialx_host_overrides/serialx_host_overrides.h

@@ -4,6 +4,11 @@
 
 #include "esphome/components/uart/uart_component_host.h"
 #include "esphome/core/component.h"
+#include "esphome/core/defines.h"
+
+#ifdef USE_NOISE
+#include "esphome/components/noise/noise.h"
+#endif
 
 #include <string>
 
@@ -29,6 +34,9 @@ class SerialxHostOverridesComponent : public Component {
   std::string right_uart_env_;
   std::string api_port_env_;
   std::string noise_psk_env_;
+#ifdef USE_NOISE
+  noise::psk_t noise_psk_{};
+#endif
   bool ready_printed_{false};
 };
 

tests/test_async_transports.py

@@ -433,6 +433,9 @@ async def test_async_rtscts_setting(serial_pair: SerialPair, rtscts: bool) -> No
     if rtscts and serial_pair.uri_scheme == "posix://":
         pytest.xfail("Strict POSIX backend does not support RTS/CTS flow control")
 
+    if rtscts and SerialBackend.ESPHOME_HOST in serial_pair.backends:
+        pytest.xfail("ESPHome host does not support RTS/CTS flow control")
+
     async with serialx.async_serial_for_url(serial_pair.right, baudrate=115200):
         async with serialx.async_serial_for_url(
             serial_pair.left, baudrate=115200, rtscts=rtscts

tests/test_sync_transports.py

@@ -424,6 +424,9 @@ def test_sync_rtscts_setting(serial_pair: SerialPair, rtscts: bool) -> None:
     if rtscts and serial_pair.uri_scheme == "posix://":
         pytest.xfail("Strict POSIX backend does not support RTS/CTS flow control")
 
+    if rtscts and SerialBackend.ESPHOME_HOST in serial_pair.backends:
+        pytest.xfail("ESPHome host does not support RTS/CTS flow control")
+
     # Open both sides: on com0com, opening right asserts DTR which raises CTS on left
     with Serial.from_url(serial_pair.right, baudrate=115200):
         with Serial.from_url(serial_pair.left, baudrate=115200, rtscts=rtscts) as left:
@@ -463,6 +466,9 @@ def test_sync_exclusive_disabled(serial_pair: SerialPair) -> None:
     if SerialBackend.SER2NET in serial_pair.backends:
         pytest.skip("ser2net only allows one connection per port")
 
+    if SerialBackend.ESPHOME_HOST in serial_pair.backends:
+        pytest.skip("ESPHome Host only allows one connection per port")
+
     with Serial.from_url(serial_pair.left, baudrate=115200, exclusive=False) as serial1:
         assert serial1.exclusive is False