fix(websocket): tighten frame-bound checks across libunit and protocol

CHANGES

@@ -14,6 +14,17 @@ Changes with FreeUnit 1.35.4                                     xx xxx 2026
     *) Bugfix: replace removed cgi module with email.parser in Python upload
        test fixture; fixes test suite compatibility with Python 3.13.
 
+    *) Bugfix: tighten WebSocket frame-bound checks across libunit and the
+       protocol layer.  Adds guards for an OOB header read on truncated
+       extended-length frames (libunit nxt_unit.c), the matching unchecked
+       hsize in nxt_unit_websocket_retain(), the unvalidated MSB of the
+       64-bit extended payload length (RFC 6455 §5.2), a frame-size
+       decrement that was a no-op (nxt_http_websocket.c) and could leak
+       cross-frame bytes into the outgoing buffer, missing buffer-capacity
+       checks on the Java sendWsFrame JNI entry points, and a uint64
+       wraparound on pending_payload_len in the Python ASGI WebSocket
+       handler.  See security-audit.md V8/V9/V10/V12.
+
 Changes with FreeUnit 1.35.3                                     07 Apr 2026
 
     *) Feature: migrate contrib package mirror from packages.nginx.org to

src/java/nxt_jni_Request.c

@@ -731,12 +731,23 @@ static void JNICALL
 nxt_java_Request_sendWsFrameBuf(JNIEnv *env, jclass cls,
     jlong req_info_ptr, jobject buf, jint pos, jint len, jbyte opCode, jboolean last)
 {
+    jlong                    cap;
     nxt_unit_request_info_t  *req;
 
     req = nxt_jlong2ptr(req_info_ptr);
     uint8_t *b = (*env)->GetDirectBufferAddress(env, buf);
 
     if (b != NULL) {
+        cap = (*env)->GetDirectBufferCapacity(env, buf);
+        if (pos < 0 || len < 0 || cap < 0
+            || (jlong) pos > cap
+            || (jlong) len > cap - (jlong) pos)
+        {
+            nxt_java_throw_IllegalStateException(env,
+                "sendWsFrame: pos/len out of buffer capacity");
+            return;
+        }
+
         nxt_unit_websocket_send(req, opCode, last, b + pos, len);
 
     } else {
@@ -749,9 +760,21 @@ static void JNICALL
 nxt_java_Request_sendWsFrameArr(JNIEnv *env, jclass cls,
     jlong req_info_ptr, jarray arr, jint pos, jint len, jbyte opCode, jboolean last)
 {
+    jsize                    cap;
     nxt_unit_request_info_t  *req;
 
     req = nxt_jlong2ptr(req_info_ptr);
+
+    cap = (*env)->GetArrayLength(env, arr);
+    if (pos < 0 || len < 0
+        || pos > cap
+        || len > cap - pos)
+    {
+        nxt_java_throw_IllegalStateException(env,
+            "sendWsFrame: pos/len out of array length");
+        return;
+    }
+
     uint8_t *b = (*env)->GetPrimitiveArrayCritical(env, arr, NULL);
 
     if (b != NULL) {

src/nxt_h1proto_websocket.c

@@ -68,6 +68,9 @@ static const nxt_ws_error_t  nxt_ws_err_invalid_opcode = {
 static const nxt_ws_error_t  nxt_ws_err_cont_expected = {
     NXT_WEBSOCKET_CR_PROTOCOL_ERROR,
     1, nxt_string("Continuation expected, but %ud opcode received") };
+static const nxt_ws_error_t  nxt_ws_err_invalid_length = {
+    NXT_WEBSOCKET_CR_PROTOCOL_ERROR,
+    0, nxt_string("Invalid extended payload length") };
 
 void
 nxt_h1p_websocket_first_frame_start(nxt_task_t *task, nxt_http_request_t *r,
@@ -267,6 +270,18 @@ nxt_h1p_conn_ws_frame_header_read(nxt_task_t *task, void *obj, void *data)
         return;
     }
 
+    /*
+     * RFC 6455 §5.2: when payload_len == 127, the most-significant bit
+     * of the 8-byte extended length MUST be 0.  Reject as protocol error
+     * before any further size arithmetic uses the value.
+     */
+    if (nxt_slow_path(wsh->payload_len == 127
+                      && (wsh->payload_len_[0] & 0x80) != 0))
+    {
+        hxt_h1p_send_ws_error(task, r, &nxt_ws_err_invalid_length);
+        return;
+    }
+
     if ((wsh->opcode & NXT_WEBSOCKET_OP_CTRL) != 0) {
         if (nxt_slow_path(wsh->fin == 0)) {
             hxt_h1p_send_ws_error(task, r, &nxt_ws_err_ctrl_fragmented);

src/nxt_http_websocket.c

@@ -82,9 +82,9 @@ nxt_http_websocket_client(nxt_task_t *task, void *obj, void *data)
             copy_size -= chunk_copy_size;
             b->mem.pos += chunk_copy_size;
             buf_free_size -= chunk_copy_size;
+            frame_size -= chunk_copy_size;
         }
 
-        frame_size -= copy_size;
         next = b->next;
         b->next = NULL;
 

src/nxt_unit.c

@@ -1679,11 +1679,30 @@ nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg)
         }
 
         ws_impl->ws.header = (void *) b->buf.start;
-        ws_impl->ws.payload_len = nxt_websocket_frame_payload_len(
-            ws_impl->ws.header);
 
         hsize = nxt_websocket_frame_header_size(ws_impl->ws.header);
 
+        /*
+         * Reject truncated frames before reading the extended length /
+         * mask fields or advancing buf.free past buf.end.  A 2-byte
+         * frame whose header advertises a 14-byte extended length would
+         * otherwise OOB-read b->buf.start + hsize - 4 (mask) and the
+         * 8-byte extended length, and break the buffer invariant.
+         */
+        if (nxt_slow_path((size_t) (b->buf.end - b->buf.start) < hsize)) {
+            nxt_unit_warn(ctx, "#%"PRIu32": truncated websocket frame: "
+                          "hsize %zu > buf size %zu",
+                          req_impl->stream, hsize,
+                          (size_t) (b->buf.end - b->buf.start));
+
+            nxt_unit_websocket_frame_release(&ws_impl->ws);
+
+            return NXT_UNIT_ERROR;
+        }
+
+        ws_impl->ws.payload_len = nxt_websocket_frame_payload_len(
+            ws_impl->ws.header);
+
         if (ws_impl->ws.header->mask) {
             ws_impl->ws.mask = (uint8_t *) b->buf.start + hsize - 4;
 
@@ -3454,6 +3473,12 @@ nxt_unit_websocket_retain(nxt_unit_websocket_frame_t *ws)
 
     hsize = nxt_websocket_frame_header_size(b);
 
+    /* Same OOB-read hazard as nxt_unit_process_websocket(). */
+    if (nxt_slow_path(hsize > size)) {
+        nxt_unit_free(ws->req->ctx, b);
+        return NXT_UNIT_ERROR;
+    }
+
     ws_impl->buf->buf.start = b;
     ws_impl->buf->buf.free = b + hsize;
     ws_impl->buf->buf.end = b + size;

src/python/nxt_python_asgi_websocket.c

@@ -706,6 +706,26 @@ nxt_py_asgi_websocket_suspend_frame(nxt_unit_websocket_frame_t *frame)
         return;
     }
 
+    /*
+     * Guard against uint64 wraparound across many fragmented frames;
+     * otherwise the eventual max_buffer_size check is bypassed.  Run
+     * the check BEFORE inserting p into the pending_frames queue so a
+     * failure exit doesn't leak the suspended-frame slot.
+     */
+    if (nxt_slow_path(frame->payload_len
+                      > UINT64_MAX - ws->pending_payload_len))
+    {
+        nxt_unit_req_alert(ws->req,
+                           "pending_payload_len overflow on suspend.");
+
+        nxt_unit_free(ws->req->ctx, p);
+        nxt_unit_websocket_done(frame);
+
+        PyErr_SetString(PyExc_RuntimeError,
+                        "pending_payload_len overflow on suspend.");
+        return;
+    }
+
     p->frame = frame;
     nxt_queue_insert_tail(&ws->pending_frames, &p->link);
 
@@ -750,6 +770,18 @@ nxt_py_asgi_websocket_pop_msg(nxt_py_asgi_websocket_t *ws,
 
     } else {
         if (frame != NULL) {
+            if (nxt_slow_path(frame->payload_len
+                              > UINT64_MAX - ws->pending_payload_len))
+            {
+                nxt_unit_req_alert(ws->req,
+                                   "pending_payload_len overflow on pop.");
+
+                nxt_unit_websocket_done(frame);
+
+                return PyErr_Format(PyExc_RuntimeError,
+                                    "pending_payload_len overflow on pop.");
+            }
+
             payload_len = ws->pending_payload_len + frame->payload_len;
             fin_frame = frame;
 

test/test_asgi_websockets.py

@@ -170,7 +170,11 @@ def test_asgi_websockets_length_long():
     ws.frame_write(sock, ws.OP_TEXT, 'fragment1', fin=False)
     ws.frame_write(sock, ws.OP_CONT, 'fragment2', length=2**64 - 1)
 
-    check_close(sock, 1009)  # 1009 - CLOSE_TOO_LARGE
+    # 1002 - CLOSE_PROTOCOL_ERROR.  RFC 6455 §5.2 requires the
+    # high bit of the 64-bit extended payload length to be 0; an
+    # all-ones length is a protocol violation and gets rejected
+    # before the policy-size check (1009 CLOSE_TOO_LARGE) runs.
+    check_close(sock, 1002)
 
 
 def test_asgi_websockets_frame_fragmentation_invalid():