-
Notifications
You must be signed in to change notification settings - Fork 229
[core] [world] Gate CBOR queue transport on specVersion #1627
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 9 commits
8399d51
b6b6dcf
dd2699b
ddc8c31
cf3dd88
50407fe
330d70d
82dcbe2
ce9ba33
57cb399
419de89
ff8a79c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| --- | ||
| "@workflow/world-vercel": patch | ||
| "@workflow/world": patch | ||
| "@workflow/core": patch | ||
| --- | ||
|
|
||
| Bump specVersion to 3 and gate CBOR queue transport on spec version. Old deployments (specVersion < 3) receive JSON queue messages; new deployments receive CBOR. Handler uses dual transport to deserialize both formats. Fixes replay/reenqueue from dashboard to older deployments. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,6 +7,8 @@ import { | |
| type QueueOptions, | ||
| type QueuePayload, | ||
| QueuePayloadSchema, | ||
| SPEC_VERSION_CURRENT, | ||
| SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, | ||
| ValidQueueName, | ||
| } from '@workflow/world'; | ||
| import { decode, encode } from 'cbor-x'; | ||
|
|
@@ -18,6 +20,8 @@ import { type APIConfig, getHeaders, getHttpUrl } from './utils.js'; | |
| * CBOR-based queue transport. Encodes values with cbor-x on send and | ||
| * decodes on receive, preserving Uint8Array values natively (workflow | ||
| * input is a Uint8Array in specVersion >= 2). | ||
| * | ||
| * Used for specVersion >= SPEC_VERSION_CURRENT (3). | ||
| */ | ||
| class CborTransport implements Transport<unknown> { | ||
| readonly contentType = 'application/cbor'; | ||
|
|
@@ -38,6 +42,58 @@ class CborTransport implements Transport<unknown> { | |
| } | ||
| } | ||
|
|
||
| /** | ||
| * JSON-based queue transport. Used for specVersion < SPEC_VERSION_CURRENT | ||
| * to maintain compatibility with older deployments that expect JSON messages. | ||
| */ | ||
| class JsonTransport implements Transport<unknown> { | ||
| readonly contentType = 'application/json'; | ||
|
|
||
| serialize(value: unknown): Buffer { | ||
| return Buffer.from(JSON.stringify(value)); | ||
| } | ||
|
|
||
| async deserialize(stream: ReadableStream<Uint8Array>): Promise<unknown> { | ||
| const chunks: Uint8Array[] = []; | ||
| const reader = stream.getReader(); | ||
| while (true) { | ||
| const { done, value } = await reader.read(); | ||
| if (done) break; | ||
| if (value) chunks.push(value); | ||
| } | ||
| return JSON.parse(Buffer.concat(chunks).toString()); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Dual transport for the queue handler. Serializes with CBOR (handler | ||
| * re-enqueues target the same new deployment) but deserializes with | ||
| * CBOR-first, falling back to JSON for messages from older deployments. | ||
| */ | ||
| class DualTransport implements Transport<unknown> { | ||
| readonly contentType = 'application/cbor'; | ||
|
|
||
| serialize(value: unknown): Buffer { | ||
| return Buffer.from(encode(value)); | ||
| } | ||
|
|
||
| async deserialize(stream: ReadableStream<Uint8Array>): Promise<unknown> { | ||
| const chunks: Uint8Array[] = []; | ||
| const reader = stream.getReader(); | ||
| while (true) { | ||
| const { done, value } = await reader.read(); | ||
| if (done) break; | ||
| if (value) chunks.push(value); | ||
| } | ||
| const buffer = Buffer.concat(chunks); | ||
| try { | ||
| return decode(buffer); | ||
| } catch { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Non-blocking observation: The CBOR-first, JSON-fallback strategy is safe because CBOR's binary encoding always starts with a type-length byte (e.g. The only theoretical concern is a corrupted message that happens to be valid CBOR but not valid JSON — but that's a data integrity issue unrelated to the dual transport. |
||
| return JSON.parse(buffer.toString()); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| const requestIdStorage = new AsyncLocalStorage<string | undefined>(); | ||
|
|
||
| const MessageWrapper = z.object({ | ||
|
|
@@ -113,11 +169,13 @@ export function createQueue(config?: APIConfig): Queue { | |
| const region = 'iad1'; | ||
|
|
||
| const cborTransport = new CborTransport(); | ||
| const jsonTransport = new JsonTransport(); | ||
| const dualTransport = new DualTransport(); | ||
|
|
||
| const clientOptions = { | ||
| region, | ||
| dispatcher: getDispatcher(), | ||
| transport: cborTransport, | ||
| transport: dualTransport, | ||
| ...(usingProxy && { | ||
| // final path will be /queues-proxy/api/v3/topic/... | ||
| // and the proxy will strip the /queues-proxy prefix before forwarding to VQS | ||
|
|
@@ -142,9 +200,17 @@ export function createQueue(config?: APIConfig): Queue { | |
| ); | ||
| } | ||
|
|
||
| // Select transport based on the target run's specVersion: | ||
| // CBOR for specVersion >= 3 (CBOR transport), JSON for older ones. | ||
| const useCbor = | ||
| (opts?.specVersion ?? SPEC_VERSION_CURRENT) >= | ||
| SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT; | ||
| const transport = useCbor ? cborTransport : jsonTransport; | ||
|
|
||
| const client = new QueueClient({ | ||
| ...clientOptions, | ||
| deploymentId, | ||
| transport, | ||
| }); | ||
|
|
||
| // The CborTransport handles CBOR encoding inside serialize(), | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,23 +15,33 @@ export type SpecVersion = number & { | |
| readonly [SpecVersionBrand]: typeof SpecVersionBrand; | ||
| }; | ||
|
|
||
| /** Legacy spec version (pre-event-sourcing). Also used for runs without specVersion. */ | ||
| /** | ||
| * Legacy spec version (pre-event-sourcing). Also used for runs without specVersion. | ||
| * This is the only true legacy version — specVersion 2+ all use the event-sourced model. | ||
| */ | ||
| export const SPEC_VERSION_LEGACY = 1 as SpecVersion; | ||
|
|
||
| /** Current spec version (event-sourced architecture). */ | ||
| export const SPEC_VERSION_CURRENT = 2 as SpecVersion; | ||
| export const SPEC_VERSION_SUPPORTS_EVENT_SOURCING = 2 as SpecVersion; | ||
| export const SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT = 3 as SpecVersion; | ||
|
|
||
| /** Current spec version (event-sourced architecture with CBOR queue transport). */ | ||
| export const SPEC_VERSION_CURRENT = | ||
| SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT as SpecVersion; | ||
|
|
||
| /** | ||
| * Check if a spec version is legacy (< SPEC_VERSION_CURRENT or undefined). | ||
| * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). | ||
| * Legacy runs require different handling - they use direct entity mutation | ||
| * instead of the event-sourced model. | ||
| * | ||
| * Checks against SPEC_VERSION_LEGACY (1), not SPEC_VERSION_CURRENT, so that | ||
| * intermediate versions (e.g. 2) are not incorrectly treated as legacy when | ||
| * SPEC_VERSION_CURRENT is bumped. | ||
| * | ||
| * @param v - The spec version number, or undefined/null for legacy runs | ||
| * @returns true if the run is a legacy run | ||
| */ | ||
| export function isLegacySpecVersion(v: number | undefined | null): boolean { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Non-blocking: This is the key correctness fix. The old |
||
| if (v === undefined || v === null) return true; | ||
| return v < SPEC_VERSION_CURRENT; | ||
| return v === undefined || v === null || v <= SPEC_VERSION_LEGACY; | ||
| } | ||
|
|
||
| /** | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.