feat(sdk): add maintainResolution publish option to hold a constant encoder resolution

.changeset/olive-donkeys-shave.md

@@ -0,0 +1,5 @@
+---
+"@millicast/sdk": minor
+---
+
+Added a `maintainResolution` publish option that keeps the encoder at a constant resolution, giving up frame rate instead when bandwidth or CPU is constrained. `contentHint` is now also carried over to the replacement track by `replaceTrack`.

packages/millicast-sdk/src/PeerConnection.js

@@ -241,6 +241,13 @@ export default class PeerConnection extends EventEmitter {
 
   /**
    * Replace current audio or video track that is being broadcasted.
+   *
+   * Any contentHint set on the outgoing track is carried over to the incoming one unless the
+   * new track already declares its own. contentHint is a property of the track rather than of
+   * the sender, so without this a replacement silently discards it - which would quietly
+   * disable the `maintainResolution` publish option the first time a camera is switched.
+   * degradationPreference does not need the same treatment; it lives on the sender and
+   * survives the replacement.
    * @param {MediaStreamTrack} mediaStreamTrack - New audio or video track to replace the current one.
    */
   replaceTrack (mediaStreamTrack) {
@@ -252,6 +259,11 @@ export default class PeerConnection extends EventEmitter {
     const currentSender = this.peer.getSenders().find(s => s.track.kind === mediaStreamTrack.kind)
 
     if (currentSender) {
+      const previousHint = currentSender.track?.contentHint
+      if (previousHint && !mediaStreamTrack.contentHint) {
+        mediaStreamTrack.contentHint = previousHint
+        logger.debug('Carried contentHint over to the replacement track: ', previousHint)
+      }
       currentSender.replaceTrack(mediaStreamTrack)
     } else {
       logger.error(`There is no ${mediaStreamTrack.kind} track in active broadcast.`)

packages/millicast-sdk/src/Publish.js

@@ -22,12 +22,62 @@ const connectOptions = {
   codec: VideoCodec.H264,
   simulcast: false,
   scalabilityMode: null,
+  maintainResolution: false,
   peerConfig: {
     autoInitStats: true,
     statsIntervalMs: 1000
   }
 }
 
+/**
+ * Ask the pipeline to keep resolution constant and sacrifice frame rate instead.
+ *
+ * A mid-stream resolution change forces the encoder to re-emit its parameter sets, and
+ * max_num_ref_frames is derived from the level's DPB capacity divided by the frame size, so it
+ * changes with the resolution. Some downstream transcoders mishandle that combination. Holding
+ * the resolution steady avoids the situation entirely.
+ *
+ * Applied in two places because neither alone is reliable across browsers: contentHint is
+ * broadly supported and steers libwebrtc's degradation preference implicitly, while
+ * degradationPreference states it explicitly but is not implemented everywhere.
+ * @param {MediaStream|Array<MediaStreamTrack>} mediaStream - Stream or tracks being published.
+ */
+const applyResolutionContentHint = (mediaStream) => {
+  const tracks = Array.isArray(mediaStream)
+    ? mediaStream.filter(track => track.kind === 'video')
+    : mediaStream?.getVideoTracks?.() ?? []
+  for (const track of tracks) {
+    track.contentHint = 'detail'
+    if (track.contentHint !== 'detail') {
+      logger.warn('maintainResolution: contentHint was not accepted for track ', track.id)
+    }
+  }
+}
+
+/**
+ * Set degradationPreference on every video sender, and verify it stuck.
+ *
+ * Browsers may accept setParameters() and silently drop the field, so the value is read back
+ * rather than assumed. A warning here means the option is resting on contentHint alone.
+ * @param {RTCPeerConnection} peer - Peer connection whose senders should be configured.
+ */
+const applyDegradationPreference = async (peer) => {
+  const senders = peer.getSenders().filter(sender => sender.track?.kind === 'video')
+  for (const sender of senders) {
+    try {
+      const parameters = sender.getParameters()
+      parameters.degradationPreference = 'maintain-resolution'
+      await sender.setParameters(parameters)
+      const applied = sender.getParameters().degradationPreference
+      if (applied !== 'maintain-resolution') {
+        logger.warn('maintainResolution: degradationPreference not honoured, read back as ', applied)
+      }
+    } catch (error) {
+      logger.warn('maintainResolution: could not set degradationPreference ', error)
+    }
+  }
+}
+
 /**
  * @class Publish
  * @extends BaseWebRTC
@@ -72,6 +122,10 @@ export default class Publish extends BaseWebRTC {
    * @param {Boolean} [options.simulcast = false] - Enable simulcast. **Only available in Chromium based browsers and with H.264 or VP8 video codecs.**
    * @param {String} [options.scalabilityMode = null] - Selected scalability mode. You can get the available capabilities using <a href="PeerConnection#.getCapabilities">PeerConnection.getCapabilities</a> method.
    * **Only available in Google Chrome.**
+   * @param {Boolean} [options.maintainResolution = false] - Hold the encoder at a constant resolution,
+   * giving up frame rate instead when bandwidth or CPU is constrained. Sets `contentHint = 'detail'` on
+   * the video track and `degradationPreference = 'maintain-resolution'` on the sender. Both are
+   * preferences, not guarantees; the browser may still adapt under sustained pressure.
    * @param {PeerConnectionConfig} [options.peerConfig = null] - Options to configure the new RTCPeerConnection.
    * @param {Boolean} [options.record = false ] - Enable stream recording. If record is not provided, use default Token configuration. **Only available in Tokens with recording enabled.**
    * @param {Array<String>} [options.events = null] - Specify which events will be delivered by the server (any of "active" | "inactive" | "viewercount").*
@@ -167,6 +221,9 @@ export default class Publish extends BaseWebRTC {
       logger.error('Error while broadcasting. MediaStream required')
       throw new Error('MediaStream required')
     }
+    if (this.options.maintainResolution) {
+      applyResolutionContentHint(this.options.mediaStream)
+    }
     if (!data.migrate && this.isActive()) {
       logger.warn('Broadcast currently working')
       throw new Error('Broadcast currently working')
@@ -221,6 +278,11 @@ export default class Publish extends BaseWebRTC {
     promises = await Promise.all([getLocalSDPPromise, signalingConnectPromise])
     const localSdp = promises[0]
 
+    // Senders only exist once getRTCLocalSDP has added the tracks above.
+    if (this.options.maintainResolution) {
+      await applyDegradationPreference(webRTCPeerInstance.getRTCPeer())
+    }
+
     if (this.options.metadata) {
       if (!this.worker) {
         this.worker = new TransformWorker()

packages/millicast-sdk/src/types/index.d.ts

@@ -1297,6 +1297,13 @@ declare module "@millicast/sdk" {
          * **Only available in Google Chrome.**
          */
         scalabilityMode?: string
+        /**
+         * - Hold the encoder at a constant resolution, giving up frame rate instead when bandwidth
+         * or CPU is constrained. Sets `contentHint = 'detail'` on the video track and
+         * `degradationPreference = 'maintain-resolution'` on the sender. Both are preferences, not
+         * guarantees; the browser may still adapt under sustained pressure.
+         */
+        maintainResolution?: boolean
         /**
          * - Options to configure the new RTCPeerConnection.
          */

packages/millicast-sdk/tests/features/ChangeMediaTrack.feature

@@ -13,4 +13,14 @@ Feature: As a user I want to change a media track so I can change one of them wh
   Scenario: Replace unexisting track to peer
     Given I have a peer connected with video track
     When I want to change the audio track
-    Then the track is not changed
+    Then the track is not changed
+
+  Scenario: Replace track keeps the content hint of the track it replaces
+    Given I have a peer connected with a hinted video track
+    When I want to change current video track
+    Then the new track keeps the content hint
+
+  Scenario: Replace track does not override an explicit content hint
+    Given I have a peer connected with a hinted video track
+    When I want to change current video track for one that is hinted for motion
+    Then the new track keeps its own content hint

packages/millicast-sdk/tests/features/MaintainResolution.feature

@@ -0,0 +1,20 @@
+Feature: As a broadcaster I want to keep a constant resolution so downstream consumers never see a mid-stream resolution change
+
+  Scenario: Broadcast with maintainResolution enabled
+    Given an instance of Publish with connection path
+    When I broadcast a stream with maintainResolution enabled
+    Then the video track is hinted for detail
+    And the video sender prefers to maintain resolution
+
+  Scenario: Broadcast without maintainResolution
+    Given an instance of Publish with connection path
+    When I broadcast a stream with media stream
+    Then the video track has no content hint
+    And the video sender has no degradation preference
+
+  Scenario: Broadcast with maintainResolution enabled and an unsupported browser
+    Given an instance of Publish with connection path
+    And a browser that ignores the degradation preference
+    When I broadcast a stream with maintainResolution enabled
+    Then the broadcast is still connected
+    And the video track is hinted for detail

packages/millicast-sdk/tests/unit/ChangeMediaTrack.steps.js

@@ -52,6 +52,49 @@ defineFeature(feature, test => {
     })
   })
 
+  // contentHint lives on the track, not the sender, so replacing a track drops it unless it is
+  // carried across. Without this the maintainResolution publish option would stop applying the
+  // first time an application switched camera.
+  test('Replace track keeps the content hint of the track it replaces', ({ given, when, then }) => {
+    const peerConnection = new PeerConnection()
+    const track = { id: 3, kind: 'video', label: 'Video2' }
+
+    given('I have a peer connected with a hinted video track', async () => {
+      await peerConnection.createRTCPeer()
+      const tracks = [{ id: 1, kind: 'audio', label: 'Audio1' }, { id: 2, kind: 'video', label: 'Video1', contentHint: 'detail' }]
+      const mediaStream = new MediaStream(tracks)
+      await peerConnection.getRTCLocalSDP({ mediaStream, disableVideo: false, disableAudio: false })
+    })
+
+    when('I want to change current video track', () => {
+      peerConnection.replaceTrack(track)
+    })
+
+    then('the new track keeps the content hint', async () => {
+      expect(track.contentHint).toEqual('detail')
+    })
+  })
+
+  test('Replace track does not override an explicit content hint', ({ given, when, then }) => {
+    const peerConnection = new PeerConnection()
+    const track = { id: 3, kind: 'video', label: 'Video2', contentHint: 'motion' }
+
+    given('I have a peer connected with a hinted video track', async () => {
+      await peerConnection.createRTCPeer()
+      const tracks = [{ id: 1, kind: 'audio', label: 'Audio1' }, { id: 2, kind: 'video', label: 'Video1', contentHint: 'detail' }]
+      const mediaStream = new MediaStream(tracks)
+      await peerConnection.getRTCLocalSDP({ mediaStream, disableVideo: false, disableAudio: false })
+    })
+
+    when('I want to change current video track for one that is hinted for motion', () => {
+      peerConnection.replaceTrack(track)
+    })
+
+    then('the new track keeps its own content hint', async () => {
+      expect(track.contentHint).toEqual('motion')
+    })
+  })
+
   test('Replace unexisting track to peer', ({ given, when, then }) => {
     const peerConnection = new PeerConnection()
     const track = { id: 2, kind: 'audio', label: 'Audio2' }

packages/millicast-sdk/tests/unit/MaintainResolution.steps.js

@@ -0,0 +1,115 @@
+import { loadFeature, defineFeature } from 'jest-cucumber'
+import Publish from '../../src/Publish'
+import './__mocks__/MockRTCPeerConnection'
+import './__mocks__/MockMediaStream'
+import './__mocks__/MockBrowser'
+
+const feature = loadFeature('../features/MaintainResolution.feature', { loadRelativePath: true, errors: true })
+
+jest.mock('../../src/Signaling')
+
+jest.mock('../../src/workers/TransformWorker.worker.js', () =>
+  jest.fn(() => ({
+    postMessage: jest.fn(),
+    terminate: jest.fn()
+  }))
+)
+
+const mockTokenGenerator = jest.fn(() => {
+  return {
+    urls: [
+      'ws://localhost:8080'
+    ],
+    jwt: process.env.JWT_TEST_TOKEN ?? 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyLCJtaWxsaWNhc3QiOnt9fQ.IqT-PLLz-X7Wn7BNo-x4pFApAbMT9mmnlupR8eD9q4U'
+  }
+})
+
+const videoSenderOf = (publisher) =>
+  publisher.getRTCPeerConnection().getSenders().find(sender => sender.track?.kind === 'video')
+
+defineFeature(feature, test => {
+  afterEach(async () => {
+    jest.restoreAllMocks()
+  })
+
+  test('Broadcast with maintainResolution enabled', ({ given, when, then, and }) => {
+    let publisher
+    let mediaStream
+
+    given('an instance of Publish with connection path', async () => {
+      publisher = new Publish('streamName', mockTokenGenerator)
+      mediaStream = new MediaStream([{ kind: 'video' }, { kind: 'audio' }])
+    })
+
+    when('I broadcast a stream with maintainResolution enabled', async () => {
+      await publisher.connect({ mediaStream, maintainResolution: true })
+    })
+
+    then('the video track is hinted for detail', async () => {
+      expect(mediaStream.getVideoTracks()[0].contentHint).toEqual('detail')
+    })
+
+    and('the video sender prefers to maintain resolution', async () => {
+      expect(videoSenderOf(publisher).getParameters().degradationPreference)
+        .toEqual('maintain-resolution')
+    })
+  })
+
+  test('Broadcast without maintainResolution', ({ given, when, then, and }) => {
+    let publisher
+    let mediaStream
+
+    given('an instance of Publish with connection path', async () => {
+      publisher = new Publish('streamName', mockTokenGenerator)
+      mediaStream = new MediaStream([{ kind: 'video' }, { kind: 'audio' }])
+    })
+
+    when('I broadcast a stream with media stream', async () => {
+      await publisher.connect({ mediaStream })
+    })
+
+    then('the video track has no content hint', async () => {
+      expect(mediaStream.getVideoTracks()[0].contentHint).toBeUndefined()
+    })
+
+    and('the video sender has no degradation preference', async () => {
+      expect(videoSenderOf(publisher).getParameters().degradationPreference).toBeUndefined()
+    })
+  })
+
+  test('Broadcast with maintainResolution enabled and an unsupported browser', ({ given, when, then, and }) => {
+    let publisher
+    let mediaStream
+
+    given('an instance of Publish with connection path', async () => {
+      publisher = new Publish('streamName', mockTokenGenerator)
+      mediaStream = new MediaStream([{ kind: 'video' }, { kind: 'audio' }])
+    })
+
+    // Browsers may accept setParameters() and drop degradationPreference, or reject the call
+    // outright. Neither may take the broadcast down: the option degrades to contentHint alone.
+    and('a browser that ignores the degradation preference', async () => {
+      jest.spyOn(global.RTCPeerConnection.prototype, 'addTransceiver')
+        .mockImplementation(function (track) {
+          this.senders.push({
+            track,
+            getParameters: () => ({}),
+            setParameters: () => Promise.reject(new Error('Not supported')),
+            replaceTrack: () => {}
+          })
+        })
+    })
+
+    when('I broadcast a stream with maintainResolution enabled', async () => {
+      await publisher.connect({ mediaStream, maintainResolution: true })
+    })
+
+    then('the broadcast is still connected', async () => {
+      expect(publisher.isActive()).toBeTruthy()
+    })
+
+    and('the video track is hinted for detail', async () => {
+      expect(mediaStream.getVideoTracks()[0].contentHint).toEqual('detail')
+    })
+  })
+})

packages/millicast-sdk/tests/unit/__mocks__/MockRTCPeerConnection.js

@@ -98,6 +98,12 @@ export default class MockRTCPeerConnection {
   addTrack (track, mediaStream) {
     this.senders.push({
       track,
+      parameters: {},
+      getParameters () { return { ...this.parameters } },
+      setParameters (parameters) {
+        this.parameters = { ...parameters }
+        return Promise.resolve()
+      },
       replaceTrack: (newTrack) => {
         for (const sender of this.senders) {
           if (sender.track.kind === newTrack.kind) {
@@ -127,6 +133,12 @@ export default class MockRTCPeerConnection {
   addTransceiver (track, options) {
     this.senders.push({
       track,
+      parameters: {},
+      getParameters () { return { ...this.parameters } },
+      setParameters (parameters) {
+        this.parameters = { ...parameters }
+        return Promise.resolve()
+      },
       replaceTrack: (newTrack) => {
         for (const sender of this.senders) {
           if (sender.track.kind === newTrack.kind) {