diff --git a/.changeset/olive-donkeys-shave.md b/.changeset/olive-donkeys-shave.md new file mode 100644 index 000000000..a88cb1664 --- /dev/null +++ b/.changeset/olive-donkeys-shave.md @@ -0,0 +1,5 @@ +--- +"@millicast/sdk": minor +--- + +Added a `maintainResolution` publish option that works around decoders which glitch on a mid-stream resolution change, by keeping the encoder at a constant resolution and giving up frame rate instead when bandwidth or CPU is constrained. `contentHint` is now also carried over to the replacement track by `replaceTrack`. diff --git a/packages/millicast-sdk/src/PeerConnection.js b/packages/millicast-sdk/src/PeerConnection.js index ac6402a96..04f55bc70 100644 --- a/packages/millicast-sdk/src/PeerConnection.js +++ b/packages/millicast-sdk/src/PeerConnection.js @@ -241,6 +241,13 @@ export default class PeerConnection extends EventEmitter { /** * Replace current audio or video track that is being broadcasted. + * + * Any contentHint set on the outgoing track is carried over to the incoming one unless the + * new track already declares its own. contentHint is a property of the track rather than of + * the sender, so without this a replacement silently discards it - which would quietly + * disable the `maintainResolution` publish option the first time a camera is switched. + * degradationPreference does not need the same treatment; it lives on the sender and + * survives the replacement. * @param {MediaStreamTrack} mediaStreamTrack - New audio or video track to replace the current one. */ replaceTrack (mediaStreamTrack) { @@ -252,6 +259,11 @@ export default class PeerConnection extends EventEmitter { const currentSender = this.peer.getSenders().find(s => s.track.kind === mediaStreamTrack.kind) if (currentSender) { + const previousHint = currentSender.track?.contentHint + if (previousHint && !mediaStreamTrack.contentHint) { + mediaStreamTrack.contentHint = previousHint + logger.debug('Carried contentHint over to the replacement track: ', previousHint) + } currentSender.replaceTrack(mediaStreamTrack) } else { logger.error(`There is no ${mediaStreamTrack.kind} track in active broadcast.`) diff --git a/packages/millicast-sdk/src/Publish.js b/packages/millicast-sdk/src/Publish.js index 2a392f3cf..f556c7d49 100644 --- a/packages/millicast-sdk/src/Publish.js +++ b/packages/millicast-sdk/src/Publish.js @@ -22,12 +22,60 @@ const connectOptions = { codec: VideoCodec.H264, simulcast: false, scalabilityMode: null, + maintainResolution: false, peerConfig: { autoInitStats: true, statsIntervalMs: 1000 } } +/** + * Ask the pipeline to keep resolution constant and sacrifice frame rate instead. + * + * This works around decoders that glitch on a mid-stream resolution change; holding the + * resolution steady avoids the change in the first place. + * + * Applied in two places because neither alone is reliable across browsers: contentHint is + * broadly supported and steers libwebrtc's degradation preference implicitly, while + * degradationPreference states it explicitly but is not implemented everywhere. + * @param {MediaStream|Array} mediaStream - Stream or tracks being published. + */ +const applyResolutionContentHint = (mediaStream) => { + const tracks = Array.isArray(mediaStream) + ? mediaStream.filter(track => track.kind === 'video') + : mediaStream?.getVideoTracks?.() ?? [] + for (const track of tracks) { + track.contentHint = 'detail' + if (track.contentHint !== 'detail') { + logger.warn('maintainResolution: contentHint was not accepted for track ', track.id) + } + } +} + +/** + * Set degradationPreference on every video sender, and verify it stuck. + * + * Browsers may accept setParameters() and silently drop the field, so the value is read back + * rather than assumed. A warning here means the option is resting on contentHint alone. + * @param {RTCPeerConnection} peer - Peer connection whose senders should be configured. + */ +const applyDegradationPreference = async (peer) => { + const senders = peer.getSenders().filter(sender => sender.track?.kind === 'video') + for (const sender of senders) { + try { + const parameters = sender.getParameters() + parameters.degradationPreference = 'maintain-resolution' + await sender.setParameters(parameters) + const applied = sender.getParameters().degradationPreference + if (applied !== 'maintain-resolution') { + logger.warn('maintainResolution: degradationPreference not honoured, read back as ', applied) + } + } catch (error) { + logger.warn('maintainResolution: could not set degradationPreference ', error) + } + } +} + /** * @class Publish * @extends BaseWebRTC @@ -72,6 +120,10 @@ export default class Publish extends BaseWebRTC { * @param {Boolean} [options.simulcast = false] - Enable simulcast. **Only available in Chromium based browsers and with H.264 or VP8 video codecs.** * @param {String} [options.scalabilityMode = null] - Selected scalability mode. You can get the available capabilities using PeerConnection.getCapabilities method. * **Only available in Google Chrome.** + * @param {Boolean} [options.maintainResolution = false] - Work around decoders that glitch on a + * mid-stream resolution change by holding the encoder at a constant resolution, giving up frame + * rate instead when bandwidth or CPU is constrained. Both settings it applies are preferences, + * not guarantees; the browser may still adapt under sustained pressure. * @param {PeerConnectionConfig} [options.peerConfig = null] - Options to configure the new RTCPeerConnection. * @param {Boolean} [options.record = false ] - Enable stream recording. If record is not provided, use default Token configuration. **Only available in Tokens with recording enabled.** * @param {Array} [options.events = null] - Specify which events will be delivered by the server (any of "active" | "inactive" | "viewercount").* @@ -171,6 +223,9 @@ export default class Publish extends BaseWebRTC { logger.warn('Broadcast currently working') throw new Error('Broadcast currently working') } + if (this.options.maintainResolution) { + applyResolutionContentHint(this.options.mediaStream) + } let publisherData try { publisherData = await this.tokenGenerator() @@ -221,6 +276,11 @@ export default class Publish extends BaseWebRTC { promises = await Promise.all([getLocalSDPPromise, signalingConnectPromise]) const localSdp = promises[0] + // Senders only exist once getRTCLocalSDP has added the tracks above. + if (this.options.maintainResolution) { + await applyDegradationPreference(webRTCPeerInstance.getRTCPeer()) + } + if (this.options.metadata) { if (!this.worker) { this.worker = new TransformWorker() diff --git a/packages/millicast-sdk/src/types/index.d.ts b/packages/millicast-sdk/src/types/index.d.ts index d5b22e506..84322e56e 100644 --- a/packages/millicast-sdk/src/types/index.d.ts +++ b/packages/millicast-sdk/src/types/index.d.ts @@ -1297,6 +1297,14 @@ declare module "@millicast/sdk" { * **Only available in Google Chrome.** */ scalabilityMode?: string + /** + * - Work around decoders that glitch on a mid-stream resolution change (seen with YouTube + * when the stream is restreamed) by holding the encoder at a constant resolution, giving up + * frame rate instead when bandwidth or CPU is constrained. Sets `contentHint = 'detail'` on + * the video track and `degradationPreference = 'maintain-resolution'` on the sender. Both + * are preferences, not guarantees; the browser may still adapt under sustained pressure. + */ + maintainResolution?: boolean /** * - Options to configure the new RTCPeerConnection. */ diff --git a/packages/millicast-sdk/tests/features/ChangeMediaTrack.feature b/packages/millicast-sdk/tests/features/ChangeMediaTrack.feature index 6c8937fff..b4ed46df4 100644 --- a/packages/millicast-sdk/tests/features/ChangeMediaTrack.feature +++ b/packages/millicast-sdk/tests/features/ChangeMediaTrack.feature @@ -13,4 +13,14 @@ Feature: As a user I want to change a media track so I can change one of them wh Scenario: Replace unexisting track to peer Given I have a peer connected with video track When I want to change the audio track - Then the track is not changed \ No newline at end of file + Then the track is not changed + + Scenario: Replace track keeps the content hint of the track it replaces + Given I have a peer connected with a hinted video track + When I want to change current video track + Then the new track keeps the content hint + + Scenario: Replace track does not override an explicit content hint + Given I have a peer connected with a hinted video track + When I want to change current video track for one that is hinted for motion + Then the new track keeps its own content hint diff --git a/packages/millicast-sdk/tests/features/MaintainResolution.feature b/packages/millicast-sdk/tests/features/MaintainResolution.feature new file mode 100644 index 000000000..5d03bc0d1 --- /dev/null +++ b/packages/millicast-sdk/tests/features/MaintainResolution.feature @@ -0,0 +1,20 @@ +Feature: As a broadcaster I want to keep a constant resolution so downstream consumers never see a mid-stream resolution change + + Scenario: Broadcast with maintainResolution enabled + Given an instance of Publish with connection path + When I broadcast a stream with maintainResolution enabled + Then the video track is hinted for detail + And the video sender prefers to maintain resolution + + Scenario: Broadcast without maintainResolution + Given an instance of Publish with connection path + When I broadcast a stream with media stream + Then the video track has no content hint + And the video sender has no degradation preference + + Scenario: Broadcast with maintainResolution enabled and an unsupported browser + Given an instance of Publish with connection path + And a browser that ignores the degradation preference + When I broadcast a stream with maintainResolution enabled + Then the broadcast is still connected + And the video track is hinted for detail diff --git a/packages/millicast-sdk/tests/unit/ChangeMediaTrack.steps.js b/packages/millicast-sdk/tests/unit/ChangeMediaTrack.steps.js index 8dc7768a9..6df490651 100644 --- a/packages/millicast-sdk/tests/unit/ChangeMediaTrack.steps.js +++ b/packages/millicast-sdk/tests/unit/ChangeMediaTrack.steps.js @@ -52,6 +52,49 @@ defineFeature(feature, test => { }) }) + // contentHint lives on the track, not the sender, so replacing a track drops it unless it is + // carried across. Without this the maintainResolution publish option would stop applying the + // first time an application switched camera. + test('Replace track keeps the content hint of the track it replaces', ({ given, when, then }) => { + const peerConnection = new PeerConnection() + const track = { id: 3, kind: 'video', label: 'Video2' } + + given('I have a peer connected with a hinted video track', async () => { + await peerConnection.createRTCPeer() + const tracks = [{ id: 1, kind: 'audio', label: 'Audio1' }, { id: 2, kind: 'video', label: 'Video1', contentHint: 'detail' }] + const mediaStream = new MediaStream(tracks) + await peerConnection.getRTCLocalSDP({ mediaStream, disableVideo: false, disableAudio: false }) + }) + + when('I want to change current video track', () => { + peerConnection.replaceTrack(track) + }) + + then('the new track keeps the content hint', async () => { + expect(track.contentHint).toEqual('detail') + }) + }) + + test('Replace track does not override an explicit content hint', ({ given, when, then }) => { + const peerConnection = new PeerConnection() + const track = { id: 3, kind: 'video', label: 'Video2', contentHint: 'motion' } + + given('I have a peer connected with a hinted video track', async () => { + await peerConnection.createRTCPeer() + const tracks = [{ id: 1, kind: 'audio', label: 'Audio1' }, { id: 2, kind: 'video', label: 'Video1', contentHint: 'detail' }] + const mediaStream = new MediaStream(tracks) + await peerConnection.getRTCLocalSDP({ mediaStream, disableVideo: false, disableAudio: false }) + }) + + when('I want to change current video track for one that is hinted for motion', () => { + peerConnection.replaceTrack(track) + }) + + then('the new track keeps its own content hint', async () => { + expect(track.contentHint).toEqual('motion') + }) + }) + test('Replace unexisting track to peer', ({ given, when, then }) => { const peerConnection = new PeerConnection() const track = { id: 2, kind: 'audio', label: 'Audio2' } diff --git a/packages/millicast-sdk/tests/unit/MaintainResolution.steps.js b/packages/millicast-sdk/tests/unit/MaintainResolution.steps.js new file mode 100644 index 000000000..4e2dbd49e --- /dev/null +++ b/packages/millicast-sdk/tests/unit/MaintainResolution.steps.js @@ -0,0 +1,115 @@ +import { loadFeature, defineFeature } from 'jest-cucumber' +import Publish from '../../src/Publish' +import './__mocks__/MockRTCPeerConnection' +import './__mocks__/MockMediaStream' +import './__mocks__/MockBrowser' + +const feature = loadFeature('../features/MaintainResolution.feature', { loadRelativePath: true, errors: true }) + +jest.mock('../../src/Signaling') + +jest.mock('../../src/workers/TransformWorker.worker.js', () => + jest.fn(() => ({ + postMessage: jest.fn(), + terminate: jest.fn() + })) +) + +const mockTokenGenerator = jest.fn(() => { + return { + urls: [ + 'ws://localhost:8080' + ], + jwt: process.env.JWT_TEST_TOKEN ?? 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyLCJtaWxsaWNhc3QiOnt9fQ.IqT-PLLz-X7Wn7BNo-x4pFApAbMT9mmnlupR8eD9q4U' + } +}) + +const videoSenderOf = (publisher) => + publisher.getRTCPeerConnection().getSenders().find(sender => sender.track?.kind === 'video') + +defineFeature(feature, test => { + afterEach(async () => { + jest.restoreAllMocks() + }) + + test('Broadcast with maintainResolution enabled', ({ given, when, then, and }) => { + let publisher + let mediaStream + + given('an instance of Publish with connection path', async () => { + publisher = new Publish('streamName', mockTokenGenerator) + mediaStream = new MediaStream([{ kind: 'video' }, { kind: 'audio' }]) + }) + + when('I broadcast a stream with maintainResolution enabled', async () => { + await publisher.connect({ mediaStream, maintainResolution: true }) + }) + + then('the video track is hinted for detail', async () => { + expect(mediaStream.getVideoTracks()[0].contentHint).toEqual('detail') + }) + + and('the video sender prefers to maintain resolution', async () => { + expect(videoSenderOf(publisher).getParameters().degradationPreference) + .toEqual('maintain-resolution') + }) + }) + + test('Broadcast without maintainResolution', ({ given, when, then, and }) => { + let publisher + let mediaStream + + given('an instance of Publish with connection path', async () => { + publisher = new Publish('streamName', mockTokenGenerator) + mediaStream = new MediaStream([{ kind: 'video' }, { kind: 'audio' }]) + }) + + when('I broadcast a stream with media stream', async () => { + await publisher.connect({ mediaStream }) + }) + + then('the video track has no content hint', async () => { + expect(mediaStream.getVideoTracks()[0].contentHint).toBeUndefined() + }) + + and('the video sender has no degradation preference', async () => { + expect(videoSenderOf(publisher).getParameters().degradationPreference).toBeUndefined() + }) + }) + + test('Broadcast with maintainResolution enabled and an unsupported browser', ({ given, when, then, and }) => { + let publisher + let mediaStream + + given('an instance of Publish with connection path', async () => { + publisher = new Publish('streamName', mockTokenGenerator) + mediaStream = new MediaStream([{ kind: 'video' }, { kind: 'audio' }]) + }) + + // Browsers may accept setParameters() and drop degradationPreference, or reject the call + // outright. Neither may take the broadcast down: the option degrades to contentHint alone. + and('a browser that ignores the degradation preference', async () => { + jest.spyOn(global.RTCPeerConnection.prototype, 'addTransceiver') + .mockImplementation(function (track) { + this.senders.push({ + track, + getParameters: () => ({}), + setParameters: () => Promise.reject(new Error('Not supported')), + replaceTrack: () => {} + }) + }) + }) + + when('I broadcast a stream with maintainResolution enabled', async () => { + await publisher.connect({ mediaStream, maintainResolution: true }) + }) + + then('the broadcast is still connected', async () => { + expect(publisher.isActive()).toBeTruthy() + }) + + and('the video track is hinted for detail', async () => { + expect(mediaStream.getVideoTracks()[0].contentHint).toEqual('detail') + }) + }) +}) diff --git a/packages/millicast-sdk/tests/unit/__mocks__/MockRTCPeerConnection.js b/packages/millicast-sdk/tests/unit/__mocks__/MockRTCPeerConnection.js index bb95012c9..ce97a01a7 100644 --- a/packages/millicast-sdk/tests/unit/__mocks__/MockRTCPeerConnection.js +++ b/packages/millicast-sdk/tests/unit/__mocks__/MockRTCPeerConnection.js @@ -98,6 +98,12 @@ export default class MockRTCPeerConnection { addTrack (track, mediaStream) { this.senders.push({ track, + parameters: {}, + getParameters () { return { ...this.parameters } }, + setParameters (parameters) { + this.parameters = { ...parameters } + return Promise.resolve() + }, replaceTrack: (newTrack) => { for (const sender of this.senders) { if (sender.track.kind === newTrack.kind) { @@ -127,6 +133,12 @@ export default class MockRTCPeerConnection { addTransceiver (track, options) { this.senders.push({ track, + parameters: {}, + getParameters () { return { ...this.parameters } }, + setParameters (parameters) { + this.parameters = { ...parameters } + return Promise.resolve() + }, replaceTrack: (newTrack) => { for (const sender of this.senders) { if (sender.track.kind === newTrack.kind) {