diff --git a/worker/.clang-format b/worker/.clang-format index 7ca24bba9b..485d42de89 100644 --- a/worker/.clang-format +++ b/worker/.clang-format @@ -17,9 +17,6 @@ AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: Yes BinPackArguments: false BinPackParameters: false -# NOTE: This doesn't do anything because it requires Cpp11BracedListStyle: true, -# which we don't want. -BreakAfterOpenBracketBracedList: true BraceWrapping: AfterClass: true AfterControlStatement: Always diff --git a/worker/include/RTC/Consumer.hpp b/worker/include/RTC/Consumer.hpp index 51b4d9e5ac..a988b0042c 100644 --- a/worker/include/RTC/Consumer.hpp +++ b/worker/include/RTC/Consumer.hpp @@ -2,12 +2,14 @@ #define MS_RTC_CONSUMER_HPP #include "common.hpp" +#include "Shared.hpp" #include "SharedInterface.hpp" #include "Channel/ChannelRequest.hpp" #include "Channel/ChannelSocket.hpp" #include "FBS/consumer.h" #include "FBS/transport.h" #include "RTC/ConsumerTypes.hpp" +#include "RTC/ProducerStreamManager.hpp" #include "RTC/RTCP/CompoundPacket.hpp" #include "RTC/RTCP/FeedbackRtpNack.hpp" #include "RTC/RTCP/ReceiverReport.hpp" @@ -17,14 +19,22 @@ #include "RTC/RTP/RtpStreamSend.hpp" #include "RTC/RTP/SharedPacket.hpp" #include "RTC/RtpDictionaries.hpp" +#include "RTC/SeqManager.hpp" #include +#include +#include #include #include namespace RTC { - class Consumer : public Channel::ChannelSocket::RequestHandler + class Consumer : public Channel::ChannelSocket::RequestHandler, + public RTC::RTP::RtpStreamSend::Listener, + public RTC::ProducerStreamManager::Listener { + using RetransmissionBuffer = + std::map::SeqLowerThan>; + public: class Listener { @@ -56,20 +66,18 @@ namespace RTC const std::string& id, const std::string& producerId, RTC::Consumer::Listener* listener, - const FBS::Transport::ConsumeRequest* data, - RTC::RtpParameters::Type type); + const FBS::Transport::ConsumeRequest* data); ~Consumer() override; public: - flatbuffers::Offset FillBuffer( + flatbuffers::Offset FillBuffer( + flatbuffers::FlatBufferBuilder& builder) const; + flatbuffers::Offset FillBufferBase( + flatbuffers::FlatBufferBuilder& builder) const; + flatbuffers::Offset FillBufferStats( + flatbuffers::FlatBufferBuilder& builder); + flatbuffers::Offset FillBufferScore( flatbuffers::FlatBufferBuilder& builder) const; - virtual flatbuffers::Offset FillBufferStats( - flatbuffers::FlatBufferBuilder& builder) = 0; - virtual flatbuffers::Offset FillBufferScore( - flatbuffers::FlatBufferBuilder& /*builder*/) const - { - return 0; - }; RTC::Media::Kind GetKind() const { return this->kind; @@ -86,12 +94,9 @@ namespace RTC { return this->type; } - virtual RTC::ConsumerTypes::VideoLayers GetPreferredLayers() const + RTC::ConsumerTypes::VideoLayers GetPreferredLayers() const { - // By default return 1:1. - RTC::ConsumerTypes::VideoLayers layers; - - return layers; + return this->producerStreamManager->GetPreferredLayers(); } const std::vector& GetMediaSsrcs() const { @@ -101,10 +106,8 @@ namespace RTC { return this->rtxSsrcs; } - virtual bool IsActive() const + bool IsActive() const override { - // The parent Consumer just checks whether Consumer and Producer are - // not paused and the transport connected. // clang-format off return ( this->transportConnected && @@ -126,32 +129,30 @@ namespace RTC } void ProducerPaused(); void ProducerResumed(); - virtual void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) = 0; - virtual void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) = 0; + void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc); + void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc); void ProducerRtpStreamScores(const std::vector* scores); - virtual void ProducerRtpStreamScore( - RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) = 0; - virtual void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) = 0; + void ProducerRtpStreamScore(RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore); + void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first); void ProducerClosed(); void SetExternallyManagedBitrate() { this->externallyManagedBitrate = true; + this->producerStreamManager->SetExternallyManagedBitrate(); } - virtual uint8_t GetBitratePriority() const = 0; - virtual uint32_t IncreaseLayer(uint32_t bitrate, bool considerLoss) = 0; - virtual void ApplyLayers() = 0; - virtual uint32_t GetDesiredBitrate() const = 0; - virtual void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) = 0; - virtual bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) = 0; - virtual const std::vector& GetRtpStreams() const = 0; - virtual void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) = 0; - virtual void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) = 0; - virtual void ReceiveKeyFrameRequest( - RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) = 0; - virtual void ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) = 0; - virtual void ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) = 0; - virtual uint32_t GetTransmissionRate(uint64_t nowMs) = 0; - virtual float GetRtt() const = 0; + uint8_t GetBitratePriority() const; + uint32_t IncreaseLayer(uint32_t bitrate, bool considerLoss); + void ApplyLayers(); + uint32_t GetDesiredBitrate() const; + void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket); + bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs); + void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost); + void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket); + void ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc); + void ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report); + void ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report); + uint32_t GetTransmissionRate(uint64_t nowMs); + float GetRtt() const; /* Methods inherited from Channel::ChannelSocket::RequestHandler. */ public: @@ -165,10 +166,35 @@ namespace RTC void EmitTraceEvent(flatbuffers::Offset& notification) const; private: - virtual void UserOnTransportConnected() = 0; - virtual void UserOnTransportDisconnected() = 0; - virtual void UserOnPaused() = 0; - virtual void UserOnResumed() = 0; + void EmitScore() const; + void EmitLayersChange() const; + + private: + void UserOnTransportConnected(); + void UserOnTransportDisconnected(); + void UserOnPaused(); + void UserOnResumed(); + + private: + void CreateRtpStreams(); + static void StorePacketInTargetLayerRetransmissionBuffer( + RetransmissionBuffer& targetLayerRetransmissionBuffer, + RTC::RTP::Packet* packet, + RTC::RTP::SharedPacket& sharedPacket); + + /* Pure virtual methods inherited from RtpStreamSend::Listener. */ + public: + void OnRtpStreamScore(RTC::RTP::RtpStream* rtpStream, uint8_t score, uint8_t previousScore) override; + void OnRtpStreamRetransmitRtpPacket( + RTC::RTP::RtpStreamSend* rtpStream, RTC::RTP::Packet* packet) override; + + /* Pure virtual methods inherited from ProducerStreamManager::Listener. */ + public: + void OnProducerStreamManagerKeyFrameRequested(uint32_t mappedSsrc) override; + void OnProducerStreamManagerNeedBitrateChange() override; + void OnProducerStreamManagerLayersChanged() override; + void OnProducerStreamManagerClearRetransmissionBuffer() override; + void OnProducerStreamManagerScore() override; public: // Passed by argument. @@ -186,8 +212,6 @@ namespace RTC struct RTC::RTP::HeaderExtensionIds rtpHeaderExtensionIds; const std::vector* producerRtpStreamScores{ nullptr }; // Others. - // Whether a payload type is supported or not is represented in the - // corresponding position of the bitset. std::bitset<128u> supportedCodecPayloadTypes; uint64_t lastRtcpSentTime{ 0u }; uint16_t maxRtcpInterval{ 0u }; @@ -196,13 +220,24 @@ namespace RTC struct TraceEventTypes traceEventTypes; private: + bool pipe{ false }; // Others. + std::vector rtpStreams; + absl::flat_hash_map mapMappedSsrcSsrc; + absl::flat_hash_map mapSsrcRtpStream; + absl::flat_hash_map> mapRtpStreamRtpSeqManager; + // Buffers to store packets that arrive earlier than the first packet of the + // video key frame. + absl::flat_hash_map + mapRtpStreamTargetLayerRetransmissionBuffer; std::vector mediaSsrcs; std::vector rtxSsrcs; bool transportConnected{ false }; bool paused{ false }; bool producerPaused{ false }; bool producerClosed{ false }; + bool lastSentPacketHasMarker{ false }; + std::unique_ptr producerStreamManager; }; } // namespace RTC diff --git a/worker/include/RTC/OldConsumer.hpp b/worker/include/RTC/OldConsumer.hpp new file mode 100644 index 0000000000..59398d526f --- /dev/null +++ b/worker/include/RTC/OldConsumer.hpp @@ -0,0 +1,210 @@ +#ifndef MS_RTC_OLD_CONSUMER_HPP +#define MS_RTC_OLD_CONSUMER_HPP + +#include "common.hpp" +#include "Shared.hpp" +#include "Channel/ChannelRequest.hpp" +#include "Channel/ChannelSocket.hpp" +#include "FBS/consumer.h" +#include "RTC/ConsumerTypes.hpp" +#include "RTC/RTCP/CompoundPacket.hpp" +#include "RTC/RTCP/FeedbackRtpNack.hpp" +#include "RTC/RTCP/ReceiverReport.hpp" +#include "RTC/RTP/HeaderExtensionIds.hpp" +#include "RTC/RTP/Packet.hpp" +#include "RTC/RTP/RtpStreamRecv.hpp" +#include "RTC/RTP/RtpStreamSend.hpp" +#include "RTC/RTP/SharedPacket.hpp" +#include "RTC/RtpDictionaries.hpp" +#include +#include +#include + +namespace RTC +{ + using namespace ConsumerTypes; + + class OldConsumer : public Channel::ChannelSocket::RequestHandler + { + public: + class Listener + { + public: + virtual ~Listener() = default; + + public: + virtual void OnConsumerSendRtpPacket(RTC::OldConsumer* consumer, RTC::RTP::Packet* packet) = 0; + virtual void OnConsumerRetransmitRtpPacket( + RTC::OldConsumer* consumer, RTC::RTP::Packet* packet) = 0; + virtual void OnConsumerKeyFrameRequested(RTC::OldConsumer* consumer, uint32_t mappedSsrc) = 0; + virtual void OnConsumerNeedBitrateChange(RTC::OldConsumer* consumer) = 0; + virtual void OnConsumerNeedZeroBitrate(RTC::OldConsumer* consumer) = 0; + virtual void OnConsumerProducerClosed(RTC::OldConsumer* consumer) = 0; + }; + + private: + struct TraceEventTypes + { + bool rtp{ false }; + bool keyframe{ false }; + bool nack{ false }; + bool pli{ false }; + bool fir{ false }; + }; + + public: + OldConsumer( + SharedInterface* shared, + const std::string& id, + const std::string& producerId, + RTC::OldConsumer::Listener* listener, + const FBS::Transport::ConsumeRequest* data, + RTC::RtpParameters::Type type); + ~OldConsumer() override; + + public: + flatbuffers::Offset FillBuffer( + flatbuffers::FlatBufferBuilder& builder) const; + virtual flatbuffers::Offset FillBufferStats( + flatbuffers::FlatBufferBuilder& builder) = 0; + virtual flatbuffers::Offset FillBufferScore( + flatbuffers::FlatBufferBuilder& /*builder*/) const + { + return 0; + }; + RTC::Media::Kind GetKind() const + { + return this->kind; + } + const RTC::RtpParameters& GetRtpParameters() const + { + return this->rtpParameters; + } + const struct RTC::RTP::HeaderExtensionIds& GetRtpHeaderExtensionIds() const + { + return this->rtpHeaderExtensionIds; + } + RTC::RtpParameters::Type GetType() const + { + return this->type; + } + virtual VideoLayers GetPreferredLayers() const + { + // By default return 1:1. + VideoLayers layers; + + return layers; + } + const std::vector& GetMediaSsrcs() const + { + return this->mediaSsrcs; + } + const std::vector& GetRtxSsrcs() const + { + return this->rtxSsrcs; + } + virtual bool IsActive() const + { + // The parent Consumer just checks whether Consumer and Producer are + // not paused and the transport connected. + // clang-format off + return ( + this->transportConnected && + !this->paused && + !this->producerPaused && + !this->producerClosed + ); + // clang-format on + } + void TransportConnected(); + void TransportDisconnected(); + bool IsPaused() const + { + return this->paused; + } + bool IsProducerPaused() const + { + return this->producerPaused; + } + void ProducerPaused(); + void ProducerResumed(); + virtual void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) = 0; + virtual void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) = 0; + void ProducerRtpStreamScores(const std::vector* scores); + virtual void ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) = 0; + virtual void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) = 0; + void ProducerClosed(); + void SetExternallyManagedBitrate() + { + this->externallyManagedBitrate = true; + } + virtual uint8_t GetBitratePriority() const = 0; + virtual uint32_t IncreaseLayer(uint32_t bitrate, bool considerLoss) = 0; + virtual void ApplyLayers() = 0; + virtual uint32_t GetDesiredBitrate() const = 0; + virtual void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) = 0; + virtual bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) = 0; + virtual void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) = 0; + virtual void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) = 0; + virtual void ReceiveKeyFrameRequest( + RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) = 0; + virtual void ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) = 0; + virtual void ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) = 0; + virtual uint32_t GetTransmissionRate(uint64_t nowMs) = 0; + virtual float GetRtt() const = 0; + + /* Methods inherited from Channel::ChannelSocket::RequestHandler. */ + public: + void HandleRequest(Channel::ChannelRequest* request) override; + + protected: + void EmitTraceEventRtpAndKeyFrameTypes(const RTC::RTP::Packet* packet, bool isRtx = false) const; + void EmitTraceEventPliType(uint32_t ssrc) const; + void EmitTraceEventFirType(uint32_t ssrc) const; + void EmitTraceEventNackType() const; + void EmitTraceEvent(flatbuffers::Offset& notification) const; + + private: + virtual void UserOnTransportConnected() = 0; + virtual void UserOnTransportDisconnected() = 0; + virtual void UserOnPaused() = 0; + virtual void UserOnResumed() = 0; + + public: + // Passed by argument. + std::string id; + std::string producerId; + + protected: + // Passed by argument. + SharedInterface* shared{ nullptr }; + RTC::OldConsumer::Listener* listener{ nullptr }; + RTC::Media::Kind kind; + RTC::RtpParameters rtpParameters; + RTC::RtpParameters::Type type; + std::vector consumableRtpEncodings; + struct RTC::RTP::HeaderExtensionIds rtpHeaderExtensionIds; + const std::vector* producerRtpStreamScores{ nullptr }; + // Others. + // Whether a payload type is supported or not is represented in the + // corresponding position of the bitset. + std::bitset<128u> supportedCodecPayloadTypes; + uint64_t lastRtcpSentTime{ 0u }; + uint16_t maxRtcpInterval{ 0u }; + bool externallyManagedBitrate{ false }; + uint8_t priority{ 1u }; + struct TraceEventTypes traceEventTypes; + + private: + // Others. + std::vector mediaSsrcs; + std::vector rtxSsrcs; + bool transportConnected{ false }; + bool paused{ false }; + bool producerPaused{ false }; + bool producerClosed{ false }; + }; +} // namespace RTC + +#endif diff --git a/worker/include/RTC/PipeConsumer.hpp b/worker/include/RTC/PipeConsumer.hpp index 28eaf7e80a..1e9bea98d1 100644 --- a/worker/include/RTC/PipeConsumer.hpp +++ b/worker/include/RTC/PipeConsumer.hpp @@ -1,13 +1,13 @@ #ifndef MS_RTC_PIPECONSUMER_HPP #define MS_RTC_PIPECONSUMER_HPP -#include "RTC/Consumer.hpp" +#include "RTC/OldConsumer.hpp" #include "RTC/SeqManager.hpp" #include namespace RTC { - class PipeConsumer : public RTC::Consumer, public RTC::RTP::RtpStreamSend::Listener + class PipeConsumer : public RTC::OldConsumer, public RTC::RTP::RtpStreamSend::Listener { private: static void StorePacketInTargetLayerRetransmissionBuffer( @@ -21,7 +21,7 @@ namespace RTC SharedInterface* shared, const std::string& id, const std::string& producerId, - RTC::Consumer::Listener* listener, + RTC::OldConsumer::Listener* listener, const FBS::Transport::ConsumeRequest* data); ~PipeConsumer() override; @@ -43,10 +43,6 @@ namespace RTC uint32_t GetDesiredBitrate() const override; void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) override; bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) override; - const std::vector& GetRtpStreams() const override - { - return this->rtpStreams; - } void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) override; void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) override; void ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) override; diff --git a/worker/include/RTC/PipeProducerStreamManager.hpp b/worker/include/RTC/PipeProducerStreamManager.hpp new file mode 100644 index 0000000000..3cd03367f2 --- /dev/null +++ b/worker/include/RTC/PipeProducerStreamManager.hpp @@ -0,0 +1,78 @@ +#ifndef MS_RTC_PIPE_PRODUCER_STREAM_MANAGER_HPP +#define MS_RTC_PIPE_PRODUCER_STREAM_MANAGER_HPP + +#include "RTC/ProducerStreamManager.hpp" +#include + +namespace RTC +{ + class PipeProducerStreamManager : public ProducerStreamManager + { + public: + PipeProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared); + + public: + RTC::ConsumerTypes::VideoLayers GetTargetLayers() const override + { + return {}; + } + int16_t GetCurrentSpatialLayer() const override + { + return -1; + } + int16_t GetCurrentTemporalLayer() const override + { + return -1; + } + RTC::RTP::RtpStreamRecv* GetProducerCurrentRtpStream() const override + { + return nullptr; + } + RTC::RTP::RtpStreamRecv* GetProducerTargetRtpStream() const override + { + return nullptr; + } + bool IsPacketForCurrentStream(const RTC::RTP::Packet* /*packet*/) const override + { + // Pipe has no concept of "current" stream — all packets belong. + return true; + } + bool IsActive() const override; + void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) override; + void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) override; + uint32_t IncreaseLayer( + uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs) override; + void ApplyLayers(uint64_t rtpStreamActiveMs) override; + uint32_t GetDesiredBitrate(uint64_t nowMs) const override; + RtpPacketProcessResult ProcessRtpPacket( + RTC::RTP::Packet* packet, + bool lastSentPacketHasMarker, + uint32_t clockRate, + uint32_t maxPacketTs) override; + void RequestKeyFrame() override; + void RequestKeyFrameForTargetSpatialLayer() override; + void RequestKeyFrameForCurrentSpatialLayer() override; + void UpdateTargetLayers(int16_t spatial, int16_t temporal) override; + bool RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const override; + void OnTransportConnected() override; + void OnTransportDisconnected() override; + void OnPaused() override; + void OnResumed() override; + + private: + // Per-stream sync state, keyed by mapped SSRC. + absl::flat_hash_map mapMappedSsrcSyncRequired; + }; +} // namespace RTC + +#endif diff --git a/worker/include/RTC/ProducerStreamManager.hpp b/worker/include/RTC/ProducerStreamManager.hpp new file mode 100644 index 0000000000..14fae16c4c --- /dev/null +++ b/worker/include/RTC/ProducerStreamManager.hpp @@ -0,0 +1,183 @@ +#ifndef MS_RTC_PRODUCER_STREAM_MANAGER_HPP +#define MS_RTC_PRODUCER_STREAM_MANAGER_HPP + +#include "common.hpp" +#include "Logger.hpp" +#include "SharedInterface.hpp" +#include "RTC/ConsumerTypes.hpp" +#include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" +#include "RTC/RTP/Packet.hpp" +#include "RTC/RTP/RtpStreamRecv.hpp" +#include "RTC/RtpDictionaries.hpp" +#include +#include + +namespace RTC +{ + class ProducerStreamManager + { + public: + class Listener + { + public: + virtual ~Listener() = default; + + public: + virtual bool IsActive() const = 0; + virtual void OnProducerStreamManagerKeyFrameRequested(uint32_t mappedSsrc) = 0; + virtual void OnProducerStreamManagerNeedBitrateChange() = 0; + virtual void OnProducerStreamManagerLayersChanged() = 0; + virtual void OnProducerStreamManagerClearRetransmissionBuffer() = 0; + virtual void OnProducerStreamManagerScore() = 0; + }; + + struct RtpPacketProcessResult + { + enum class Type : uint8_t + { + FORWARD, + DROP, + SILENT_DROP, + BUFFER + }; + + Type type{ Type::FORWARD }; + + // Valid when type == FORWARD: + uint32_t tsOffset{ 0u }; + bool isSyncPacket{ false }; + uint16_t syncSeqValue{ 0u }; + bool shouldSyncEncodingContext{ false }; + bool spatialLayerSwitched{ false }; + bool temporalLayerChanged{ false }; + bool marker{ false }; + bool sendBufferedPackets{ false }; + }; + + public: + ProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared) + : listener(listener), + shared(shared), + keyFrameSupported(keyFrameSupported), + kind(kind), + consumableRtpEncodings(consumableRtpEncodings), + encodingContext(std::move(encodingContext)), + preferredLayers(preferredLayers) + { + } + virtual ~ProducerStreamManager() = default; + + public: + virtual RTC::ConsumerTypes::VideoLayers GetTargetLayers() const = 0; + virtual int16_t GetCurrentSpatialLayer() const = 0; + virtual int16_t GetCurrentTemporalLayer() const = 0; + virtual RTC::RTP::RtpStreamRecv* GetProducerCurrentRtpStream() const = 0; + virtual RTC::RTP::RtpStreamRecv* GetProducerTargetRtpStream() const = 0; + // Returns true if the given packet belongs to the stream currently being + // forwarded to the consumer. Used by Consumer to decide whether to account + // a discarded packet in the RTP sequence manager. + virtual bool IsPacketForCurrentStream(const RTC::RTP::Packet* packet) const = 0; + RTC::RTP::Codecs::EncodingContext* GetEncodingContext() const + { + return this->encodingContext.get(); + } + const RTC::ConsumerTypes::VideoLayers& GetPreferredLayers() const + { + return this->preferredLayers; + } + void SetPreferredLayers(const RTC::ConsumerTypes::VideoLayers& layers) + { + this->preferredLayers = layers; + } + + virtual void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) = 0; + virtual void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) = 0; + virtual void ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) = 0; + virtual void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) = 0; + + void SetExternallyManagedBitrate() + { + this->externallyManagedBitrate = true; + } + + virtual uint32_t IncreaseLayer( + uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs) = 0; + virtual void ApplyLayers(uint64_t rtpStreamActiveMs) = 0; + virtual uint32_t GetDesiredBitrate(uint64_t nowMs) const = 0; + + virtual RtpPacketProcessResult ProcessRtpPacket( + RTC::RTP::Packet* packet, + bool lastSentPacketHasMarker, + uint32_t clockRate, + uint32_t maxPacketTs) = 0; + + virtual void RequestKeyFrame() = 0; + virtual void RequestKeyFrameForTargetSpatialLayer() = 0; + virtual void RequestKeyFrameForCurrentSpatialLayer() = 0; + + virtual void UpdateTargetLayers(int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer) = 0; + virtual bool RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const = 0; + + void MayChangeLayers(bool force) + { + RTC::ConsumerTypes::VideoLayers newTargetLayers; + + if (RecalculateTargetLayers(newTargetLayers)) + { + // If bitrate externally managed, don't bother the transport unless + // the newTargetSpatialLayer has changed (or force is true). + // This is because, if bitrate is externally managed, the target temporal + // layer is managed by the available given bitrate so the transport + // will let us change it when it considers. + if (this->externallyManagedBitrate) + { + if (newTargetLayers.spatial != GetTargetLayers().spatial || force) + { + this->listener->OnProducerStreamManagerNeedBitrateChange(); + } + } + else + { + UpdateTargetLayers(newTargetLayers.spatial, newTargetLayers.temporal); + } + } + } + virtual void OnTransportConnected() = 0; + virtual void OnTransportDisconnected() = 0; + virtual void OnPaused() = 0; + virtual void OnResumed() = 0; + + protected: + virtual bool IsActive() const = 0; + + // Passed by argument. + Listener* listener{ nullptr }; + SharedInterface* shared{ nullptr }; + bool keyFrameSupported{ false }; + RTC::Media::Kind kind{}; + std::vector consumableRtpEncodings; + + // Encoding context. + std::unique_ptr encodingContext; + + // Externally managed bitrate. + bool externallyManagedBitrate{ false }; + + // Layer preferences. + RTC::ConsumerTypes::VideoLayers preferredLayers; + RTC::ConsumerTypes::VideoLayers provisionalTargetLayers; + + // Sync state. + bool syncRequired{ false }; + }; +} // namespace RTC + +#endif diff --git a/worker/include/RTC/SimpleConsumer.hpp b/worker/include/RTC/SimpleConsumer.hpp index 77ac9a97f2..ad47fb80de 100644 --- a/worker/include/RTC/SimpleConsumer.hpp +++ b/worker/include/RTC/SimpleConsumer.hpp @@ -1,20 +1,20 @@ #ifndef MS_RTC_SIMPLE_CONSUMER_HPP #define MS_RTC_SIMPLE_CONSUMER_HPP -#include "RTC/Consumer.hpp" +#include "RTC/OldConsumer.hpp" #include "RTC/SeqManager.hpp" #include namespace RTC { - class SimpleConsumer : public RTC::Consumer, public RTC::RTP::RtpStreamSend::Listener + class SimpleConsumer : public RTC::OldConsumer, public RTC::RTP::RtpStreamSend::Listener { public: SimpleConsumer( SharedInterface* shared, const std::string& id, const std::string& producerId, - RTC::Consumer::Listener* listener, + RTC::OldConsumer::Listener* listener, const FBS::Transport::ConsumeRequest* data); ~SimpleConsumer() override; @@ -29,7 +29,7 @@ namespace RTC { // clang-format off return ( - RTC::Consumer::IsActive() && + RTC::OldConsumer::IsActive() && this->producerRtpStream && // If there is no RTP inactivity check do not consider the stream // inactive despite it has score 0. @@ -47,10 +47,6 @@ namespace RTC void ApplyLayers() override; uint32_t GetDesiredBitrate() const override; void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) override; - const std::vector& GetRtpStreams() const override - { - return this->rtpStreams; - } bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) override; void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) override; void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) override; diff --git a/worker/include/RTC/SimpleProducerStreamManager.hpp b/worker/include/RTC/SimpleProducerStreamManager.hpp new file mode 100644 index 0000000000..570dbc2918 --- /dev/null +++ b/worker/include/RTC/SimpleProducerStreamManager.hpp @@ -0,0 +1,72 @@ +#ifndef MS_RTC_SIMPLE_CONSUMER_STREAM_HPP +#define MS_RTC_SIMPLE_CONSUMER_STREAM_HPP + +#include "RTC/ProducerStreamManager.hpp" + +namespace RTC +{ + class SimpleProducerStreamManager : public ProducerStreamManager + { + public: + SimpleProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared); + + public: + RTC::ConsumerTypes::VideoLayers GetTargetLayers() const override + { + return {}; + } + int16_t GetCurrentSpatialLayer() const override + { + return 0; + } + int16_t GetCurrentTemporalLayer() const override + { + return 0; + } + RTC::RTP::RtpStreamRecv* GetProducerCurrentRtpStream() const override; + RTC::RTP::RtpStreamRecv* GetProducerTargetRtpStream() const override; + bool IsPacketForCurrentStream(const RTC::RTP::Packet* /*packet*/) const override + { + return true; + } + bool IsActive() const override; + void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) override; + void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) override; + uint32_t IncreaseLayer( + uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs) override; + void ApplyLayers(uint64_t rtpStreamActiveMs) override; + uint32_t GetDesiredBitrate(uint64_t nowMs) const override; + RtpPacketProcessResult ProcessRtpPacket( + RTC::RTP::Packet* packet, + bool lastSentPacketHasMarker, + uint32_t clockRate, + uint32_t maxPacketTs) override; + void RequestKeyFrame() override; + void RequestKeyFrameForTargetSpatialLayer() override; + void RequestKeyFrameForCurrentSpatialLayer() override; + void UpdateTargetLayers(int16_t spatial, int16_t temporal) override; + bool RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const override; + void OnTransportConnected() override; + void OnTransportDisconnected() override; + void OnPaused() override; + void OnResumed() override; + + private: + // Producer RTP stream (single stream for Simple). + RTC::RTP::RtpStreamRecv* producerRtpStream{ nullptr }; + // Prevents double IncreaseLayer in one BWE round. + bool managingBitrate{ false }; + }; +} // namespace RTC + +#endif diff --git a/worker/include/RTC/SimulcastConsumer.hpp b/worker/include/RTC/SimulcastConsumer.hpp index 0a9d37beb8..208e99de3e 100644 --- a/worker/include/RTC/SimulcastConsumer.hpp +++ b/worker/include/RTC/SimulcastConsumer.hpp @@ -1,21 +1,21 @@ #ifndef MS_RTC_SIMULCAST_CONSUMER_HPP #define MS_RTC_SIMULCAST_CONSUMER_HPP -#include "RTC/Consumer.hpp" +#include "RTC/OldConsumer.hpp" #include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" #include "RTC/SeqManager.hpp" #include namespace RTC { - class SimulcastConsumer : public RTC::Consumer, public RTC::RTP::RtpStreamSend::Listener + class SimulcastConsumer : public RTC::OldConsumer, public RTC::RTP::RtpStreamSend::Listener { public: SimulcastConsumer( SharedInterface* shared, const std::string& id, const std::string& producerId, - RTC::Consumer::Listener* listener, + RTC::OldConsumer::Listener* listener, const FBS::Transport::ConsumeRequest* data); ~SimulcastConsumer() override; @@ -39,7 +39,7 @@ namespace RTC { // clang-format off return ( - RTC::Consumer::IsActive() && + RTC::OldConsumer::IsActive() && std::any_of( this->producerRtpStreams.begin(), this->producerRtpStreams.end(), @@ -64,10 +64,6 @@ namespace RTC uint32_t GetDesiredBitrate() const override; void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) override; bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) override; - const std::vector& GetRtpStreams() const override - { - return this->rtpStreams; - } void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) override; void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) override; void ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) override; diff --git a/worker/include/RTC/SimulcastProducerStreamManager.hpp b/worker/include/RTC/SimulcastProducerStreamManager.hpp new file mode 100644 index 0000000000..72cbf6a053 --- /dev/null +++ b/worker/include/RTC/SimulcastProducerStreamManager.hpp @@ -0,0 +1,93 @@ +#ifndef MS_RTC_SIMULCAST_CONSUMER_STREAM_HPP +#define MS_RTC_SIMULCAST_CONSUMER_STREAM_HPP + +#include "RTC/ProducerStreamManager.hpp" +#include + +namespace RTC +{ + class SimulcastProducerStreamManager : public ProducerStreamManager + { + public: + SimulcastProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared); + + public: + RTC::ConsumerTypes::VideoLayers GetTargetLayers() const override + { + return this->targetLayers; + } + int16_t GetCurrentSpatialLayer() const override + { + return this->currentSpatialLayer; + } + int16_t GetCurrentTemporalLayer() const override + { + return this->encodingContext->GetCurrentTemporalLayer(); + } + RTC::RTP::RtpStreamRecv* GetProducerCurrentRtpStream() const override; + RTC::RTP::RtpStreamRecv* GetProducerTargetRtpStream() const override; + bool IsPacketForCurrentStream(const RTC::RTP::Packet* packet) const override + { + const auto it = this->mapMappedSsrcSpatialLayer.find(packet->GetSsrc()); + if (it == this->mapMappedSsrcSpatialLayer.end()) + { + return false; + } + return it->second == this->currentSpatialLayer; + } + bool IsActive() const override; + void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) override; + void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) override; + uint32_t IncreaseLayer( + uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs) override; + void ApplyLayers(uint64_t rtpStreamActiveMs) override; + uint32_t GetDesiredBitrate(uint64_t nowMs) const override; + RtpPacketProcessResult ProcessRtpPacket( + RTC::RTP::Packet* packet, + bool lastSentPacketHasMarker, + uint32_t clockRate, + uint32_t maxPacketTs) override; + void RequestKeyFrame() override; + void RequestKeyFrameForTargetSpatialLayer() override; + void RequestKeyFrameForCurrentSpatialLayer() override; + void UpdateTargetLayers(int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer) override; + bool RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const override; + void OnTransportConnected() override; + void OnTransportDisconnected() override; + void OnPaused() override; + void OnResumed() override; + + private: + bool CanSwitchToSpatialLayer(int16_t spatialLayer) const; + RTC::RTP::RtpStreamRecv* GetProducerTsReferenceRtpStream() const; + + private: + // Producer RTP streams (multiple for Simulcast). + std::vector producerRtpStreams; + absl::flat_hash_map mapMappedSsrcSpatialLayer; + RTC::ConsumerTypes::VideoLayers targetLayers; + int16_t currentSpatialLayer{ -1 }; + int16_t spatialLayerToSync{ -1 }; + // Timestamp synchronization. + int16_t tsReferenceSpatialLayer{ -1 }; + uint32_t tsOffset{ 0u }; + bool keyFrameForTsOffsetRequested{ false }; + // Old-packet filtering after spatial switch. + uint16_t snReferenceSpatialLayer{ 0u }; + bool checkingForOldPacketsInSpatialLayer{ false }; + // BWE downgrade tracking. + uint64_t lastBweDowngradeAtMs{ 0u }; + }; +} // namespace RTC + +#endif diff --git a/worker/include/RTC/SvcConsumer.hpp b/worker/include/RTC/SvcConsumer.hpp index 0f628fe326..a18f024826 100644 --- a/worker/include/RTC/SvcConsumer.hpp +++ b/worker/include/RTC/SvcConsumer.hpp @@ -1,21 +1,21 @@ #ifndef MS_RTC_SVC_CONSUMER_HPP #define MS_RTC_SVC_CONSUMER_HPP -#include "RTC/Consumer.hpp" +#include "RTC/OldConsumer.hpp" #include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" #include "RTC/SeqManager.hpp" #include namespace RTC { - class SvcConsumer : public RTC::Consumer, public RTC::RTP::RtpStreamSend::Listener + class SvcConsumer : public RTC::OldConsumer, public RTC::RTP::RtpStreamSend::Listener { public: SvcConsumer( SharedInterface* shared, const std::string& id, const std::string& producerId, - RTC::Consumer::Listener* listener, + RTC::OldConsumer::Listener* listener, const FBS::Transport::ConsumeRequest* data); ~SvcConsumer() override; @@ -39,7 +39,7 @@ namespace RTC { // clang-format off return ( - RTC::Consumer::IsActive() && + RTC::OldConsumer::IsActive() && this->producerRtpStream && // If there is no RTP inactivity check do not consider the stream // inactive despite it has score 0. @@ -58,10 +58,6 @@ namespace RTC uint32_t GetDesiredBitrate() const override; void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) override; bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) override; - const std::vector& GetRtpStreams() const override - { - return this->rtpStreams; - } void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) override; void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) override; void ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) override; diff --git a/worker/include/RTC/SvcProducerStreamManager.hpp b/worker/include/RTC/SvcProducerStreamManager.hpp new file mode 100644 index 0000000000..1b72b32f6b --- /dev/null +++ b/worker/include/RTC/SvcProducerStreamManager.hpp @@ -0,0 +1,72 @@ +#ifndef MS_RTC_SVC_CONSUMER_STREAM_HPP +#define MS_RTC_SVC_CONSUMER_STREAM_HPP + +#include "RTC/ProducerStreamManager.hpp" + +namespace RTC +{ + class SvcProducerStreamManager : public ProducerStreamManager + { + public: + SvcProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared); + + public: + RTC::ConsumerTypes::VideoLayers GetTargetLayers() const override + { + return this->encodingContext->GetTargetLayers(); + } + int16_t GetCurrentSpatialLayer() const override + { + return this->encodingContext->GetCurrentSpatialLayer(); + } + int16_t GetCurrentTemporalLayer() const override + { + return this->encodingContext->GetCurrentTemporalLayer(); + } + RTC::RTP::RtpStreamRecv* GetProducerCurrentRtpStream() const override; + RTC::RTP::RtpStreamRecv* GetProducerTargetRtpStream() const override; + bool IsPacketForCurrentStream(const RTC::RTP::Packet* /*packet*/) const override + { + return true; + } + bool IsActive() const override; + void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) override; + void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) override; + uint32_t IncreaseLayer( + uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs) override; + void ApplyLayers(uint64_t rtpStreamActiveMs) override; + uint32_t GetDesiredBitrate(uint64_t nowMs) const override; + RtpPacketProcessResult ProcessRtpPacket( + RTC::RTP::Packet* packet, + bool lastSentPacketHasMarker, + uint32_t clockRate, + uint32_t maxPacketTs) override; + void RequestKeyFrame() override; + void RequestKeyFrameForTargetSpatialLayer() override; + void RequestKeyFrameForCurrentSpatialLayer() override; + void UpdateTargetLayers(int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer) override; + bool RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const override; + void OnTransportConnected() override; + void OnTransportDisconnected() override; + void OnPaused() override; + void OnResumed() override; + + private: + // Producer RTP stream (single stream for SVC). + RTC::RTP::RtpStreamRecv* producerRtpStream{ nullptr }; + // BWE downgrade tracking. + uint64_t lastBweDowngradeAtMs{ 0u }; + }; +} // namespace RTC + +#endif diff --git a/worker/meson.build b/worker/meson.build index b78c51c009..11b1361b57 100644 --- a/worker/meson.build +++ b/worker/meson.build @@ -121,6 +121,7 @@ common_sources = [ 'src/RTC/DtlsTransport.cpp', 'src/RTC/KeyFrameRequestManager.cpp', 'src/RTC/NackGenerator.cpp', + 'src/RTC/OldConsumer.cpp', 'src/RTC/PipeConsumer.cpp', 'src/RTC/PipeTransport.cpp', 'src/RTC/PlainTransport.cpp', @@ -138,9 +139,13 @@ common_sources = [ 'src/RTC/SeqManager.cpp', 'src/RTC/Serializable.cpp', 'src/RTC/SimpleConsumer.cpp', + 'src/RTC/PipeProducerStreamManager.cpp', + 'src/RTC/SimpleProducerStreamManager.cpp', 'src/RTC/SimulcastConsumer.cpp', + 'src/RTC/SimulcastProducerStreamManager.cpp', 'src/RTC/SrtpSession.cpp', 'src/RTC/SvcConsumer.cpp', + 'src/RTC/SvcProducerStreamManager.cpp', 'src/RTC/TcpConnection.cpp', 'src/RTC/TcpServer.cpp', 'src/RTC/Transport.cpp', @@ -456,12 +461,16 @@ mock_sources = [ test_sources = [ 'test/src/tests.cpp', 'test/src/testHelpers.cpp', + 'test/src/RTC/TestConsumer.cpp', + 'test/src/RTC/TestPipeProducerStreamManager.cpp', + 'test/src/RTC/TestSimpleProducerStreamManager.cpp', + 'test/src/RTC/TestSimulcastProducerStreamManager.cpp', + 'test/src/RTC/TestSvcProducerStreamManager.cpp', 'test/src/RTC/TestKeyFrameRequestManager.cpp', 'test/src/RTC/TestNackGenerator.cpp', 'test/src/RTC/TestRateCalculator.cpp', 'test/src/RTC/TestRtpEncodingParameters.cpp', 'test/src/RTC/TestSeqManager.cpp', - 'test/src/RTC/TestSimpleConsumer.cpp', 'test/src/RTC/TestTransportCongestionControlServer.cpp', 'test/src/RTC/TestTransportTuple.cpp', 'test/src/RTC/TestTrendCalculator.cpp', diff --git a/worker/src/RTC/Consumer.cpp b/worker/src/RTC/Consumer.cpp index ade190311f..488f88e396 100644 --- a/worker/src/RTC/Consumer.cpp +++ b/worker/src/RTC/Consumer.cpp @@ -4,27 +4,42 @@ #include "RTC/Consumer.hpp" #include "Logger.hpp" #include "MediaSoupErrors.hpp" +#include "Utils.hpp" +#include "RTC/PipeProducerStreamManager.hpp" +#include "RTC/RTP/Codecs/Tools.hpp" +#include "RTC/SimpleProducerStreamManager.hpp" +#include "RTC/SimulcastProducerStreamManager.hpp" +#include "RTC/SvcProducerStreamManager.hpp" +#ifdef MS_RTC_LOGGER_RTP +#include "RTC/RtcLogger.hpp" +#endif +#include // std::numeric_limits namespace RTC { + /* Static. */ + + static constexpr size_t TargetLayerRetransmissionBufferSize{ 30u }; + /* Instance methods. */ Consumer::Consumer( SharedInterface* shared, const std::string& id, const std::string& producerId, - Listener* listener, - const FBS::Transport::ConsumeRequest* data, - RTC::RtpParameters::Type type) + RTC::Consumer::Listener* listener, + const FBS::Transport::ConsumeRequest* data) : id(id), producerId(producerId), shared(shared), listener(listener), kind(RTC::Media::Kind(data->kind())), - type(type) + type(RTC::RtpParameters::Type(data->type())) { MS_TRACE(); + this->pipe = this->type == RTC::RtpParameters::Type::PIPE; + // This may throw. this->rtpParameters = RTC::RtpParameters(data->rtpParameters()); @@ -69,6 +84,12 @@ namespace RTC } } + // Ensure there are as many encodings as consumable encodings for pipe. + if (pipe && this->rtpParameters.encodings.size() != this->consumableRtpEncodings.size()) + { + MS_THROW_TYPE_ERROR("number of rtpParameters.encodings and consumableRtpEncodings do not match"); + } + // Fill RTP header extension ids and their mapped values. // This may throw. for (auto& exten : this->rtpParameters.headerExtensions) @@ -120,11 +141,6 @@ namespace RTC this->rtpHeaderExtensionIds.videoOrientation = exten.id; } - if (this->rtpHeaderExtensionIds.videoOrientation == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::VIDEO_ORIENTATION) - { - this->rtpHeaderExtensionIds.videoOrientation = exten.id; - } - if (this->rtpHeaderExtensionIds.absCaptureTime == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::ABS_CAPTURE_TIME) { this->rtpHeaderExtensionIds.absCaptureTime = exten.id; @@ -177,14 +193,294 @@ namespace RTC { this->maxRtcpInterval = RTC::RTCP::MaxVideoIntervalMs; } + + auto& encoding = this->rtpParameters.encodings[0]; + + // Determine keyFrameSupported. + const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding); + const bool keyFrameSupported = RTC::RTP::Codecs::Tools::CanBeKeyFrame(mediaCodec->mimeType); + + // Build preferred layers from FBS data. + RTC::ConsumerTypes::VideoLayers preferredLayers; + + // Create the appropriate ProducerStreamManager subclass based on type. + switch (this->type) + { + case RTC::RtpParameters::Type::SIMPLE: + { + // Ensure there is a single encoding. + if (this->consumableRtpEncodings.size() != 1u) + { + MS_THROW_TYPE_ERROR("invalid consumableRtpEncodings with size != 1"); + } + + // Create the encoding context for Opus (DTX filtering). + std::unique_ptr encodingContext; + + if ( + mediaCodec->mimeType.type == RTC::RtpCodecMimeType::Type::AUDIO && + (mediaCodec->mimeType.subtype == RTC::RtpCodecMimeType::Subtype::OPUS || + mediaCodec->mimeType.subtype == RTC::RtpCodecMimeType::Subtype::MULTIOPUS)) + { + RTC::RTP::Codecs::EncodingContext::Params params; + + encodingContext.reset( + RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params)); + + // ignoreDtx is set to false by default. + encodingContext->SetIgnoreDtx(data->ignoreDtx()); + } + + this->producerStreamManager = std::make_unique( + this->consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + this->kind, + keyFrameSupported, + this, + this->shared); + + break; + } + + case RTC::RtpParameters::Type::SIMULCAST: + { + // We allow a single encoding in simulcast (so we can enable temporal + // layers with a single simulcast stream). + + // Ensure there are as many spatial layers as encodings. + if (encoding.spatialLayers != this->consumableRtpEncodings.size()) + { + MS_THROW_TYPE_ERROR( + "encoding.spatialLayers does not match number of consumableRtpEncodings"); + } + + // Set preferredLayers. + if (flatbuffers::IsFieldPresent(data, FBS::Transport::ConsumeRequest::VT_PREFERREDLAYERS)) + { + const auto* fbsPreferredLayers = data->preferredLayers(); + + preferredLayers.spatial = fbsPreferredLayers->spatialLayer(); + + if (preferredLayers.spatial > encoding.spatialLayers - 1) + { + preferredLayers.spatial = static_cast(encoding.spatialLayers - 1); + } + + if ( + auto preferredTemporalLayer = fbsPreferredLayers->temporalLayer(); + preferredTemporalLayer.has_value()) + { + preferredLayers.temporal = preferredTemporalLayer.value(); + + if (preferredLayers.temporal > encoding.temporalLayers - 1) + { + preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + } + else + { + preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + } + else + { + // Initially set preferredSpatialLayer and preferredTemporalLayer + // to the maximum value. + preferredLayers.spatial = static_cast(encoding.spatialLayers - 1); + preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + + // Create the encoding context. + if (!RTC::RTP::Codecs::Tools::IsValidTypeForCodec(this->type, mediaCodec->mimeType)) + { + MS_THROW_TYPE_ERROR( + "%s codec not supported for simulcast", mediaCodec->mimeType.ToString().c_str()); + } + + RTC::RTP::Codecs::EncodingContext::Params params; + + params.spatialLayers = encoding.spatialLayers; + params.temporalLayers = encoding.temporalLayers; + + std::unique_ptr encodingContext( + RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params)); + + MS_ASSERT(encodingContext, "no encoding context for this codec"); + + this->producerStreamManager = std::make_unique( + this->consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + this->kind, + keyFrameSupported, + this, + this->shared); + + break; + } + + case RTC::RtpParameters::Type::SVC: + { + // Ensure there is a single encoding. + if (this->consumableRtpEncodings.size() != 1u) + { + MS_THROW_TYPE_ERROR("invalid consumableRtpEncodings with size != 1"); + } + + // Ensure there are multiple spatial or temporal layers. + if (encoding.spatialLayers < 2u && encoding.temporalLayers < 2u) + { + MS_THROW_TYPE_ERROR("invalid number of layers"); + } + + // Set preferredLayers. + if (flatbuffers::IsFieldPresent(data, FBS::Transport::ConsumeRequest::VT_PREFERREDLAYERS)) + { + preferredLayers.spatial = data->preferredLayers()->spatialLayer(); + + if (preferredLayers.spatial > encoding.spatialLayers - 1) + { + preferredLayers.spatial = static_cast(encoding.spatialLayers - 1); + } + + // preferredTemporalLayer is optional. + auto preferredTemporalLayer = data->preferredLayers()->temporalLayer(); + + if (preferredTemporalLayer) + { + preferredLayers.temporal = preferredTemporalLayer.value(); + + if (preferredLayers.temporal > encoding.temporalLayers - 1) + { + preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + } + else + { + preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + } + else + { + // Initially set preferredSpatialLayer and preferredTemporalLayer + // to the maximum value. + preferredLayers.spatial = static_cast(encoding.spatialLayers - 1); + preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + + // Create the encoding context. + if (!RTC::RTP::Codecs::Tools::IsValidTypeForCodec(this->type, mediaCodec->mimeType)) + { + MS_THROW_TYPE_ERROR( + "%s codec not supported for svc", mediaCodec->mimeType.ToString().c_str()); + } + + RTC::RTP::Codecs::EncodingContext::Params params; + + params.spatialLayers = encoding.spatialLayers; + params.temporalLayers = encoding.temporalLayers; + params.ksvc = encoding.ksvc; + + std::unique_ptr encodingContext( + RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params)); + + MS_ASSERT(encodingContext, "no encoding context for this codec"); + + this->producerStreamManager = std::make_unique( + this->consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + this->kind, + keyFrameSupported, + this, + this->shared); + + break; + } + + case RTC::RtpParameters::Type::PIPE: + { + // Pipe consumer: no layer management, N streams. + this->producerStreamManager = std::make_unique( + this->consumableRtpEncodings, + preferredLayers, + nullptr, + this->kind, + keyFrameSupported, + this, + this->shared); + + break; + } + + default: + { + MS_THROW_TYPE_ERROR("invalid consumer type"); + } + } + + // Create RtpStreamSend instances. + CreateRtpStreams(); + + // NOTE: This may throw. + this->shared->GetChannelMessageRegistrator()->RegisterHandler( + this->id, + /*channelRequestHandler*/ this, + /*channelNotificationHandler*/ nullptr); } Consumer::~Consumer() { MS_TRACE(); + + this->shared->GetChannelMessageRegistrator()->UnregisterHandler(this->id); + + for (auto* rtpStream : this->rtpStreams) + { + delete rtpStream; + } + + this->rtpStreams.clear(); + this->mapMappedSsrcSsrc.clear(); + this->mapSsrcRtpStream.clear(); + this->mapRtpStreamRtpSeqManager.clear(); + this->mapRtpStreamTargetLayerRetransmissionBuffer.clear(); + } + + flatbuffers::Offset Consumer::FillBuffer( + flatbuffers::FlatBufferBuilder& builder) const + { + MS_TRACE(); + + // Call the base method. + auto base = FillBufferBase(builder); + // Add rtpStreams. + std::vector> rtpStreams; + rtpStreams.reserve(this->rtpStreams.size()); + + for (const auto* rtpStream : this->rtpStreams) + { + rtpStreams.emplace_back(rtpStream->FillBuffer(builder)); + } + + auto targetLayers = this->producerStreamManager->GetTargetLayers(); + + auto dump = FBS::Consumer::CreateConsumerDumpDirect( + builder, + base, + &rtpStreams, + this->producerStreamManager->GetPreferredLayers().spatial, + targetLayers.spatial, + this->producerStreamManager->GetCurrentSpatialLayer(), + this->producerStreamManager->GetPreferredLayers().temporal, + targetLayers.temporal, + this->producerStreamManager->GetCurrentTemporalLayer()); + + return FBS::Consumer::CreateDumpResponse(builder, dump); } - flatbuffers::Offset Consumer::FillBuffer( + flatbuffers::Offset Consumer::FillBufferBase( flatbuffers::FlatBufferBuilder& builder) const { MS_TRACE(); @@ -252,12 +548,173 @@ namespace RTC this->priority); } + flatbuffers::Offset Consumer::FillBufferStats( + flatbuffers::FlatBufferBuilder& builder) + { + MS_TRACE(); + + std::vector> rtpStreams; + rtpStreams.reserve(this->rtpStreams.size()); + + // Add stats of our send streams. + for (auto* rtpStream : this->rtpStreams) + { + rtpStreams.emplace_back(rtpStream->FillBufferStats(builder)); + } + + // Add stats of the current recv stream. + auto* producerCurrentRtpStream = this->producerStreamManager->GetProducerCurrentRtpStream(); + + if (producerCurrentRtpStream) + { + rtpStreams.emplace_back(producerCurrentRtpStream->FillBufferStats(builder)); + } + + return FBS::Consumer::CreateGetStatsResponseDirect(builder, &rtpStreams); + } + + flatbuffers::Offset Consumer::FillBufferScore( + flatbuffers::FlatBufferBuilder& builder) const + { + MS_TRACE(); + + MS_ASSERT(this->producerRtpStreamScores, "producerRtpStreamScores not set"); + + // NOTE: Hardcoded values in PipeTransport. + if (this->pipe) + { + return FBS::Consumer::CreateConsumerScoreDirect(builder, 10, 10, this->producerRtpStreamScores); + } + + auto* rtpStream = this->mapSsrcRtpStream.begin()->second; + uint8_t producerScore{ 0 }; + + auto* producerCurrentRtpStream = this->producerStreamManager->GetProducerCurrentRtpStream(); + + if (producerCurrentRtpStream) + { + producerScore = producerCurrentRtpStream->GetScore(); + } + + return FBS::Consumer::CreateConsumerScoreDirect( + builder, rtpStream->GetScore(), producerScore, this->producerRtpStreamScores); + } + void Consumer::HandleRequest(Channel::ChannelRequest* request) { MS_TRACE(); switch (request->method) { + case Channel::ChannelRequest::Method::CONSUMER_DUMP: + { + auto dumpOffset = FillBuffer(request->GetBufferBuilder()); + + request->Accept(FBS::Response::Body::Consumer_DumpResponse, dumpOffset); + + break; + } + + case Channel::ChannelRequest::Method::CONSUMER_REQUEST_KEY_FRAME: + { + if (IsActive()) + { + if (this->pipe) + { + if (this->kind != RTC::Media::Kind::VIDEO) + { + return; + } + + for (auto& consumableRtpEncoding : this->consumableRtpEncodings) + { + auto mappedSsrc = consumableRtpEncoding.ssrc; + + this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); + } + } + else + { + this->producerStreamManager->RequestKeyFrame(); + } + } + + request->Accept(); + + break; + } + + case Channel::ChannelRequest::Method::CONSUMER_SET_PREFERRED_LAYERS: + { + // Simple consumers and pipes have no layers concept. + if (this->type == RTC::RtpParameters::Type::SIMPLE || this->pipe) + { + // Accept with empty preferred layers object. + auto responseOffset = + FBS::Consumer::CreateSetPreferredLayersResponse(request->GetBufferBuilder()); + + request->Accept(FBS::Response::Body::Consumer_SetPreferredLayersResponse, responseOffset); + + break; + } + + auto* rtpStream = this->mapSsrcRtpStream.begin()->second; + auto previousPreferredLayers = this->producerStreamManager->GetPreferredLayers(); + + const auto* body = request->data->body_as(); + const auto* preferredLayers = body->preferredLayers(); + + RTC::ConsumerTypes::VideoLayers newPreferredLayers; + + // Spatial layer. + newPreferredLayers.spatial = preferredLayers->spatialLayer(); + + if (newPreferredLayers.spatial > rtpStream->GetSpatialLayers() - 1) + { + newPreferredLayers.spatial = static_cast(rtpStream->GetSpatialLayers() - 1); + } + + // preferredTemporalLayer is optional. + auto preferredTemporalLayer = preferredLayers->temporalLayer(); + + if (preferredTemporalLayer.has_value()) + { + newPreferredLayers.temporal = preferredTemporalLayer.value(); + + if (newPreferredLayers.temporal > rtpStream->GetTemporalLayers() - 1) + { + newPreferredLayers.temporal = static_cast(rtpStream->GetTemporalLayers() - 1); + } + } + else + { + newPreferredLayers.temporal = static_cast(rtpStream->GetTemporalLayers() - 1); + } + + this->producerStreamManager->SetPreferredLayers(newPreferredLayers); + + MS_DEBUG_DEV( + "preferred layers changed [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]", + newPreferredLayers.spatial, + newPreferredLayers.temporal, + this->id.c_str()); + + preferredTemporalLayer = newPreferredLayers.temporal; + auto preferredLayersOffset = FBS::Consumer::CreateConsumerLayers( + request->GetBufferBuilder(), newPreferredLayers.spatial, preferredTemporalLayer); + auto responseOffset = FBS::Consumer::CreateSetPreferredLayersResponse( + request->GetBufferBuilder(), preferredLayersOffset); + + request->Accept(FBS::Response::Body::Consumer_SetPreferredLayersResponse, responseOffset); + + if (IsActive() && newPreferredLayers != previousPreferredLayers) + { + this->producerStreamManager->MayChangeLayers(/*force*/ true); + } + + break; + } + case Channel::ChannelRequest::Method::CONSUMER_GET_STATS: { auto responseOffset = FillBufferStats(request->GetBufferBuilder()); @@ -470,6 +927,20 @@ namespace RTC this->id, FBS::Notification::Event::CONSUMER_PRODUCER_RESUME); } + void Consumer::ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) + { + MS_TRACE(); + + this->producerStreamManager->ProducerRtpStream(rtpStream, mappedSsrc); + } + + void Consumer::ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) + { + MS_TRACE(); + + this->producerStreamManager->ProducerNewRtpStream(rtpStream, mappedSsrc); + } + void Consumer::ProducerRtpStreamScores(const std::vector* scores) { MS_TRACE(); @@ -478,6 +949,23 @@ namespace RTC this->producerRtpStreamScores = scores; } + void Consumer::ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) + { + MS_TRACE(); + + EmitScore(); + + this->producerStreamManager->ProducerRtpStreamScore(rtpStream, score, previousScore); + } + + void Consumer::ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) + { + MS_TRACE(); + + this->producerStreamManager->ProducerRtcpSenderReport(rtpStream, first); + } + // The caller (Router) is supposed to proceed with the deletion of this Consumer // right after calling this method. Otherwise ugly things may happen. void Consumer::ProducerClosed() @@ -494,31 +982,846 @@ namespace RTC this->listener->OnConsumerProducerClosed(this); } - void Consumer::EmitTraceEventRtpAndKeyFrameTypes(const RTC::RTP::Packet* packet, bool isRtx) const + uint8_t Consumer::GetBitratePriority() const { MS_TRACE(); - if (this->traceEventTypes.keyframe && packet->IsKeyFrame()) - { - auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder()); - auto traceInfo = FBS::Consumer::CreateKeyFrameTraceInfo( - this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx); - - auto notification = FBS::Consumer::CreateTraceNotification( - this->shared->GetChannelNotifier()->GetBufferBuilder(), - FBS::Consumer::TraceEventType::KEYFRAME, - this->shared->GetTimeMs(), - FBS::Common::TraceDirection::DIRECTION_OUT, - FBS::Consumer::TraceInfo::KeyFrameTraceInfo, - traceInfo.Union()); + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - EmitTraceEvent(notification); - } - else if (this->traceEventTypes.rtp) + // Audio does not play the BWE game. + if (this->kind != RTC::Media::Kind::VIDEO) { - auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder()); - auto traceInfo = FBS::Consumer::CreateRtpTraceInfo( - this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx); + return 0u; + } + + if (!IsActive()) + { + return 0u; + } + + return this->priority; + } + + uint32_t Consumer::IncreaseLayer(uint32_t bitrate, bool considerLoss) + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + MS_ASSERT(IsActive(), "should be active"); + + // Pipe does not play the BWE game. + if (this->pipe) + { + return 0u; + } + + float lossPercentage{ 0.0f }; + + auto* rtpStream = this->mapSsrcRtpStream.begin()->second; + + if (considerLoss) + { + lossPercentage = rtpStream->GetLossPercentage(); + } + + auto nowMs = DepLibUV::GetTimeMs(); + + return this->producerStreamManager->IncreaseLayer(bitrate, considerLoss, lossPercentage, nowMs); + } + + void Consumer::ApplyLayers() + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + MS_ASSERT(IsActive(), "should be active"); + + // Pipe does not play the BWE game. + if (this->pipe) + { + return; + } + + auto* rtpStream = this->mapSsrcRtpStream.begin()->second; + + this->producerStreamManager->ApplyLayers(rtpStream->GetActiveMs()); + } + + uint32_t Consumer::GetDesiredBitrate() const + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + + // Pipe does not play the BWE game. + if (this->pipe) + { + return 0u; + } + + // Audio does not play the BWE game. + if (this->kind != RTC::Media::Kind::VIDEO) + { + return 0u; + } + + if (!IsActive()) + { + return 0u; + } + + auto nowMs = DepLibUV::GetTimeMs(); + auto desiredBitrate = this->producerStreamManager->GetDesiredBitrate(nowMs); + + // If consumer.rtpParameters.encodings[0].maxBitrate was given and it's + // greater than computed one, then use it. + auto maxBitrate = this->rtpParameters.encodings[0].maxBitrate; + + desiredBitrate = std::max(maxBitrate, desiredBitrate); + + return desiredBitrate; + } + + // NOLINTNEXTLINE(misc-no-recursion) + void Consumer::SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) + { + MS_TRACE(); + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.consumerId = this->id; +#endif + + RTC::RTP::RtpStreamSend* rtpStream; + RTC::SeqManager* rtpSeqManager; + RetransmissionBuffer* targetLayerRetransmissionBuffer; + + if (this->pipe) + { + auto ssrc = this->mapMappedSsrcSsrc.at(packet->GetSsrc()); + rtpStream = this->mapSsrcRtpStream.at(ssrc); + rtpSeqManager = &this->mapRtpStreamRtpSeqManager.at(rtpStream); + targetLayerRetransmissionBuffer = + &this->mapRtpStreamTargetLayerRetransmissionBuffer.at(rtpStream); + } + else + { + rtpStream = this->mapSsrcRtpStream.begin()->second; + rtpSeqManager = &this->mapRtpStreamRtpSeqManager.at(rtpStream); + targetLayerRetransmissionBuffer = + &this->mapRtpStreamTargetLayerRetransmissionBuffer.at(rtpStream); + } + + if (!IsActive()) + { + if (this->producerStreamManager->IsPacketForCurrentStream(packet)) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::CONSUMER_INACTIVE); +#endif + rtpSeqManager->Drop(packet->GetSequenceNumber()); + } + + return; + } + + auto payloadType = packet->GetPayloadType(); + + // NOTE: This may happen if this Consumer supports just some codecs of those + // in the corresponding Producer. + if (!this->supportedCodecPayloadTypes[payloadType]) + { + if (this->producerStreamManager->IsPacketForCurrentStream(packet)) + { + MS_WARN_DEV("payload type not supported [payloadType:%" PRIu8 "]", payloadType); + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::UNSUPPORTED_PAYLOAD_TYPE); +#endif + rtpSeqManager->Drop(packet->GetSequenceNumber()); + } + + return; + } + + // Ask the ProducerStreamManager to process the packet. + auto action = this->producerStreamManager->ProcessRtpPacket( + packet, this->lastSentPacketHasMarker, rtpStream->GetClockRate(), rtpStream->GetMaxPacketTs()); + + switch (action.type) + { + case ProducerStreamManager::RtpPacketProcessResult::Type::DROP: + { + rtpSeqManager->Drop(packet->GetSequenceNumber()); + return; + } + + case ProducerStreamManager::RtpPacketProcessResult::Type::SILENT_DROP: + { + // Don't account in seq manager. + return; + } + + case ProducerStreamManager::RtpPacketProcessResult::Type::BUFFER: + { + StorePacketInTargetLayerRetransmissionBuffer( + *targetLayerRetransmissionBuffer, packet, sharedPacket); + + return; + } + + case ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD: + { + // Continue below. + break; + } + } + + // Handle sync. + if (action.isSyncPacket) + { + rtpSeqManager->Sync(action.syncSeqValue); + } + + // Handle spatial layer switch. + if (action.spatialLayerSwitched) + { + // Reset the score of our RtpStream to 10. + rtpStream->ResetScore(10u, /*notify*/ false); + + // Emit the layersChange event. + EmitLayersChange(); + + // Emit the score event. + EmitScore(); + } + + // Handle temporal layer change. + if (action.temporalLayerChanged) + { + EmitLayersChange(); + } + + // Update RTP seq number and timestamp based on offset. + uint16_t seq; + const uint32_t timestamp = packet->GetTimestamp() - action.tsOffset; + + rtpSeqManager->Input(packet->GetSequenceNumber(), seq); + + // Save original packet fields. + auto origSsrc = packet->GetSsrc(); + auto origSeq = packet->GetSequenceNumber(); + auto origTimestamp = packet->GetTimestamp(); + const bool origMarker = packet->HasMarker(); + + // Rewrite packet. + // For pipe each stream has its own SSRC; for non-pipe always encodings[0]. + packet->SetSsrc(this->pipe ? rtpStream->GetSsrc() : this->rtpParameters.encodings[0].ssrc); + packet->SetSequenceNumber(seq); + packet->SetTimestamp(timestamp); + packet->SetMarker(action.marker); + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.sendRtpTimestamp = timestamp; + packet->logger.sendSeqNumber = seq; +#endif + + if (action.isSyncPacket) + { + MS_DEBUG_TAG( + rtp, + "sending sync packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 + "] from original [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp(), + origSsrc, + origSeq, + origTimestamp); + } + + const RTC::RTP::RtpStreamSend::ReceivePacketResult result = + rtpStream->ReceivePacket(packet, sharedPacket); + + if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED) + { + if (rtpSeqManager->GetMaxOutput() == packet->GetSequenceNumber()) + { + this->lastSentPacketHasMarker = packet->HasMarker(); + } + + // Send the packet. + this->listener->OnConsumerSendRtpPacket(this, packet); + + // May emit 'trace' event. + EmitTraceEventRtpAndKeyFrameTypes(packet); + } + else + { + MS_WARN_TAG( + rtp, + "failed to send packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 + "] from original [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp(), + origSsrc, + origSeq, + origTimestamp); + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::SEND_RTP_STREAM_DISCARDED); +#endif + } + + // Restore packet fields. + packet->SetSsrc(origSsrc); + packet->SetSequenceNumber(origSeq); + packet->SetTimestamp(origTimestamp); + packet->SetMarker(origMarker); + + // Restore the original payload if needed. + packet->RestorePayload(); + + // If sharedPacket doesn't have a packet inside and it has been stored we + // need to clone the packet into it. + if (!sharedPacket.HasPacket() && result == RTC::RTP::RtpStreamSend::ReceivePacketResult::ACCEPTED_AND_STORED) + { + sharedPacket.Assign(packet); + } + + // If sent packet was the first packet of a key frame, let's send buffered + // packets belonging to the same key frame that arrived earlier due to + // packet misorder. + if (action.sendBufferedPackets) + { + // NOTE: Only send buffered packets if the first packet containing the + // key frame was sent. + if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED) + { + for (auto& kv : *targetLayerRetransmissionBuffer) + { + auto& bufferedSharedPacket = kv.second; + auto* bufferedPacket = bufferedSharedPacket.GetPacket(); + + if (bufferedPacket->GetSequenceNumber() > origSeq) + { + MS_DEBUG_DEV( + "sending packet buffered in the target layer retransmission buffer " + "[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 + "] after sending first packet of the key frame [ssrc:%" PRIu32 ", seq:%" PRIu16 + ", ts:%" PRIu32 "]", + bufferedPacket->GetSsrc(), + bufferedPacket->GetSequenceNumber(), + bufferedPacket->GetTimestamp(), + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + SendRtpPacket(bufferedPacket, bufferedSharedPacket); + + // Be sure that the target layer retransmission buffer has not + // been emptied as a result of sending this packet. If so, exit + // the loop. + if (targetLayerRetransmissionBuffer->empty()) + { + MS_DEBUG_DEV( + "target layer retransmission buffer emptied while iterating " + "it, exiting the loop"); + + break; + } + } + } + } + + targetLayerRetransmissionBuffer->clear(); + } + } + + bool Consumer::GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) + { + MS_TRACE(); + + // Special condition for pipe consumer since this method will be called in a + // loop for each stream. + if (this->pipe) + { + if ( + nowMs != this->lastRtcpSentTime && + static_cast((nowMs - this->lastRtcpSentTime) * 1.15) < this->maxRtcpInterval) + { + return true; + } + } + else if (static_cast((nowMs - this->lastRtcpSentTime) * 1.15) < this->maxRtcpInterval) + { + return true; + } + + std::vector senderReports; + std::vector sdesChunks; + std::vector delaySinceLastRrSsrcInfos; + + for (auto* rtpStream : this->rtpStreams) + { + auto* report = rtpStream->GetRtcpSenderReport(nowMs); + + if (!report) + { + continue; + } + + senderReports.push_back(report); + + // Build SDES chunk for this sender. + auto* sdesChunk = rtpStream->GetRtcpSdesChunk(); + sdesChunks.push_back(sdesChunk); + + auto* delaySinceLastRrSsrcInfo = rtpStream->GetRtcpXrDelaySinceLastRrSsrcInfo(nowMs); + + if (delaySinceLastRrSsrcInfo) + { + delaySinceLastRrSsrcInfos.push_back(delaySinceLastRrSsrcInfo); + } + } + + // RTCP Compound packet buffer cannot hold the data. + if (!packet->Add(senderReports, sdesChunks, delaySinceLastRrSsrcInfos)) + { + return false; + } + + this->lastRtcpSentTime = nowMs; + + return true; + } + + void Consumer::NeedWorstRemoteFractionLost(uint32_t /*mappedSsrc*/, uint8_t& worstRemoteFractionLost) + { + MS_TRACE(); + + if (!IsActive()) + { + return; + } + + for (auto* rtpStream : this->rtpStreams) + { + auto fractionLost = rtpStream->GetFractionLost(); + + // If our fraction lost is worse than the given one, update it. + worstRemoteFractionLost = std::max(fractionLost, worstRemoteFractionLost); + } + } + + void Consumer::ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) + { + MS_TRACE(); + + if (!IsActive()) + { + return; + } + + // May emit 'trace' event. + EmitTraceEventNackType(); + + RTC::RTP::RtpStreamSend* rtpStream; + + if (this->pipe) + { + auto ssrc = nackPacket->GetMediaSsrc(); + rtpStream = this->mapSsrcRtpStream.at(ssrc); + } + else + { + rtpStream = this->mapSsrcRtpStream.begin()->second; + } + + rtpStream->ReceiveNack(nackPacket); + } + + void Consumer::ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) + { + MS_TRACE(); + + if (this->kind != RTC::Media::Kind::VIDEO) + { + return; + } + + switch (messageType) + { + case RTC::RTCP::FeedbackPs::MessageType::PLI: + { + EmitTraceEventPliType(ssrc); + + break; + } + + case RTC::RTCP::FeedbackPs::MessageType::FIR: + { + EmitTraceEventFirType(ssrc); + + break; + } + + default:; + } + + auto* rtpStream = this->mapSsrcRtpStream.at(ssrc); + + rtpStream->ReceiveKeyFrameRequest(messageType); + + if (IsActive()) + { + if (this->pipe) + { + for (auto& consumableRtpEncoding : this->consumableRtpEncodings) + { + auto mappedSsrc = consumableRtpEncoding.ssrc; + + this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); + } + } + else + { + this->producerStreamManager->RequestKeyFrameForCurrentSpatialLayer(); + } + } + } + + void Consumer::ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) + { + MS_TRACE(); + + auto* rtpStream = this->mapSsrcRtpStream.at(report->GetSsrc()); + + rtpStream->ReceiveRtcpReceiverReport(report); + } + + void Consumer::ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) + { + MS_TRACE(); + + for (auto* rtpStream : this->rtpStreams) + { + rtpStream->ReceiveRtcpXrReceiverReferenceTime(report); + } + } + + uint32_t Consumer::GetTransmissionRate(uint64_t nowMs) + { + MS_TRACE(); + + if (!IsActive()) + { + return 0u; + } + + uint32_t rate{ 0u }; + + for (auto* rtpStream : this->rtpStreams) + { + rate += rtpStream->GetBitrate(nowMs); + } + + return rate; + } + + float Consumer::GetRtt() const + { + MS_TRACE(); + + float rtt{ 0 }; + + for (auto* rtpStream : this->rtpStreams) + { + rtt = std::max(rtpStream->GetRtt(), rtt); + } + + return rtt; + } + + void Consumer::UserOnTransportConnected() + { + MS_TRACE(); + + this->producerStreamManager->OnTransportConnected(); + } + + void Consumer::UserOnTransportDisconnected() + { + MS_TRACE(); + + for (auto* rtpStream : this->rtpStreams) + { + rtpStream->Pause(); + } + + for (auto& kv : this->mapRtpStreamTargetLayerRetransmissionBuffer) + { + auto& targetLayerRetransmissionBuffer = kv.second; + + targetLayerRetransmissionBuffer.clear(); + } + + this->producerStreamManager->OnTransportDisconnected(); + } + + void Consumer::UserOnPaused() + { + MS_TRACE(); + + for (auto* rtpStream : this->rtpStreams) + { + rtpStream->Pause(); + } + + for (auto& kv : this->mapRtpStreamTargetLayerRetransmissionBuffer) + { + auto& targetLayerRetransmissionBuffer = kv.second; + + targetLayerRetransmissionBuffer.clear(); + } + + this->producerStreamManager->OnPaused(); + + if (this->externallyManagedBitrate) + { + // Audio does not play the BWE game. + if (this->kind != RTC::Media::Kind::VIDEO) + { + return; + } + + this->listener->OnConsumerNeedZeroBitrate(this); + } + } + + void Consumer::UserOnResumed() + { + MS_TRACE(); + + this->producerStreamManager->OnResumed(); + } + + void Consumer::CreateRtpStreams() + { + MS_TRACE(); + + // NOTE: For non-pipe consumers, all spatial layers are multiplexed through + // a single RtpStreamSend (encodings[0]). Pipe consumers need one stream per + // encoding. Here we know that SSRCs in Consumer's rtpParameters must be the + // same as in the given consumableRtpEncodings. + const size_t numStreams = this->pipe ? this->rtpParameters.encodings.size() : 1u; + + for (size_t idx{ 0u }; idx < numStreams; ++idx) + { + auto& encoding = this->rtpParameters.encodings[idx]; + const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding); + auto& consumableEncoding = this->consumableRtpEncodings[idx]; + + MS_DEBUG_TAG( + rtp, "[ssrc:%" PRIu32 ", payloadType:%" PRIu8 "]", encoding.ssrc, mediaCodec->payloadType); + + // Set stream params. + RTC::RTP::RtpStream::Params params; + + params.encodingIdx = idx; + params.ssrc = encoding.ssrc; + params.payloadType = mediaCodec->payloadType; + params.mimeType = mediaCodec->mimeType; + params.clockRate = mediaCodec->clockRate; + params.cname = this->rtpParameters.rtcp.cname; + params.spatialLayers = encoding.spatialLayers; + params.temporalLayers = encoding.temporalLayers; + + // Check in band FEC in codec parameters. + if (mediaCodec->parameters.HasInteger("useinbandfec") && mediaCodec->parameters.GetInteger("useinbandfec") == 1) + { + MS_DEBUG_TAG(rtp, "in band FEC enabled"); + + params.useInBandFec = true; + } + + // Check DTX in codec parameters. + if (mediaCodec->parameters.HasInteger("usedtx") && mediaCodec->parameters.GetInteger("usedtx") == 1) + { + MS_DEBUG_TAG(rtp, "DTX enabled"); + + params.useDtx = true; + } + + // Check DTX in the encoding. + if (encoding.dtx) + { + MS_DEBUG_TAG(rtp, "DTX enabled"); + + params.useDtx = true; + } + + for (const auto& fb : mediaCodec->rtcpFeedback) + { + if (!params.useNack && fb.type == "nack" && fb.parameter.empty()) + { + MS_DEBUG_2TAGS(rtp, rtcp, "NACK supported"); + + params.useNack = true; + } + else if (!params.usePli && fb.type == "nack" && fb.parameter == "pli") + { + MS_DEBUG_2TAGS(rtp, rtcp, "PLI supported"); + + params.usePli = true; + } + else if (!params.useFir && fb.type == "ccm" && fb.parameter == "fir") + { + MS_DEBUG_2TAGS(rtp, rtcp, "FIR supported"); + + params.useFir = true; + } + } + + auto* rtpStream = + new RTC::RTP::RtpStreamSend(this, this->shared, params, this->rtpParameters.mid); + + // If the Consumer is paused, tell the RtpStreamSend. + if (IsPaused() || IsProducerPaused()) + { + rtpStream->Pause(); + } + + const auto* rtxCodec = this->rtpParameters.GetRtxCodecForEncoding(encoding); + + if (rtxCodec && encoding.hasRtx) + { + rtpStream->SetRtx(rtxCodec->payloadType, encoding.rtx.ssrc); + } + + this->rtpStreams.push_back(rtpStream); + this->mapMappedSsrcSsrc[consumableEncoding.ssrc] = encoding.ssrc; + this->mapSsrcRtpStream[encoding.ssrc] = rtpStream; + + // Let's choose an initial output seq number between 1000 and 32768 to avoid + // libsrtp bug: + // https://github.com/versatica/mediasoup/issues/1437 + const auto initialOutputSeq = + Utils::Crypto::GetRandomUInt(1000u, std::numeric_limits::max() / 2); + + this->mapRtpStreamRtpSeqManager[rtpStream] = RTC::SeqManager(initialOutputSeq); + + this->mapRtpStreamTargetLayerRetransmissionBuffer[rtpStream]; + } + } + + void Consumer::StorePacketInTargetLayerRetransmissionBuffer( + std::map::SeqLowerThan>& + targetLayerRetransmissionBuffer, + RTC::RTP::Packet* packet, + RTC::RTP::SharedPacket& sharedPacket) + { + MS_TRACE(); + + MS_DEBUG_DEV( + "storing packet in target layer retransmission buffer [ssrc:%" PRIu32 ", seq:%" PRIu16 + ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + // Store original packet into the buffer. Only clone once and only if + // necessary. + if (!sharedPacket.HasPacket()) + { + sharedPacket.Assign(packet); + } + // Assert that, if sharedPacket was already filled, both packet and + // sharedPacket are the very same RTP packet. + else + { + sharedPacket.AssertSamePacket(packet); + } + + targetLayerRetransmissionBuffer[packet->GetSequenceNumber()] = sharedPacket; + + if (targetLayerRetransmissionBuffer.size() > TargetLayerRetransmissionBufferSize) + { + targetLayerRetransmissionBuffer.erase(targetLayerRetransmissionBuffer.begin()); + } + } + + void Consumer::EmitScore() const + { + MS_TRACE(); + + auto scoreOffset = FillBufferScore(this->shared->GetChannelNotifier()->GetBufferBuilder()); + + auto notificationOffset = FBS::Consumer::CreateScoreNotification( + this->shared->GetChannelNotifier()->GetBufferBuilder(), scoreOffset); + + this->shared->GetChannelNotifier()->Emit( + this->id, + FBS::Notification::Event::CONSUMER_SCORE, + FBS::Notification::Body::Consumer_ScoreNotification, + notificationOffset); + } + + void Consumer::EmitLayersChange() const + { + MS_TRACE(); + + MS_DEBUG_DEV( + "current layers changed to [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]", + this->producerStreamManager->GetCurrentSpatialLayer(), + this->producerStreamManager->GetCurrentTemporalLayer(), + this->id.c_str()); + + flatbuffers::Offset layersOffset; + + if (this->producerStreamManager->GetCurrentSpatialLayer() >= 0) + { + layersOffset = FBS::Consumer::CreateConsumerLayers( + this->shared->GetChannelNotifier()->GetBufferBuilder(), + this->producerStreamManager->GetCurrentSpatialLayer(), + this->producerStreamManager->GetCurrentTemporalLayer()); + } + + auto notificationOffset = FBS::Consumer::CreateLayersChangeNotification( + this->shared->GetChannelNotifier()->GetBufferBuilder(), layersOffset); + + this->shared->GetChannelNotifier()->Emit( + this->id, + FBS::Notification::Event::CONSUMER_LAYERS_CHANGE, + FBS::Notification::Body::Consumer_LayersChangeNotification, + notificationOffset); + } + + void Consumer::EmitTraceEventRtpAndKeyFrameTypes(const RTC::RTP::Packet* packet, bool isRtx) const + { + MS_TRACE(); + + if (this->traceEventTypes.keyframe && packet->IsKeyFrame()) + { + auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder()); + auto traceInfo = FBS::Consumer::CreateKeyFrameTraceInfo( + this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx); + + auto notification = FBS::Consumer::CreateTraceNotification( + this->shared->GetChannelNotifier()->GetBufferBuilder(), + FBS::Consumer::TraceEventType::KEYFRAME, + this->shared->GetTimeMs(), + FBS::Common::TraceDirection::DIRECTION_OUT, + FBS::Consumer::TraceInfo::KeyFrameTraceInfo, + traceInfo.Union()); + + EmitTraceEvent(notification); + } + else if (this->traceEventTypes.rtp) + { + auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder()); + auto traceInfo = FBS::Consumer::CreateRtpTraceInfo( + this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx); auto notification = FBS::Consumer::CreateTraceNotification( this->shared->GetChannelNotifier()->GetBufferBuilder(), @@ -606,4 +1909,75 @@ namespace RTC FBS::Notification::Body::Consumer_TraceNotification, notification); } + + void Consumer::OnRtpStreamScore( + RTC::RTP::RtpStream* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) + { + MS_TRACE(); + + // Emit the score event. + EmitScore(); + + // NOTE @jmillan: Previously present in Simulcast and SVC. Does it make sense to move it here? + // Previously present in Simulcast and SVC. Does it make sense to move it here? + if (IsActive()) + { + // Just check target layers if our bitrate is not externally managed. + if (!this->externallyManagedBitrate) + { + this->producerStreamManager->MayChangeLayers(/*force*/ false); + } + } + } + + void Consumer::OnRtpStreamRetransmitRtpPacket( + RTC::RTP::RtpStreamSend* rtpStream, RTC::RTP::Packet* packet) + { + MS_TRACE(); + + this->listener->OnConsumerRetransmitRtpPacket(this, packet); + + // May emit 'trace' event. + EmitTraceEventRtpAndKeyFrameTypes(packet, rtpStream->HasRtx()); + } + + /* ProducerStreamManager::Listener methods. */ + + void Consumer::OnProducerStreamManagerKeyFrameRequested(uint32_t mappedSsrc) + { + MS_TRACE(); + + this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); + } + + void Consumer::OnProducerStreamManagerNeedBitrateChange() + { + MS_TRACE(); + + this->listener->OnConsumerNeedBitrateChange(this); + } + + void Consumer::OnProducerStreamManagerLayersChanged() + { + MS_TRACE(); + + EmitLayersChange(); + } + + void Consumer::OnProducerStreamManagerClearRetransmissionBuffer() + { + MS_TRACE(); + + for (auto& kv : this->mapRtpStreamTargetLayerRetransmissionBuffer) + { + kv.second.clear(); + } + } + + void Consumer::OnProducerStreamManagerScore() + { + MS_TRACE(); + + EmitScore(); + } } // namespace RTC diff --git a/worker/src/RTC/OldConsumer.cpp b/worker/src/RTC/OldConsumer.cpp new file mode 100644 index 0000000000..a750366257 --- /dev/null +++ b/worker/src/RTC/OldConsumer.cpp @@ -0,0 +1,610 @@ +#define MS_CLASS "RTC::OldConsumer" +// #define MS_LOG_DEV_LEVEL 3 + +#include "RTC/OldConsumer.hpp" +#include "DepLibUV.hpp" +#include "Logger.hpp" +#include "MediaSoupErrors.hpp" + +namespace RTC +{ + /* Instance methods. */ + + OldConsumer::OldConsumer( + SharedInterface* shared, + const std::string& id, + const std::string& producerId, + Listener* listener, + const FBS::Transport::ConsumeRequest* data, + RTC::RtpParameters::Type type) + : id(id), + producerId(producerId), + shared(shared), + listener(listener), + kind(RTC::Media::Kind(data->kind())), + type(type) + { + MS_TRACE(); + + // This may throw. + this->rtpParameters = RTC::RtpParameters(data->rtpParameters()); + + if (this->rtpParameters.encodings.empty()) + { + MS_THROW_TYPE_ERROR("empty rtpParameters.encodings"); + } + + // All encodings must have SSRCs. + for (auto& encoding : this->rtpParameters.encodings) + { + if (encoding.ssrc == 0) + { + MS_THROW_TYPE_ERROR("invalid encoding in rtpParameters (missing ssrc)"); + } + else if (encoding.hasRtx && encoding.rtx.ssrc == 0) + { + MS_THROW_TYPE_ERROR("invalid encoding in rtpParameters (missing rtx.ssrc)"); + } + } + + if (data->consumableRtpEncodings()->size() == 0) + { + MS_THROW_TYPE_ERROR("empty consumableRtpEncodings"); + } + + this->consumableRtpEncodings.reserve(data->consumableRtpEncodings()->size()); + + for (size_t i{ 0 }; i < data->consumableRtpEncodings()->size(); ++i) + { + const auto* entry = data->consumableRtpEncodings()->Get(i); + + // This may throw due the constructor of RTC::RtpEncodingParameters. + this->consumableRtpEncodings.emplace_back(entry); + + // Verify that it has ssrc field. + auto& encoding = this->consumableRtpEncodings[i]; + + if (encoding.ssrc == 0u) + { + MS_THROW_TYPE_ERROR("wrong encoding in consumableRtpEncodings (missing ssrc)"); + } + } + + // Fill RTP header extension ids and their mapped values. + // This may throw. + for (auto& exten : this->rtpParameters.headerExtensions) + { + if (exten.id == 0u) + { + MS_THROW_TYPE_ERROR("RTP extension id cannot be 0"); + } + + if (this->rtpHeaderExtensionIds.mid == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::MID) + { + this->rtpHeaderExtensionIds.mid = exten.id; + } + + if (this->rtpHeaderExtensionIds.rid == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::RTP_STREAM_ID) + { + this->rtpHeaderExtensionIds.rid = exten.id; + } + + if (this->rtpHeaderExtensionIds.rrid == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::REPAIRED_RTP_STREAM_ID) + { + this->rtpHeaderExtensionIds.rrid = exten.id; + } + + if (this->rtpHeaderExtensionIds.absSendTime == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::ABS_SEND_TIME) + { + this->rtpHeaderExtensionIds.absSendTime = exten.id; + } + + if (this->rtpHeaderExtensionIds.transportWideCc01 == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::TRANSPORT_WIDE_CC_01) + { + this->rtpHeaderExtensionIds.transportWideCc01 = exten.id; + } + + if (this->rtpHeaderExtensionIds.ssrcAudioLevel == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::SSRC_AUDIO_LEVEL) + { + this->rtpHeaderExtensionIds.ssrcAudioLevel = exten.id; + } + + if ( + this->rtpHeaderExtensionIds.dependencyDescriptor == 0u && + exten.type == RTC::RtpHeaderExtensionUri::Type::DEPENDENCY_DESCRIPTOR) + { + this->rtpHeaderExtensionIds.dependencyDescriptor = exten.id; + } + + if (this->rtpHeaderExtensionIds.videoOrientation == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::VIDEO_ORIENTATION) + { + this->rtpHeaderExtensionIds.videoOrientation = exten.id; + } + + if (this->rtpHeaderExtensionIds.videoOrientation == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::VIDEO_ORIENTATION) + { + this->rtpHeaderExtensionIds.videoOrientation = exten.id; + } + + if (this->rtpHeaderExtensionIds.absCaptureTime == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::ABS_CAPTURE_TIME) + { + this->rtpHeaderExtensionIds.absCaptureTime = exten.id; + } + + if (this->rtpHeaderExtensionIds.playoutDelay == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::PLAYOUT_DELAY) + { + this->rtpHeaderExtensionIds.playoutDelay = exten.id; + } + + if (this->rtpHeaderExtensionIds.mediasoupPacketId == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::MEDIASOUP_PACKET_ID) + { + this->rtpHeaderExtensionIds.mediasoupPacketId = exten.id; + } + } + + // paused is set to false by default. + this->paused = data->paused(); + + // Fill supported codec payload types. + for (auto& codec : this->rtpParameters.codecs) + { + if (codec.mimeType.IsMediaCodec()) + { + this->supportedCodecPayloadTypes[codec.payloadType] = true; + } + } + + // Fill media SSRCs vector. + for (auto& encoding : this->rtpParameters.encodings) + { + this->mediaSsrcs.push_back(encoding.ssrc); + } + + // Fill RTX SSRCs vector. + for (auto& encoding : this->rtpParameters.encodings) + { + if (encoding.hasRtx) + { + this->rtxSsrcs.push_back(encoding.rtx.ssrc); + } + } + + // Set the RTCP report generation interval. + if (this->kind == RTC::Media::Kind::AUDIO) + { + this->maxRtcpInterval = RTC::RTCP::MaxAudioIntervalMs; + } + else + { + this->maxRtcpInterval = RTC::RTCP::MaxVideoIntervalMs; + } + } + + OldConsumer::~OldConsumer() + { + MS_TRACE(); + } + + flatbuffers::Offset OldConsumer::FillBuffer( + flatbuffers::FlatBufferBuilder& builder) const + { + MS_TRACE(); + + // Add rtpParameters. + auto rtpParameters = this->rtpParameters.FillBuffer(builder); + + // Add consumableRtpEncodings. + std::vector> consumableRtpEncodings; + consumableRtpEncodings.reserve(this->consumableRtpEncodings.size()); + + for (const auto& encoding : this->consumableRtpEncodings) + { + consumableRtpEncodings.emplace_back(encoding.FillBuffer(builder)); + } + + // Add supportedCodecPayloadTypes. + std::vector supportedCodecPayloadTypes; + + for (auto i = 0; i < 128; ++i) + { + if (this->supportedCodecPayloadTypes[i]) + { + supportedCodecPayloadTypes.push_back(i); + } + } + + // Add traceEventTypes. + std::vector traceEventTypes; + + if (this->traceEventTypes.rtp) + { + traceEventTypes.emplace_back(FBS::Consumer::TraceEventType::RTP); + } + if (this->traceEventTypes.keyframe) + { + traceEventTypes.emplace_back(FBS::Consumer::TraceEventType::KEYFRAME); + } + if (this->traceEventTypes.nack) + { + traceEventTypes.emplace_back(FBS::Consumer::TraceEventType::NACK); + } + if (this->traceEventTypes.pli) + { + traceEventTypes.emplace_back(FBS::Consumer::TraceEventType::PLI); + } + if (this->traceEventTypes.fir) + { + traceEventTypes.emplace_back(FBS::Consumer::TraceEventType::FIR); + } + + return FBS::Consumer::CreateBaseConsumerDumpDirect( + builder, + this->id.c_str(), + RTC::RtpParameters::TypeToFbs(this->type), + this->producerId.c_str(), + this->kind == RTC::Media::Kind::AUDIO ? FBS::RtpParameters::MediaKind::AUDIO + : FBS::RtpParameters::MediaKind::VIDEO, + rtpParameters, + &consumableRtpEncodings, + &supportedCodecPayloadTypes, + &traceEventTypes, + this->paused, + this->producerPaused, + this->priority); + } + + void OldConsumer::HandleRequest(Channel::ChannelRequest* request) + { + MS_TRACE(); + + switch (request->method) + { + case Channel::ChannelRequest::Method::CONSUMER_GET_STATS: + { + auto responseOffset = FillBufferStats(request->GetBufferBuilder()); + + request->Accept(FBS::Response::Body::Consumer_GetStatsResponse, responseOffset); + + break; + } + + case Channel::ChannelRequest::Method::CONSUMER_PAUSE: + { + if (this->paused) + { + request->Accept(); + + break; + } + + const bool wasActive = IsActive(); + + this->paused = true; + + MS_DEBUG_DEV("Consumer paused [consumerId:%s]", this->id.c_str()); + + if (wasActive) + { + UserOnPaused(); + } + + request->Accept(); + + break; + } + + case Channel::ChannelRequest::Method::CONSUMER_RESUME: + { + if (!this->paused) + { + request->Accept(); + + break; + } + + this->paused = false; + + MS_DEBUG_DEV("Consumer resumed [consumerId:%s]", this->id.c_str()); + + if (IsActive()) + { + UserOnResumed(); + } + + request->Accept(); + + break; + } + + case Channel::ChannelRequest::Method::CONSUMER_SET_PRIORITY: + { + const auto* body = request->data->body_as(); + + if (body->priority() < 1u) + { + MS_THROW_TYPE_ERROR("wrong priority (must be higher than 0)"); + } + + this->priority = body->priority(); + + auto responseOffset = + FBS::Consumer::CreateSetPriorityResponse(request->GetBufferBuilder(), this->priority); + + request->Accept(FBS::Response::Body::Consumer_SetPriorityResponse, responseOffset); + + break; + } + + case Channel::ChannelRequest::Method::CONSUMER_ENABLE_TRACE_EVENT: + { + const auto* body = request->data->body_as(); + + // Reset traceEventTypes. + struct TraceEventTypes newTraceEventTypes; + + for (const auto& type : *body->events()) + { + switch (type) + { + case FBS::Consumer::TraceEventType::KEYFRAME: + { + newTraceEventTypes.keyframe = true; + + break; + } + case FBS::Consumer::TraceEventType::FIR: + { + newTraceEventTypes.fir = true; + + break; + } + case FBS::Consumer::TraceEventType::NACK: + { + newTraceEventTypes.nack = true; + + break; + } + case FBS::Consumer::TraceEventType::PLI: + { + newTraceEventTypes.pli = true; + + break; + } + case FBS::Consumer::TraceEventType::RTP: + { + newTraceEventTypes.rtp = true; + + break; + } + } + } + + this->traceEventTypes = newTraceEventTypes; + + request->Accept(); + + break; + } + + default: + { + MS_THROW_ERROR("unknown method '%s'", request->methodCStr); + } + } + } + + void OldConsumer::TransportConnected() + { + MS_TRACE(); + + if (this->transportConnected) + { + return; + } + + this->transportConnected = true; + + MS_DEBUG_DEV("Transport connected [consumerId:%s]", this->id.c_str()); + + UserOnTransportConnected(); + } + + void OldConsumer::TransportDisconnected() + { + MS_TRACE(); + + if (!this->transportConnected) + { + return; + } + + this->transportConnected = false; + + MS_DEBUG_DEV("Transport disconnected [consumerId:%s]", this->id.c_str()); + + UserOnTransportDisconnected(); + } + + void OldConsumer::ProducerPaused() + { + MS_TRACE(); + + if (this->producerPaused) + { + return; + } + + const bool wasActive = IsActive(); + + this->producerPaused = true; + + MS_DEBUG_DEV("Producer paused [consumerId:%s]", this->id.c_str()); + + if (wasActive) + { + UserOnPaused(); + } + + this->shared->GetChannelNotifier()->Emit( + this->id, FBS::Notification::Event::CONSUMER_PRODUCER_PAUSE); + } + + void OldConsumer::ProducerResumed() + { + MS_TRACE(); + + if (!this->producerPaused) + { + return; + } + + this->producerPaused = false; + + MS_DEBUG_DEV("Producer resumed [consumerId:%s]", this->id.c_str()); + + if (IsActive()) + { + UserOnResumed(); + } + + this->shared->GetChannelNotifier()->Emit( + this->id, FBS::Notification::Event::CONSUMER_PRODUCER_RESUME); + } + + void OldConsumer::ProducerRtpStreamScores(const std::vector* scores) + { + MS_TRACE(); + + // This is gonna be a constant pointer. + this->producerRtpStreamScores = scores; + } + + // The caller (Router) is supposed to proceed with the deletion of this Consumer + // right after calling this method. Otherwise ugly things may happen. + void OldConsumer::ProducerClosed() + { + MS_TRACE(); + + this->producerClosed = true; + + MS_DEBUG_DEV("Producer closed [consumerId:%s]", this->id.c_str()); + + this->shared->GetChannelNotifier()->Emit( + this->id, FBS::Notification::Event::CONSUMER_PRODUCER_CLOSE); + + this->listener->OnConsumerProducerClosed(this); + } + + void OldConsumer::EmitTraceEventRtpAndKeyFrameTypes(const RTC::RTP::Packet* packet, bool isRtx) const + { + MS_TRACE(); + + if (this->traceEventTypes.keyframe && packet->IsKeyFrame()) + { + auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder()); + auto traceInfo = FBS::Consumer::CreateKeyFrameTraceInfo( + this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx); + + auto notification = FBS::Consumer::CreateTraceNotification( + this->shared->GetChannelNotifier()->GetBufferBuilder(), + FBS::Consumer::TraceEventType::KEYFRAME, + DepLibUV::GetTimeMs(), + FBS::Common::TraceDirection::DIRECTION_OUT, + FBS::Consumer::TraceInfo::KeyFrameTraceInfo, + traceInfo.Union()); + + EmitTraceEvent(notification); + } + else if (this->traceEventTypes.rtp) + { + auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder()); + auto traceInfo = FBS::Consumer::CreateRtpTraceInfo( + this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx); + + auto notification = FBS::Consumer::CreateTraceNotification( + this->shared->GetChannelNotifier()->GetBufferBuilder(), + FBS::Consumer::TraceEventType::RTP, + DepLibUV::GetTimeMs(), + FBS::Common::TraceDirection::DIRECTION_OUT, + FBS::Consumer::TraceInfo::RtpTraceInfo, + traceInfo.Union()); + + EmitTraceEvent(notification); + } + } + + void OldConsumer::EmitTraceEventPliType(uint32_t ssrc) const + { + MS_TRACE(); + + if (!this->traceEventTypes.pli) + { + return; + } + + auto traceInfo = FBS::Consumer::CreatePliTraceInfo( + this->shared->GetChannelNotifier()->GetBufferBuilder(), ssrc); + + auto notification = FBS::Consumer::CreateTraceNotification( + this->shared->GetChannelNotifier()->GetBufferBuilder(), + FBS::Consumer::TraceEventType::PLI, + DepLibUV::GetTimeMs(), + FBS::Common::TraceDirection::DIRECTION_IN, + FBS::Consumer::TraceInfo::PliTraceInfo, + traceInfo.Union()); + + EmitTraceEvent(notification); + } + + void OldConsumer::EmitTraceEventFirType(uint32_t ssrc) const + { + MS_TRACE(); + + if (!this->traceEventTypes.fir) + { + return; + } + + auto traceInfo = FBS::Consumer::CreateFirTraceInfo( + this->shared->GetChannelNotifier()->GetBufferBuilder(), ssrc); + + auto notification = FBS::Consumer::CreateTraceNotification( + this->shared->GetChannelNotifier()->GetBufferBuilder(), + FBS::Consumer::TraceEventType::FIR, + DepLibUV::GetTimeMs(), + FBS::Common::TraceDirection::DIRECTION_IN, + FBS::Consumer::TraceInfo::FirTraceInfo, + traceInfo.Union()); + + EmitTraceEvent(notification); + } + + void OldConsumer::EmitTraceEventNackType() const + { + MS_TRACE(); + + if (!this->traceEventTypes.nack) + { + return; + } + + auto notification = FBS::Consumer::CreateTraceNotification( + this->shared->GetChannelNotifier()->GetBufferBuilder(), + FBS::Consumer::TraceEventType::NACK, + DepLibUV::GetTimeMs(), + FBS::Common::TraceDirection::DIRECTION_IN); + + EmitTraceEvent(notification); + } + + void OldConsumer::EmitTraceEvent(flatbuffers::Offset& notification) const + { + MS_TRACE(); + + this->shared->GetChannelNotifier()->Emit( + this->id, + FBS::Notification::Event::CONSUMER_TRACE, + FBS::Notification::Body::Consumer_TraceNotification, + notification); + } +} // namespace RTC diff --git a/worker/src/RTC/PipeConsumer.cpp b/worker/src/RTC/PipeConsumer.cpp index 0d1aff834e..e7775117ca 100644 --- a/worker/src/RTC/PipeConsumer.cpp +++ b/worker/src/RTC/PipeConsumer.cpp @@ -61,9 +61,10 @@ namespace RTC SharedInterface* shared, const std::string& id, const std::string& producerId, - RTC::Consumer::Listener* listener, + RTC::OldConsumer::Listener* listener, const FBS::Transport::ConsumeRequest* data) - : RTC::Consumer::Consumer(shared, id, producerId, listener, data, RTC::RtpParameters::Type::PIPE) + : RTC::OldConsumer::OldConsumer( + shared, id, producerId, listener, data, RTC::RtpParameters::Type::PIPE) { MS_TRACE(); @@ -113,7 +114,7 @@ namespace RTC MS_TRACE(); // Call the parent method. - auto base = RTC::Consumer::FillBuffer(builder); + auto base = RTC::OldConsumer::FillBuffer(builder); // Add rtpStreams. std::vector> rtpStreams; @@ -199,7 +200,7 @@ namespace RTC default: { // Pass it to the parent class. - RTC::Consumer::HandleRequest(request); + RTC::OldConsumer::HandleRequest(request); } } } @@ -827,7 +828,7 @@ namespace RTC // Let's choose an initial output seq number between 1000 and 32768 to avoid // libsrtp bug: // https://github.com/versatica/mediasoup/issues/1437 - const uint16_t initialOutputSeq = + const auto initialOutputSeq = Utils::Crypto::GetRandomUInt(1000u, std::numeric_limits::max() / 2); this->mapRtpStreamRtpSeqManager[rtpStream] = RTC::SeqManager(initialOutputSeq); diff --git a/worker/src/RTC/PipeProducerStreamManager.cpp b/worker/src/RTC/PipeProducerStreamManager.cpp new file mode 100644 index 0000000000..0e97b0c137 --- /dev/null +++ b/worker/src/RTC/PipeProducerStreamManager.cpp @@ -0,0 +1,266 @@ +#define MS_CLASS "RTC::PipeProducerStreamManager" +// #define MS_LOG_DEV_LEVEL 3 + +#include "RTC/PipeProducerStreamManager.hpp" +#include "Logger.hpp" + +namespace RTC +{ + /* Instance methods. */ + + PipeProducerStreamManager::PipeProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared) + : ProducerStreamManager( + consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + kind, + keyFrameSupported, + listener, + shared) + { + MS_TRACE(); + + // Initialize per-stream sync state for each consumable encoding. + for (const auto& encoding : this->consumableRtpEncodings) + { + this->mapMappedSsrcSyncRequired[encoding.ssrc] = false; + } + } + + bool PipeProducerStreamManager::IsActive() const + { + MS_TRACE(); + + return this->listener->IsActive(); + } + + void PipeProducerStreamManager::ProducerRtpStream( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint32_t /*mappedSsrc*/) + { + MS_TRACE(); + + // Do nothing. + } + + void PipeProducerStreamManager::ProducerNewRtpStream( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint32_t /*mappedSsrc*/) + { + MS_TRACE(); + + // Do nothing. + } + + void PipeProducerStreamManager::ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) + { + MS_TRACE(); + + // Do nothing. + } + + void PipeProducerStreamManager::ProducerRtcpSenderReport( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, bool /*first*/) + { + MS_TRACE(); + + // Do nothing. + } + + uint32_t PipeProducerStreamManager::IncreaseLayer( + uint32_t /*bitrate*/, bool /*considerLoss*/, float /*lossPercentage*/, uint64_t /*nowMs*/) + { + MS_TRACE(); + + // Pipe does not play the BWE game. + return 0u; + } + + void PipeProducerStreamManager::ApplyLayers(uint64_t /*rtpStreamActiveMs*/) + { + MS_TRACE(); + + // Pipe does not play the BWE game. + } + + uint32_t PipeProducerStreamManager::GetDesiredBitrate(uint64_t /*nowMs*/) const + { + MS_TRACE(); + + // Pipe does not play the BWE game. + return 0u; + } + + ProducerStreamManager::RtpPacketProcessResult PipeProducerStreamManager::ProcessRtpPacket( + RTC::RTP::Packet* packet, + bool /*lastSentPacketHasMarker*/, + uint32_t /*clockRate*/, + uint32_t /*maxPacketTs*/) + { + MS_TRACE(); + + RtpPacketProcessResult result; + + auto it = this->mapMappedSsrcSyncRequired.find(packet->GetSsrc()); + + MS_ASSERT(it != this->mapMappedSsrcSyncRequired.end(), "unknown mapped SSRC"); + + bool& syncRequired = it->second; + + // If sync required and key frames supported, buffer non-key-frame packets. + if (syncRequired && this->keyFrameSupported && !packet->IsKeyFrame()) + { + result.type = RtpPacketProcessResult::Type::BUFFER; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); +#endif + + return result; + } + + // Packets with only padding are not forwarded. + if (packet->GetPayloadLength() == 0) + { + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); +#endif + + return result; + } + + // Whether this is the first packet after re-sync. + const bool isSyncPacket = syncRequired; + + if (isSyncPacket) + { + if (packet->IsKeyFrame()) + { + MS_DEBUG_TAG( + rtp, + "sync key frame received [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + result.sendBufferedPackets = true; + } + + result.isSyncPacket = true; + result.syncSeqValue = packet->GetSequenceNumber() - 1; + + syncRequired = false; + } + + // Pipe passes timestamp and marker through unchanged. + result.type = RtpPacketProcessResult::Type::FORWARD; + result.tsOffset = 0u; + result.marker = packet->HasMarker(); + + return result; + } + + void PipeProducerStreamManager::RequestKeyFrame() + { + MS_TRACE(); + + if (this->kind != RTC::Media::Kind::VIDEO) + { + return; + } + + for (const auto& encoding : this->consumableRtpEncodings) + { + this->listener->OnProducerStreamManagerKeyFrameRequested(encoding.ssrc); + } + } + + void PipeProducerStreamManager::RequestKeyFrameForTargetSpatialLayer() + { + MS_TRACE(); + + RequestKeyFrame(); + } + + void PipeProducerStreamManager::RequestKeyFrameForCurrentSpatialLayer() + { + MS_TRACE(); + + RequestKeyFrame(); + } + + void PipeProducerStreamManager::UpdateTargetLayers(int16_t /*spatial*/, int16_t /*temporal*/) + { + MS_TRACE(); + + // No layer management for pipe. + } + + bool PipeProducerStreamManager::RecalculateTargetLayers( + RTC::ConsumerTypes::VideoLayers& /*newTargetLayers*/) const + { + MS_TRACE(); + + // No layer changes for pipe. + return false; + } + + void PipeProducerStreamManager::OnTransportConnected() + { + MS_TRACE(); + + for (auto& kv : this->mapMappedSsrcSyncRequired) + { + kv.second = true; + } + + if (IsActive()) + { + RequestKeyFrame(); + } + } + + void PipeProducerStreamManager::OnTransportDisconnected() + { + MS_TRACE(); + + for (auto& kv : this->mapMappedSsrcSyncRequired) + { + kv.second = false; + } + } + + void PipeProducerStreamManager::OnPaused() + { + MS_TRACE(); + + for (auto& kv : this->mapMappedSsrcSyncRequired) + { + kv.second = false; + } + } + + void PipeProducerStreamManager::OnResumed() + { + MS_TRACE(); + + for (auto& kv : this->mapMappedSsrcSyncRequired) + { + kv.second = true; + } + + if (IsActive()) + { + RequestKeyFrame(); + } + } + +} // namespace RTC diff --git a/worker/src/RTC/SimpleConsumer.cpp b/worker/src/RTC/SimpleConsumer.cpp index e789e93dcc..184476afb2 100644 --- a/worker/src/RTC/SimpleConsumer.cpp +++ b/worker/src/RTC/SimpleConsumer.cpp @@ -24,9 +24,10 @@ namespace RTC SharedInterface* shared, const std::string& id, const std::string& producerId, - RTC::Consumer::Listener* listener, + RTC::OldConsumer::Listener* listener, const FBS::Transport::ConsumeRequest* data) - : RTC::Consumer::Consumer(shared, id, producerId, listener, data, RTC::RtpParameters::Type::SIMPLE) + : RTC::OldConsumer::OldConsumer( + shared, id, producerId, listener, data, RTC::RtpParameters::Type::SIMPLE) { MS_TRACE(); @@ -90,7 +91,7 @@ namespace RTC MS_TRACE(); // Call the parent method. - auto base = RTC::Consumer::FillBuffer(builder); + auto base = RTC::OldConsumer::FillBuffer(builder); // Add rtpStream. std::vector> rtpStreams; rtpStreams.emplace_back(this->rtpStream->FillBuffer(builder)); @@ -179,7 +180,7 @@ namespace RTC default: { // Pass it to the parent class. - RTC::Consumer::HandleRequest(request); + RTC::OldConsumer::HandleRequest(request); } } } diff --git a/worker/src/RTC/SimpleProducerStreamManager.cpp b/worker/src/RTC/SimpleProducerStreamManager.cpp new file mode 100644 index 0000000000..6f02bd97dc --- /dev/null +++ b/worker/src/RTC/SimpleProducerStreamManager.cpp @@ -0,0 +1,343 @@ +#define MS_CLASS "RTC::SimpleProducerStreamManager" +// #define MS_LOG_DEV_LEVEL 3 + +#include "RTC/SimpleProducerStreamManager.hpp" +#include "Logger.hpp" + +namespace RTC +{ + /* Instance methods. */ + + SimpleProducerStreamManager::SimpleProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared) + : ProducerStreamManager( + consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + kind, + keyFrameSupported, + listener, + shared) + { + MS_TRACE(); + } + + RTC::RTP::RtpStreamRecv* SimpleProducerStreamManager::GetProducerCurrentRtpStream() const + { + MS_TRACE(); + + return this->producerRtpStream; + } + + RTC::RTP::RtpStreamRecv* SimpleProducerStreamManager::GetProducerTargetRtpStream() const + { + MS_TRACE(); + + return this->producerRtpStream; + } + + bool SimpleProducerStreamManager::IsActive() const + { + MS_TRACE(); + + // clang-format off + return ( + this->listener->IsActive() && + this->producerRtpStream && + // If there is no RTP inactivity check do not consider the stream + // inactive despite it has score 0. + (this->producerRtpStream->GetScore() > 0u || !this->producerRtpStream->HasRtpInactivityCheckEnabled()) + ); + // clang-format on + } + + void SimpleProducerStreamManager::ProducerRtpStream( + RTC::RTP::RtpStreamRecv* rtpStream, uint32_t /*mappedSsrc*/) + { + MS_TRACE(); + + this->producerRtpStream = rtpStream; + } + + void SimpleProducerStreamManager::ProducerNewRtpStream( + RTC::RTP::RtpStreamRecv* rtpStream, uint32_t /*mappedSsrc*/) + { + MS_TRACE(); + + this->producerRtpStream = rtpStream; + + // Emit the score event. + this->listener->OnProducerStreamManagerScore(); + } + + void SimpleProducerStreamManager::ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) + { + MS_TRACE(); + } + + void SimpleProducerStreamManager::ProducerRtcpSenderReport( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, bool /*first*/) + { + MS_TRACE(); + + // Do nothing. + } + + uint32_t SimpleProducerStreamManager::IncreaseLayer( + uint32_t bitrate, bool /*considerLoss*/, float /*lossPercentage*/, uint64_t nowMs) + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + MS_ASSERT(this->kind == RTC::Media::Kind::VIDEO, "should be video"); + MS_ASSERT(IsActive(), "should be active"); + + // If this is not the first time this method is called within the same + // iteration, return 0 since a video Simple consumer does not keep state + // about this. + if (this->managingBitrate) + { + return 0u; + } + + this->managingBitrate = true; + + if (!this->producerRtpStream) + { + return 0u; + } + + // Video Simple consumer does not really play the BWE game. However, let's + // be honest and try to be nice. + auto desiredBitrate = this->producerRtpStream->GetBitrate(nowMs); + + if (desiredBitrate < bitrate) + { + return desiredBitrate; + } + else + { + return bitrate; + } + } + + void SimpleProducerStreamManager::ApplyLayers(uint64_t /*rtpStreamActiveMs*/) + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + MS_ASSERT(this->kind == RTC::Media::Kind::VIDEO, "should be video"); + MS_ASSERT(IsActive(), "should be active"); + + this->managingBitrate = false; + + // Simple does not play the BWE game (even if video kind). + } + + uint32_t SimpleProducerStreamManager::GetDesiredBitrate(uint64_t nowMs) const + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + + // Audio does not play the BWE game. + if (this->kind != RTC::Media::Kind::VIDEO) + { + return 0u; + } + + if (!IsActive()) + { + return 0u; + } + + return this->producerRtpStream->GetBitrate(nowMs); + } + + ProducerStreamManager::RtpPacketProcessResult SimpleProducerStreamManager::ProcessRtpPacket( + RTC::RTP::Packet* packet, + bool /*lastSentPacketHasMarker*/, + uint32_t /*clockRate*/, + uint32_t /*maxPacketTs*/) + { + MS_TRACE(); + + RtpPacketProcessResult result; + + // If we need to sync, support key frames and this is not a key frame, + // ignore the packet. + if (this->syncRequired && this->keyFrameSupported && !packet->IsKeyFrame()) + { + // NOTE: No need to drop the packet in the RTP sequence manager since here + // we are blocking all packets but the key frame that would trigger sync + // below. + + // Store the packet for the scenario in which this packet is part of the + // key frame and it arrived before the first packet of the key frame. + result.type = RtpPacketProcessResult::Type::BUFFER; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); +#endif + + return result; + } + + // Packets with only padding are not forwarded. + if (packet->GetPayloadLength() == 0) + { + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); +#endif + + return result; + } + + // Preserve the original marker bit. ProcessPayload may update it. + bool marker{ packet->HasMarker() }; + + // Process the payload if needed. Drop packet if necessary. + if (this->encodingContext && !packet->ProcessPayload(this->encodingContext.get(), marker)) + { + MS_DEBUG_DEV( + "discarding packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::DROPPED_BY_CODEC); +#endif + + return result; + } + + // Whether this is the first packet after re-sync. + const bool isSyncPacket = this->syncRequired; + + // Whether packets stored in the target layer retransmission buffer must be + // sent once this packet is sent. + bool sendBufferedPackets{ false }; + + // Sync sequence number and timestamp if required. + if (isSyncPacket) + { + if (packet->IsKeyFrame()) + { + MS_DEBUG_TAG( + rtp, + "sync key frame received [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + sendBufferedPackets = true; + } + + result.isSyncPacket = true; + result.syncSeqValue = packet->GetSequenceNumber() - 1; + + this->syncRequired = false; + } + + // Set forward action. + result.type = RtpPacketProcessResult::Type::FORWARD; + result.tsOffset = 0u; + result.marker = marker; + result.sendBufferedPackets = sendBufferedPackets; + + return result; + } + + void SimpleProducerStreamManager::RequestKeyFrame() + { + MS_TRACE(); + + if (this->kind != RTC::Media::Kind::VIDEO) + { + return; + } + + auto mappedSsrc = this->consumableRtpEncodings[0].ssrc; + + this->listener->OnProducerStreamManagerKeyFrameRequested(mappedSsrc); + } + + void SimpleProducerStreamManager::RequestKeyFrameForTargetSpatialLayer() + { + MS_TRACE(); + + RequestKeyFrame(); + } + + void SimpleProducerStreamManager::RequestKeyFrameForCurrentSpatialLayer() + { + MS_TRACE(); + + RequestKeyFrame(); + } + + void SimpleProducerStreamManager::UpdateTargetLayers(int16_t /*spatial*/, int16_t /*temporal*/) + { + MS_TRACE(); + } + + bool SimpleProducerStreamManager::RecalculateTargetLayers( + RTC::ConsumerTypes::VideoLayers& /*newTargetLayers*/) const + { + MS_TRACE(); + + // No layer changes possible for Simple. + return false; + } + + void SimpleProducerStreamManager::OnTransportConnected() + { + MS_TRACE(); + + this->syncRequired = true; + + if (IsActive()) + { + RequestKeyFrame(); + } + } + + void SimpleProducerStreamManager::OnTransportDisconnected() + { + MS_TRACE(); + + // Nothing specific for Simple. + } + + void SimpleProducerStreamManager::OnPaused() + { + MS_TRACE(); + + // Nothing specific for Simple. + } + + void SimpleProducerStreamManager::OnResumed() + { + MS_TRACE(); + + this->syncRequired = true; + + if (IsActive()) + { + RequestKeyFrame(); + } + } + +} // namespace RTC diff --git a/worker/src/RTC/SimulcastConsumer.cpp b/worker/src/RTC/SimulcastConsumer.cpp index 000a1b1c8e..672cd0f392 100644 --- a/worker/src/RTC/SimulcastConsumer.cpp +++ b/worker/src/RTC/SimulcastConsumer.cpp @@ -27,9 +27,9 @@ namespace RTC SharedInterface* shared, const std::string& id, const std::string& producerId, - RTC::Consumer::Listener* listener, + RTC::OldConsumer::Listener* listener, const FBS::Transport::ConsumeRequest* data) - : RTC::Consumer::Consumer( + : RTC::OldConsumer::OldConsumer( shared, id, producerId, listener, data, RTC::RtpParameters::Type::SIMULCAST) { MS_TRACE(); @@ -147,7 +147,7 @@ namespace RTC MS_TRACE(); // Call the parent method. - auto base = RTC::Consumer::FillBuffer(builder); + auto base = RTC::OldConsumer::FillBuffer(builder); // Add rtpStream. std::vector> rtpStreams; rtpStreams.emplace_back(this->rtpStream->FillBuffer(builder)); @@ -298,7 +298,7 @@ namespace RTC default: { // Pass it to the parent class. - RTC::Consumer::HandleRequest(request); + RTC::OldConsumer::HandleRequest(request); } } } @@ -345,7 +345,7 @@ namespace RTC // Emit the score event. EmitScore(); - if (RTC::Consumer::IsActive()) + if (RTC::OldConsumer::IsActive()) { // All Producer streams are dead. if (!IsActive()) diff --git a/worker/src/RTC/SimulcastProducerStreamManager.cpp b/worker/src/RTC/SimulcastProducerStreamManager.cpp new file mode 100644 index 0000000000..e0b9f2d376 --- /dev/null +++ b/worker/src/RTC/SimulcastProducerStreamManager.cpp @@ -0,0 +1,1069 @@ +#define MS_CLASS "RTC::SimulcastProducerStreamManager" +// #define MS_LOG_DEV_LEVEL 3 + +#include "RTC/SimulcastProducerStreamManager.hpp" +#include "Logger.hpp" +#include + +namespace RTC +{ + /* Static. */ + + static constexpr uint64_t StreamMinActiveMs{ 2000u }; + static constexpr uint64_t BweDowngradeConservativeMs{ 10000u }; + static constexpr uint64_t BweDowngradeMinActiveMs{ 8000u }; + static constexpr uint16_t MaxSequenceNumberGap{ 100u }; + + /* Instance methods. */ + + SimulcastProducerStreamManager::SimulcastProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared) + : ProducerStreamManager( + consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + kind, + keyFrameSupported, + listener, + shared) + { + MS_TRACE(); + + // Build mapMappedSsrcSpatialLayer and reserve producerRtpStreams. + for (size_t idx{ 0u }; idx < this->consumableRtpEncodings.size(); ++idx) + { + auto& encoding = this->consumableRtpEncodings[idx]; + + this->mapMappedSsrcSpatialLayer[encoding.ssrc] = static_cast(idx); + } + + this->producerRtpStreams.resize(this->consumableRtpEncodings.size(), nullptr); + } + + RTC::RTP::RtpStreamRecv* SimulcastProducerStreamManager::GetProducerCurrentRtpStream() const + { + MS_TRACE(); + + if (this->currentSpatialLayer == -1) + { + return nullptr; + } + + // This may return nullptr. + return this->producerRtpStreams.at(this->currentSpatialLayer); + } + + RTC::RTP::RtpStreamRecv* SimulcastProducerStreamManager::GetProducerTargetRtpStream() const + { + MS_TRACE(); + + if (this->targetLayers.spatial == -1) + { + return nullptr; + } + + // This may return nullptr. + return this->producerRtpStreams.at(this->targetLayers.spatial); + } + + bool SimulcastProducerStreamManager::IsActive() const + { + MS_TRACE(); + + // clang-format off + return ( + this->listener->IsActive() && + std::ranges::any_of( + this->producerRtpStreams, + [](const RTC::RTP::RtpStreamRecv* rtpStream) + { + return ( + rtpStream != nullptr && + (rtpStream->GetScore() > 0u || !rtpStream->HasRtpInactivityCheckEnabled()) + ); + } + ) + ); + // clang-format on + } + + void SimulcastProducerStreamManager::ProducerRtpStream( + RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) + { + MS_TRACE(); + + auto it = this->mapMappedSsrcSpatialLayer.find(mappedSsrc); + + MS_ASSERT(it != this->mapMappedSsrcSpatialLayer.end(), "unknown mappedSsrc"); + + const int16_t spatialLayer = it->second; + + this->producerRtpStreams[spatialLayer] = rtpStream; + } + + void SimulcastProducerStreamManager::ProducerNewRtpStream( + RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) + { + MS_TRACE(); + + auto it = this->mapMappedSsrcSpatialLayer.find(mappedSsrc); + + MS_ASSERT(it != this->mapMappedSsrcSpatialLayer.end(), "unknown mappedSsrc"); + + const int16_t spatialLayer = it->second; + + this->producerRtpStreams[spatialLayer] = rtpStream; + + // Emit the score event. + this->listener->OnProducerStreamManagerScore(); + + if (IsActive()) + { + MayChangeLayers(/*force*/ false); + + // If this stream is the target spatial layer and we are still waiting + // for a key frame (i.e. the previous RequestKeyFrameForTargetSpatialLayer + // call was a no-op because the stream wasn't set yet), request it now. + if ( + spatialLayer == this->targetLayers.spatial && + this->currentSpatialLayer != this->targetLayers.spatial) + { + RequestKeyFrameForTargetSpatialLayer(); + } + } + } + + void SimulcastProducerStreamManager::ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t score, uint8_t previousScore) + { + MS_TRACE(); + + if (this->listener->IsActive()) + { + // All Producer streams are dead. + if (!IsActive()) + { + UpdateTargetLayers(-1, -1); + } + // Just check target layers if the stream has died or reborned or if + // bitrate is not externally managed. + else if (!this->externallyManagedBitrate || (score == 0u || previousScore == 0u)) + { + MayChangeLayers(/*force*/ false); + } + } + } + + void SimulcastProducerStreamManager::ProducerRtcpSenderReport( + RTC::RTP::RtpStreamRecv* rtpStream, bool first) + { + MS_TRACE(); + + // Just interested if this is the first Sender Report for a RTP stream. + if (!first) + { + return; + } + + MS_DEBUG_TAG(simulcast, "first SenderReport [ssrc:%" PRIu32 "]", rtpStream->GetSsrc()); + + // If our RTP timestamp reference stream does not yet have SR, do nothing + // since we know we won't be able to switch. + auto* producerTsReferenceRtpStream = GetProducerTsReferenceRtpStream(); + + if (!producerTsReferenceRtpStream || !producerTsReferenceRtpStream->GetSenderReportNtpMs()) + { + return; + } + + if (IsActive()) + { + MayChangeLayers(/*force*/ false); + } + } + + uint32_t SimulcastProducerStreamManager::IncreaseLayer( + uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs) + { + MS_TRACE(); + + // If already in the preferred layers, do nothing. + if (this->provisionalTargetLayers == this->preferredLayers) + { + return 0u; + } + + uint32_t virtualBitrate; + + if (considerLoss) + { + // Calculate virtual available bitrate based on given bitrate and our + // packet lost. + if (lossPercentage < 2) + { + virtualBitrate = 1.08 * bitrate; + } + else if (lossPercentage > 10) + { + virtualBitrate = (1 - 0.5 * (lossPercentage / 100)) * bitrate; + } + else + { + virtualBitrate = bitrate; + } + } + else + { + virtualBitrate = bitrate; + } + + uint32_t requiredBitrate{ 0u }; + int16_t spatialLayer{ 0 }; + int16_t temporalLayer{ 0 }; + + for (size_t sIdx{ 0u }; sIdx < this->producerRtpStreams.size(); ++sIdx) + { + spatialLayer = static_cast(sIdx); + + // If this is higher than current spatial layer and we moved to current + // spatial layer due to BWE limitations, check how much it has elapsed + // since then. + if (nowMs - this->lastBweDowngradeAtMs < BweDowngradeConservativeMs) + { + if (this->provisionalTargetLayers.spatial > -1 && spatialLayer > this->currentSpatialLayer) + { + MS_DEBUG_DEV( + "avoid upgrading to spatial layer %" PRIi16 " due to recent BWE downgrade", spatialLayer); + + goto done; + } + } + + // Ignore spatial layers lower than the one we already have. + if (spatialLayer < this->provisionalTargetLayers.spatial) + { + continue; + } + // If this is higher than preferred spatial layer, abort. + else if (spatialLayer > this->preferredLayers.spatial) + { + MS_DEBUG_DEV( + "avoid upgrading to spatial layer %" PRIi16 + " since it's higher than preferred spatial layer %" PRIi16, + spatialLayer, + this->preferredLayers.spatial); + + goto done; + } + + // This can be null. + auto* producerRtpStream = this->producerRtpStreams.at(spatialLayer); + + // Producer stream does not exist. Ignore. + if (!producerRtpStream) + { + continue; + } + + // Ignore spatial layers (streams) with score 0. + if (producerRtpStream->GetScore() == 0) + { + continue; + } + + // If the stream has not been active time enough and we have an active one + // already, move to the next spatial layer. + if ( + spatialLayer != this->provisionalTargetLayers.spatial && + this->provisionalTargetLayers.spatial != -1 && + producerRtpStream->GetActiveMs() < StreamMinActiveMs) + { + const auto* provisionalProducerRtpStream = + this->producerRtpStreams.at(this->provisionalTargetLayers.spatial); + + // The stream for the current provisional spatial layer has been active + // for enough time, move to the next spatial layer. + if (provisionalProducerRtpStream->GetActiveMs() >= StreamMinActiveMs) + { + continue; + } + } + + // We may not yet switch to this spatial layer. + if (!CanSwitchToSpatialLayer(spatialLayer)) + { + continue; + } + + temporalLayer = 0; + + // Check bitrate of every temporal layer. + for (; std::cmp_less(temporalLayer, producerRtpStream->GetTemporalLayers()); ++temporalLayer) + { + // Ignore temporal layers lower than the one we already have (taking + // into account the spatial layer too). + if ( + spatialLayer == this->provisionalTargetLayers.spatial && + temporalLayer <= this->provisionalTargetLayers.temporal) + { + continue; + } + + requiredBitrate = producerRtpStream->GetLayerBitrate(nowMs, 0, temporalLayer); + + // This is simulcast so we must subtract the bitrate of the current + // temporal spatial layer if this is the temporal layer 0 of a higher + // spatial layer. + if ( + requiredBitrate && temporalLayer == 0 && this->provisionalTargetLayers.spatial > -1 && + spatialLayer > this->provisionalTargetLayers.spatial) + { + auto* provisionalProducerRtpStream = + this->producerRtpStreams.at(this->provisionalTargetLayers.spatial); + auto provisionalRequiredBitrate = provisionalProducerRtpStream->GetBitrate( + nowMs, 0, this->provisionalTargetLayers.temporal); + + if (requiredBitrate > provisionalRequiredBitrate) + { + requiredBitrate -= provisionalRequiredBitrate; + } + else + { + requiredBitrate = 1u; // Don't set 0 since it would be ignored. + } + } + + MS_DEBUG_DEV( + "testing layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32 + ", required bitrate:%" PRIu32 "]", + spatialLayer, + temporalLayer, + virtualBitrate, + requiredBitrate); + + // If active layer, end iterations here. Otherwise move to next spatial + // layer. + if (requiredBitrate) + { + goto done; + } + else + { + break; + } + } + + // If this is the preferred spatial layer or higher, take it and exit. + if (spatialLayer >= this->preferredLayers.spatial) + { + break; + } + } + + done: + + // No higher active layers found. + if (!requiredBitrate) + { + return 0u; + } + + // No luck. + if (requiredBitrate > virtualBitrate) + { + return 0u; + } + + // Set provisional layers. + this->provisionalTargetLayers.spatial = spatialLayer; + this->provisionalTargetLayers.temporal = temporalLayer; + + MS_DEBUG_DEV( + "setting provisional layers to %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32 + ", required bitrate:%" PRIu32 "]", + this->provisionalTargetLayers.spatial, + this->provisionalTargetLayers.temporal, + virtualBitrate, + requiredBitrate); + + if (requiredBitrate <= bitrate) + { + return requiredBitrate; + } + else if (requiredBitrate <= virtualBitrate) + { + return bitrate; + } + else + { + return requiredBitrate; // NOTE: This cannot happen. + } + } + + void SimulcastProducerStreamManager::ApplyLayers(uint64_t rtpStreamActiveMs) + { + MS_TRACE(); + + auto provisionalTargetLayers = this->provisionalTargetLayers; + + // Reset provisional target layers. + this->provisionalTargetLayers.Reset(); + + if (provisionalTargetLayers != this->targetLayers) + { + UpdateTargetLayers(provisionalTargetLayers.spatial, provisionalTargetLayers.temporal); + + // If this looks like a spatial layer downgrade due to BWE limitations, set + // member. + if ( + rtpStreamActiveMs > BweDowngradeMinActiveMs && + this->targetLayers.spatial < this->currentSpatialLayer && + this->currentSpatialLayer <= this->preferredLayers.spatial) + { + MS_DEBUG_DEV( + "possible target spatial layer downgrade (from %" PRIi16 " to %" PRIi16 + ") due to BWE limitation", + this->currentSpatialLayer, + this->targetLayers.spatial); + + this->lastBweDowngradeAtMs = this->shared->GetTimeMs(); + } + } + } + + uint32_t SimulcastProducerStreamManager::GetDesiredBitrate(uint64_t nowMs) const + { + MS_TRACE(); + + uint32_t desiredBitrate{ 0u }; + + // Let's iterate all streams of the Producer (from highest to lowest) and + // obtain their bitrate. Choose the highest one. + // NOTE: When the Producer enables a higher stream, initially the bitrate of + // it could be less than the bitrate of a lower stream. That's why we + // iterate all streams here anyway. + for (auto sIdx{ static_cast(this->producerRtpStreams.size() - 1) }; sIdx >= 0; --sIdx) + { + auto* producerRtpStream = this->producerRtpStreams.at(sIdx); + + if (!producerRtpStream) + { + continue; + } + + auto streamBitrate = producerRtpStream->GetBitrate(nowMs); + + desiredBitrate = std::max(streamBitrate, desiredBitrate); + } + + return desiredBitrate; + } + + ProducerStreamManager::RtpPacketProcessResult SimulcastProducerStreamManager::ProcessRtpPacket( + RTC::RTP::Packet* packet, bool lastSentPacketHasMarker, uint32_t clockRate, uint32_t maxPacketTs) + { + MS_TRACE(); + + RtpPacketProcessResult result; + + auto spatialLayer = this->mapMappedSsrcSpatialLayer.at(packet->GetSsrc()); + + // Check target layers validity. + if (this->targetLayers.temporal == -1) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::INVALID_TARGET_LAYER); +#endif + + if (spatialLayer == this->currentSpatialLayer) + { + result.type = RtpPacketProcessResult::Type::DROP; + } + else + { + result.type = RtpPacketProcessResult::Type::SILENT_DROP; + } + + return result; + } + + bool shouldSwitchCurrentSpatialLayer{ false }; + + // Check whether this is the packet we are waiting for in order to update + // the current spatial layer. + if ( + this->currentSpatialLayer != this->targetLayers.spatial && + spatialLayer == this->targetLayers.spatial) + { + // Ignore if not a key frame. + if (this->keyFrameSupported && !packet->IsKeyFrame()) + { + result.type = RtpPacketProcessResult::Type::BUFFER; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); +#endif + + return result; + } + + shouldSwitchCurrentSpatialLayer = true; + + // Need to resync the stream. + this->syncRequired = true; + this->spatialLayerToSync = spatialLayer; + } + // If the packet belongs to different spatial layer than the one being sent, + // drop it. + else if (spatialLayer != this->currentSpatialLayer) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::SPATIAL_LAYER_MISMATCH); +#endif + result.type = RtpPacketProcessResult::Type::SILENT_DROP; + + return result; + } + + // If we need to sync and this is not a key frame, ignore the packet. + if (this->syncRequired && this->keyFrameSupported && !packet->IsKeyFrame()) + { + result.type = RtpPacketProcessResult::Type::BUFFER; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); +#endif + + return result; + } + + // Packets with only padding are not forwarded. + if (packet->GetPayloadLength() == 0) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); +#endif + if (spatialLayer == this->currentSpatialLayer) + { + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); +#endif + } + else + { + result.type = RtpPacketProcessResult::Type::SILENT_DROP; + } + + return result; + } + + // Whether this is the first packet after re-sync. + const bool isSyncPacket = this->syncRequired; + + // Sync sequence number and timestamp if required. + if (isSyncPacket && (this->spatialLayerToSync == -1 || spatialLayer == this->spatialLayerToSync)) + { + if (packet->IsKeyFrame()) + { + MS_DEBUG_TAG( + rtp, + "sync key frame received [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + result.sendBufferedPackets = true; + } + + uint32_t tsOffset{ 0u }; + + // Sync our RTP stream's RTP timestamp. + if (spatialLayer == this->tsReferenceSpatialLayer) + { + tsOffset = 0u; + } + // If this is not the RTP stream we use as TS reference, do NTP based RTP + // TS synchronization. + else + { + auto* producerTsReferenceRtpStream = GetProducerTsReferenceRtpStream(); + auto* producerTargetRtpStream = GetProducerTargetRtpStream(); + + // NOTE: If we are here is because we have Sender Reports for both the + // TS reference stream and the target one. + MS_ASSERT( + producerTsReferenceRtpStream->GetSenderReportNtpMs(), + "no Sender Report for TS reference RTP stream"); + MS_ASSERT( + producerTargetRtpStream->GetSenderReportNtpMs(), "no Sender Report for current RTP stream"); + + // Calculate NTP and TS stuff. + auto ntpMs1 = producerTsReferenceRtpStream->GetSenderReportNtpMs(); + auto ts1 = producerTsReferenceRtpStream->GetSenderReportTs(); + auto ntpMs2 = producerTargetRtpStream->GetSenderReportNtpMs(); + auto ts2 = producerTargetRtpStream->GetSenderReportTs(); + int64_t diffMs; + + if (ntpMs2 >= ntpMs1) + { + diffMs = ntpMs2 - ntpMs1; + } + else + { + diffMs = -1 * (ntpMs1 - ntpMs2); + } + + const int64_t diffTs = diffMs * clockRate / 1000; + const uint32_t newTs2 = ts2 - diffTs; + + // Apply offset. This is the difference that later must be removed from + // the sending RTP packet. + tsOffset = newTs2 - ts1; + } + + // When switching to a new stream it may happen that the timestamp of this + // key frame is lower than the highest timestamp sent to the remote endpoint. + // If so, apply an extra offset to "fix" it for the whole life of this + // selected Producer stream. + if (shouldSwitchCurrentSpatialLayer && (packet->GetTimestamp() - tsOffset <= maxPacketTs)) + { + // Max delay in ms we allow for the stream when switching. + // https://en.wikipedia.org/wiki/Audio-to-video_synchronization#Recommendations + static const uint32_t MaxExtraOffsetMs{ 75u }; + + // Outgoing packet matches the highest timestamp seen in the previous + // stream. Apply an expected offset for a new frame in a 30fps stream. + static const uint8_t MsOffset{ 33u }; // (1 / 30 * 1000). + + const int64_t maxTsExtraOffset = MaxExtraOffsetMs * clockRate / 1000; + uint32_t tsExtraOffset = + maxPacketTs - packet->GetTimestamp() + tsOffset + (MsOffset * clockRate / 1000); + + // NOTE: Don't ask for a key frame if already done. + if (this->keyFrameForTsOffsetRequested) + { + // Give up and use the theoretical offset. + if (std::cmp_greater(tsExtraOffset, maxTsExtraOffset)) + { + MS_WARN_TAG( + simulcast, + "giving up on proper stream switching after got a requested keyframe for " + "which still too high RTP timestamp extra offset is needed (%" PRIu32 ")", + tsExtraOffset); + + tsExtraOffset = 1u; + } + } + else if (std::cmp_greater(tsExtraOffset, maxTsExtraOffset)) + { + MS_WARN_TAG( + simulcast, + "cannot switch stream due to too high RTP timestamp extra offset needed " + "(%" PRIu32 "), requesting keyframe", + tsExtraOffset); + + RequestKeyFrameForTargetSpatialLayer(); + + this->keyFrameForTsOffsetRequested = true; + + // Reset flags since we are discarding this key frame. + this->syncRequired = false; + this->spatialLayerToSync = -1; + + result.type = RtpPacketProcessResult::Type::SILENT_DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded( + RTC::RtcLogger::RtpPacket::DiscardReason::TOO_HIGH_TIMESTAMP_EXTRA_NEEDED); +#endif + + return result; + } + + if (tsExtraOffset > 0u) + { + MS_DEBUG_TAG( + simulcast, + "RTP timestamp extra offset generated for stream switching: %" PRIu32, + tsExtraOffset); + + // Increase the timestamp offset for the whole life of this Producer + // stream (until switched to a different one). + tsOffset -= tsExtraOffset; + } + } + + this->tsOffset = tsOffset; + + // Sync our RTP stream's sequence number. + // If previous frame has not been sent completely when we switch layer, + // we can tell libwebrtc that previous frame is incomplete by skipping + // one RTP sequence number. + // https://github.com/versatica/mediasoup/issues/408 + result.isSyncPacket = true; + result.syncSeqValue = packet->GetSequenceNumber() - (lastSentPacketHasMarker ? 1 : 2); + result.shouldSyncEncodingContext = true; + + // Sync the encoding context now, before ProcessPayload runs below. + // This must happen before ProcessPayload so the codec handler resets + // its state before processing the first (key frame) packet. + if (this->encodingContext) + { + this->encodingContext->SyncRequired(); + } + + this->syncRequired = false; + this->spatialLayerToSync = -1; + this->keyFrameForTsOffsetRequested = false; + } + + // Old-packet filtering after spatial layer switch. + if (!shouldSwitchCurrentSpatialLayer && this->checkingForOldPacketsInSpatialLayer) + { + if (SeqManager::IsSeqLowerThan(packet->GetSequenceNumber(), this->snReferenceSpatialLayer)) + { + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded( + RTC::RtcLogger::RtpPacket::DiscardReason::PACKET_PREVIOUS_TO_SPATIAL_LAYER_SWITCH); +#endif + + return result; + } + else if ( + SeqManager::IsSeqHigherThan( + packet->GetSequenceNumber(), this->snReferenceSpatialLayer + MaxSequenceNumberGap)) + { + this->checkingForOldPacketsInSpatialLayer = false; + } + } + + // Preserve the original marker bit. ProcessPayload may update it. + bool marker{ packet->HasMarker() }; + + if (shouldSwitchCurrentSpatialLayer) + { + // Update current spatial layer. + this->currentSpatialLayer = this->targetLayers.spatial; + + this->snReferenceSpatialLayer = packet->GetSequenceNumber(); + this->checkingForOldPacketsInSpatialLayer = true; + + // Update target and current temporal layer. + this->encodingContext->SetTargetTemporalLayer(this->targetLayers.temporal); + this->encodingContext->SetCurrentTemporalLayer(packet->GetTemporalLayer()); + + // Rewrite payload if needed. + packet->ProcessPayload(this->encodingContext.get(), marker); + + result.spatialLayerSwitched = true; + } + else + { + auto previousTemporalLayer = this->encodingContext->GetCurrentTemporalLayer(); + + // Rewrite payload if needed. Drop packet if necessary. + if (!packet->ProcessPayload(this->encodingContext.get(), marker)) + { + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::DROPPED_BY_CODEC); +#endif + + return result; + } + + if (previousTemporalLayer != this->encodingContext->GetCurrentTemporalLayer()) + { + result.temporalLayerChanged = true; + } + } + + // Set forward action. + result.type = RtpPacketProcessResult::Type::FORWARD; + result.tsOffset = this->tsOffset; + result.marker = marker; + + return result; + } + + void SimulcastProducerStreamManager::RequestKeyFrame() + { + MS_TRACE(); + + auto* producerTargetRtpStream = GetProducerTargetRtpStream(); + auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); + + if (producerTargetRtpStream) + { + auto mappedSsrc = this->consumableRtpEncodings[this->targetLayers.spatial].ssrc; + + this->listener->OnProducerStreamManagerKeyFrameRequested(mappedSsrc); + } + + if (producerCurrentRtpStream && producerCurrentRtpStream != producerTargetRtpStream) + { + auto mappedSsrc = this->consumableRtpEncodings[this->currentSpatialLayer].ssrc; + + this->listener->OnProducerStreamManagerKeyFrameRequested(mappedSsrc); + } + } + + void SimulcastProducerStreamManager::RequestKeyFrameForTargetSpatialLayer() + { + MS_TRACE(); + + auto* producerTargetRtpStream = GetProducerTargetRtpStream(); + + if (!producerTargetRtpStream) + { + return; + } + + auto mappedSsrc = this->consumableRtpEncodings[this->targetLayers.spatial].ssrc; + + this->listener->OnProducerStreamManagerKeyFrameRequested(mappedSsrc); + } + + void SimulcastProducerStreamManager::RequestKeyFrameForCurrentSpatialLayer() + { + MS_TRACE(); + + auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); + + if (!producerCurrentRtpStream) + { + return; + } + + auto mappedSsrc = this->consumableRtpEncodings[this->currentSpatialLayer].ssrc; + + this->listener->OnProducerStreamManagerKeyFrameRequested(mappedSsrc); + } + + void SimulcastProducerStreamManager::UpdateTargetLayers( + int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer) + { + MS_TRACE(); + + // If we don't have yet a RTP timestamp reference, set it now. + if ( + newTargetSpatialLayer != -1 && (this->tsReferenceSpatialLayer == -1 || + !GetProducerTsReferenceRtpStream()->GetSenderReportNtpMs())) + { + MS_DEBUG_TAG( + simulcast, "using spatial layer %" PRIi16 " as RTP timestamp reference", newTargetSpatialLayer); + + this->tsReferenceSpatialLayer = newTargetSpatialLayer; + } + + // If the new target spatial layer doesn't match the current one, clear the + // target layer retransmission buffer. + if (newTargetSpatialLayer != this->targetLayers.spatial) + { + this->listener->OnProducerStreamManagerClearRetransmissionBuffer(); + } + + if (newTargetSpatialLayer == -1) + { + // Unset current and target layers. + this->targetLayers.spatial = -1; + this->targetLayers.temporal = -1; + this->currentSpatialLayer = -1; + + this->encodingContext->SetTargetTemporalLayer(-1); + this->encodingContext->SetCurrentTemporalLayer(-1); + + MS_DEBUG_TAG(simulcast, "target layers changed [spatial:-1, temporal:-1]"); + + this->listener->OnProducerStreamManagerLayersChanged(); + + return; + } + + this->targetLayers.spatial = newTargetSpatialLayer; + this->targetLayers.temporal = newTargetTemporalLayer; + + // If the new target spatial layer matches the current one, apply the new + // target temporal layer now. + if (this->targetLayers.spatial == this->currentSpatialLayer) + { + this->encodingContext->SetTargetTemporalLayer(this->targetLayers.temporal); + } + + MS_DEBUG_TAG( + simulcast, + "target layers changed [spatial:%" PRIi16 ", temporal:%" PRIi16 "]", + this->targetLayers.spatial, + this->targetLayers.temporal); + + // If the target spatial layer is different than the current one, request + // a key frame. + if (this->targetLayers.spatial != this->currentSpatialLayer) + { + RequestKeyFrameForTargetSpatialLayer(); + } + } + + bool SimulcastProducerStreamManager::RecalculateTargetLayers( + RTC::ConsumerTypes::VideoLayers& newTargetLayers) const + { + MS_TRACE(); + + // Start with no layers. + newTargetLayers.Reset(); + + auto nowMs = this->shared->GetTimeMs(); + + for (size_t sIdx{ 0u }; sIdx < this->producerRtpStreams.size(); ++sIdx) + { + auto spatialLayer = static_cast(sIdx); + auto* producerRtpStream = this->producerRtpStreams.at(sIdx); + auto producerScore = producerRtpStream ? producerRtpStream->GetScore() : 0u; + + // If this is higher than current spatial layer and we moved to current + // spatial layer due to BWE limitations, check how much it has elapsed + // since then. + if (nowMs - this->lastBweDowngradeAtMs < BweDowngradeConservativeMs) + { + if (newTargetLayers.spatial > -1 && spatialLayer > this->currentSpatialLayer) + { + continue; + } + } + + // Ignore spatial layers for non existing Producer streams or for those + // with score 0. + if (producerScore == 0u) + { + continue; + } + + // If the stream has not been active time enough and we have an active one + // already, move to the next spatial layer. + // NOTE: Require bitrate externally managed for this. + if (this->externallyManagedBitrate && newTargetLayers.spatial != -1 && producerRtpStream->GetActiveMs() < StreamMinActiveMs) + { + continue; + } + + // We may not yet switch to this spatial layer. + if (!CanSwitchToSpatialLayer(spatialLayer)) + { + continue; + } + + newTargetLayers.spatial = spatialLayer; + + // If this is the preferred or higher spatial layer take it and exit. + if (spatialLayer >= this->preferredLayers.spatial) + { + break; + } + } + + if (newTargetLayers.spatial != -1) + { + if (newTargetLayers.spatial == this->preferredLayers.spatial) + { + newTargetLayers.temporal = this->preferredLayers.temporal; + } + else if (newTargetLayers.spatial < this->preferredLayers.spatial) + { + newTargetLayers.temporal = + static_cast(this->encodingContext->GetTemporalLayers() - 1); + } + else + { + newTargetLayers.temporal = 0; + } + } + + // Return true if any target layer changed. + return (newTargetLayers != this->targetLayers); + } + + void SimulcastProducerStreamManager::OnTransportConnected() + { + MS_TRACE(); + + this->syncRequired = true; + this->spatialLayerToSync = -1; + this->keyFrameForTsOffsetRequested = false; + + if (IsActive()) + { + MayChangeLayers(/*force*/ false); + } + } + + void SimulcastProducerStreamManager::OnTransportDisconnected() + { + MS_TRACE(); + + this->lastBweDowngradeAtMs = 0u; + + UpdateTargetLayers(-1, -1); + } + + void SimulcastProducerStreamManager::OnPaused() + { + MS_TRACE(); + + this->lastBweDowngradeAtMs = 0u; + + UpdateTargetLayers(-1, -1); + } + + void SimulcastProducerStreamManager::OnResumed() + { + MS_TRACE(); + + this->syncRequired = true; + this->spatialLayerToSync = -1; + this->keyFrameForTsOffsetRequested = false; + this->checkingForOldPacketsInSpatialLayer = false; + + if (IsActive()) + { + MayChangeLayers(/*force*/ false); + } + } + + bool SimulcastProducerStreamManager::CanSwitchToSpatialLayer(int16_t spatialLayer) const + { + MS_TRACE(); + + MS_ASSERT( + this->producerRtpStreams.at(spatialLayer), + "no Producer RtpStream for the given spatialLayer:%" PRIi16, + spatialLayer); + + return ( + this->tsReferenceSpatialLayer == -1 || spatialLayer == this->tsReferenceSpatialLayer || + this->producerRtpStreams.at(spatialLayer)->GetSenderReportNtpMs()); + } + + RTC::RTP::RtpStreamRecv* SimulcastProducerStreamManager::GetProducerTsReferenceRtpStream() const + { + MS_TRACE(); + + if (this->tsReferenceSpatialLayer == -1) + { + return nullptr; + } + + // This may return nullptr. + return this->producerRtpStreams.at(this->tsReferenceSpatialLayer); + } +} // namespace RTC diff --git a/worker/src/RTC/SvcConsumer.cpp b/worker/src/RTC/SvcConsumer.cpp index c2d62f290d..b233303e14 100644 --- a/worker/src/RTC/SvcConsumer.cpp +++ b/worker/src/RTC/SvcConsumer.cpp @@ -25,9 +25,10 @@ namespace RTC SharedInterface* shared, const std::string& id, const std::string& producerId, - RTC::Consumer::Listener* listener, + RTC::OldConsumer::Listener* listener, const FBS::Transport::ConsumeRequest* data) - : RTC::Consumer::Consumer(shared, id, producerId, listener, data, RTC::RtpParameters::Type::SVC) + : RTC::OldConsumer::OldConsumer( + shared, id, producerId, listener, data, RTC::RtpParameters::Type::SVC) { MS_TRACE(); @@ -128,7 +129,7 @@ namespace RTC MS_TRACE(); // Call the parent method. - auto base = RTC::Consumer::FillBuffer(builder); + auto base = RTC::OldConsumer::FillBuffer(builder); // Add rtpStream. std::vector> rtpStreams; rtpStreams.emplace_back(this->rtpStream->FillBuffer(builder)); @@ -274,7 +275,7 @@ namespace RTC default: { // Pass it to the parent class. - RTC::Consumer::HandleRequest(request); + RTC::OldConsumer::HandleRequest(request); } } } @@ -309,7 +310,7 @@ namespace RTC // Emit score event. EmitScore(); - if (RTC::Consumer::IsActive()) + if (RTC::OldConsumer::IsActive()) { // Just check target layers if the stream has died or reborned. if (!this->externallyManagedBitrate || (score == 0u || previousScore == 0u)) diff --git a/worker/src/RTC/SvcProducerStreamManager.cpp b/worker/src/RTC/SvcProducerStreamManager.cpp new file mode 100644 index 0000000000..074366ba43 --- /dev/null +++ b/worker/src/RTC/SvcProducerStreamManager.cpp @@ -0,0 +1,664 @@ +#define MS_CLASS "RTC::SvcProducerStreamManager" +// #define MS_LOG_DEV_LEVEL 3 + +#include "RTC/SvcProducerStreamManager.hpp" +#include "Logger.hpp" + +namespace RTC +{ + /* Static. */ + + static constexpr uint64_t BweDowngradeConservativeMs{ 10000u }; + static constexpr uint64_t BweDowngradeMinActiveMs{ 8000u }; + + /* Instance methods. */ + + SvcProducerStreamManager::SvcProducerStreamManager( + const std::vector& consumableRtpEncodings, + const RTC::ConsumerTypes::VideoLayers& preferredLayers, + std::unique_ptr encodingContext, + RTC::Media::Kind kind, + bool keyFrameSupported, + Listener* listener, + SharedInterface* shared) + : ProducerStreamManager( + consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + kind, + keyFrameSupported, + listener, + shared) + { + MS_TRACE(); + } + + RTC::RTP::RtpStreamRecv* SvcProducerStreamManager::GetProducerCurrentRtpStream() const + { + MS_TRACE(); + + // SVC has a single producer stream. Return it if we have a current layer. + if (this->encodingContext->GetCurrentSpatialLayer() == -1) + { + return nullptr; + } + + return this->producerRtpStream; + } + + RTC::RTP::RtpStreamRecv* SvcProducerStreamManager::GetProducerTargetRtpStream() const + { + MS_TRACE(); + + // SVC has a single producer stream. Return it if we have a target layer. + if (this->encodingContext->GetTargetSpatialLayer() == -1) + { + return nullptr; + } + + return this->producerRtpStream; + } + + bool SvcProducerStreamManager::IsActive() const + { + MS_TRACE(); + + // clang-format off + return ( + this->listener->IsActive() && + this->producerRtpStream && + // If there is no RTP inactivity check do not consider the stream + // inactive despite it has score 0. + (this->producerRtpStream->GetScore() > 0u || !this->producerRtpStream->HasRtpInactivityCheckEnabled()) + ); + // clang-format on + } + + void SvcProducerStreamManager::ProducerRtpStream( + RTC::RTP::RtpStreamRecv* rtpStream, uint32_t /*mappedSsrc*/) + { + MS_TRACE(); + + this->producerRtpStream = rtpStream; + } + + void SvcProducerStreamManager::ProducerNewRtpStream( + RTC::RTP::RtpStreamRecv* rtpStream, uint32_t /*mappedSsrc*/) + { + MS_TRACE(); + + this->producerRtpStream = rtpStream; + + // Emit the score event. + this->listener->OnProducerStreamManagerScore(); + + if (IsActive()) + { + MayChangeLayers(/*force*/ false); + } + } + + void SvcProducerStreamManager::ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t score, uint8_t previousScore) + { + MS_TRACE(); + + if (this->listener->IsActive()) + { + // Just check target layers if the stream has died or reborned or if + // bitrate is not externally managed. + if (!this->externallyManagedBitrate || (score == 0u || previousScore == 0u)) + { + MayChangeLayers(/*force*/ false); + } + } + } + + void SvcProducerStreamManager::ProducerRtcpSenderReport( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, bool /*first*/) + { + MS_TRACE(); + + // Do nothing. + } + + uint32_t SvcProducerStreamManager::IncreaseLayer( + uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs) + { + MS_TRACE(); + + if (!this->producerRtpStream || this->producerRtpStream->GetScore() == 0u) + { + return 0u; + } + + // If already in the preferred layers, do nothing. + if (this->provisionalTargetLayers == this->preferredLayers) + { + return 0u; + } + + uint32_t virtualBitrate; + + if (considerLoss) + { + // Calculate virtual available bitrate based on given bitrate and our + // packet lost. + if (lossPercentage < 2) + { + virtualBitrate = 1.08 * bitrate; + } + else if (lossPercentage > 10) + { + virtualBitrate = (1 - 0.5 * (lossPercentage / 100)) * bitrate; + } + else + { + virtualBitrate = bitrate; + } + } + else + { + virtualBitrate = bitrate; + } + + uint32_t requiredBitrate{ 0u }; + int16_t spatialLayer{ 0 }; + int16_t temporalLayer{ 0 }; + + for (; std::cmp_less(spatialLayer, this->producerRtpStream->GetSpatialLayers()); ++spatialLayer) + { + // If this is higher than current spatial layer and we moved to current + // spatial layer due to BWE limitations, check how much it has elapsed + // since then. + if (nowMs - this->lastBweDowngradeAtMs < BweDowngradeConservativeMs) + { + if (this->provisionalTargetLayers.spatial > -1 && spatialLayer > this->encodingContext->GetCurrentSpatialLayer()) + { + MS_DEBUG_DEV( + "avoid upgrading to spatial layer %" PRIi16 " due to recent BWE downgrade", spatialLayer); + + goto done; + } + } + + // Ignore spatial layers lower than the one we already have. + if (spatialLayer < this->provisionalTargetLayers.spatial) + { + continue; + } + + temporalLayer = 0; + + // Check bitrate of every temporal layer. + for (; std::cmp_less(temporalLayer, this->producerRtpStream->GetTemporalLayers()); + ++temporalLayer) + { + // Ignore temporal layers lower than the one we already have (taking + // into account the spatial layer too). + if ( + spatialLayer == this->provisionalTargetLayers.spatial && + temporalLayer <= this->provisionalTargetLayers.temporal) + { + continue; + } + + requiredBitrate = + this->producerRtpStream->GetLayerBitrate(nowMs, spatialLayer, temporalLayer); + + // When using K-SVC we must subtract the bitrate of the current used + // layer if the new layer is the temporal layer 0 of a higher spatial + // layer. + if ( + this->encodingContext->IsKSvc() && requiredBitrate && temporalLayer == 0 && + this->provisionalTargetLayers.spatial > -1 && + spatialLayer > this->provisionalTargetLayers.spatial) + { + auto provisionalRequiredBitrate = this->producerRtpStream->GetSpatialLayerBitrate( + nowMs, this->provisionalTargetLayers.spatial); + + if (requiredBitrate > provisionalRequiredBitrate) + { + requiredBitrate -= provisionalRequiredBitrate; + } + else + { + requiredBitrate = 1u; // Don't set 0 since it would be ignored. + } + } + + MS_DEBUG_DEV( + "testing layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32 + ", required bitrate:%" PRIu32 "]", + spatialLayer, + temporalLayer, + virtualBitrate, + requiredBitrate); + + // If active layer, end iterations here. Otherwise move to next spatial + // layer. + if (requiredBitrate) + { + goto done; + } + else + { + break; + } + } + + // If this is the preferred or higher spatial layer, take it and exit. + if (spatialLayer >= this->preferredLayers.spatial) + { + break; + } + } + + done: + + // No higher active layers found. + if (!requiredBitrate) + { + return 0u; + } + + // No luck. + if (requiredBitrate > virtualBitrate) + { + return 0u; + } + + // Set provisional layers. + this->provisionalTargetLayers.spatial = spatialLayer; + this->provisionalTargetLayers.temporal = temporalLayer; + + MS_DEBUG_DEV( + "upgrading to layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32 + ", required bitrate:%" PRIu32 "]", + this->provisionalTargetLayers.spatial, + this->provisionalTargetLayers.temporal, + virtualBitrate, + requiredBitrate); + + if (requiredBitrate <= bitrate) + { + return requiredBitrate; + } + else if (requiredBitrate <= virtualBitrate) + { + return bitrate; + } + else + { + return requiredBitrate; // NOTE: This cannot happen. + } + } + + void SvcProducerStreamManager::ApplyLayers(uint64_t rtpStreamActiveMs) + { + MS_TRACE(); + + auto provisionalTargetLayers = this->provisionalTargetLayers; + + // Reset provisional target layers. + this->provisionalTargetLayers.Reset(); + + if (provisionalTargetLayers != this->encodingContext->GetTargetLayers()) + { + UpdateTargetLayers(provisionalTargetLayers.spatial, provisionalTargetLayers.temporal); + + // If this looks like a spatial layer downgrade due to BWE limitations, + // set member. + if ( + rtpStreamActiveMs > BweDowngradeMinActiveMs && + this->encodingContext->GetTargetSpatialLayer() < + this->encodingContext->GetCurrentSpatialLayer() && + this->encodingContext->GetCurrentSpatialLayer() <= this->preferredLayers.spatial) + { + MS_DEBUG_DEV( + "possible target spatial layer downgrade (from %" PRIi16 " to %" PRIi16 + ") due to BWE limitation", + this->encodingContext->GetCurrentSpatialLayer(), + this->encodingContext->GetTargetSpatialLayer()); + + this->lastBweDowngradeAtMs = this->shared->GetTimeMs(); + } + } + } + + uint32_t SvcProducerStreamManager::GetDesiredBitrate(uint64_t nowMs) const + { + MS_TRACE(); + + if (!this->producerRtpStream) + { + return 0u; + } + + uint32_t desiredBitrate{ 0u }; + + // When using K-SVC each spatial layer is independent of the others. + if (this->encodingContext->IsKSvc()) + { + // Let's iterate all spatial layers of the Producer (from highest to lowest) + // and obtain their bitrate. Choose the highest one. + // NOTE: When the Producer enables a higher spatial layer, initially the + // bitrate of it could be less than the bitrate of a lower one. That's why + // we iterate all spatial layers here anyway. + for (auto spatialLayer{ this->producerRtpStream->GetSpatialLayers() - 1 }; spatialLayer >= 0; + --spatialLayer) + { + auto spatialLayerBitrate = + this->producerRtpStream->GetSpatialLayerBitrate(nowMs, spatialLayer); + + desiredBitrate = std::max(spatialLayerBitrate, desiredBitrate); + } + } + else + { + desiredBitrate = this->producerRtpStream->GetBitrate(nowMs); + } + + return desiredBitrate; + } + + ProducerStreamManager::RtpPacketProcessResult SvcProducerStreamManager::ProcessRtpPacket( + RTC::RTP::Packet* packet, + bool /*lastSentPacketHasMarker*/, + uint32_t /*clockRate*/, + uint32_t /*maxPacketTs*/) + { + MS_TRACE(); + + RtpPacketProcessResult result; + + if (this->encodingContext->GetTargetSpatialLayer() == -1 || this->encodingContext->GetTargetTemporalLayer() == -1) + { + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::INVALID_TARGET_LAYER); +#endif + + return result; + } + + // If we need to sync and this is not a key frame, ignore the packet. + if (this->syncRequired && !packet->IsKeyFrame()) + { + // NOTE: No need to drop the packet in the RTP sequence manager since here + // we are blocking all packets but the key frame that would trigger sync + // below. + + // Store the packet for the scenario in which this packet is part of the + // key frame and it arrived before the first packet of the key frame. + result.type = RtpPacketProcessResult::Type::BUFFER; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); +#endif + + return result; + } + + // Packets with only padding are not forwarded. + if (packet->GetPayloadLength() == 0) + { + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); +#endif + + return result; + } + + // Whether this is the first packet after re-sync. + const bool isSyncPacket = this->syncRequired; + + // Whether packets stored in the target layer retransmission buffer must be + // sent once this packet is sent. + bool sendBufferedPackets{ false }; + + // Sync sequence number and timestamp if required. + if (isSyncPacket) + { + if (packet->IsKeyFrame()) + { + MS_DEBUG_TAG( + rtp, + "sync key frame received [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + sendBufferedPackets = true; + } + + result.isSyncPacket = true; + result.syncSeqValue = packet->GetSequenceNumber() - 1; + result.shouldSyncEncodingContext = true; + + // Sync the encoding context before ProcessPayload runs below. + this->encodingContext->SyncRequired(); + + this->syncRequired = false; + } + + auto previousLayers = this->encodingContext->GetCurrentLayers(); + + bool marker{ false }; + + if (!packet->ProcessPayload(this->encodingContext.get(), marker)) + { + result.type = RtpPacketProcessResult::Type::DROP; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::DROPPED_BY_CODEC); +#endif + + return result; + } + + if (previousLayers != this->encodingContext->GetCurrentLayers()) + { + result.temporalLayerChanged = true; + } + + // Set forward action. + result.type = RtpPacketProcessResult::Type::FORWARD; + result.tsOffset = 0u; + result.marker = marker; + result.sendBufferedPackets = sendBufferedPackets; + + return result; + } + + void SvcProducerStreamManager::RequestKeyFrame() + { + MS_TRACE(); + + auto mappedSsrc = this->consumableRtpEncodings[0].ssrc; + + this->listener->OnProducerStreamManagerKeyFrameRequested(mappedSsrc); + } + + void SvcProducerStreamManager::RequestKeyFrameForTargetSpatialLayer() + { + MS_TRACE(); + + RequestKeyFrame(); + } + + void SvcProducerStreamManager::RequestKeyFrameForCurrentSpatialLayer() + { + MS_TRACE(); + + RequestKeyFrame(); + } + + void SvcProducerStreamManager::UpdateTargetLayers( + int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer) + { + MS_TRACE(); + + if (newTargetSpatialLayer == -1) + { + // Unset current and target layers. + this->encodingContext->SetTargetSpatialLayer(-1); + this->encodingContext->SetCurrentSpatialLayer(-1); + this->encodingContext->SetTargetTemporalLayer(-1); + this->encodingContext->SetCurrentTemporalLayer(-1); + + MS_DEBUG_TAG(svc, "target layers changed [spatial:-1, temporal:-1]"); + + this->listener->OnProducerStreamManagerLayersChanged(); + + return; + } + + this->encodingContext->SetTargetSpatialLayer(newTargetSpatialLayer); + this->encodingContext->SetTargetTemporalLayer(newTargetTemporalLayer); + + MS_DEBUG_TAG( + svc, + "target layers changed [spatial:%" PRIi16 ", temporal:%" PRIi16 "]", + newTargetSpatialLayer, + newTargetTemporalLayer); + + // Target spatial layer has changed. + if (newTargetSpatialLayer != this->encodingContext->GetCurrentSpatialLayer()) + { + // In K-SVC always ask for a keyframe when changing target spatial layer. + if (this->encodingContext->IsKSvc()) + { + MS_DEBUG_DEV("K-SVC: requesting keyframe to target spatial change"); + + RequestKeyFrame(); + } + // In full SVC just ask for a keyframe when upgrading target spatial layer. + // NOTE: This is because nobody implements RTCP LRR yet. + else if (newTargetSpatialLayer > this->encodingContext->GetCurrentSpatialLayer()) + { + MS_DEBUG_DEV("full SVC: requesting keyframe to target spatial upgrade"); + + RequestKeyFrame(); + } + } + } + + bool SvcProducerStreamManager::RecalculateTargetLayers( + RTC::ConsumerTypes::VideoLayers& newTargetLayers) const + { + MS_TRACE(); + + // Start with no layers. + newTargetLayers.Reset(); + + auto nowMs = this->shared->GetTimeMs(); + int16_t spatialLayer{ 0 }; + + if (!this->producerRtpStream) + { + goto done; + } + + if (this->producerRtpStream->GetScore() == 0u) + { + goto done; + } + + for (; std::cmp_less(spatialLayer, this->producerRtpStream->GetSpatialLayers()); ++spatialLayer) + { + // If this is higher than current spatial layer and we moved to current + // spatial layer due to BWE limitations, check how much it has elapsed + // since then. + if (nowMs - this->lastBweDowngradeAtMs < BweDowngradeConservativeMs) + { + if (newTargetLayers.spatial > -1 && spatialLayer > this->encodingContext->GetCurrentSpatialLayer()) + { + continue; + } + } + + if (!this->producerRtpStream->GetSpatialLayerBitrate(nowMs, spatialLayer)) + { + continue; + } + + newTargetLayers.spatial = spatialLayer; + + // If this is the preferred or higher spatial layer and has bitrate, + // take it and exit. + if (spatialLayer >= this->preferredLayers.spatial) + { + break; + } + } + + if (newTargetLayers.spatial != -1) + { + if (newTargetLayers.spatial == this->preferredLayers.spatial) + { + newTargetLayers.temporal = this->preferredLayers.temporal; + } + else if (newTargetLayers.spatial < this->preferredLayers.spatial) + { + newTargetLayers.temporal = + static_cast(this->encodingContext->GetTemporalLayers() - 1); + } + else + { + newTargetLayers.temporal = 0; + } + } + + done: + + // Return true if any target layer changed. + return newTargetLayers != this->encodingContext->GetTargetLayers(); + } + + void SvcProducerStreamManager::OnTransportConnected() + { + MS_TRACE(); + + this->syncRequired = true; + + if (IsActive()) + { + MayChangeLayers(/*force*/ false); + } + } + + void SvcProducerStreamManager::OnTransportDisconnected() + { + MS_TRACE(); + + this->lastBweDowngradeAtMs = 0u; + + UpdateTargetLayers(-1, -1); + } + + void SvcProducerStreamManager::OnPaused() + { + MS_TRACE(); + + this->lastBweDowngradeAtMs = 0u; + + UpdateTargetLayers(-1, -1); + } + + void SvcProducerStreamManager::OnResumed() + { + MS_TRACE(); + + this->syncRequired = true; + + if (IsActive()) + { + MayChangeLayers(/*force*/ false); + } + } + +} // namespace RTC diff --git a/worker/src/RTC/Transport.cpp b/worker/src/RTC/Transport.cpp index baed0d8bba..cfdf151608 100644 --- a/worker/src/RTC/Transport.cpp +++ b/worker/src/RTC/Transport.cpp @@ -12,7 +12,8 @@ #include "FBS/transport.h" #include "RTC/BweType.hpp" #include "RTC/Consts.hpp" -#include "RTC/PipeConsumer.hpp" +#include "RTC/Consumer.hpp" +// #include "RTC/PipeConsumer.hpp" // TODO: PipeConsumer not yet ported to new Consumer. #include "RTC/RTCP/FeedbackPs.hpp" #include "RTC/RTCP/FeedbackPsAfb.hpp" #include "RTC/RTCP/FeedbackPsRemb.hpp" @@ -22,9 +23,6 @@ #include "RTC/RtpDictionaries.hpp" #include "RTC/SCTP/association/Association.hpp" #include "RTC/SCTP/public/SctpOptions.hpp" -#include "RTC/SimpleConsumer.hpp" -#include "RTC/SimulcastConsumer.hpp" -#include "RTC/SvcConsumer.hpp" #ifdef MS_RTC_LOGGER_RTP #include "RTC/RtcLogger.hpp" #endif @@ -909,44 +907,8 @@ namespace RTC MS_THROW_ERROR("a Consumer with same consumerId already exists"); } - auto type = RTC::RtpParameters::Type(body->type()); - - RTC::Consumer* consumer{ nullptr }; - - switch (type) - { - case RTC::RtpParameters::Type::SIMPLE: - { - // This may throw. - consumer = new RTC::SimpleConsumer(this->shared, consumerId, producerId, this, body); - - break; - } - - case RTC::RtpParameters::Type::SIMULCAST: - { - // This may throw. - consumer = new RTC::SimulcastConsumer(this->shared, consumerId, producerId, this, body); - - break; - } - - case RTC::RtpParameters::Type::SVC: - { - // This may throw. - consumer = new RTC::SvcConsumer(this->shared, consumerId, producerId, this, body); - - break; - } - - case RTC::RtpParameters::Type::PIPE: - { - // This may throw. - consumer = new RTC::PipeConsumer(this->shared, consumerId, producerId, this, body); - - break; - } - } + // This may throw. + auto* consumer = new RTC::Consumer(this->shared, consumerId, producerId, this, body); // Notify the listener. // This may throw if no Producer is found. diff --git a/worker/test/src/RTC/TestSimpleConsumer.cpp b/worker/test/src/RTC/TestConsumer.cpp similarity index 96% rename from worker/test/src/RTC/TestSimpleConsumer.cpp rename to worker/test/src/RTC/TestConsumer.cpp index 318989a6ae..c8721786dd 100644 --- a/worker/test/src/RTC/TestSimpleConsumer.cpp +++ b/worker/test/src/RTC/TestConsumer.cpp @@ -2,12 +2,13 @@ #include "mocks/include/MockShared.hpp" #include "FBS/rtpParameters.h" #include "FBS/transport.h" +#include "RTC/Consumer.hpp" #include "RTC/RTP/Packet.hpp" #include "RTC/RTP/RtpStream.hpp" #include "RTC/RTP/RtpStreamRecv.hpp" #include "RTC/RTP/SharedPacket.hpp" #include "RTC/RtpDictionaries.hpp" -#include "RTC/SimpleConsumer.hpp" + #include namespace @@ -112,7 +113,7 @@ namespace return rtpParameters.FillBuffer(builder); }; - std::unique_ptr createConsumer(ConsumerListener* listener) + std::unique_ptr createConsumer(ConsumerListener* listener) { flatbuffers::FlatBufferBuilder bufferBuilder; @@ -140,8 +141,8 @@ namespace const auto* consumeRequest = flatbuffers::GetRoot(buf); - return std::make_unique( - std::addressof(shared), + return std::make_unique( + &shared, consumeRequest->consumerId()->str(), consumeRequest->producerId()->str(), listener, @@ -181,12 +182,12 @@ namespace } std::unique_ptr listener; - std::unique_ptr consumer; + std::unique_ptr consumer; std::unique_ptr rtpStream; }; } // namespace -SCENARIO("SimpleConsumer", "[rtp][consumer]") +SCENARIO("Consumer with SimpleProducerStreamManager", "[rtp][consumer]") { // TODO: We should NOT parse RTP packets for tests anymore. We should use // RTC::RTP::Packet::Factory() instead. @@ -230,7 +231,6 @@ SCENARIO("SimpleConsumer", "[rtp][consumer]") 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, }; // clang-format on diff --git a/worker/test/src/RTC/TestPipeProducerStreamManager.cpp b/worker/test/src/RTC/TestPipeProducerStreamManager.cpp new file mode 100644 index 0000000000..fce3fc793b --- /dev/null +++ b/worker/test/src/RTC/TestPipeProducerStreamManager.cpp @@ -0,0 +1,473 @@ +#include "DepLibUV.hpp" +#include "mocks/include/MockShared.hpp" +#include "RTC/PipeProducerStreamManager.hpp" +#include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" +#include "RTC/RTP/rtpCommon.hpp" +#include + +namespace +{ + using RtpPacketProcessResult = RTC::ProducerStreamManager::RtpPacketProcessResult; + + // NOLINTBEGIN(readability-identifier-naming) + constexpr uint32_t mappedSsrc0 = 1000; + constexpr uint32_t mappedSsrc1 = 2000; + constexpr uint32_t mappedSsrc2 = 3000; + // NOLINTEND(readability-identifier-naming) + + class MockListener : public RTC::ProducerStreamManager::Listener + { + public: + bool IsActive() const override + { + return this->isActive; + } + + void OnProducerStreamManagerKeyFrameRequested(uint32_t mappedSsrc) override + { + this->keyFrameRequestCount++; + this->keyFrameRequestedSsrcs.push_back(mappedSsrc); + } + + void OnProducerStreamManagerNeedBitrateChange() override + { + } + + void OnProducerStreamManagerLayersChanged() override + { + } + + void OnProducerStreamManagerClearRetransmissionBuffer() override + { + } + + void OnProducerStreamManagerScore() override + { + } + + public: + bool isActive{ true }; + int keyFrameRequestCount{ 0 }; + std::vector keyFrameRequestedSsrcs; + }; + + class MockPayloadDescriptorHandler : public RTC::RTP::Codecs::PayloadDescriptorHandler + { + public: + explicit MockPayloadDescriptorHandler() = default; + ~MockPayloadDescriptorHandler() override = default; + void Dump(int /*indentation*/) const override + { + } + bool Process( + RTC::RTP::Codecs::EncodingContext* /*context*/, + RTC::RTP::Packet* /*packet*/, + bool& /*marker*/) override + { + return true; + } + void RtpPacketChanged(RTC::RTP::Packet* /*packet*/) override + { + } + std::unique_ptr GetEncoder() const override + { + return nullptr; + } + void Encode( + RTC::RTP::Packet* /*packet*/, RTC::RTP::Codecs::PayloadDescriptor::Encoder* /*encoder*/) override + { + } + void Restore(RTC::RTP::Packet* /*packet*/) override + { + } + uint8_t GetSpatialLayer() const override + { + return 0; + } + uint8_t GetTemporalLayer() const override + { + return 0; + } + bool IsKeyFrame() const override + { + return this->isKeyFrame; + } + + public: + bool isKeyFrame{ false }; + }; + + // NOLINTNEXTLINE(readability-identifier-naming) + mocks::MockShared shared(/*getTimeMs*/ + []() + { + return DepLibUV::GetTimeMs(); + }); + + std::unique_ptr createManager( + MockListener* listener, + const std::vector& ssrcs = { mappedSsrc0, mappedSsrc1, mappedSsrc2 }, + bool keyFrameSupported = true, + RTC::Media::Kind kind = RTC::Media::Kind::VIDEO) + { + std::vector consumableRtpEncodings; + + for (auto ssrc : ssrcs) + { + RTC::RtpEncodingParameters encoding; + encoding.ssrc = ssrc; + consumableRtpEncodings.push_back(encoding); + } + + const RTC::ConsumerTypes::VideoLayers preferredLayers; + + return std::make_unique( + consumableRtpEncodings, preferredLayers, nullptr, kind, keyFrameSupported, listener, &shared); + } +} // namespace + +SCENARIO("PipeProducerStreamManager", "[rtp][producer-stream-manager][pipe]") +{ + std::unique_ptr packet( + RTC::RTP::Packet::Factory(rtpCommon::FactoryBuffer, sizeof(rtpCommon::FactoryBuffer))); + + packet->SetPayloadType(1); + packet->SetSsrc(mappedSsrc0); + packet->SetPayloadLength(40); + + SECTION("IsActive() returns listener state") + { + MockListener listener; + auto manager = createManager(&listener); + + REQUIRE(manager->IsActive() == true); + + listener.isActive = false; + + REQUIRE(manager->IsActive() == false); + } + + SECTION("ProcessRtpPacket() returns FORWARD with isSyncPacket on first packet after connect") + { + MockListener listener; + auto manager = createManager(&listener, { mappedSsrc0 }, /*keyFrameSupported*/ false); + + manager->OnTransportConnected(); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(10); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.syncSeqValue == 9); + REQUIRE(result.tsOffset == 0u); + + // Second packet forwards normally, no sync. + packet->SetSequenceNumber(11); + + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == false); + } + + SECTION("ProcessRtpPacket() returns BUFFER when sync required and packet is not a key frame") + { + MockListener listener; + auto manager = createManager(&listener); + + manager->OnTransportConnected(); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(10); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::BUFFER); + } + + SECTION("ProcessRtpPacket() syncs independently per stream") + { + MockListener listener; + auto manager = + createManager(&listener, { mappedSsrc0, mappedSsrc1 }, /*keyFrameSupported*/ false); + + manager->OnTransportConnected(); + + // Sync stream 0. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(100); + + auto result0 = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result0.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result0.isSyncPacket == true); + REQUIRE(result0.syncSeqValue == 99); + + // Stream 1 syncs independently. + packet->SetSsrc(mappedSsrc1); + packet->SetSequenceNumber(200); + + auto result1 = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result1.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result1.isSyncPacket == true); + REQUIRE(result1.syncSeqValue == 199); + + // Stream 0 next packet is no longer a sync packet. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(101); + + auto result2 = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result2.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result2.isSyncPacket == false); + } + + SECTION("ProcessRtpPacket() returns FORWARD with sendBufferedPackets on key frame sync") + { + MockListener listener; + auto manager = createManager(&listener); + + manager->OnTransportConnected(); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(50); + + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + + auto result = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.syncSeqValue == 49); + REQUIRE(result.sendBufferedPackets == true); + } + + SECTION("ProcessRtpPacket() returns DROP for empty payload packets") + { + MockListener listener; + auto manager = createManager(&listener, { mappedSsrc0 }, /*keyFrameSupported*/ false); + + manager->OnTransportConnected(); + + // Complete sync first. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + // Padding-only packet. + packet->SetSequenceNumber(2); + packet->RemovePayload(); + + auto result = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::DROP); + } + + SECTION("ProcessRtpPacket() preserves original marker bit") + { + MockListener listener; + auto manager = createManager(&listener, { mappedSsrc0 }, /*keyFrameSupported*/ false); + + manager->OnTransportConnected(); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + packet->SetMarker(true); + + auto result = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.marker == true); + + packet->SetSequenceNumber(2); + packet->SetMarker(false); + + result = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result.marker == false); + } + + SECTION("OnTransportConnected() sets sync required for all streams and requests key frames") + { + MockListener listener; + auto manager = createManager(&listener, { mappedSsrc0, mappedSsrc1, mappedSsrc2 }); + + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount == 3); + REQUIRE( + std::find( + listener.keyFrameRequestedSsrcs.begin(), listener.keyFrameRequestedSsrcs.end(), mappedSsrc0) != + listener.keyFrameRequestedSsrcs.end()); + REQUIRE( + std::find( + listener.keyFrameRequestedSsrcs.begin(), listener.keyFrameRequestedSsrcs.end(), mappedSsrc1) != + listener.keyFrameRequestedSsrcs.end()); + REQUIRE( + std::find( + listener.keyFrameRequestedSsrcs.begin(), listener.keyFrameRequestedSsrcs.end(), mappedSsrc2) != + listener.keyFrameRequestedSsrcs.end()); + + // All streams require a key frame before forwarding. + packet->SetSequenceNumber(1); + + for (auto ssrc : { mappedSsrc0, mappedSsrc1, mappedSsrc2 }) + { + packet->SetSsrc(ssrc); + + auto result = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::BUFFER); + } + } + + SECTION("OnTransportConnected() does not request key frames when not active") + { + MockListener listener; + listener.isActive = false; + auto manager = createManager(&listener); + + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount == 0); + } + + SECTION("OnTransportConnected() does not request key frames for audio") + { + MockListener listener; + auto manager = createManager( + &listener, { mappedSsrc0 }, /*keyFrameSupported*/ false, RTC::Media::Kind::AUDIO); + + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount == 0); + } + + SECTION("OnTransportDisconnected() clears sync required for all streams") + { + MockListener listener; + auto manager = + createManager(&listener, { mappedSsrc0, mappedSsrc1 }, /*keyFrameSupported*/ false); + + manager->OnTransportConnected(); + manager->OnTransportDisconnected(); + + // Sync cleared — packets forward immediately without waiting for key frame. + packet->SetSequenceNumber(1); + + for (auto ssrc : { mappedSsrc0, mappedSsrc1 }) + { + packet->SetSsrc(ssrc); + + auto result = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == false); + } + } + + SECTION("OnPaused() clears sync required for all streams") + { + MockListener listener; + auto manager = createManager(&listener, { mappedSsrc0 }, /*keyFrameSupported*/ false); + + manager->OnTransportConnected(); + manager->OnPaused(); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + + auto result = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == false); + } + + SECTION("OnResumed() sets sync required and requests key frames for all streams") + { + MockListener listener; + auto manager = createManager(&listener, { mappedSsrc0, mappedSsrc1 }); + + manager->OnTransportConnected(); + + const int countBefore = listener.keyFrameRequestCount; + + manager->OnResumed(); + + REQUIRE(listener.keyFrameRequestCount == countBefore + 2); + + // All streams need sync again. + packet->SetSequenceNumber(1); + + for (auto ssrc : { mappedSsrc0, mappedSsrc1 }) + { + packet->SetSsrc(ssrc); + + auto result = manager->ProcessRtpPacket(packet.get(), false, 0, 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::BUFFER); + } + } + + SECTION("RequestKeyFrame() requests for all video streams") + { + MockListener listener; + auto manager = createManager(&listener, { mappedSsrc0, mappedSsrc1, mappedSsrc2 }); + + manager->RequestKeyFrame(); + + REQUIRE(listener.keyFrameRequestCount == 3); + } + + SECTION("RequestKeyFrame() is no-op for audio") + { + MockListener listener; + auto manager = createManager( + &listener, { mappedSsrc0 }, /*keyFrameSupported*/ false, RTC::Media::Kind::AUDIO); + + manager->RequestKeyFrame(); + + REQUIRE(listener.keyFrameRequestCount == 0); + } + + SECTION("RecalculateTargetLayers() always returns false") + { + MockListener listener; + auto manager = createManager(&listener); + + RTC::ConsumerTypes::VideoLayers newTargetLayers; + + REQUIRE(manager->RecalculateTargetLayers(newTargetLayers) == false); + } + + SECTION("IncreaseLayer() returns 0 (no BWE)") + { + MockListener listener; + auto manager = createManager(&listener); + + manager->SetExternallyManagedBitrate(); + + const auto used = manager->IncreaseLayer(1000000u, false, 0.0f, DepLibUV::GetTimeMs()); + + REQUIRE(used == 0u); + } + + SECTION("GetDesiredBitrate() returns 0 (no BWE)") + { + MockListener listener; + auto manager = createManager(&listener); + + manager->SetExternallyManagedBitrate(); + + REQUIRE(manager->GetDesiredBitrate(DepLibUV::GetTimeMs()) == 0u); + } +} diff --git a/worker/test/src/RTC/TestSimpleProducerStreamManager.cpp b/worker/test/src/RTC/TestSimpleProducerStreamManager.cpp new file mode 100644 index 0000000000..324da1cdb6 --- /dev/null +++ b/worker/test/src/RTC/TestSimpleProducerStreamManager.cpp @@ -0,0 +1,533 @@ +#include "DepLibUV.hpp" +#include "Utils.hpp" +#include "mocks/include/MockShared.hpp" +#include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" +#include "RTC/RTP/RtpStreamRecv.hpp" +#include "RTC/RTP/rtpCommon.hpp" +#include "RTC/SimpleProducerStreamManager.hpp" +#include + +namespace +{ + using RtpPacketProcessResult = RTC::ProducerStreamManager::RtpPacketProcessResult; + + // NOLINTBEGIN(readability-identifier-naming) + constexpr uint32_t mappedSsrc = 1234567890; + // NOLINTEND(readability-identifier-naming) + + class MockListener : public RTC::ProducerStreamManager::Listener + { + public: + bool isActive{ true }; + int keyFrameRequestCount{ 0 }; + uint32_t lastKeyFrameRequestedMappedSsrc{ 0 }; + + bool IsActive() const override + { + return this->isActive; + } + + void OnProducerStreamManagerKeyFrameRequested(uint32_t mappedSsrc) override + { + this->keyFrameRequestCount++; + this->lastKeyFrameRequestedMappedSsrc = mappedSsrc; + } + + void OnProducerStreamManagerNeedBitrateChange() override + { + } + + void OnProducerStreamManagerLayersChanged() override + { + } + + void OnProducerStreamManagerClearRetransmissionBuffer() override + { + } + + void OnProducerStreamManagerScore() override + { + } + }; + + class RtpStreamRecvListener : public RTC::RTP::RtpStreamRecv::Listener + { + public: + void OnRtpStreamScore( + RTC::RTP::RtpStream* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) override + { + } + + void OnRtpStreamSendRtcpPacket( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, RTC::RTCP::Packet* /*packet*/) override + { + } + + void OnRtpStreamNeedWorstRemoteFractionLost( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t& /*worstRemoteFractionLost*/) override + { + } + }; + + class MockEncodingContext : public RTC::RTP::Codecs::EncodingContext + { + public: + MockEncodingContext() : RTC::RTP::Codecs::EncodingContext(MockEncodingContext::params) + { + } + void SyncRequired() override + { + } + + private: + static RTC::RTP::Codecs::EncodingContext::Params params; + }; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) + RTC::RTP::Codecs::EncodingContext::Params MockEncodingContext::params; + + class MockPayloadDescriptorHandler : public RTC::RTP::Codecs::PayloadDescriptorHandler + { + public: + explicit MockPayloadDescriptorHandler() = default; + ~MockPayloadDescriptorHandler() override = default; + void Dump(int /*indentation*/) const override + { + } + bool Process( + RTC::RTP::Codecs::EncodingContext* /*context*/, + RTC::RTP::Packet* /*packet*/, + bool& /*marker*/) override + { + return this->processResult; + } + void RtpPacketChanged(RTC::RTP::Packet* /*packet*/) override + { + } + std::unique_ptr GetEncoder() const override + { + return nullptr; + } + void Encode( + RTC::RTP::Packet* /*packet*/, RTC::RTP::Codecs::PayloadDescriptor::Encoder* /*encoder*/) override + { + } + void Restore(RTC::RTP::Packet* /*packet*/) override + { + } + uint8_t GetSpatialLayer() const override + { + return 0; + } + uint8_t GetTemporalLayer() const override + { + return 0; + } + bool IsKeyFrame() const override + { + return this->isKeyFrame; + } + + public: + bool isKeyFrame{ false }; + bool processResult{ true }; + }; + + // RtpStreamRecvListener must outlive the RtpStreamRecv. + RtpStreamRecvListener streamRecvListener; // NOLINT(readability-identifier-naming) + + // NOLINTNEXTLINE(readability-identifier-naming) + mocks::MockShared shared(/*getTimeMs*/ + []() + { + return DepLibUV::GetTimeMs(); + }); // NOLINT(readability-identifier-naming) + + std::unique_ptr createManager( + MockListener* listener, + bool keyFrameSupported = true, + RTC::Media::Kind kind = RTC::Media::Kind::VIDEO, + std::unique_ptr encodingContext = nullptr) + { + RTC::RtpEncodingParameters encoding; + encoding.ssrc = mappedSsrc; + + const std::vector consumableRtpEncodings{ encoding }; + const RTC::ConsumerTypes::VideoLayers preferredLayers; + + return std::make_unique( + consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + kind, + keyFrameSupported, + listener, + &shared); + } + + std::unique_ptr createRtpStreamRecv() + { + RTC::RTP::RtpStream::Params params; + + params.ssrc = mappedSsrc; + params.clockRate = 90000; + + return std::make_unique(&streamRecvListener, &shared, params, 0u, false); + } + + // Feed packets into the RtpStreamRecv so GetBitrate() returns non-zero. + void feedRtpStreamRecv(RTC::RTP::RtpStreamRecv* rtpStream, RTC::RTP::Packet* packet, uint16_t count) + { + auto firstSeq = static_cast(packet->GetSequenceNumber() + 1); + auto lastSeq = static_cast(firstSeq + count); + + for (uint16_t seq = firstSeq; Utils::Number::IsLowerThan(seq, lastSeq); ++seq) + { + packet->SetSequenceNumber(seq); + rtpStream->ReceivePacket(packet); + } + + auto nowMs = DepLibUV::GetTimeMs(); + + // bitrate (bps) = totalBytes * 8000 / windowSizeMs. + // windowSizeMs for RtpStreamRecv is 2500. + auto expectedBitrate = + static_cast(std::trunc((count * packet->GetLength() * 8000.0f / 2500) + 0.5f)); + + REQUIRE(rtpStream->GetBitrate(nowMs) == expectedBitrate); + } +} // namespace + +SCENARIO("SimpleProducerStreamManager", "[rtp][producer-stream-manager][simple]") +{ + std::unique_ptr packet( + RTC::RTP::Packet::Factory(rtpCommon::FactoryBuffer, sizeof(rtpCommon::FactoryBuffer))); + + packet->SetPayloadType(1); + packet->SetSsrc(mappedSsrc); + packet->SetPayloadLength(40); + + SECTION("returns BUFFER when sync required and packet is not a keyframe") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->OnTransportConnected(); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::BUFFER); + } + + SECTION("returns FORWARD with sendBufferedPackets when syncing with a keyframe") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->OnTransportConnected(); + const uint16_t seq{ 100 }; + + packet->SetSequenceNumber(seq); + + // packet->SetPayloadDescriptorHandler() will take the ownership. + auto* payloadDescriptorHandler = new MockPayloadDescriptorHandler(); + payloadDescriptorHandler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(payloadDescriptorHandler); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.syncSeqValue == seq - 1); + REQUIRE(result.sendBufferedPackets == true); + } + + SECTION("returns DROP for empty payload packets") + { + MockListener listener; + auto manager = createManager(&listener, /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->OnTransportConnected(); + + packet->SetSequenceNumber(1); + packet->RemovePayload(); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::DROP); + } + + SECTION("returns DROP when ProcessPayload fails") + { + MockListener listener; + auto encodingContext = std::make_unique(); + auto manager = createManager( + &listener, + /*keyFrameSupported*/ false, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->OnTransportConnected(); + + packet->SetSequenceNumber(1); + + // Set a handler whose Process() returns false. + auto* handler = new MockPayloadDescriptorHandler(); + handler->processResult = false; + packet->SetPayloadDescriptorHandler(handler); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::DROP); + } + + SECTION("returns FORWARD and completes sync when keyFrameSupported is false for the first packet") + { + MockListener listener; + auto manager = createManager(&listener, /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->OnTransportConnected(); + + const uint16_t seq{ 100 }; + + packet->SetSequenceNumber(seq); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.syncSeqValue == seq - 1); + REQUIRE(result.sendBufferedPackets == false); + REQUIRE(result.tsOffset == 0u); + } + + SECTION("returns FORWARD for normal packets after sync") + { + MockListener listener; + auto manager = createManager(&listener, /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->OnTransportConnected(); + + // Complete sync with first packet. + + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + // Send a second normal packet. + packet->SetSequenceNumber(2); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == false); + REQUIRE(result.sendBufferedPackets == false); + REQUIRE(result.tsOffset == 0u); + } + + SECTION("OnTransportConnected() requests keyframe when active") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // OnTransportConnected should request a keyframe since the manager is active. + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount == 1); + REQUIRE(listener.lastKeyFrameRequestedMappedSsrc == mappedSsrc); + } + + SECTION("OnTransportConnected() does not request keyframe when not active") + { + MockListener listener; + listener.isActive = false; + + auto manager = createManager(&listener); + + // Don't wire producerRtpStream — manager is not active. + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount == 0); + } + + SECTION("OnResumed() sets syncRequired and requests keyframe") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->OnTransportConnected(); + + const int keyFrameCount = listener.keyFrameRequestCount; + + // Call OnResumed — should set syncRequired and request keyframe again. + manager->OnResumed(); + + REQUIRE(listener.keyFrameRequestCount == keyFrameCount + 1); + + // Prove syncRequired was set: sending a non-keyframe returns BUFFER. + + packet->SetSequenceNumber(1); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 0, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::BUFFER); + } + + SECTION("IncreaseLayer() returns producer bitrate when it is less than available bitrate") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + manager->OnTransportConnected(); + + // Feed packets so the stream has non-zero bitrate. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + auto steamBitrate = rtpStream->GetBitrate(nowMs); + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ steamBitrate + 1u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate == steamBitrate); + } + + SECTION("IncreaseLayer() returns available bitrate when it is less than producer bitrate") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + manager->OnTransportConnected(); + + // Feed packets so the stream has non-zero bitrate. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + const auto streamBitrate = rtpStream->GetBitrate(nowMs); + const uint32_t availableBitrate = streamBitrate - 1; + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ availableBitrate, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate == availableBitrate); + } + + SECTION("IncreaseLayer() returns 0 on second call in same iteration") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + manager->OnTransportConnected(); + + // Feed packets so the stream has non-zero bitrate. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + + // First call claims bitrate. + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate > 0u); + + // Second call in same iteration should return 0. + auto usedBitrate2 = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate2 == 0u); + } + + SECTION("IncreaseLayer() works again after ApplyLayers()") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + manager->OnTransportConnected(); + + // Feed packets so the stream has non-zero bitrate. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + + // First iteration: claim bitrate and apply. + manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + manager->ApplyLayers(/*rtpStreamActiveMs*/ 0u); + + // After ApplyLayers, IncreaseLayer should work again. + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate > 0u); + } + + SECTION("GetDesiredBitrate() returns producer bitrate for video") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + manager->OnTransportConnected(); + + // Feed packets so the stream has non-zero bitrate. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + auto steamBitrate = rtpStream->GetBitrate(nowMs); + auto desiredBitrate = manager->GetDesiredBitrate(nowMs); + + REQUIRE(desiredBitrate == steamBitrate); + } + + SECTION("GetDesiredBitrate() returns 0 for audio kind") + { + MockListener listener; + auto manager = createManager(&listener, /*keyFrameSupported*/ false, RTC::Media::Kind::AUDIO); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + manager->OnTransportConnected(); + + // Feed packets so the stream has non-zero bitrate. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + auto desiredBitrate = manager->GetDesiredBitrate(nowMs); + + REQUIRE(desiredBitrate == 0u); + } +} diff --git a/worker/test/src/RTC/TestSimulcastProducerStreamManager.cpp b/worker/test/src/RTC/TestSimulcastProducerStreamManager.cpp new file mode 100644 index 0000000000..2c94a5891f --- /dev/null +++ b/worker/test/src/RTC/TestSimulcastProducerStreamManager.cpp @@ -0,0 +1,1042 @@ +#include "DepLibUV.hpp" +#include "Utils.hpp" +#include "mocks/include/MockShared.hpp" +#include "RTC/RTCP/SenderReport.hpp" +#include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" +#include "RTC/RTP/RtpStreamRecv.hpp" +#include "RTC/RTP/rtpCommon.hpp" +#include "RTC/SimulcastProducerStreamManager.hpp" + +namespace +{ + // NOLINTBEGIN(readability-identifier-naming) + constexpr uint32_t mappedSsrc0 = 1000; + constexpr uint32_t mappedSsrc1 = 2000; + constexpr uint32_t mappedSsrc2 = 3000; + + const std::vector threeSsrcs = { mappedSsrc0, mappedSsrc1, mappedSsrc2 }; + const std::vector twoSsrcs = { mappedSsrc0, mappedSsrc1 }; + const std::vector oneSsrc = { mappedSsrc0 }; + // NOLINTEND(readability-identifier-naming) + + class MockListener : public RTC::ProducerStreamManager::Listener + { + public: + bool IsActive() const override + { + return this->isActive; + } + + void OnProducerStreamManagerKeyFrameRequested(uint32_t mappedSsrc) override + { + this->keyFrameRequestCount++; + this->lastKeyFrameRequestedMappedSsrc = mappedSsrc; + } + + void OnProducerStreamManagerNeedBitrateChange() override + { + this->needBitrateChangeCount++; + } + + void OnProducerStreamManagerLayersChanged() override + { + this->layersChangedCount++; + } + + void OnProducerStreamManagerClearRetransmissionBuffer() override + { + } + + void OnProducerStreamManagerScore() override + { + } + + public: + bool isActive{ true }; + int keyFrameRequestCount{ 0 }; + uint32_t lastKeyFrameRequestedMappedSsrc{ 0 }; + int layersChangedCount{ 0 }; + int needBitrateChangeCount{ 0 }; + }; + + class RtpStreamRecvListener : public RTC::RTP::RtpStreamRecv::Listener + { + public: + void OnRtpStreamScore( + RTC::RTP::RtpStream* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) override + { + } + + void OnRtpStreamSendRtcpPacket( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, RTC::RTCP::Packet* /*packet*/) override + { + } + + void OnRtpStreamNeedWorstRemoteFractionLost( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t& /*worstRemoteFractionLost*/) override + { + } + }; + + class MockEncodingContext : public RTC::RTP::Codecs::EncodingContext + { + public: + MockEncodingContext() : RTC::RTP::Codecs::EncodingContext(MockEncodingContext::params) + { + } + void SyncRequired() override + { + } + + private: + static RTC::RTP::Codecs::EncodingContext::Params params; + }; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) + RTC::RTP::Codecs::EncodingContext::Params MockEncodingContext::params; + + class MockPayloadDescriptorHandler : public RTC::RTP::Codecs::PayloadDescriptorHandler + { + public: + explicit MockPayloadDescriptorHandler() = default; + ~MockPayloadDescriptorHandler() override = default; + void Dump(int /*indentation*/) const override + { + } + bool Process( + RTC::RTP::Codecs::EncodingContext* /*context*/, + RTC::RTP::Packet* /*packet*/, + bool& /*marker*/) override + { + return this->processResult; + } + void RtpPacketChanged(RTC::RTP::Packet* /*packet*/) override + { + } + std::unique_ptr GetEncoder() const override + { + return nullptr; + } + void Encode( + RTC::RTP::Packet* /*packet*/, RTC::RTP::Codecs::PayloadDescriptor::Encoder* /*encoder*/) override + { + } + void Restore(RTC::RTP::Packet* /*packet*/) override + { + } + uint8_t GetSpatialLayer() const override + { + return 0; + } + uint8_t GetTemporalLayer() const override + { + return 0; + } + bool IsKeyFrame() const override + { + return this->isKeyFrame; + } + + public: + bool isKeyFrame{ false }; + bool processResult{ true }; + }; + + // RtpStreamRecvListener must outlive the RtpStreamRecv. + RtpStreamRecvListener streamRecvListener; // NOLINT(readability-identifier-naming) + + // NOLINTNEXTLINE(readability-identifier-naming) + mocks::MockShared shared(/*getTimeMs*/ + []() + { + return DepLibUV::GetTimeMs(); + }); // NOLINT(readability-identifier-naming) + + std::unique_ptr createManager( + MockListener* listener, + const std::vector& ssrcs = threeSsrcs, + RTC::ConsumerTypes::VideoLayers preferredLayers = { 2, 2 }, + bool keyFrameSupported = true, + RTC::Media::Kind kind = RTC::Media::Kind::VIDEO, + std::unique_ptr encodingContext = nullptr) + { + std::vector consumableRtpEncodings; + + for (auto ssrc : ssrcs) + { + RTC::RtpEncodingParameters encoding; + encoding.ssrc = ssrc; + consumableRtpEncodings.push_back(encoding); + } + + if (!encodingContext) + { + encodingContext = std::make_unique(); + } + + return std::make_unique( + consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + kind, + keyFrameSupported, + listener, + &shared); + } + + std::unique_ptr createRtpStreamRecv(uint32_t ssrc) + { + RTC::RTP::RtpStream::Params params; + + params.ssrc = ssrc; + params.clockRate = 90000; + + return std::make_unique(&streamRecvListener, &shared, params, 0u, false); + } + + // Feed packets into the RtpStreamRecv so GetBitrate() returns non-zero. + void feedRtpStreamRecv(RTC::RTP::RtpStreamRecv* rtpStream, RTC::RTP::Packet* packet, uint16_t count) + { + auto firstSeq = static_cast(packet->GetSequenceNumber() + 1); + auto lastSeq = static_cast(firstSeq + count); + + for (uint16_t seq = firstSeq; Utils::Number::IsLowerThan(seq, lastSeq); ++seq) + { + packet->SetSequenceNumber(seq); + rtpStream->ReceivePacket(packet); + } + + auto nowMs = DepLibUV::GetTimeMs(); + + // bitrate (bps) = totalBytes * 8000 / windowSizeMs. + // windowSizeMs for RtpStreamRecv is 2500. + auto expectedBitrate = + static_cast(std::trunc((count * packet->GetLength() * 8000.0f / 2500) + 0.5f)); + + REQUIRE(rtpStream->GetBitrate(nowMs) == expectedBitrate); + } +} // namespace + +SCENARIO("SimulcastProducerStreamManager", "[rtp][producer-stream-manager][simulcast]") +{ + std::unique_ptr packet( + RTC::RTP::Packet::Factory(rtpCommon::FactoryBuffer, sizeof(rtpCommon::FactoryBuffer))); + + packet->SetPayloadType(1); + packet->SetSsrc(mappedSsrc0); + packet->SetTimestamp(1000); + packet->SetPayloadLength(40); + + SECTION("ProcessRtpPacket() returns DROP when target temporal layer is -1") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + + // Set target layer to 0, complete sync. + manager->UpdateTargetLayers(0, 0); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Reset target layers to -1,-1 (simulates pause/disconnect). + manager->UpdateTargetLayers(-1, -1); + + // Packet from previous current layer should be dropped. + packet->SetSequenceNumber(2); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::SILENT_DROP); + } + + SECTION( + "ProcessRtpPacket() returns BUFFER when sync required and packet is from target spatial layer but not keyframe") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + + // Set target layer to 0. This sets syncRequired since target != current. + manager->UpdateTargetLayers(0, 0); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::BUFFER); + } + + SECTION("ProcessRtpPacket() requires keyframe for spatial layer switch") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + + // Set target layer to 0 and complete sync. + manager->UpdateTargetLayers(0, 0); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + + auto* handler0 = new MockPayloadDescriptorHandler(); + handler0->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler0); + + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Now switch target layer to 1. + manager->UpdateTargetLayers(1, 0); + + // Send non-keyframe from layer 1 — should be buffered. + packet->SetSsrc(mappedSsrc1); + packet->SetSequenceNumber(1); + + // Explicitly set a non-keyframe handler. + auto* handler1 = new MockPayloadDescriptorHandler(); + handler1->isKeyFrame = false; + packet->SetPayloadDescriptorHandler(handler1); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::BUFFER); + + // Send keyframe from layer 1 — should switch and forward. + packet->SetSequenceNumber(2); + + auto* handler2 = new MockPayloadDescriptorHandler(); + handler2->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler2); + + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.spatialLayerSwitched == true); + REQUIRE(result.sendBufferedPackets == true); + REQUIRE(manager->GetCurrentSpatialLayer() == 1); + } + + SECTION("ProcessRtpPacket() returns SILENT_DROP for non-target and non-current spatial layer packet") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + + // Set target to spatial layer 0. + manager->UpdateTargetLayers(0, 0); + + // Sync on layer 0 with a keyframe. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + + auto* handler0 = new MockPayloadDescriptorHandler(); + handler0->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler0); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + + // Now send a packet from spatial layer 1 — should be silently dropped. + packet->SetSsrc(mappedSsrc1); + packet->SetSequenceNumber(1); + + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::SILENT_DROP); + } + + SECTION( + "ProcessRtpPacket() returns SILENT_DROP when sync required and packet is from non-target spatial layer") + { + MockListener listener; + auto manager = createManager(&listener, /*ssrcs*/ twoSsrcs); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + + // Set target layer to 0. This sets syncRequired since target != current. + manager->UpdateTargetLayers(0, 0); + + // Send a packet from layer 1 (non-target) while sync is pending. + packet->SetSsrc(mappedSsrc1); + packet->SetSequenceNumber(1); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::SILENT_DROP); + } + + SECTION("ProcessRtpPacket() returns FORWARD with sendBufferedPackets when syncing with a keyframe") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->UpdateTargetLayers(0, 0); + + const uint16_t seq{ 100 }; + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(seq); + + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.sendBufferedPackets == true); + } + + SECTION("ProcessRtpPacket() returns FORWARD and completes sync when keyFrameSupported is false") + { + MockListener listener; + auto manager = createManager( + &listener, + /*ssrcs*/ threeSsrcs, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->UpdateTargetLayers(0, 0); + + const uint16_t seq{ 100 }; + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(seq); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.tsOffset == 0u); + } + + SECTION("ProcessRtpPacket() returns DROP for empty payload packets on current spatial layer") + { + MockListener listener; + auto manager = createManager( + &listener, + /*ssrcs*/ threeSsrcs, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->UpdateTargetLayers(0, 0); + + // Complete sync. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Send empty payload packet on current layer. + packet->SetSequenceNumber(2); + packet->RemovePayload(); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::DROP); + } + + SECTION( + "ProcessRtpPacket() returns SILENT_DROP for empty payload packets on non-current spatial layer") + { + MockListener listener; + auto manager = createManager( + &listener, + /*ssrcs*/ threeSsrcs, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->UpdateTargetLayers(0, 0); + + // Complete sync on layer 0. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Send empty payload packet on non-current layer. + packet->SetSsrc(mappedSsrc1); + packet->SetSequenceNumber(1); + packet->RemovePayload(); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::SILENT_DROP); + } + + SECTION("ProcessRtpPacket() returns FORWARD for current layer packets after sync") + { + MockListener listener; + auto manager = createManager( + &listener, + /*ssrcs*/ threeSsrcs, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->UpdateTargetLayers(0, 0); + + // Complete sync with first packet. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Send a second normal packet. + packet->SetSequenceNumber(2); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == false); + REQUIRE(result.sendBufferedPackets == false); + REQUIRE(result.tsOffset == 0u); + } + + SECTION("ProcessRtpPacket() forwards packets from current layer after spatial switch") + { + MockListener listener; + auto manager = createManager( + &listener, + /*ssrcs*/ twoSsrcs, + /*preferredLayers*/ { 1, 0 }, + /*keyFrameSupported*/ false); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + + // Set target layer to 0 and sync. + manager->UpdateTargetLayers(0, 0); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(manager->GetCurrentSpatialLayer() == 0); + + // Switch to layer 1 — keyFrameSupported=false so first packet syncs. + manager->UpdateTargetLayers(1, 0); + + packet->SetSsrc(mappedSsrc1); + packet->SetSequenceNumber(1); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.spatialLayerSwitched == true); + REQUIRE(manager->GetCurrentSpatialLayer() == 1); + + // Subsequent packets on layer 1 should be forwarded normally. + packet->SetSequenceNumber(2); + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.spatialLayerSwitched == false); + + // Packets from old layer 0 should be silently dropped. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(2); + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::SILENT_DROP); + } + + SECTION("OnTransportConnected() sets syncRequired") + { + MockListener listener; + auto manager = createManager( + &listener, + /*ssrcs*/ oneSsrc, + /*preferredLayers*/ { 0, 0 }, + /*keyFrameSupported*/ false); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + + // Give score > 0 so RecalculateTargetLayers keeps target at layer 0. + manager->ProducerRtpStreamScore(rtpStream0.get(), /*score*/ 7, /*previousScore*/ 0); + + // Set target and sync on layer 0. + manager->UpdateTargetLayers(0, 0); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Verify we are synced: normal packet is forwarded without sync flag. + packet->SetSequenceNumber(2); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == false); + + // Now OnTransportConnected resets syncRequired. + manager->OnTransportConnected(); + + // Next packet should be a sync packet. + packet->SetSequenceNumber(3); + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + } + + SECTION("OnTransportConnected() requests keyframe when active and target changes") + { + MockListener listener; + auto manager = createManager(&listener, /*ssrcs*/ twoSsrcs, /*preferredLayers*/ { 1, 0 }); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + + // Both streams have score > 0 so RecalculateTargetLayers can pick layers. + manager->ProducerRtpStreamScore(rtpStream0.get(), /*score*/ 7, /*previousScore*/ 0); + manager->ProducerRtpStreamScore(rtpStream1.get(), /*score*/ 7, /*previousScore*/ 0); + + // Sync on layer 1 with a keyframe. + manager->UpdateTargetLayers(1, 0); + + packet->SetSsrc(mappedSsrc1); + packet->SetSequenceNumber(1); + + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // current=1, target=1. Disconnect resets everything to -1. + manager->OnTransportDisconnected(); + + auto keyFrameCountBefore = listener.keyFrameRequestCount; + + // Reconnect: RecalculateTargetLayers picks preferred layer 1, + // which differs from current (-1), so a keyframe is requested. + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount > keyFrameCountBefore); + } + + SECTION("OnTransportConnected() does not request keyframe when not active") + { + MockListener listener; + listener.isActive = false; + + auto manager = createManager(&listener); + + // Don't wire producerRtpStream — manager is not active. + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount == 0); + } + + SECTION("RecalculateTargetLayers() skips spatial layer without Sender Report") + { + MockListener listener; + auto manager = createManager(&listener, /*ssrcs*/ twoSsrcs, /*preferredLayers*/ { 1, 0 }); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + + // Set target layer to 0. This sets tsReferenceSpatialLayer = 0. + manager->UpdateTargetLayers(0, 0); + + // Sync on layer 0 with a keyframe so current = 0. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(manager->GetCurrentSpatialLayer() == 0); + REQUIRE(manager->GetTargetLayers().temporal == 0); + + // Give both streams score > 0. RecalculateTargetLayers will be called + // but CanSwitchToSpatialLayer(1) returns false because layer 1 has no + // Sender Report and tsReferenceSpatialLayer is 0 (not -1). + manager->ProducerRtpStreamScore(rtpStream0.get(), /*score*/ 7, /*previousScore*/ 0); + + REQUIRE(manager->GetTargetLayers().spatial == 0); + REQUIRE(manager->GetTargetLayers().temporal == 0); + + manager->ProducerRtpStreamScore(rtpStream1.get(), /*score*/ 7, /*previousScore*/ 0); + + // Despite preferred being layer 1, target stays at layer 0. + REQUIRE(manager->GetTargetLayers().spatial == 0); + REQUIRE(manager->GetTargetLayers().temporal == 0); + + // Verify packets on layer 0 are still forwarded with tsOffset 0. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(2); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.tsOffset == 0u); + } + + SECTION("RecalculateTargetLayers() switches to preferred layer after Sender Report") + { + MockListener listener; + auto manager = createManager(&listener, /*ssrcs*/ twoSsrcs, /*preferredLayers*/ { 1, 0 }); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + + // Set target layer to 0. This sets tsReferenceSpatialLayer = 0. + manager->UpdateTargetLayers(0, 0); + + // Sync on layer 0 with a keyframe so current = 0. + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Target stays at 0 because layer 1 has no Sender Report. + REQUIRE(manager->GetTargetLayers().spatial == 0); + + // Feed packets to both streams so ReceiveRtcpSenderReport's UpdateScore + // doesn't drop the score to 0. + packet->SetSsrc(mappedSsrc0); + feedRtpStreamRecv(rtpStream0.get(), packet.get(), 10); + + packet->SetSsrc(mappedSsrc1); + feedRtpStreamRecv(rtpStream1.get(), packet.get(), 10); + + // Provide Sender Reports for both streams. + RTC::RTCP::SenderReport sr0; + sr0.SetSsrc(mappedSsrc0); + sr0.SetNtpSec(1000); + sr0.SetNtpFrac(0); + sr0.SetRtpTs(90000); + rtpStream0->ReceiveRtcpSenderReport(&sr0); + + RTC::RTCP::SenderReport sr1; + sr1.SetSsrc(mappedSsrc1); + sr1.SetNtpSec(1000); + sr1.SetNtpFrac(0); + sr1.SetRtpTs(90000); + rtpStream1->ReceiveRtcpSenderReport(&sr1); + + // Notify the manager about the first Sender Report for layer 1. + // This triggers MayChangeLayers → RecalculateTargetLayers. + // Now CanSwitchToSpatialLayer(1) returns true, so preferred layer 1 is picked. + manager->ProducerRtcpSenderReport(rtpStream1.get(), /*first*/ true); + + REQUIRE(manager->GetTargetLayers().spatial == 1); + + // Send a keyframe from layer 1 to complete the spatial switch. + packet->SetSsrc(mappedSsrc1); + packet->SetSequenceNumber(100); + + auto* handler1 = new MockPayloadDescriptorHandler(); + handler1->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler1); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.spatialLayerSwitched == true); + // Both SRs have the same NTP/RTP values, so tsOffset is 0. + REQUIRE(result.tsOffset == 0u); + REQUIRE(manager->GetCurrentSpatialLayer() == 1); + } + + SECTION("OnResumed() sets syncRequired") + { + MockListener listener; + auto manager = createManager( + &listener, + /*ssrcs*/ threeSsrcs, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + + // Set target and sync on layer 0. + manager->UpdateTargetLayers(0, 0); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Verify we are synced: normal packet is forwarded without sync flag. + packet->SetSequenceNumber(2); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == false); + + // Call OnResumed — should set syncRequired. + manager->OnResumed(); + + // Reset target since OnResumed may have reset it via MayChangeLayers. + manager->UpdateTargetLayers(0, 0); + + // Prove syncRequired was set: next packet is a sync packet. + packet->SetSequenceNumber(3); + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RTC::ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + } + + SECTION("OnTransportDisconnected() resets target layers") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->UpdateTargetLayers(0, 0); + + manager->OnTransportDisconnected(); + + auto targetLayers = manager->GetTargetLayers(); + + REQUIRE(targetLayers.spatial == -1); + REQUIRE(targetLayers.temporal == -1); + } + + SECTION("OnPaused() resets target layers") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->UpdateTargetLayers(0, 0); + + manager->OnPaused(); + + auto targetLayers = manager->GetTargetLayers(); + + REQUIRE(targetLayers.spatial == -1); + REQUIRE(targetLayers.temporal == -1); + } + + SECTION("IncreaseLayer() returns 0 on second call in same iteration") + { + MockListener listener; + auto manager = createManager(&listener, /*ssrcs*/ oneSsrc, /*preferredLayers*/ { 0, 0 }); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->SetExternallyManagedBitrate(); + + // Feed packets so the stream has non-zero bitrate. + packet->SetSsrc(mappedSsrc0); + feedRtpStreamRecv(rtpStream0.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + + // First call claims bitrate. + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate > 0u); + + // Second call in same iteration should return 0 (already at preferred layers). + auto usedBitrate2 = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate2 == 0u); + } + + SECTION("IncreaseLayer() works again after ApplyLayers()") + { + MockListener listener; + auto manager = createManager(&listener, /*ssrcs*/ twoSsrcs, /*preferredLayers*/ { 1, 0 }); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + manager->SetExternallyManagedBitrate(); + + // Feed packets to layer 0. + packet->SetSsrc(mappedSsrc0); + feedRtpStreamRecv(rtpStream0.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + + // First iteration: claim layer 0. + manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + manager->ApplyLayers(/*rtpStreamActiveMs*/ 0u); + + // Feed packets to layer 1. + packet->SetSsrc(mappedSsrc1); + feedRtpStreamRecv(rtpStream1.get(), packet.get(), 100); + + // After ApplyLayers, IncreaseLayer should work again for layer 1. + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate > 0u); + } + + SECTION("GetDesiredBitrate() returns max bitrate across all producer streams") + { + MockListener listener; + auto manager = createManager(&listener, /*ssrcs*/ twoSsrcs, /*preferredLayers*/ { 1, 0 }); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + manager->SetExternallyManagedBitrate(); + + // Feed packets to both streams with different counts to get different bitrates. + packet->SetSsrc(mappedSsrc0); + feedRtpStreamRecv(rtpStream0.get(), packet.get(), 50); + + packet->SetSsrc(mappedSsrc1); + feedRtpStreamRecv(rtpStream1.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + auto bitrate0 = rtpStream0->GetBitrate(nowMs); + auto bitrate1 = rtpStream1->GetBitrate(nowMs); + + REQUIRE(bitrate1 > bitrate0); + + auto desiredBitrate = manager->GetDesiredBitrate(nowMs); + + REQUIRE(desiredBitrate == bitrate1); + } + + SECTION("GetProducerCurrentRtpStream() returns nullptr before sync") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + + REQUIRE(manager->GetProducerCurrentRtpStream() == nullptr); + REQUIRE(manager->GetCurrentSpatialLayer() == -1); + } + + SECTION("GetProducerTargetRtpStream() returns correct stream after UpdateTargetLayers()") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + + manager->UpdateTargetLayers(1, 0); + + REQUIRE(manager->GetProducerTargetRtpStream() == rtpStream1.get()); + REQUIRE(manager->GetTargetLayers().spatial == 1); + REQUIRE(manager->GetTargetLayers().temporal == 0); + } + + SECTION("RequestKeyFrame() requests for both target and current spatial layers") + { + MockListener listener; + auto manager = createManager( + &listener, + /*ssrcs*/ twoSsrcs, + /*preferredLayers*/ { 1, 0 }, + /*keyFrameSupported*/ false); + auto rtpStream0 = createRtpStreamRecv(mappedSsrc0); + auto rtpStream1 = createRtpStreamRecv(mappedSsrc1); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc0); + manager->ProducerRtpStream(rtpStream1.get(), mappedSsrc1); + + // Set target and sync on layer 0. + manager->UpdateTargetLayers(0, 0); + + packet->SetSsrc(mappedSsrc0); + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Change target layer to 1 (current stays at 0 until keyframe). + manager->UpdateTargetLayers(1, 0); + + listener.keyFrameRequestCount = 0; + + // RequestKeyFrame should request for both target (1) and current (0). + manager->RequestKeyFrame(); + + REQUIRE(listener.keyFrameRequestCount == 2); + } +} diff --git a/worker/test/src/RTC/TestSvcProducerStreamManager.cpp b/worker/test/src/RTC/TestSvcProducerStreamManager.cpp new file mode 100644 index 0000000000..d61b50ef8f --- /dev/null +++ b/worker/test/src/RTC/TestSvcProducerStreamManager.cpp @@ -0,0 +1,1280 @@ +#include "DepLibUV.hpp" +#include "Utils.hpp" +#include "mocks/include/MockShared.hpp" +#include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" +#include "RTC/RTP/RtpStreamRecv.hpp" +#include "RTC/RTP/rtpCommon.hpp" +#include "RTC/SvcProducerStreamManager.hpp" +#include + +namespace +{ + using RtpPacketProcessResult = RTC::ProducerStreamManager::RtpPacketProcessResult; + + // NOLINTBEGIN(readability-identifier-naming) + constexpr uint32_t mappedSsrc = 1234567890; + // NOLINTEND(readability-identifier-naming) + + class MockListener : public RTC::ProducerStreamManager::Listener + { + public: + bool IsActive() const override + { + return this->isActive; + } + + void OnProducerStreamManagerKeyFrameRequested(uint32_t mappedSsrc) override + { + this->keyFrameRequestCount++; + this->lastKeyFrameRequestedMappedSsrc = mappedSsrc; + } + + void OnProducerStreamManagerNeedBitrateChange() override + { + this->needBitrateChangeCount++; + } + + void OnProducerStreamManagerLayersChanged() override + { + this->layersChangedCount++; + } + + void OnProducerStreamManagerClearRetransmissionBuffer() override + { + } + + void OnProducerStreamManagerScore() override + { + } + + public: + bool isActive{ true }; + size_t keyFrameRequestCount{ 0 }; + uint32_t lastKeyFrameRequestedMappedSsrc{ 0 }; + size_t layersChangedCount{ 0 }; + size_t needBitrateChangeCount{ 0 }; + }; + + class RtpStreamRecvListener : public RTC::RTP::RtpStreamRecv::Listener + { + public: + void OnRtpStreamScore( + RTC::RTP::RtpStream* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) override + { + } + + void OnRtpStreamSendRtcpPacket( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, RTC::RTCP::Packet* /*packet*/) override + { + } + + void OnRtpStreamNeedWorstRemoteFractionLost( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t& /*worstRemoteFractionLost*/) override + { + } + }; + + class MockEncodingContext : public RTC::RTP::Codecs::EncodingContext + { + public: + explicit MockEncodingContext( + uint8_t spatialLayers = 3u, uint8_t temporalLayers = 3u, bool ksvc = false) + : RTC::RTP::Codecs::EncodingContext( + MockEncodingContext::BuildParams(spatialLayers, temporalLayers, ksvc)) + { + } + void SyncRequired() override + { + } + + private: + static RTC::RTP::Codecs::EncodingContext::Params& BuildParams( + uint8_t spatialLayers, uint8_t temporalLayers, bool ksvc) + { + // Use a thread-local so each call gets its own params instance. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) + static thread_local RTC::RTP::Codecs::EncodingContext::Params params; + params.spatialLayers = spatialLayers; + params.temporalLayers = temporalLayers; + params.ksvc = ksvc; + return params; + } + }; + + class MockPayloadDescriptorHandler : public RTC::RTP::Codecs::PayloadDescriptorHandler + { + public: + explicit MockPayloadDescriptorHandler() = default; + ~MockPayloadDescriptorHandler() override = default; + void Dump(int /*indentation*/) const override + { + } + bool Process( + RTC::RTP::Codecs::EncodingContext* /*context*/, + RTC::RTP::Packet* /*packet*/, + bool& /*marker*/) override + { + return this->processResult; + } + void RtpPacketChanged(RTC::RTP::Packet* /*packet*/) override + { + } + std::unique_ptr GetEncoder() const override + { + return nullptr; + } + void Encode( + RTC::RTP::Packet* /*packet*/, RTC::RTP::Codecs::PayloadDescriptor::Encoder* /*encoder*/) override + { + } + void Restore(RTC::RTP::Packet* /*packet*/) override + { + } + uint8_t GetSpatialLayer() const override + { + return 0; + } + uint8_t GetTemporalLayer() const override + { + return 0; + } + bool IsKeyFrame() const override + { + return this->isKeyFrame; + } + + public: + bool isKeyFrame{ false }; + bool processResult{ true }; + }; + + // RtpStreamRecvListener must outlive the RtpStreamRecv. + RtpStreamRecvListener streamRecvListener; // NOLINT(readability-identifier-naming) + + // NOLINTNEXTLINE(readability-identifier-naming) + mocks::MockShared shared(/*getTimeMs*/ + []() + { + return DepLibUV::GetTimeMs(); + }); // NOLINT(readability-identifier-naming) + + std::unique_ptr createManager( + MockListener* listener, + RTC::ConsumerTypes::VideoLayers preferredLayers = { 2, 2 }, + bool keyFrameSupported = true, + RTC::Media::Kind kind = RTC::Media::Kind::VIDEO, + std::unique_ptr encodingContext = nullptr) + { + RTC::RtpEncodingParameters encoding; + encoding.ssrc = mappedSsrc; + const std::vector consumableRtpEncodings{ encoding }; + + if (!encodingContext) + { + encodingContext = std::make_unique(); + } + + return std::make_unique( + consumableRtpEncodings, + preferredLayers, + std::move(encodingContext), + kind, + keyFrameSupported, + listener, + &shared); + } + + std::unique_ptr createRtpStreamRecv( + uint32_t ssrc = mappedSsrc, uint8_t spatialLayers = 3u, uint8_t temporalLayers = 3u) + { + RTC::RTP::RtpStream::Params params; + + params.ssrc = ssrc; + params.clockRate = 90000; + params.spatialLayers = spatialLayers; + params.temporalLayers = temporalLayers; + + return std::make_unique(&streamRecvListener, &shared, params, 0u, false); + } + + // Feed packets into the RtpStreamRecv so GetBitrate() returns non-zero. + void feedRtpStreamRecv(RTC::RTP::RtpStreamRecv* rtpStream, RTC::RTP::Packet* packet, uint16_t count) + { + auto firstSeq = static_cast(packet->GetSequenceNumber() + 1); + auto lastSeq = static_cast(firstSeq + count); + + for (uint16_t seq = firstSeq; Utils::Number::IsLowerThan(seq, lastSeq); ++seq) + { + packet->SetSequenceNumber(seq); + rtpStream->ReceivePacket(packet); + } + + auto nowMs = DepLibUV::GetTimeMs(); + + // bitrate (bps) = totalBytes * 8000 / windowSizeMs. + // windowSizeMs for RtpStreamRecv is 2500. + auto expectedBitrate = + static_cast(std::trunc((count * packet->GetLength() * 8000.0f / 2500) + 0.5f)); + + REQUIRE(rtpStream->GetBitrate(nowMs) == expectedBitrate); + } +} // namespace + +SCENARIO("SvcProducerStreamManager", "[rtp][producer-stream-manager][svc]") +{ + std::unique_ptr packet( + RTC::RTP::Packet::Factory(rtpCommon::FactoryBuffer, sizeof(rtpCommon::FactoryBuffer))); + + packet->SetPayloadType(1); + packet->SetSsrc(mappedSsrc); + packet->SetTimestamp(1000); + packet->SetPayloadLength(40); + + SECTION("ProcessRtpPacket() returns DROP when target spatial layer is -1") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // Default target layers are -1,-1 — packet must be dropped. + packet->SetSequenceNumber(1); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::DROP); + } + + SECTION("ProcessRtpPacket() returns DROP when target temporal layer is -1") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // Set only spatial — temporal stays -1. + manager->UpdateTargetLayers(0, -1); + + packet->SetSequenceNumber(1); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::DROP); + } + + SECTION("ProcessRtpPacket() returns BUFFER when sync required and packet is not a keyframe") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // In SVC, syncRequired is set by OnTransportConnected/OnResumed, not by + // UpdateTargetLayers. Trigger it explicitly. + manager->OnTransportConnected(); + + // Re-apply target since MayChangeLayers may have reset it. + manager->UpdateTargetLayers(0, 0); + + packet->SetSequenceNumber(1); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::BUFFER); + } + + SECTION( + "ProcessRtpPacket() returns FORWARD with isSyncPacket and sendBufferedPackets on keyframe sync") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // syncRequired is only set by OnTransportConnected/OnResumed in SVC. + manager->OnTransportConnected(); + + // Re-apply target since MayChangeLayers may have reset it. + manager->UpdateTargetLayers(0, 0); + + const uint16_t seq{ 100 }; + + packet->SetSequenceNumber(seq); + + // packet->SetPayloadDescriptorHandler() takes ownership. + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.syncSeqValue == seq - 1); + REQUIRE(result.shouldSyncEncodingContext == true); + REQUIRE(result.sendBufferedPackets == true); + } + + SECTION("ProcessRtpPacket() always requires a keyframe for sync regardless of keyFrameSupported flag") + { + MockListener listener; + // keyFrameSupported=false has no effect in SVC: sync always needs a keyframe. + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // syncRequired is only set by OnTransportConnected/OnResumed in SVC. + manager->OnTransportConnected(); + + // Re-apply target since MayChangeLayers may have reset it. + manager->UpdateTargetLayers(0, 0); + + // A non-keyframe must still be buffered even with keyFrameSupported=false. + packet->SetSequenceNumber(1); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::BUFFER); + + // Now a keyframe completes the sync. + const uint16_t seq{ 100 }; + packet->SetSequenceNumber(seq); + + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + REQUIRE(result.syncSeqValue == seq - 1); + REQUIRE(result.tsOffset == 0u); + REQUIRE(result.sendBufferedPackets == true); + } + + SECTION("ProcessRtpPacket() returns DROP for empty payload packets") + { + MockListener listener; + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // Trigger sync via OnTransportConnected, then re-apply target. + manager->OnTransportConnected(); + manager->UpdateTargetLayers(0, 0); + + // Complete sync with a keyframe (SVC always requires keyframe for sync). + packet->SetSequenceNumber(1); + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Now send a packet with empty payload. + packet->SetSequenceNumber(2); + packet->RemovePayload(); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::DROP); + } + + SECTION("ProcessRtpPacket() returns DROP when ProcessPayload fails") + { + MockListener listener; + auto encodingContext = std::make_unique(); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // Trigger sync via OnTransportConnected, then re-apply target. + manager->OnTransportConnected(); + manager->UpdateTargetLayers(0, 0); + + // Complete sync with a keyframe (SVC always requires keyframe for sync). + packet->SetSequenceNumber(1); + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Set a handler whose Process() returns false. + packet->SetSequenceNumber(2); + handler = new MockPayloadDescriptorHandler(); + handler->processResult = false; + packet->SetPayloadDescriptorHandler(handler); + + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::DROP); + } + + SECTION("ProcessRtpPacket() returns FORWARD for normal packets after sync") + { + MockListener listener; + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // Trigger sync via OnTransportConnected, then re-apply target. + manager->OnTransportConnected(); + manager->UpdateTargetLayers(0, 0); + + // Complete sync with a keyframe (SVC always requires keyframe for sync). + packet->SetSequenceNumber(1); + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Send a second normal packet — should be a plain FORWARD. + packet->SetSequenceNumber(2); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == false); + REQUIRE(result.sendBufferedPackets == false); + REQUIRE(result.tsOffset == 0u); + } + + SECTION("ProcessRtpPacket() always sets tsOffset to 0 (SVC uses single stream)") + { + MockListener listener; + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // Trigger sync, then re-apply target. + manager->OnTransportConnected(); + manager->UpdateTargetLayers(0, 0); + + // Sync with a keyframe — SVC always requires one. + packet->SetSequenceNumber(1); + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.tsOffset == 0u); + + // Subsequent normal packets also have tsOffset == 0. + packet->SetSequenceNumber(2); + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.tsOffset == 0u); + } + + SECTION("UpdateTargetLayers() to -1 resets current and target layers") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(1, 1); + + // Reset. + manager->UpdateTargetLayers(-1, -1); + + auto targetLayers = manager->GetTargetLayers(); + + REQUIRE(targetLayers.spatial == -1); + REQUIRE(targetLayers.temporal == -1); + REQUIRE(manager->GetCurrentSpatialLayer() == -1); + } + + SECTION("UpdateTargetLayers() fires OnProducerStreamManagerLayersChanged() when setting -1") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + const auto countBefore = listener.layersChangedCount; + + manager->UpdateTargetLayers(-1, -1); + + REQUIRE(listener.layersChangedCount == countBefore + 1); + } + + SECTION("UpdateTargetLayers() requests keyframe for full-SVC spatial upgrade") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // Start at layer 0. + manager->UpdateTargetLayers(0, 0); + + const auto keyFrameRequestCountBefore = listener.keyFrameRequestCount; + + // Upgrade spatial: full-SVC requests a keyframe. + manager->UpdateTargetLayers(1, 0); + + REQUIRE(listener.keyFrameRequestCount > keyFrameRequestCountBefore); + REQUIRE(listener.lastKeyFrameRequestedMappedSsrc == mappedSsrc); + } + + SECTION("GetProducerCurrentRtpStream() returns nullptr when current spatial layer is -1") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // No target set yet — current spatial layer is -1. + REQUIRE(manager->GetProducerCurrentRtpStream() == nullptr); + REQUIRE(manager->GetCurrentSpatialLayer() == -1); + } + + SECTION("GetProducerTargetRtpStream() returns nullptr when target spatial layer is -1") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + REQUIRE(manager->GetProducerTargetRtpStream() == nullptr); + } + + SECTION("GetProducerTargetRtpStream() returns the producer stream after UpdateTargetLayers()") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + REQUIRE(manager->GetProducerTargetRtpStream() == rtpStream.get()); + REQUIRE(manager->GetTargetLayers().spatial == 0); + REQUIRE(manager->GetTargetLayers().temporal == 0); + } + + SECTION("IsActive() returns false when listener is not active") + { + MockListener listener; + listener.isActive = false; + + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + REQUIRE(manager->IsActive() == false); + } + + SECTION("IsActive() returns false when no producer stream is set") + { + MockListener listener; + auto manager = createManager(&listener); + + // No ProducerRtpStream call. + REQUIRE(manager->IsActive() == false); + } + + SECTION("RequestKeyFrame() always uses the single SVC mapped SSRC") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + listener.keyFrameRequestCount = 0; + manager->RequestKeyFrame(); + + REQUIRE(listener.keyFrameRequestCount == 1); + REQUIRE(listener.lastKeyFrameRequestedMappedSsrc == mappedSsrc); + } + + SECTION("RequestKeyFrameForTargetSpatialLayer() delegates to single-SSRC RequestKeyFrame()") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(1, 0); + + listener.keyFrameRequestCount = 0; + manager->RequestKeyFrameForTargetSpatialLayer(); + + REQUIRE(listener.keyFrameRequestCount == 1); + REQUIRE(listener.lastKeyFrameRequestedMappedSsrc == mappedSsrc); + } + + SECTION("RequestKeyFrameForCurrentSpatialLayer() delegates to single-SSRC RequestKeyFrame()") + { + MockListener listener; + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // Sync so we have a current layer. + packet->SetSequenceNumber(1); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + listener.keyFrameRequestCount = 0; + manager->RequestKeyFrameForCurrentSpatialLayer(); + + REQUIRE(listener.keyFrameRequestCount == 1); + REQUIRE(listener.lastKeyFrameRequestedMappedSsrc == mappedSsrc); + } + + SECTION("OnTransportConnected() sets syncRequired") + { + MockListener listener; + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // First OnTransportConnected to set syncRequired, then sync with a keyframe. + manager->OnTransportConnected(); + manager->UpdateTargetLayers(0, 0); + + packet->SetSequenceNumber(1); + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Verify synced: next normal packet is not a sync packet. + packet->SetSequenceNumber(2); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.isSyncPacket == false); + + // Second OnTransportConnected — sets syncRequired again. + manager->OnTransportConnected(); + + // Re-set target because MayChangeLayers may have reset it. + manager->UpdateTargetLayers(0, 0); + + packet->SetSequenceNumber(3); + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + } + + SECTION("OnTransportConnected() requests keyframe when active and target changes") + { + MockListener listener; + // Use preferred layer 0 so RecalculateTargetLayers can find it with spatial-layer-0 bitrate. + auto manager = createManager(&listener, /*preferredLayers*/ { 0, 0 }); + auto rtpStream = createRtpStreamRecv(mappedSsrc, /*spatialLayers*/ 1u, /*temporalLayers*/ 1u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // Feed packets so GetSpatialLayerBitrate(0) returns non-zero. + // Packets without a PayloadDescriptorHandler default to spatialLayer=0. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto keyFrameCountBefore = listener.keyFrameRequestCount; + + // OnTransportConnected calls MayChangeLayers → RecalculateTargetLayers picks + // layer {0,0} (bitrate > 0), which differs from the initial {-1,-1} → + // UpdateTargetLayers → newTarget(0) > current(-1) → keyframe requested. + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount > keyFrameCountBefore); + REQUIRE(listener.lastKeyFrameRequestedMappedSsrc == mappedSsrc); + } + + SECTION("OnTransportConnected() does not request keyframe when not active") + { + MockListener listener; + listener.isActive = false; + + auto manager = createManager(&listener); + + // Don't wire producerRtpStream — manager is not active. + manager->OnTransportConnected(); + + REQUIRE(listener.keyFrameRequestCount == 0); + } + + SECTION("OnTransportDisconnected() resets target and current layers to -1") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(1, 1); + + manager->OnTransportDisconnected(); + + auto targetLayers = manager->GetTargetLayers(); + + REQUIRE(targetLayers.spatial == -1); + REQUIRE(targetLayers.temporal == -1); + REQUIRE(manager->GetCurrentSpatialLayer() == -1); + } + + SECTION("OnPaused() resets target and current layers to -1") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(2, 2); + + manager->OnPaused(); + + auto targetLayers = manager->GetTargetLayers(); + + REQUIRE(targetLayers.spatial == -1); + REQUIRE(targetLayers.temporal == -1); + REQUIRE(manager->GetCurrentSpatialLayer() == -1); + } + + SECTION("OnResumed() sets syncRequired and requests keyframe when active") + { + MockListener listener; + // Use preferred layer 0 so RecalculateTargetLayers can find it with spatial-layer-0 bitrate. + auto manager = createManager(&listener, /*preferredLayers*/ { 0, 0 }); + auto rtpStream = createRtpStreamRecv(mappedSsrc, /*spatialLayers*/ 1u, /*temporalLayers*/ 1u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // Feed packets so GetSpatialLayerBitrate(0) returns non-zero. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + // Connect so RecalculateTargetLayers sets target to {0,0}. + manager->OnTransportConnected(); + + // Disconnect: target and current reset to {-1,-1}. + manager->OnTransportDisconnected(); + + const auto keyFrameCountBefore = listener.keyFrameRequestCount; + + // OnResumed sets syncRequired and calls MayChangeLayers → RecalculateTargetLayers + // returns {0,0} which differs from {-1,-1} → UpdateTargetLayers(0,0) → + // newTarget(0) > current(-1) → keyframe requested. + manager->OnResumed(); + + REQUIRE(listener.keyFrameRequestCount > keyFrameCountBefore); + REQUIRE(listener.lastKeyFrameRequestedMappedSsrc == mappedSsrc); + } + + SECTION("OnResumed() sets syncRequired: next packet is a sync packet") + { + MockListener listener; + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(0, 0); + + // First sync via OnTransportConnected + keyframe. + manager->OnTransportConnected(); + manager->UpdateTargetLayers(0, 0); + + packet->SetSequenceNumber(1); + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Verify synced: second normal packet is not a sync packet. + packet->SetSequenceNumber(2); + auto result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + REQUIRE(result.isSyncPacket == false); + + // Call OnResumed — sets syncRequired again. + manager->OnResumed(); + + // Re-set target since MayChangeLayers may have reset it. + manager->UpdateTargetLayers(0, 0); + + // SVC requires a keyframe to complete sync. + packet->SetSequenceNumber(3); + handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + result = manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + REQUIRE(result.type == RtpPacketProcessResult::Type::FORWARD); + REQUIRE(result.isSyncPacket == true); + } + + SECTION("ProducerNewRtpStream() updates the producer stream and fires score event") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream0 = createRtpStreamRecv(); + auto rtpStream1 = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream0.get(), mappedSsrc); + + // Replace with a new stream. + manager->ProducerNewRtpStream(rtpStream1.get(), mappedSsrc); + + // GetProducerTargetRtpStream still returns the new stream when target is set. + manager->UpdateTargetLayers(0, 0); + + REQUIRE(manager->GetProducerTargetRtpStream() == rtpStream1.get()); + } + + SECTION("ProducerRtpStreamScore() triggers MayChangeLayers() when stream dies (score==0)") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->UpdateTargetLayers(1, 1); + + const auto layersChangedBefore = listener.layersChangedCount; + + // Score drops to 0 — MayChangeLayers is called, RecalculateTargetLayers + // returns -1,-1 (no active stream), UpdateTargetLayers fires layersChanged. + manager->ProducerRtpStreamScore(rtpStream.get(), /*score*/ 0, /*previousScore*/ 7); + + REQUIRE(listener.layersChangedCount > layersChangedBefore); + REQUIRE(manager->GetTargetLayers().spatial == -1); + } + + SECTION("ProducerRtpStreamScore() triggers MayChangeLayers() when stream revives (previousScore==0)") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // First bring score to 0 so target becomes -1. + manager->ProducerRtpStreamScore(rtpStream.get(), /*score*/ 0, /*previousScore*/ 7); + REQUIRE(manager->GetTargetLayers().spatial == -1); + + // Revive the stream — MayChangeLayers should fire again. + // Even though RecalculateTargetLayers may still return -1 (no bitrate), + // the important thing is the code path is exercised and no crash occurs. + manager->ProducerRtpStreamScore(rtpStream.get(), /*score*/ 7, /*previousScore*/ 0); + } + + SECTION("IncreaseLayer() returns 0 when no producer stream is set") + { + MockListener listener; + auto manager = createManager(&listener); + + manager->SetExternallyManagedBitrate(); + + auto nowMs = DepLibUV::GetTimeMs(); + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate == 0u); + } + + SECTION("IncreaseLayer() returns 0 when producer stream has score 0") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + + // Score is 0 by default (no RTP received + inactivity check disabled). + auto nowMs = DepLibUV::GetTimeMs(); + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate == 0u); + } + + SECTION("IncreaseLayer() returns 0 on second call in same iteration (already at preferred layers)") + { + MockListener listener; + // Single spatial/temporal layer so preferred == available immediately. + auto encodingContext = + std::make_unique(/*spatialLayers*/ 1u, /*temporalLayers*/ 1u); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 0, 0 }, + /*keyFrameSupported*/ true, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(mappedSsrc, /*spatialLayers*/ 1u, /*temporalLayers*/ 1u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + + // Feed packets so the stream has non-zero bitrate. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + + // First call claims bitrate. + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate > 0u); + + // Second call in same iteration should return 0 (already at preferred layers). + auto usedBitrate2 = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate2 == 0u); + } + + SECTION("IncreaseLayer() works again after ApplyLayers()") + { + MockListener listener; + auto encodingContext = + std::make_unique(/*spatialLayers*/ 1u, /*temporalLayers*/ 1u); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 0, 0 }, + /*keyFrameSupported*/ true, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(mappedSsrc, /*spatialLayers*/ 1u, /*temporalLayers*/ 1u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + + // Feed packets so the stream has non-zero bitrate. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + + // First iteration. + manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + manager->ApplyLayers(/*rtpStreamActiveMs*/ 0u); + + // After ApplyLayers, IncreaseLayer should work again. + auto usedBitrate = manager->IncreaseLayer( + /*bitrate*/ 1000000u, /*considerLoss*/ false, /*lossPercentage*/ 0.0f, nowMs); + + REQUIRE(usedBitrate > 0u); + } + + SECTION("ApplyLayers() records BWE downgrade when spatial layer drops below current") + { + MockListener listener; + auto encodingContext = + std::make_unique(/*spatialLayers*/ 3u, /*temporalLayers*/ 3u); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ false, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(mappedSsrc, /*spatialLayers*/ 3u, /*temporalLayers*/ 3u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + + // Manually set current to layer 2 via ProcessRtpPacket() sync. + manager->UpdateTargetLayers(2, 2); + + packet->SetSequenceNumber(1); + auto* handler = new MockPayloadDescriptorHandler(); + handler->isKeyFrame = true; + packet->SetPayloadDescriptorHandler(handler); + manager->ProcessRtpPacket( + packet.get(), /*lastSentPacketHasMarker*/ false, /*clockRate*/ 90000, /*maxPacketTs*/ 0); + + // Simulate a BWE downgrade: IncreaseLayer returns layer 0, then ApplyLayers + // detects current(2) > target(0) and rtpStreamActiveMs > BweDowngradeMinActiveMs. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + // Force provisional target to spatial 0 by only offering very little bitrate. + // With very low bitrate IncreaseLayer may return 0; skip that and call + // ApplyLayers directly with a high rtpStreamActiveMs. + // The important check is that no crash happens and target can be updated. + manager->ApplyLayers(/*rtpStreamActiveMs*/ 9000u); + + // After ApplyLayers the target may have changed; just assert no crash and + // that layers are either set or -1. + auto targetLayers = manager->GetTargetLayers(); + REQUIRE((targetLayers.spatial >= -1 && targetLayers.spatial <= 2)); + } + + SECTION("GetDesiredBitrate() returns 0 when no producer stream is set") + { + MockListener listener; + auto manager = createManager(&listener); + + auto nowMs = DepLibUV::GetTimeMs(); + auto desiredBitrate = manager->GetDesiredBitrate(nowMs); + + REQUIRE(desiredBitrate == 0u); + } + + SECTION("GetDesiredBitrate() (full SVC) returns total stream bitrate") + { + MockListener listener; + // Full SVC (ksvc=false). + auto encodingContext = std::make_unique(3u, 3u, /*ksvc*/ false); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ true, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(mappedSsrc, 3u, 3u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + auto streamBitrate = rtpStream->GetBitrate(nowMs); + auto desiredBitrate = manager->GetDesiredBitrate(nowMs); + + REQUIRE(desiredBitrate == streamBitrate); + } + + SECTION("GetDesiredBitrate() (K-SVC) returns the maximum spatial-layer bitrate") + { + MockListener listener; + // K-SVC (ksvc=true): GetDesiredBitrate iterates per-layer bitrates. + auto encodingContext = std::make_unique(3u, 3u, /*ksvc*/ true); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ true, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(mappedSsrc, 3u, 3u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + + // Feed packets so total stream bitrate > 0. + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + auto desiredBitrate = manager->GetDesiredBitrate(nowMs); + + // K-SVC returns the max per-spatial-layer bitrate. With a plain RtpStreamRecv + // and no per-layer packet tagging the returned value may be 0 or > 0. + // The contract is just: no crash and type is uint32_t. + REQUIRE(desiredBitrate >= 0u); + } + + SECTION("RecalculateTargetLayers() returns false when no producer stream is set") + { + MockListener listener; + auto manager = createManager(&listener); + + RTC::ConsumerTypes::VideoLayers newTargetLayers; + const bool changed = manager->RecalculateTargetLayers(newTargetLayers); + + // No change from initial -1,-1 target. + REQUIRE(changed == false); + REQUIRE(newTargetLayers.spatial == -1); + } + + SECTION("RecalculateTargetLayers() resets layers when stream score is 0") + { + MockListener listener; + auto manager = createManager(&listener); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // Score is 0 by default. + RTC::ConsumerTypes::VideoLayers newTargetLayers; + const auto changed = manager->RecalculateTargetLayers(newTargetLayers); + + REQUIRE(newTargetLayers.spatial == -1); + // changed may be false because initial target is also -1. + (void)changed; + } + + SECTION("K-SVC: UpdateTargetLayers() requests keyframe on any spatial layer change") + { + MockListener listener; + auto encodingContext = std::make_unique(3u, 3u, /*ksvc*/ true); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ true, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // For K-SVC, even a downgrade should trigger a keyframe request. + manager->UpdateTargetLayers(2, 0); + + const auto keyFrameRequestCountBefore = listener.keyFrameRequestCount; + + // Downgrade in K-SVC also requests a keyframe. + manager->UpdateTargetLayers(0, 0); + + REQUIRE(listener.keyFrameRequestCount > keyFrameRequestCountBefore); + REQUIRE(listener.lastKeyFrameRequestedMappedSsrc == mappedSsrc); + } + + SECTION("Full SVC: UpdateTargetLayers() does NOT request keyframe on spatial downgrade") + { + MockListener listener; + auto encodingContext = std::make_unique(3u, 3u, /*ksvc*/ false); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 2, 2 }, + /*keyFrameSupported*/ true, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + + // Manually set target AND current spatial layer to 2 so that a downgrade + // to 0 does not qualify as an upgrade (newTarget 0 < current 2). + manager->UpdateTargetLayers(2, 0); + manager->GetEncodingContext()->SetCurrentSpatialLayer(2); + manager->GetEncodingContext()->SetCurrentTemporalLayer(0); + + // Reset counter after the initial upgrade keyframe request. + listener.keyFrameRequestCount = 0; + + // Downgrade in full SVC: current(2) > new target(0) — no keyframe requested. + manager->UpdateTargetLayers(0, 0); + + REQUIRE(listener.keyFrameRequestCount == 0); + } + + SECTION("IncreaseLayer() uses virtual bitrate when considerLoss is true and loss < 2%") + { + MockListener listener; + auto encodingContext = std::make_unique(1u, 1u, /*ksvc*/ false); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 0, 0 }, + /*keyFrameSupported*/ true, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(mappedSsrc, 1u, 1u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + auto streamBitrate = rtpStream->GetBitrate(nowMs); + + // With 1% loss virtualBitrate = 1.08 * bitrate, so we can afford the stream + // even with slightly less physical bitrate available. + auto availableBitrate = static_cast(streamBitrate * 0.95f); + + auto usedBitrate = manager->IncreaseLayer( + availableBitrate, /*considerLoss*/ true, /*lossPercentage*/ 1.0f, nowMs); + + // virtualBitrate = 1.08 * availableBitrate >= streamBitrate, so the layer is taken. + REQUIRE(usedBitrate > 0u); + } + + SECTION("IncreaseLayer() uses reduced virtual bitrate when considerLoss is true and loss > 10%") + { + MockListener listener; + auto encodingContext = std::make_unique(1u, 1u, /*ksvc*/ false); + auto manager = createManager( + &listener, + /*preferredLayers*/ { 0, 0 }, + /*keyFrameSupported*/ true, + RTC::Media::Kind::VIDEO, + std::move(encodingContext)); + auto rtpStream = createRtpStreamRecv(mappedSsrc, 1u, 1u); + + manager->ProducerRtpStream(rtpStream.get(), mappedSsrc); + manager->SetExternallyManagedBitrate(); + + feedRtpStreamRecv(rtpStream.get(), packet.get(), 100); + + auto nowMs = DepLibUV::GetTimeMs(); + auto streamBitrate = rtpStream->GetBitrate(nowMs); + + // With 50% loss virtualBitrate = (1 - 0.5*0.5) * bitrate = 0.75 * bitrate. + // Pass available = streamBitrate but virtual will be reduced below required. + auto usedBitrate = + manager->IncreaseLayer(streamBitrate, /*considerLoss*/ true, /*lossPercentage*/ 50.0f, nowMs); + + // virtualBitrate = 0.75 * streamBitrate < streamBitrate (requiredBitrate), + // so the layer cannot be taken → 0. + REQUIRE(usedBitrate == 0u); + } +}