Request configuration headers from encoder

alvr/client_core/src/c_api.rs

@@ -443,6 +443,12 @@ pub extern "C" fn alvr_request_idr() {
     crate::request_idr();
 }
 
+/// Call only with external decoder
+#[no_mangle]
+pub extern "C" fn alvr_request_config_nal() {
+    crate::request_config_nal();
+}
+
 /// Call only with external decoder
 #[no_mangle]
 pub extern "C" fn alvr_report_frame_decoded(target_timestamp_ns: u64) {

alvr/client_core/src/decoder.rs

@@ -85,7 +85,7 @@ pub extern "C" fn push_nal(buffer: *const c_char, length: i32, timestamp_ns: u64
             show_err(decoder.push_frame_nal(timestamp, &nal, Duration::from_millis(500)));
         } else if let Some(sender) = &*crate::CONTROL_CHANNEL_SENDER.lock() {
             sender
-                .send(alvr_sockets::ClientControlPacket::RequestIdr)
+                .send(alvr_sockets::ClientControlPacket::RequestConfigNAL)
                 .ok();
         }
     }

alvr/client_core/src/lib.rs

@@ -221,6 +221,13 @@ pub fn request_idr() {
     }
 }
 
+/// Call only with external decoder
+pub fn request_config_nal() {
+    if let Some(sender) = &*CONTROL_CHANNEL_SENDER.lock() {
+        sender.send(ClientControlPacket::RequestConfigNAL).ok();
+    }
+}
+
 /// Call only with external decoder
 pub fn report_frame_decoded(target_timestamp: Duration) {
     if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {

alvr/server/cpp/alvr_server/ClientConnection.cpp

@@ -8,9 +8,6 @@
 #include "Utils.h"
 #include "Settings.h"
 
-static const uint8_t NAL_TYPE_SPS = 7;
-static const uint8_t H265_NAL_TYPE_VPS = 32;
-
 ClientConnection::ClientConnection() {
 
 	m_Statistics = std::make_shared<Statistics>();
@@ -21,31 +18,6 @@ ClientConnection::ClientConnection() {
 	m_fecPercentage = INITIAL_FEC_PERCENTAGE;
 }
 
-int findVPSSPS(const uint8_t *frameBuffer, int frameByteSize) {
-    int zeroes = 0;
-    int foundNals = 0;
-    for (int i = 0; i < frameByteSize; i++) {
-        if (frameBuffer[i] == 0) {
-            zeroes++;
-        } else if (frameBuffer[i] == 1) {
-            if (zeroes >= 2) {
-                foundNals++;
-                if (Settings::Instance().m_codec == ALVR_CODEC_H264 && foundNals >= 3) {
-                    // Find end of SPS+PPS on H.264.
-                    return i - 3;
-                } else if (Settings::Instance().m_codec == ALVR_CODEC_H265 && foundNals >= 4) {
-                    // Find end of VPS+SPS+PPS on H.264.
-                    return i - 3;
-                }
-            }
-            zeroes = 0;
-        } else {
-            zeroes = 0;
-        }
-    }
-    return -1;
-}
-
 void ClientConnection::FECSend(uint8_t *buf, int len, uint64_t targetTimestampNs, uint64_t videoFrameIndex) {
 	int shardPackets = CalculateFECShardPackets(len, m_fecPercentage);
 
@@ -133,31 +105,6 @@ void ClientConnection::SendVideo(uint8_t *buf, int len, uint64_t targetTimestamp
 	// Report before the frame is packetized
 	ReportEncoded(targetTimestampNs);
 
-	uint8_t NALType;
-	if (Settings::Instance().m_codec == ALVR_CODEC_H264)
-		NALType = buf[4] & 0x1F;
-	else
-		NALType = (buf[4] >> 1) & 0x3F;
-
-	if ((Settings::Instance().m_codec == ALVR_CODEC_H264 && NALType == NAL_TYPE_SPS) ||
-		(Settings::Instance().m_codec == ALVR_CODEC_H265 && NALType == H265_NAL_TYPE_VPS)) {
-		// This frame contains (VPS + )SPS + PPS + IDR on NVENC H.264 (H.265) stream.
-		// (VPS + )SPS + PPS has short size (8bytes + 28bytes in some environment), so we can
-		// assume SPS + PPS is contained in first fragment.
-
-		int end = findVPSSPS(buf, len);
-		if (end == -1) {
-			// Invalid frame.
-			return;
-		}
-
-		InitializeDecoder((const unsigned char *)buf, end);
-
-		// move the cursor forward excluding config NALs
-		buf = &buf[end];
-		len = len - end;
-	}
-
 	if (Settings::Instance().m_enableFec) {
 		FECSend(buf, len, targetTimestampNs, mVideoFrameIndex);
 	} else {

alvr/server/cpp/alvr_server/alvr_server.cpp

@@ -242,6 +242,12 @@ void RequestIDR() {
     }
 }
 
+void RequestConfigNAL() {
+    if (g_driver_provider.hmd && g_driver_provider.hmd->m_encoder) {
+        g_driver_provider.hmd->m_encoder->GetConfigNAL();
+    }
+}
+
 void SetTracking(unsigned long long targetTimestampNs,
                  float controllerPoseTimeOffsetS,
                  const AlvrDeviceMotion *deviceMotions,

alvr/server/cpp/alvr_server/bindings.h

@@ -140,6 +140,7 @@ extern "C" void InitializeStreaming();
 extern "C" void DeinitializeStreaming();
 extern "C" void SendVSync(float frameIntervalS);
 extern "C" void RequestIDR();
+extern "C" void RequestConfigNAL();
 extern "C" void SetTracking(unsigned long long targetTimestampNs,
                             float controllerPoseTimeOffsetS,
                             const AlvrDeviceMotion *deviceMotions,

alvr/server/cpp/platform/linux/CEncoder.cpp

@@ -220,7 +220,7 @@ void CEncoder::Run() {
 
       alvr::VkFrameCtx vk_frame_ctx(vk_ctx, output.imageInfo);
       alvr::VkFrame frame(vk_ctx, output.image, output.imageInfo, output.size, output.memory, output.drm);
-      auto encode_pipeline = alvr::EncodePipeline::Create(&render, vk_ctx, frame, vk_frame_ctx, render.GetEncodingWidth(), render.GetEncodingHeight());
+      m_encodePipeline = alvr::EncodePipeline::Create(&render, vk_ctx, frame, vk_frame_ctx, render.GetEncodingWidth(), render.GetEncodingHeight());
 
       fprintf(stderr, "CEncoder starting to read present packets");
       present_packet frame_info;
@@ -229,7 +229,7 @@ void CEncoder::Run() {
         read_latest(client, (char *)&frame_info, sizeof(frame_info), m_exiting);
 
         if (m_listener->GetStatistics()->CheckBitrateUpdated()) {
-          encode_pipeline->SetBitrate(m_listener->GetStatistics()->GetBitrate() * 1000000L); // in bits;
+          m_encodePipeline->SetBitrate(m_listener->GetStatistics()->GetBitrate() * 1000000L); // in bits;
         }
 
         auto pose = m_poseHistory->GetBestPoseMatch((const vr::HmdMatrix34_t&)frame_info.pose);
@@ -246,19 +246,19 @@ void CEncoder::Run() {
 
         render.Render(frame_info.image, frame_info.semaphore_value);
 
-        encode_pipeline->PushFrame(pose->targetTimestampNs, m_scheduler.CheckIDRInsertion());
+        m_encodePipeline->PushFrame(pose->targetTimestampNs, m_scheduler.CheckIDRInsertion());
 
         static_assert(sizeof(frame_info.pose) == sizeof(vr::HmdMatrix34_t&));
 
         encoded_data.clear();
         uint64_t pts;
-        if (!encode_pipeline->GetEncoded(encoded_data, &pts)) {
+        if (!m_encodePipeline->GetEncoded(encoded_data, &pts)) {
           Error("Failed to get encoded data!");
           continue;
         }
 
         auto render_timestamps = render.GetTimestamps();
-        auto encode_timestamp = encode_pipeline->GetTimestamp();
+        auto encode_timestamp = m_encodePipeline->GetTimestamp();
 
         uint64_t present_offset = render_timestamps.now - render_timestamps.renderBegin;
         uint64_t composed_offset = 0;
@@ -306,4 +306,6 @@ void CEncoder::OnPacketLoss() { m_scheduler.OnPacketLoss(); }
 
 void CEncoder::InsertIDR() { m_scheduler.InsertIDR(); }
 
+void CEncoder::GetConfigNAL() { m_encodePipeline->GetConfigNAL(); }
+
 void CEncoder::CaptureFrame() { m_captureFrame = true; }

alvr/server/cpp/platform/linux/CEncoder.h

@@ -20,13 +20,15 @@ class CEncoder : public CThread {
     void Stop();
     void OnPacketLoss();
     void InsertIDR();
+    void GetConfigNAL();
     bool IsConnected() { return m_connected; }
     void CaptureFrame();
 
   private:
     void GetFds(int client, int (*fds)[6]);
     std::shared_ptr<ClientConnection> m_listener;
     std::shared_ptr<PoseHistory> m_poseHistory;
+    std::unique_ptr<alvr::EncodePipeline> m_encodePipeline;
     std::atomic_bool m_exiting{false};
     IDRScheduler m_scheduler;
     pollfd m_socket;

alvr/server/cpp/platform/linux/EncodePipeline.h

@@ -26,6 +26,7 @@ class EncodePipeline
 
   virtual void PushFrame(uint64_t targetTimestampNs, bool idr) = 0;
   virtual bool GetEncoded(std::vector<uint8_t> & out, uint64_t *pts);
+  virtual void GetConfigNAL() = 0;
   virtual Timestamp GetTimestamp() { return timestamp; }
 
   virtual void SetBitrate(int64_t bitrate);

alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp

@@ -500,18 +500,12 @@ void EncodePipelineAMF::ApplyFrameProperties(const amf::AMFSurfacePtr &surface,
     case ALVR_CODEC_H264:
         surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_AUD, false);
         if (insertIDR) {
-            Debug("Inserting IDR frame for H.264.\n");
-            surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_SPS, true);
-            surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_PPS, true);
             surface->SetProperty(AMF_VIDEO_ENCODER_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_PICTURE_TYPE_IDR);
         }
         break;
     case ALVR_CODEC_H265:
         surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_AUD, false);
         if (insertIDR) {
-            Debug("Inserting IDR frame for H.265.\n");
-            // Insert VPS,SPS,PPS
-            surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_HEADER, true);
             surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_HEVC_PICTURE_TYPE_IDR);
         }
         break;
@@ -520,4 +514,15 @@ void EncodePipelineAMF::ApplyFrameProperties(const amf::AMFSurfacePtr &surface,
     }
 }
 
+void EncodePipelineAMF::GetConfigNAL() {
+	amf::AMFVariant var;
+	if (m_codec == ALVR_CODEC_H264) {
+		m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_EXTRADATA, &var);
+	} else {
+		m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_HEVC_EXTRADATA, &var);
+	}
+	amf::AMFBufferPtr buffer(var.pInterface);
+	InitializeDecoder(reinterpret_cast<unsigned char *>(buffer->GetNative()), buffer->GetSize());
+}
+
 };

alvr/server/cpp/platform/linux/EncodePipelineAMF.h

@@ -70,7 +70,7 @@ class EncodePipelineAMF : public EncodePipeline
     void PushFrame(uint64_t targetTimestampNs, bool idr) override;
     bool GetEncoded(std::vector<uint8_t> &out, uint64_t *pts) override;
     void SetBitrate(int64_t bitrate) override;
-
+    void GetConfigNAL() override;
 private:
     amf::AMFComponentPtr MakeConverter(amf::AMF_SURFACE_FORMAT inputFormat, int width, int height, amf::AMF_SURFACE_FORMAT outputFormat);
     amf::AMFComponentPtr MakePreprocessor(amf::AMF_SURFACE_FORMAT inputFormat, int width, int height);

alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp

@@ -120,6 +120,8 @@ alvr::EncodePipelineNvEnc::EncodePipelineNvEnc(Renderer *render,
     encoder_ctx->gop_size = INT16_MAX;
     encoder_ctx->bit_rate = settings.mEncodeBitrateMBs * 1000 * 1000;
 
+    encoder_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+
     err = avcodec_open2(encoder_ctx, codec, NULL);
     if (err < 0) {
         throw alvr::AvException("Cannot open video encoder codec:", err);
@@ -148,3 +150,7 @@ void alvr::EncodePipelineNvEnc::PushFrame(uint64_t targetTimestampNs, bool idr)
         throw alvr::AvException("avcodec_send_frame failed:", err);
     }
 }
+
+void alvr::EncodePipelineNvEnc::GetConfigNAL() {
+	InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size);
+}

alvr/server/cpp/platform/linux/EncodePipelineNvEnc.h

@@ -19,6 +19,7 @@ class EncodePipelineNvEnc: public EncodePipeline
   EncodePipelineNvEnc(Renderer *render, VkFrame &input_frame, VkFrameCtx& vk_frame_ctx, uint32_t width, uint32_t height);
 
   void PushFrame(uint64_t targetTimestampNs, bool idr) override;
+  void GetConfigNAL() override;
 
 private:
   Renderer *r = nullptr;

alvr/server/cpp/platform/linux/EncodePipelineSW.cpp

@@ -92,6 +92,8 @@ alvr::EncodePipelineSW::EncodePipelineSW(Renderer *render, uint32_t width, uint3
   encoder_ctx->thread_type = FF_THREAD_SLICE;
   encoder_ctx->thread_count = settings.m_swThreadCount;
 
+  encoder_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+
   int err = avcodec_open2(encoder_ctx, codec, &opt);
   if (err < 0) {
     throw alvr::AvException("Cannot open video encoder codec:", err);
@@ -127,3 +129,7 @@ void alvr::EncodePipelineSW::PushFrame(uint64_t targetTimestampNs, bool idr)
     throw alvr::AvException("avcodec_send_frame failed:", err);
   }
 }
+
+void alvr::EncodePipelineSW::GetConfigNAL() {
+	InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size);
+}

alvr/server/cpp/platform/linux/EncodePipelineSW.h

@@ -16,6 +16,7 @@ class EncodePipelineSW: public EncodePipeline
   EncodePipelineSW(Renderer *render, uint32_t width, uint32_t height);
 
   void PushFrame(uint64_t targetTimestampNs, bool idr) override;
+  void GetConfigNAL() override;
 
 private:
   AVFrame *encoder_frame = nullptr;

alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp

@@ -196,6 +196,8 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(Renderer *render, VkContext &vk_c
   encoder_ctx->rc_max_rate = encoder_ctx->bit_rate;
   encoder_ctx->rc_buffer_size = encoder_ctx->bit_rate / settings.m_refreshRate;
 
+  encoder_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+
   vlVaQualityBits quality = {};
   quality.valid_setting = 1;
   quality.vbaq_mode = Settings::Instance().m_enableVbaq;  //No noticable performance difference and should improve subjective quality by allocating more bits to smooth areas
@@ -318,3 +320,7 @@ void alvr::EncodePipelineVAAPI::PushFrame(uint64_t targetTimestampNs, bool idr)
   }
   av_frame_unref(encoder_frame);
 }
+
+void alvr::EncodePipelineVAAPI::GetConfigNAL() {
+	InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size);
+}

alvr/server/cpp/platform/linux/EncodePipelineVAAPI.h

@@ -30,6 +30,7 @@ class EncodePipelineVAAPI: public EncodePipeline
   EncodePipelineVAAPI(Renderer *render, VkContext &vk_ctx, VkFrame &input_frame, uint32_t width, uint32_t height);
 
   void PushFrame(uint64_t targetTimestampNs, bool idr) override;
+  void GetConfigNAL() override;
 
 private:
   Renderer *r = nullptr;

alvr/server/cpp/platform/win32/CEncoder.cpp

@@ -137,5 +137,9 @@
 			m_scheduler.InsertIDR();
 		}
 
+		void CEncoder::GetConfigNAL() {
+			m_videoEncoder->GetConfigNAL();
+		}
+
 		void CEncoder::CaptureFrame() {
 		}

alvr/server/cpp/platform/win32/CEncoder.h

@@ -55,6 +55,8 @@
 
 		void CaptureFrame();
 
+		void GetConfigNAL();
+
 	private:
 		CThreadEvent m_newFrameReady, m_encodeFinished;
 		std::shared_ptr<VideoEncoder> m_videoEncoder;

alvr/server/cpp/platform/win32/VideoEncoder.h

@@ -12,4 +12,5 @@ class VideoEncoder
 	virtual void Shutdown() = 0;
 
 	virtual void Transmit(ID3D11Texture2D *pTexture, uint64_t presentationTime, uint64_t targetTimestampNs, bool insertIDR) = 0;
+	virtual void GetConfigNAL() = 0;
 };

alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp

@@ -268,4 +268,10 @@ void VideoEncoderNVENC::FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializePar
 	if (Settings::Instance().m_nvencRcAverageBitrate != -1) {
 		encodeConfig.rcParams.averageBitRate = Settings::Instance().m_nvencRcAverageBitrate;
 	}
-}
+}
+
+void VideoEncoderNVENC::GetConfigNAL() {
+	std::vector<uint8_t> header;
+	m_NvNecoder->GetSequenceParams(header);
+	InitializeDecoder(header.data(), header.size());
+}

alvr/server/cpp/platform/win32/VideoEncoderNVENC.h

@@ -24,6 +24,8 @@ class VideoEncoderNVENC : public VideoEncoder
 	void Shutdown();
 
 	void Transmit(ID3D11Texture2D *pTexture, uint64_t presentationTime, uint64_t targetTimestampNs, bool insertIDR);
+
+	void GetConfigNAL();
 private:
 	void FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializeParams, int refreshRate, int renderWidth, int renderHeight, uint64_t bitrateBits);
 

alvr/server/cpp/platform/win32/VideoEncoderSW.cpp

@@ -93,8 +93,7 @@ void VideoEncoderSW::Initialize() {
 			av_dict_set(&opt, "nal-hrd", "vbr", 0);
 			break;
 	}
-	m_codecContext->rc_max_rate = Settings::Instance().mEncodeBitrateMBs * 1'000'000L;
-	m_codecContext->thread_count = Settings::Instance().m_swThreadCount;
+	m_codecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
 
 	if((err = avcodec_open2(m_codecContext, codec, &opt))) throw MakeException("Cannot open video encoder codec: %d", err);
 
@@ -297,4 +296,8 @@ AVCodecID VideoEncoderSW::ToFFMPEGCodec(ALVR_CODEC codec) {
 	}
 }
 
+void VideoEncoderSW::GetConfigNAL() {
+	InitializeDecoder(m_codecContext->extradata, m_codecContext->extradata_size);
+}
+
 #endif // ALVR_GPL

alvr/server/cpp/platform/win32/VideoEncoderSW.h

@@ -40,6 +40,8 @@ class VideoEncoderSW : public VideoEncoder
 	void Transmit(ID3D11Texture2D *pTexture, uint64_t presentationTime, uint64_t targetTimestampNs, bool insertIDR);
 	HRESULT SetupStagingTexture(ID3D11Texture2D *pTexture);
 	HRESULT CopyTexture(ID3D11Texture2D *pTexture);
+
+	void GetConfigNAL();
 private:
     std::shared_ptr<CD3DRender> m_d3dRender;
 	std::shared_ptr<ClientConnection> m_Listener;