From e6f8aa1e6410e46482edc6069f657aba71b18578 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Fri, 12 Apr 2024 19:00:00 +0700 Subject: [PATCH] Addressing streaming issues --- .../src/bc_protocol/connection/udpsource.rs | 14 ++++++ crates/core/src/bcudp/xml.rs | 15 ++++++ src/rtsp/factory.rs | 31 +++++++----- src/rtsp/stream.rs | 48 ++++++++----------- 4 files changed, 67 insertions(+), 41 deletions(-) diff --git a/crates/core/src/bc_protocol/connection/udpsource.rs b/crates/core/src/bc_protocol/connection/udpsource.rs index 4a22f91..66dc72b 100644 --- a/crates/core/src/bc_protocol/connection/udpsource.rs +++ b/crates/core/src/bc_protocol/connection/udpsource.rs @@ -518,6 +518,20 @@ impl UdpPayloadInner { let (item, addr) = v.ok_or(Error::DroppedConnection)?; if addr == camera_addr { match item { + BcUdp::Discovery(UdpDiscovery{ + payload: UdpXml::D2cHb(D2cHb { + cid, + did, + }), + .. + }) => { + if cid != self.client_id { + log::info!("Camera sent different client ID in HB"); + } + if did != self.camera_id { + log::info!("Camera sent different device ID in HB"); + } + }, BcUdp::Discovery(_disc) => {}, BcUdp::Ack(ack) => { if ack.connection_id == self.client_id { diff --git a/crates/core/src/bcudp/xml.rs b/crates/core/src/bcudp/xml.rs index 397e707..4f6231e 100644 --- a/crates/core/src/bcudp/xml.rs +++ b/crates/core/src/bcudp/xml.rs @@ -60,6 +60,9 @@ pub enum UdpXml { /// C2D_HB xml client to device heartbeat. This is the keep alive #[serde(rename = "C2R_HB")] C2rHb(C2rHb), + /// D2D_HB xml client to device heartbeat. This is the keep alive + #[serde(rename = "D2C_HB")] + D2cHb(D2cHb), } /// The top level holder for P2P we auto add/remove this at serde @@ -424,3 +427,15 @@ pub struct C2rHb { /// The camera connection ID pub did: i32, } + +/// D2C_HB xml +/// +/// Device to client heart beat. +/// Seems to act as a keep alive +#[derive(PartialEq, Eq, Default, Debug, Deserialize, Serialize, Clone)] +pub struct D2cHb { + /// The client connection ID + pub cid: i32, + /// The camera connection ID + pub did: i32, +} diff --git a/src/rtsp/factory.rs b/src/rtsp/factory.rs index a37c248..ef2720d 100644 --- a/src/rtsp/factory.rs +++ b/src/rtsp/factory.rs @@ -156,7 +156,11 @@ fn build_unknown(bin: &Element, pattern: &str) -> Result<()> { } fn build_h264(bin: &Element, stream_config: &StreamConfig) -> Result { - let buffer_size = buffer_size(stream_config.bitrate / 10 / 8); + let buffer_size = buffer_size(stream_config.bitrate); + log::info!( + "buffer_size: {buffer_size}, bitrate: {}", + stream_config.bitrate + ); let bin = bin .clone() .dynamic_cast::() @@ -179,10 +183,10 @@ fn build_h264(bin: &Element, stream_config: &StreamConfig) -> Result { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_queue("source_queue", buffer_size)?; let parser = make_element("h264parse", "parser")?; - let stamper = make_element("h264timestamper", "stamper")?; + // let stamper = make_element("h264timestamper", "stamper")?; let payload = make_element("rtph264pay", "pay0")?; - bin.add_many([&source, &queue, &parser, &stamper, &payload])?; - Element::link_many([&source, &queue, &parser, &stamper, &payload])?; + bin.add_many([&source, &queue, &parser, &payload])?; + Element::link_many([&source, &queue, &parser, &payload])?; let source = source .dynamic_cast::() @@ -191,7 +195,7 @@ fn build_h264(bin: &Element, stream_config: &StreamConfig) -> Result { } fn build_h265(bin: &Element, stream_config: &StreamConfig) -> Result { - let buffer_size = buffer_size(stream_config.bitrate / 8 / 10); + let buffer_size = buffer_size(stream_config.bitrate); let bin = bin .clone() .dynamic_cast::() @@ -213,10 +217,10 @@ fn build_h265(bin: &Element, stream_config: &StreamConfig) -> Result { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_queue("source_queue", buffer_size)?; let parser = make_element("h265parse", "parser")?; - let stamper = make_element("h265timestamper", "stamper")?; + // let stamper = make_element("h265timestamper", "stamper")?; let payload = make_element("rtph265pay", "pay0")?; - bin.add_many([&source, &queue, &parser, &stamper, &payload])?; - Element::link_many([&source, &queue, &parser, &stamper, &payload])?; + bin.add_many([&source, &queue, &parser, &payload])?; + Element::link_many([&source, &queue, &parser, &payload])?; let source = source .dynamic_cast::() @@ -224,8 +228,8 @@ fn build_h265(bin: &Element, stream_config: &StreamConfig) -> Result { Ok(source) } -fn build_aac(bin: &Element, stream_config: &StreamConfig) -> Result { - let buffer_size = buffer_size(512 * 10); +fn build_aac(bin: &Element, _stream_config: &StreamConfig) -> Result { + let buffer_size = 512 * 6; let bin = bin .clone() .dynamic_cast::() @@ -289,8 +293,8 @@ fn build_aac(bin: &Element, stream_config: &StreamConfig) -> Result { Ok(source) } -fn build_adpcm(bin: &Element, block_size: u32, stream_config: &StreamConfig) -> Result { - let buffer_size = buffer_size(512 * 6); +fn build_adpcm(bin: &Element, block_size: u32, _stream_config: &StreamConfig) -> Result { + let buffer_size = 512 * 6; let bin = bin .clone() .dynamic_cast::() @@ -442,5 +446,6 @@ fn make_queue(name: &str, buffer_size: u32) -> AnyResult { } fn buffer_size(bitrate: u32) -> u32 { - std::cmp::max(bitrate * 15u32 / 8u32, 4u32 * 1024u32 * 1024u32) + // 0.1 seconds (according to bitrate) or 4kb what ever is larger + std::cmp::max(bitrate / 10u32 / 8u32, 4u32 * 1024u32) } diff --git a/src/rtsp/stream.rs b/src/rtsp/stream.rs index 8a758f0..ce34c43 100644 --- a/src/rtsp/stream.rs +++ b/src/rtsp/stream.rs @@ -413,8 +413,8 @@ async fn stream_run( let mut thread_client_count = client_count.subscribe(); let thread_format = stream_config.vid_format; // let fallback_time = Duration::from_secs(3); - let framerate = - Duration::from_millis(1000u64 / std::cmp::max(stream_config.fps as u64, 5u64)); + // let framerate = + // Duration::from_millis(1000u64 / std::cmp::max(stream_config.fps as u64, 5u64)); if let Some(thread_vid) = thread_vid { set.spawn(async move { thread_client_count.activate().await?; @@ -424,14 +424,14 @@ async fn stream_run( }, v = send_to_appsrc( pad_vid( - frametime_stream( - ensure_order( + // frametime_stream( + // ensure_order( wait_for_keyframe( vid_data_rx, - ) - ), - framerate - ), + ), + // ), + // framerate + // ), thread_format, ), &thread_vid @@ -449,8 +449,8 @@ async fn stream_run( let thread_stream_cancel = stream_cancel.clone(); let aud_data_rx = BroadcastStream::new(aud_data_rx).filter(|f| f.is_ok()); // Filter to ignore lagged let thread_aud = aud.clone(); - let aud_framerate = - Duration::from_millis(1000u64 / std::cmp::max(stream_config.fps as u64, 5u64)); + // let aud_framerate = + // Duration::from_millis(1000u64 / std::cmp::max(stream_config.fps as u64, 5u64)); if let Some(thread_aud) = thread_aud { set.spawn(async move { let r = tokio::select! { @@ -458,14 +458,14 @@ async fn stream_run( AnyResult::Ok(()) }, v = send_to_appsrc( - frametime_stream( - ensure_order( + // frametime_stream( + // ensure_order( wait_for_keyframe( aud_data_rx - ) - ), - aud_framerate - ), &thread_aud) => { + ), + // ), + // aud_framerate), + &thread_aud) => { v }, }; @@ -551,6 +551,7 @@ fn hold_stream> + Unpin>( // Take a stream of stamped data and reorder it // in case they are out of order // this also releases frames in waves of keyframe so it should replace `hold_stream` +#[allow(dead_code)] fn ensure_order> + Unpin>( mut stream: T, ) -> impl Stream> + Unpin { @@ -565,6 +566,7 @@ fn ensure_order> + Unpin>( reorder_buffer.push(frame_buffer.pop().unwrap()); } frame_buffer.push(frame); + while let Some(frame) = reorder_buffer.pop() { frame_buffer.push(frame); } @@ -582,6 +584,7 @@ fn ensure_order> + Unpin>( // Take a stream of stamped data pause until // it is time to display it +#[allow(dead_code)] fn frametime_stream> + Unpin>( mut stream: T, expected_frame_rate: Duration, @@ -719,9 +722,8 @@ async fn send_to_appsrc> + Unpin>( let mut rt = Duration::ZERO; let mut wait_for_iframe = true; let mut pools: HashMap = Default::default(); - let mut dts: u64 = 0; let mut buffer_inited = false; - log::info!("Setting stream to pause"); + let mut dts = 1; appsrc.set_state(gstreamer::State::Paused).unwrap(); let (tx, mut rx) = tokio::sync::mpsc::channel::(2000); @@ -744,17 +746,14 @@ async fn send_to_appsrc> + Unpin>( // let mut gst_buf = pool.acquire_buffer(None).unwrap(); let msg_size = data.data.len(); let pool = pools.entry(msg_size).or_insert_with_key(|size| { - log::info!("new pool: {}", size); let pool = gstreamer::BufferPool::new(); let mut pool_config = pool.config(); pool_config.set_params(None, (*size) as u32, 8, 0); pool.set_config(pool_config).unwrap(); // let (allocator, alloc_parms) = pool.allocator().unwrap(); pool.set_active(true).unwrap(); - log::info!("Options: {:?}", pool.options()); pool }); - log::info!("buffer size: {}", data.data.len()); let mut gst_buf = pool.acquire_buffer(None).unwrap(); // let mut gst_buf = gstreamer::Buffer::with_size(data.data.len()).unwrap(); { @@ -781,15 +780,8 @@ async fn send_to_appsrc> + Unpin>( Err(e) => Err(anyhow!("Error in streaming: {e:?}")), }?; if !buffer_inited && appsrc.current_level_bytes() >= appsrc.max_bytes() { - log::info!("Setting stream to play"); appsrc.set_state(gstreamer::State::Playing).unwrap(); buffer_inited = true; - } else { - log::info!( - "Not yet ready: {}/{}", - appsrc.current_level_bytes(), - appsrc.max_bytes() - ); } } AnyResult::Ok(())