From d24a4d12e213062bc7958239d7e65f5602be85e2 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 1 May 2023 12:26:00 +0700 Subject: [PATCH] More adjustments to how the pre-buffer is sent --- crates/core/src/bcmedia/codex.rs | 21 ++- src/rtsp/gst/factory.rs | 284 +++++++++++++++---------------- src/rtsp/gst/sender.rs | 63 ++++--- src/rtsp/gst/server.rs | 43 ++++- src/rtsp/mod.rs | 11 +- 5 files changed, 232 insertions(+), 190 deletions(-) diff --git a/crates/core/src/bcmedia/codex.rs b/crates/core/src/bcmedia/codex.rs index 4fc40bc..a8aa28e 100644 --- a/crates/core/src/bcmedia/codex.rs +++ b/crates/core/src/bcmedia/codex.rs @@ -12,11 +12,15 @@ pub struct BcMediaCodex { /// If true we will not search for the start of the next packet /// in the event that the stream appears to be corrupted strict: bool, + amount_skipped: usize, } impl BcMediaCodex { pub(crate) fn new(strict: bool) -> Self { - Self { strict } + Self { + strict, + amount_skipped: 0, + } } } @@ -46,18 +50,19 @@ impl Decoder for BcMediaCodex { } fn decode(&mut self, src: &mut BytesMut) -> Result> { - let mut amount_skipped: usize = 0; loop { match { BcMedia::deserialize(src) } { Ok(bc) => { - if amount_skipped > 0 { - debug!("Amount skipped to restore stream: {}", amount_skipped); + if self.amount_skipped > 0 { + debug!("Amount skipped to restore stream: {}", self.amount_skipped); + self.amount_skipped = 0; } return Ok(Some(bc)); } Err(Error::NomIncomplete(_)) => { - if amount_skipped > 0 { - debug!("Amount skipped to restore stream: {}", amount_skipped); + if self.amount_skipped > 0 { + debug!("Amount skipped to restore stream: {}", self.amount_skipped); + self.amount_skipped = 0; } return Ok(None); } @@ -67,12 +72,12 @@ impl Decoder for BcMediaCodex { } else if src.is_empty() { return Ok(None); } else { - if amount_skipped == 0 { + if self.amount_skipped == 0 { debug!("Error in stream attempting to restore"); trace!(" Stream Error: {:?}", e); } // Drop the whole packet and wait for a packet that starts with magic - amount_skipped += src.len(); + self.amount_skipped += src.len(); src.clear(); continue; } diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 34823ab..0d0f94f 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -26,7 +26,10 @@ use std::{ sync::{atomic::Ordering, Arc}, }; use tokio::{ - sync::mpsc::{channel, Sender}, + sync::{ + mpsc::{channel, Sender}, + RwLock, + }, task::JoinSet, time::Duration, }; @@ -104,6 +107,10 @@ impl NeoMediaFactory { pub(crate) fn buffer_ready(&self) -> bool { self.imp().buffer_ready() } + + pub(crate) async fn join(&self) -> AnyResult<()> { + self.imp().join().await + } } unsafe impl Send for NeoMediaFactory {} @@ -114,7 +121,7 @@ pub(crate) struct NeoMediaFactoryImpl { clientsender: Sender, shared: Arc, #[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj - threads: JoinSet>, + threads: RwLock>>, } impl Drop for NeoMediaFactoryImpl { @@ -149,14 +156,15 @@ impl Default for NeoMediaFactoryImpl { } } } - Ok(()) + unreachable!(); + // Ok(()) }); Self { sender: datasender, clientsender, shared, - threads, + threads: RwLock::new(threads), } } } @@ -294,12 +302,6 @@ impl NeoMediaFactoryImpl { // source.set_property("max-time", Duration::from_secs_f32(2.25).as_nanos() as u64); source.set_do_timestamp(false); source.set_stream_type(AppStreamType::RandomAccess); - let clock = source.clock(); - if let Some(clock) = clock { - debug!("Clock: {:?}", clock.time()); - } else { - debug!("No clock"); - } let need_command = client_data.get_commader(); let enough_command = client_data.get_commader(); @@ -391,154 +393,136 @@ impl NeoMediaFactoryImpl { } } - let do_aud = false; - if do_aud { - match *self.shared.aud_format.blocking_read() { - AudFormats::Unknown => {} - AudFormats::Aac => { - debug!("Building Aac pipeline"); - let source = make_element("appsrc", "audsrc")? - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + // let do_aud = false; + // if do_aud { + match *self.shared.aud_format.blocking_read() { + AudFormats::Unknown => {} + AudFormats::Aac => { + debug!("Building Aac pipeline"); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_is_live(true); - source.set_block(false); - source.set_property("emit-signals", false); - source.set_max_bytes(52428800); - source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Seekable); - let need_command = client_data.get_commader(); - let enough_command = client_data.get_commader(); - let seek_command = client_data.get_commader(); - source.set_callbacks( - AppSrcCallbacks::builder() - .need_data(move |_appsrc, _amt| { - let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); - }) - .enough_data(move |_appsrc| { - let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); - }) - .seek_data(move |_appsrc, seek_pos| { - debug!("Send seek AAC"); - let _ = seek_command - .blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); - true - }) - .build(), - ); + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + // let seek_command = client_data.get_commader(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); + }) + .enough_data(move |_appsrc| { + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); + }) + .seek_data(move |_appsrc, _seek_pos| true) + .build(), + ); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast back"))?; + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); - let parser = make_element("aacparse", "audparser")?; - let decoder = make_element("decodebin", "auddecoder")?; - let encoder = make_element("audioconvert", "audencoder")?; - let payload = make_element("rtpL16pay", "audpayload")?; - let storage = make_element("rtpstorage", "audstorage")?; - storage - .set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64); - let jitter = make_element("rtpjitterbuffer", "pay1")?; - jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32); + let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); + let parser = make_element("aacparse", "audparser")?; + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; - bin.add_many(&[ - &source, &queue, &parser, &decoder, &encoder, &payload, &storage, &jitter, - ])?; - Element::link_many(&[&source, &queue, &parser, &decoder])?; - Element::link_many(&[&encoder, &payload, &storage, &jitter])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link AAC decoder to encoder"); - }); + bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &parser, &decoder])?; + Element::link_many(&[&encoder, &payload])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link AAC decoder to encoder"); + }); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.update_aud(source); - } - AudFormats::Adpcm(block_size) => { - debug!("Building Adpcm pipeline"); - // Original command line - // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 - // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 - // ! adpcmdec - // ! audioconvert - // ! rtpL16pay name=pay1 + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + client_data.update_aud(source); + } + AudFormats::Adpcm(block_size) => { + debug!("Building Adpcm pipeline"); + // Original command line + // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 + // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 + // ! adpcmdec + // ! audioconvert + // ! rtpL16pay name=pay1 - let source = make_element("appsrc", "audsrc")? - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_is_live(true); - source.set_block(false); - source.set_property("emit-signals", false); - source.set_max_bytes(52428800); - source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::RandomAccess); - source.set_caps(Some( - &Caps::builder("audio/x-adpcm") - .field("layout", "div") - .field("block_align", block_size as i32) - .field("channels", 1i32) - .field("rate", 8000i32) - .build(), - )); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::RandomAccess); + source.set_caps(Some( + &Caps::builder("audio/x-adpcm") + .field("layout", "div") + .field("block_align", block_size as i32) + .field("channels", 1i32) + .field("rate", 8000i32) + .build(), + )); - let need_command = client_data.get_commader(); - let enough_command = client_data.get_commader(); - let seek_command = client_data.get_commader(); - source.set_callbacks( - AppSrcCallbacks::builder() - .need_data(move |_appsrc, _amt| { - let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); - }) - .enough_data(move |_appsrc| { - let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); - }) - .seek_data(move |_appsrc, seek_pos| { - debug!("Send seek Adpcm"); - let _ = seek_command - .blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); - true - }) - .build(), - ); + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + // let seek_command = client_data.get_commader(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); + }) + .enough_data(move |_appsrc| { + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); + }) + .seek_data(move |_appsrc, _seek_pos| true) + .build(), + ); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast back"))?; + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); - let decoder = make_element("decodebin", "auddecoder")?; - let encoder = make_element("audioconvert", "audencoder")?; - let payload = make_element("rtpL16pay", "pay1")?; + let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; - bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &decoder])?; - Element::link_many(&[&encoder, &payload])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link ADPCM decoder to encoder"); - }); + bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &decoder])?; + Element::link_many(&[&encoder, &payload])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link ADPCM decoder to encoder"); + }); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.update_aud(source); - } + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + client_data.update_aud(source); } } @@ -547,4 +531,12 @@ impl NeoMediaFactoryImpl { bin.dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back")) } + + async fn join(&self) -> AnyResult<()> { + let mut threads = self.threads.write().await; + while let Some(thread) = threads.join_next().await { + thread??; + } + Ok(()) + } } diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 4bbee00..0737ba2 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -27,8 +27,9 @@ use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; -const LATENCY: Duration = Duration::from_millis(200); +const LATENCY: Duration = Duration::from_millis(20); const BUFFER_TIME: Duration = Duration::from_secs(10); +const BUFFER_SIZE: usize = 2; #[derive(Default)] struct NeoBuffer { @@ -68,7 +69,7 @@ impl NeoBuffer { if let Some(last_frame_time) = self.last_iframe_time() { let mut time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff); while let Some(time) = time_delta { - if time > BUFFER_TIME && self.buf.len() > 2 { + if time > BUFFER_TIME && self.buf.len() > BUFFER_SIZE { let _ = self.buf.pop_front(); debug!("Popping frame with time difference of {:?}", time); time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff); @@ -242,6 +243,9 @@ impl NeoMediaSenders { _ => {} } self.buffer.push(data); + if self.buffer.buf.len() >= BUFFER_SIZE { + self.shared.buffer_ready.store(true, Ordering::Relaxed); + } Ok(()) } @@ -320,7 +324,7 @@ pub(super) struct NeoMediaSender { impl NeoMediaSender { pub(super) fn new() -> Self { - let (tx, rx) = channel(3); + let (tx, rx) = channel(30); Self { start_time: Duration::ZERO, last_sent_time: Duration::ZERO, @@ -347,25 +351,24 @@ impl NeoMediaSender { async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { - if let Some(runtime) = self.get_runtime() { - let (fronts, backs) = buffer.buf.as_slices(); - let frames = fronts.iter().chain(backs.iter()).collect::>(); - let idx = frames.len().saturating_div(2); - let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| { - if let BcMedia::Iframe(data) = f { - Some(Duration::from_micros(data.microseconds as u64)) - } else { - None - } - }); - if let Some(target_time) = target_time { - self.start_time = target_time - runtime; - self.last_sent_time = target_time; - debug!( - "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", - target_time, self.start_time, target_time - self.start_time, runtime - ); + let runtime = self.get_runtime().unwrap_or(Duration::ZERO); + let (fronts, backs) = buffer.buf.as_slices(); + let frames = fronts.iter().chain(backs.iter()).collect::>(); + let idx = frames.len().saturating_div(2); + let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| { + if let BcMedia::Iframe(data) = f { + Some(Duration::from_micros(data.microseconds as u64)) + } else { + None } + }); + if let Some(target_time) = target_time { + self.start_time = target_time - runtime; + self.last_sent_time = target_time; + debug!( + "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", + target_time, self.start_time, target_time - self.start_time, runtime + ); } } @@ -484,7 +487,25 @@ impl NeoMediaSender { } } else { // Not live. Send the WHOLE buffer for analysis + // Up to the last sent frame + debug!("Sending pre buffer"); for frame in buf.iter() { + let frame_time = match frame { + BcMedia::Iframe(data) => { + Some(Duration::from_micros(data.microseconds as u64)) + } + BcMedia::Pframe(data) => { + Some(Duration::from_micros(data.microseconds as u64)) + } + _ => None, + }; + debug!(" - Maybe: {:?} > {:?}", frame_time, self.last_sent_time); + if let Some(frame_time) = frame_time { + if frame_time > self.last_sent_time { + return Ok(()); + } + } + debug!(" - Send: {:?}", frame_time); self.send_buffer(frame).await?; } } diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 3a85da0..15e2bad 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -24,7 +24,7 @@ use std::{ }; use tokio::{ sync::{mpsc::Sender, RwLock}, - task::JoinHandle, + task::JoinSet, }; glib::wrapper! { @@ -83,11 +83,7 @@ impl NeoRtspServer { self.imp().add_permitted_roles(tag, permitted_users).await } - pub(crate) async fn run( - &self, - bind_addr: &str, - bind_port: u16, - ) -> AnyResult<(JoinHandle>, Arc)> { + pub(crate) async fn run(&self, bind_addr: &str, bind_port: u16) -> AnyResult<()> { let server = self; server.set_address(bind_addr); server.set_service(&format!("{}", bind_port)); @@ -96,10 +92,32 @@ impl NeoRtspServer { let main_loop = Arc::new(MainLoop::new(None, false)); // Run the Glib main loop. let main_loop_thread = main_loop.clone(); - #[allow(clippy::unit_arg)] - let handle = tokio::task::spawn_blocking(move || Ok(main_loop_thread.run())); + let handle = tokio::task::spawn_blocking(move || { + main_loop_thread.run(); + AnyResult::Ok(()) + }); + self.imp() + .threads + .write() + .await + .spawn(async move { handle.await? }); + self.imp().main_loop.write().await.replace(main_loop); + Ok(()) + } - Ok((handle, main_loop)) + pub(crate) async fn quit(&self) -> AnyResult<()> { + if let Some(main_loop) = self.imp().main_loop.read().await.as_ref() { + main_loop.quit(); + } + Ok(()) + } + + pub(crate) async fn join(&self) -> AnyResult<()> { + let mut threads = self.imp().threads.write().await; + while let Some(thread) = threads.join_next().await { + thread??; + } + Ok(()) } pub(crate) fn set_up_tls(&self, config: &Config) { @@ -163,6 +181,8 @@ struct FactoryData { #[derive(Default)] pub(crate) struct NeoRtspServerImpl { medias: RwLock>, + threads: RwLock>>, + main_loop: RwLock>>, } impl ObjectImpl for NeoRtspServerImpl {} @@ -182,6 +202,11 @@ impl NeoRtspServerImpl { Entry::Occupied(_occ) => {} Entry::Vacant(vac) => { let media = NeoMediaFactory::new(); + let thread_media = media.clone(); + self.threads + .write() + .await + .spawn(async move { thread_media.join().await }); vac.insert(FactoryData { factory: media, paths: Default::default(), diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index e07d11d..bf4bc70 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -131,14 +131,15 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { let bind_addr = config.bind_addr.clone(); let bind_port = config.bind_port; - let (handle, main_loop) = rtsp.run(&bind_addr, bind_port).await?; - set.spawn(async move { handle.await? }); + rtsp.run(&bind_addr, bind_port).await?; + let thread_rtsp = rtsp.clone(); + set.spawn(async move { thread_rtsp.join().await }); while let Some(joined) = set.join_next().await { match &joined { Err(_) | Ok(Err(_)) => { // Panicked or error in task - main_loop.quit(); + rtsp.quit().await?; } Ok(Ok(_)) => { // All good @@ -200,9 +201,7 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi break; } } - if streaming.get_config().pause.on_motion || streaming.get_config().pause.on_disconnect { - log::info!("Pause buffer prepared"); - } + log::info!("{}: Buffers prepared", name); // tokio::task::spawn(async move { // let mut inter = tokio::time::interval(tokio::time::Duration::from_secs_f32(0.01));