From 824206faaac196fc0253454e8e76747620be20b3 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 16 Apr 2023 15:54:47 +0700 Subject: [PATCH] Reducing blocking on gstraemer pushes --- .../core/src/bc_protocol/connection/bcconn.rs | 3 +- .../src/bc_protocol/connection/udpsource.rs | 14 +- crates/core/src/bc_protocol/stream.rs | 13 +- src/rtsp/gst/factory.rs | 14 +- src/rtsp/gst/sender.rs | 141 ++++++++++++------ src/rtsp/states/streaming.rs | 5 +- 6 files changed, 133 insertions(+), 57 deletions(-) diff --git a/crates/core/src/bc_protocol/connection/bcconn.rs b/crates/core/src/bc_protocol/connection/bcconn.rs index 790473b..d4b2160 100644 --- a/crates/core/src/bc_protocol/connection/bcconn.rs +++ b/crates/core/src/bc_protocol/connection/bcconn.rs @@ -8,6 +8,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use tokio::sync::mpsc::{channel, Sender}; use tokio::sync::Mutex; +use tokio::task::yield_now; use tokio::{sync::RwLock, task::JoinSet}; @@ -46,12 +47,12 @@ impl BcConnection { let sink_thread = sink.clone(); rx_thread.spawn(async move { loop { + yield_now().await; // Give other tasks a chance to run before pulling next packet let packet = source.next().await; let bc = match packet { Some(res) => res, None => continue, }; - Self::poll(bc, &subs, &sink_thread).await? } }); diff --git a/crates/core/src/bc_protocol/connection/udpsource.rs b/crates/core/src/bc_protocol/connection/udpsource.rs index a05553f..b57ae0f 100644 --- a/crates/core/src/bc_protocol/connection/udpsource.rs +++ b/crates/core/src/bc_protocol/connection/udpsource.rs @@ -199,9 +199,10 @@ impl Sink<(BcUdp, SocketAddr)> for BcUdpSource { } enum State { - Normal, - Flushing, - Closed, + Normal, // Normal recieve + Flushing, // Used to send ack packets and things in the buffer + Closed, // Used to shutdown + YieldNow, // Used to ensure we rest between polling packets so as to not starve the runtime } pub(crate) struct UdpPayloadSource { @@ -297,6 +298,7 @@ impl Stream for UdpPayloadSource { let mut this = self.get_mut(); match this.state { State::Normal => { + this.state = State::YieldNow; // Data ready to go if let Some(payload) = this.recieved.remove(&this.packets_want) { this.packets_want += 1; @@ -384,6 +386,9 @@ impl Stream for UdpPayloadSource { State::Closed => { return Poll::Ready(Some(Err(IoError::from(ErrorKind::ConnectionAborted)))); } + State::YieldNow => { + this.state = State::Normal; + } } cx.waker().wake_by_ref(); Poll::Pending @@ -399,7 +404,7 @@ impl Sink> for UdpPayloadSource { ) -> Poll> { let this = self.get_mut(); match this.state { - State::Normal => this + State::Normal | State::YieldNow => this .inner .poll_ready_unpin(cx) .map_err(|e| IoError::new(ErrorKind::Other, e)), @@ -476,6 +481,7 @@ impl Sink> for UdpPayloadSource { cx.waker().wake_by_ref(); Poll::Pending } + fn poll_close( self: Pin<&mut Self>, cx: &mut Context<'_>, diff --git a/crates/core/src/bc_protocol/stream.rs b/crates/core/src/bc_protocol/stream.rs index 6457d36..01424dd 100644 --- a/crates/core/src/bc_protocol/stream.rs +++ b/crates/core/src/bc_protocol/stream.rs @@ -4,6 +4,7 @@ use crate::{ bcmedia::model::*, }; use futures::stream::StreamExt; +use log::*; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -50,9 +51,14 @@ impl StreamData { self.abort_handle.store(true, Ordering::Relaxed); return Err(Error::DroppedConnection); } + debug!("StreamData: Get"); match self.rx.recv().await { - Some(data) => Ok(data), + Some(data) => { + debug!("StreamData: Got"); + Ok(data) + } None => { + debug!("StreamData: Drop"); self.abort_handle.store(true, Ordering::Relaxed); Err(Error::DroppedConnection) } @@ -176,11 +182,16 @@ impl BcCamera { let mut media_sub = sub_video.bcmedia_stream(strict); while !abort_handle_thread.load(Ordering::Relaxed) { + debug!("Stream: Get"); if let Some(bc_media) = media_sub.next().await { + debug!("Stream: Got"); // We now have a complete interesting packet. Send it to on the callback + debug!("Stream: Send"); if tx.send(bc_media).await.is_err() { + debug!("Stream: Dropped"); break; // Connection dropped } + debug!("Stream: Sent"); } else { break; } diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index af318e6..956b25b 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -230,7 +230,7 @@ impl NeoMediaFactoryImpl { self.shared.microseconds.load(Ordering::Relaxed), )); source.set_is_live(true); - source.set_block(true); + source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); @@ -242,6 +242,7 @@ impl NeoMediaFactoryImpl { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; + queue.set_property_from_str("leaky", "downstream"); let parser = make_element("h265parse", "parser")?; let payload = make_element("rtph265pay", "pay0")?; bin.add_many(&[&source, &queue, &parser, &payload])?; @@ -261,7 +262,7 @@ impl NeoMediaFactoryImpl { self.shared.microseconds.load(Ordering::Relaxed), )); source.set_is_live(true); - source.set_block(true); + source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); @@ -273,6 +274,8 @@ impl NeoMediaFactoryImpl { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; + queue.set_property_from_str("leaky", "downstream"); + let parser = make_element("h264parse", "parser")?; let payload = make_element("rtph264pay", "pay0")?; bin.add_many(&[&source, &queue, &parser, &payload])?; @@ -288,6 +291,7 @@ impl NeoMediaFactoryImpl { let source = make_element("videotestsrc", "vidsrc")?; source.set_property_from_str("pattern", "snow"); let queue = make_element("queue", "queue0")?; + queue.set_property_from_str("leaky", "downstream"); let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -324,7 +328,7 @@ impl NeoMediaFactoryImpl { self.shared.microseconds.load(Ordering::Relaxed), )); source.set_is_live(true); - source.set_block(true); + source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); @@ -334,6 +338,7 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); let parser = make_element("aacparse", "audparser")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; @@ -372,7 +377,7 @@ impl NeoMediaFactoryImpl { self.shared.microseconds.load(Ordering::Relaxed), )); source.set_is_live(true); - source.set_block(true); + source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); @@ -390,6 +395,7 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 9cab641..8afc8dc 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -5,8 +5,9 @@ use futures::stream::StreamExt; use gstreamer::ClockTime; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; +use log::*; use neolink_core::bcmedia::model::*; -use std::collections::HashMap; +use std::collections::{hash_map::Entry, HashMap}; use std::sync::{ atomic::{AtomicU64, Ordering}, Arc, @@ -54,8 +55,10 @@ impl NeoMediaSender { self.shared .buffer_ready .store(!buffer_read.is_empty(), Ordering::Relaxed); + debug!("Sender: Get"); tokio::select! { v = self.data_source.next() => { + debug!("Sender: Got Data"); // debug!("data_source recieved"); if let Some(bc_media) = v { if ! self.skip_bcmedia(&bc_media)? { @@ -74,7 +77,7 @@ impl NeoMediaSender { ms: frame.microseconds as u64, data: frame.data, }; - self.process_vidbuffer(&new_data)?; + self.process_vidbuffer(&new_data).await?; buffer_write.push(new_data); } BcMedia::Pframe(frame) => { @@ -82,7 +85,7 @@ impl NeoMediaSender { ms: frame.microseconds as u64, data: frame.data, }; - self.process_vidbuffer(&new_data)?; + self.process_vidbuffer(&new_data).await?; buffer_write.push(new_data); } BcMedia::Aac(aac) => { @@ -91,7 +94,18 @@ impl NeoMediaSender { ms: last.ms, data: aac.data, }; - self.process_audbuffer(&new_data)?; + self.process_audbuffer(&new_data).await?; + audbuffer_write.push(new_data); + } + + } + BcMedia::Adpcm(adpcm) => { + if let Some(last) = buffer_write.last().as_ref() { + let new_data = StampedData { + ms: last.ms, + data: adpcm.data, + }; + self.process_audbuffer(&new_data).await?; audbuffer_write.push(new_data); } @@ -104,6 +118,7 @@ impl NeoMediaSender { } } v = self.clientsource.next() => { + debug!("Sender: Got Client"); if let Some(mut clientdata) = v { // Resend the video and audio buffers if !clientdata.inited { @@ -119,7 +134,7 @@ impl NeoMediaSender { next_vid.data.as_slice(), *idx as u64, 0, - ).is_err() { + ).await.is_err() { break; }; } @@ -132,7 +147,7 @@ impl NeoMediaSender { next_aud.data.as_slice(), *idx as u64, 0, - ).is_err() { + ).await.is_err() { break; }; } @@ -161,47 +176,77 @@ impl NeoMediaSender { Ok(()) } - fn process_vidbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { - self.clientdata.retain(|_, data| { - if let Some(vidsrc) = data.vidsrc.as_ref() { - Self::send_buffer( - vidsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.start_time, - ) - .is_ok() - // If ok retain is true - } else { - // Audio only - true + async fn process_vidbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { + for key in self + .clientdata + .keys() + .cloned() + .collect::>() + .drain(..) + { + match self.clientdata.entry(key) { + Entry::Occupied(data) => { + if let Some(vidsrc) = data.get().vidsrc.as_ref() { + if Self::send_buffer( + vidsrc, + stamped_data.data.as_slice(), + stamped_data.ms, + data.get().start_time, + ) + .await + .is_err() + { + data.remove(); + } + } else { + data.remove(); + } + } + Entry::Vacant(_) => {} } - }); + } + Ok(()) + } + + async fn process_audbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { + for key in self + .clientdata + .keys() + .cloned() + .collect::>() + .drain(..) + { + match self.clientdata.entry(key) { + Entry::Occupied(data) => { + if let Some(audsrc) = data.get().audsrc.as_ref() { + if Self::send_buffer( + audsrc, + stamped_data.data.as_slice(), + stamped_data.ms, + data.get().start_time, + ) + .await + .is_err() + { + data.remove(); + } + } else { + data.remove(); + } + } + Entry::Vacant(_) => {} + } + } Ok(()) } - fn process_audbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { - self.clientdata.retain(|_, data| { - if let Some(audsrc) = data.audsrc.as_ref() { - Self::send_buffer( - audsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.start_time, - ) - .is_ok() - // If ok retain is true - } else { - // video only - true - } - }); - - Ok(()) - } - - fn send_buffer(appsrc: &AppSrc, buf: &[u8], frame_ms: u64, start_time: u64) -> AnyResult<()> { + async fn send_buffer( + appsrc: &AppSrc, + buf: &[u8], + frame_ms: u64, + start_time: u64, + ) -> AnyResult<()> { let micros = if frame_ms > start_time { frame_ms - start_time } else { @@ -222,10 +267,14 @@ impl NeoMediaSender { gst_buf_data.copy_from_slice(buf); } // debug!("Buffer pushed"); - appsrc - .push_buffer(gst_buf.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) + let thread_appsrc = appsrc.clone(); // GObjects are refcounted + tokio::task::spawn_blocking(move || { + thread_appsrc + .push_buffer(gst_buf.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await? } fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult { diff --git a/src/rtsp/states/streaming.rs b/src/rtsp/states/streaming.rs index f6b5394..36ce2df 100644 --- a/src/rtsp/states/streaming.rs +++ b/src/rtsp/states/streaming.rs @@ -52,8 +52,9 @@ impl Camera { .await?; set.spawn(async move { loop { - debug!("{}: BcMediaStreamRecv", &tag_thread); + debug!("Straming: Get"); let data = timeout(Duration::from_secs(15), stream_data.get_data()).await??; + debug!("Straming: Got"); match &data { Ok(BcMedia::InfoV1(_)) => debug!("{}: - InfoV1", &tag_thread), Ok(BcMedia::InfoV2(_)) => debug!("{}: - InfoV2", &tag_thread), @@ -63,7 +64,9 @@ impl Camera { Ok(BcMedia::Adpcm(_)) => debug!("{}: - Adpcm", &tag_thread), Err(_) => debug!(" - Error"), } + debug!("Straming: Send"); sender.send(data?).await?; + debug!("Straming: Sent"); } }); }