diff --git a/crates/core/src/bc_protocol/connection/udpsource.rs b/crates/core/src/bc_protocol/connection/udpsource.rs index b1e2ceb..f236078 100644 --- a/crates/core/src/bc_protocol/connection/udpsource.rs +++ b/crates/core/src/bc_protocol/connection/udpsource.rs @@ -163,7 +163,7 @@ impl BcUdpSource { recieved: Default::default(), state: State::Normal, send_buffer: Default::default(), - ack_interval: interval(Duration::from_millis(100)), // Offical Client does ack every 10ms + ack_interval: interval(Duration::from_millis(10)), // Offical Client does ack every 10ms resend_interval: interval(Duration::from_millis(500)), // Offical Client does resend every 500ms } } diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 2e2e6b2..d8e1528 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -100,8 +100,7 @@ unsafe impl Sync for NeoMediaFactory {} pub(crate) struct NeoMediaFactoryImpl { sender: Sender, - vidsender: Sender, - audsender: Sender, + clientsender: Sender, shared: Arc, #[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj threads: JoinSet>, @@ -117,28 +116,24 @@ impl Default for NeoMediaFactoryImpl { fn default() -> Self { warn!("Constructing Factor Impl"); let (datasender, datarx) = channel(3); - let (vidsender, rx_vidsender) = channel(3); - let (audsender, rx_audsender) = channel(3); + let (clientsender, rx_clientsender) = channel(3); let shared: Arc = Default::default(); // Prepare thread that sends data into the appsrcs let mut threads: JoinSet> = Default::default(); let mut sender = NeoMediaSender { data_source: ReceiverStream::new(datarx), - vidsource: ReceiverStream::new(rx_vidsender), - audsource: ReceiverStream::new(rx_audsender), + clientsource: ReceiverStream::new(rx_clientsender), shared: shared.clone(), uid: Default::default(), - vidsrcs: Default::default(), - audsrcs: Default::default(), + clientdata: Default::default(), waiting_for_iframe: true, }; threads.spawn(async move { sender.run().await }); Self { sender: datasender, - vidsender, - audsender, + clientsender, shared, threads, } @@ -201,6 +196,8 @@ impl NeoMediaFactoryImpl { bin.remove(&element)?; } + let mut client_data: ClientPipelineData = Default::default(); + // Now contruct the actual ones match VidFormats::from(self.shared.vid_format.load(Ordering::Relaxed)) { VidFormats::H265 => { @@ -232,7 +229,7 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - self.vidsender.blocking_send(source)?; + client_data.vidsrc.replace(source); } VidFormats::H264 => { debug!("Building H264 Pipeline"); @@ -263,7 +260,7 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - self.vidsender.blocking_send(source)?; + client_data.vidsrc.replace(source); } VidFormats::Unknown => { debug!("Building Unknown Pipeline"); @@ -274,7 +271,7 @@ impl NeoMediaFactoryImpl { overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); overlay.set_property_from_str("halignment", "left"); - overlay.set_property("font-desc", "Sans, 32"); + overlay.set_property("font-desc", "Sans, 16"); let encoder = make_element("jpegenc", "encoder")?; let payload = make_element("rtpjpegpay", "pay0")?; @@ -345,11 +342,13 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - self.audsender.blocking_send(source)?; + client_data.audsrc.replace(source); } AudFormats::Adpcm => {} } + self.clientsender.blocking_send(client_data)?; + bin.dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back")) } diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 0ff3ba7..cc385a5 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -14,24 +14,15 @@ use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; -#[derive(Hash, PartialEq, Eq)] -pub(super) struct ClientData { - pub(super) appsrc: AppSrc, - pub(super) start_time: u64, - pub(super) inited: bool, -} - pub(super) struct NeoMediaSender { pub(super) data_source: ReceiverStream, - pub(super) vidsource: ReceiverStream, - pub(super) audsource: ReceiverStream, + pub(super) clientsource: ReceiverStream, pub(super) shared: Arc, // Used to generate the key for the hashmaps pub(super) uid: AtomicU64, // Hashmap so we can alter the data in ClientData while also allowing for remove at any point // If you know a better collections pm me - pub(super) vidsrcs: HashMap, - pub(super) audsrcs: HashMap, + pub(super) clientdata: HashMap, pub(super) waiting_for_iframe: bool, } @@ -102,24 +93,9 @@ impl NeoMediaSender { break; } } - v = self.vidsource.next() => { - if let Some(appsrc) = v { - self.vidsrcs.insert(self.uid.fetch_add(1, Ordering::Relaxed) , ClientData { - appsrc, - start_time: self.shared.microseconds.load(Ordering::Relaxed), - inited: false, - }); - } else { - break; - } - }, - v = self.audsource.next() => { - if let Some(appsrc) = v { - self.audsrcs.insert(self.uid.fetch_add(1, Ordering::Relaxed), ClientData { - appsrc, - start_time: self.shared.microseconds.load(Ordering::Relaxed), - inited: false, - }); + v = self.clientsource.next() => { + if let Some(clientdata) = v { + self.clientdata.insert(self.uid.fetch_add(1, Ordering::Relaxed) , clientdata); } else { break; } @@ -138,53 +114,63 @@ impl NeoMediaSender { } fn process_vidbuffer(&mut self, process_data: &ProcessData) -> AnyResult<()> { - self.vidsrcs.retain(|_, data| { - // If ! inited then we just joined the stream - // Send all data from the last iframe to catch up - // This should eliminate visual artifact that arise from recieving a pframe - // before an iframe - if !data.inited { - data.inited = true; - for (idx, resend_buffer) in process_data.resend.iter().enumerate() { - if Self::send_buffer(&data.appsrc, resend_buffer.data.as_slice(), idx as u64, 0) - .is_err() - { - return false; // If fails then this appsrc is dead + self.clientdata.retain(|_, data| { + if let Some(vidsrc) = data.vidsrc.as_ref() { + // If ! inited then we just joined the stream + // Send all data from the last iframe to catch up + // This should eliminate visual artifact that arise from recieving a pframe + // before an iframe + if !data.inited { + data.inited = true; + for (idx, resend_buffer) in process_data.resend.iter().enumerate() { + if Self::send_buffer(vidsrc, resend_buffer.data.as_slice(), idx as u64, 0) + .is_err() + { + return false; // If fails then this appsrc is dead + } } } + Self::send_buffer( + vidsrc, + process_data.live.data.as_slice(), + process_data.live.ms, + data.start_time, + ) + .is_ok() // If ok retain is true + } else { + // Audio only + true } - Self::send_buffer( - &data.appsrc, - process_data.live.data.as_slice(), - process_data.live.ms, - data.start_time, - ) - .is_ok() // If ok retain is true }); Ok(()) } fn process_audbuffer(&mut self, buf: &[u8], frame_ms: u64) -> AnyResult<()> { - self.audsrcs.retain(|_, data| { - Self::send_buffer(&data.appsrc, buf, frame_ms, data.start_time).is_ok() - // If ok retain is true + self.clientdata.retain(|_, data| { + if let Some(audsrc) = data.vidsrc.as_ref() { + Self::send_buffer(audsrc, buf, frame_ms, data.start_time).is_ok() + // If ok retain is true + } else { + // video only + true + } }); Ok(()) } fn send_buffer(appsrc: &AppSrc, buf: &[u8], frame_ms: u64, start_time: u64) -> AnyResult<()> { + let micros = if frame_ms > start_time { + frame_ms - start_time + } else { + start_time + }; + let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { let gst_buf_mut = gst_buf.get_mut().unwrap(); - let micros = if frame_ms > start_time { - frame_ms - start_time - } else { - start_time - }; - let time = ClockTime::from_useconds(micros); gst_buf_mut.set_pts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); diff --git a/src/rtsp/gst/shared.rs b/src/rtsp/gst/shared.rs index 9470a1e..0b47426 100644 --- a/src/rtsp/gst/shared.rs +++ b/src/rtsp/gst/shared.rs @@ -1,5 +1,6 @@ //! Data shared between the various //! components that manage a media stream +use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use std::convert::{From, Into}; use std::sync::atomic::AtomicU64; @@ -75,3 +76,11 @@ impl Default for NeoMediaShared { } } } + +#[derive(Default, Debug)] +pub(super) struct ClientPipelineData { + pub(super) vidsrc: Option, + pub(super) audsrc: Option, + pub(super) start_time: u64, + pub(super) inited: bool, +} diff --git a/src/rtsp/states/streaming.rs b/src/rtsp/states/streaming.rs index 54b861e..01e581c 100644 --- a/src/rtsp/states/streaming.rs +++ b/src/rtsp/states/streaming.rs @@ -2,28 +2,25 @@ // // Data is streamed into a gstreamer source -use anyhow::{anyhow, Context, Error, Result}; +use anyhow::{anyhow, Error, Result}; use async_trait::async_trait; +use futures::future::FutureExt; use log::*; -use std::collections::HashMap; -use tokio::task::{self, JoinHandle}; +use tokio::task::JoinSet; +use tokio::time::{timeout, Duration}; use neolink_core::{bc_protocol::StreamKind as Stream, bcmedia::model::BcMedia}; use super::{CameraState, Shared}; -use crate::rtsp::abort::AbortHandle; - #[derive(Default)] pub(crate) struct Streaming { - handles: HashMap>>, - abort_handle: AbortHandle, + set: JoinSet>, } #[async_trait] impl CameraState for Streaming { async fn setup(&mut self, shared: &Shared) -> Result<(), Error> { - self.abort_handle.reset(); // Create new gst outputs // // Otherwise use those already present @@ -31,7 +28,7 @@ impl CameraState for Streaming { let tag = shared.get_tag_for_stream(stream); let sender = shared .rtsp - .get_sender(tag) + .get_sender(&tag) .await .ok_or_else(|| anyhow!("Stream has not been created"))?; @@ -46,70 +43,69 @@ impl CameraState for Streaming { ); let thread_camera = shared.camera.clone(); - let thread_abort_handle = self.abort_handle.clone(); let stream_thead = *stream; let strict_thread = shared.strict; - let handle = task::spawn(async move { + let tag_thread = tag.clone(); + let handle = self.set.spawn(async move { let mut stream_data = thread_camera .start_video(stream_thead, 0, strict_thread) .await?; - while thread_abort_handle.is_live() { - debug!("BcMediaStreamRecv"); - let data = stream_data.get_data().await?; + loop { + debug!("{}: BcMediaStreamRecv", &tag_thread); + let data = timeout(Duration::from_secs(15), stream_data.get_data()).await??; match &data { - Ok(BcMedia::InfoV1(_)) => debug!(" - InfoV1"), - Ok(BcMedia::InfoV2(_)) => debug!(" - InfoV2"), - Ok(BcMedia::Iframe(_)) => debug!(" - Iframe"), - Ok(BcMedia::Pframe(_)) => debug!(" - Pframe"), - Ok(BcMedia::Aac(_)) => debug!(" - Aac"), - Ok(BcMedia::Adpcm(_)) => debug!(" - Adpcm"), + Ok(BcMedia::InfoV1(_)) => debug!("{}: - InfoV1", &tag_thread), + Ok(BcMedia::InfoV2(_)) => debug!("{}: - InfoV2", &tag_thread), + Ok(BcMedia::Iframe(_)) => debug!("{}: - Iframe", &tag_thread), + Ok(BcMedia::Pframe(_)) => debug!("{}: - Pframe", &tag_thread), + Ok(BcMedia::Aac(_)) => debug!("{}: - Aac", &tag_thread), + Ok(BcMedia::Adpcm(_)) => debug!("{}: - Adpcm", &tag_thread), Err(_) => debug!(" - Error"), } sender.send(data?).await?; } - debug!("Shutting down streaming thread"); - Ok(()) }); - - self.handles.entry(*stream).or_insert_with(|| handle); } Ok(()) } async fn tear_down(&mut self, _shared: &Shared) -> Result<(), Error> { - self.abort_handle.abort(); - - if !self.handles.is_empty() { - for (stream, handle) in self.handles.drain() { - match handle.await { - Ok(Err(e)) => return Err(e), - Err(_) => return Err(anyhow!("Panicked while streaming {:?}", stream)), - Ok(Ok(_)) => {} - } - } - } - + self.set.shutdown().await; Ok(()) } } impl Drop for Streaming { fn drop(&mut self) { - self.abort_handle.abort(); + self.set.abort_all(); } } impl Streaming { pub(crate) async fn is_running(&mut self) -> Result<()> { - if self.handles.iter().all(|(_, h)| !h.is_finished()) && self.abort_handle.is_live() { - return Ok(()); + if self.set.is_empty() { + return Err(anyhow!("Streaming has no active tasks")); } - if !self.abort_handle.is_live() { - return Err(anyhow!("Stream aborted")); - } - for (s, h) in self.handles.drain() { - h.await?.context(format!("On stream: {:?}", s))?; + if let Some(res) = self.set.join_next().now_or_never() { + match res { + None => { + return Err(anyhow!("Streaming has no active tasks")); + } + Some(Ok(Ok(()))) => { + unreachable!(); + } + Some(Ok(Err(e))) => { + // Error in tasks + self.set.abort_all(); + return Err(e); + } + Some(Err(e)) => { + // Panic in tasks + self.set.abort_all(); + return Err(anyhow!("Panic in streaming task: {:?}", e)); + } + } } Ok(()) }