diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index dfe2531..34823ab 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -11,7 +11,7 @@ use gstreamer::{ glib::{self, Object}, Structure, }; -use gstreamer::{Bin, Caps, ClockTime, Element, ElementFactory}; +use gstreamer::{Bin, Caps, Element, ElementFactory}; use gstreamer_app::{AppSrc, AppSrcCallbacks, AppStreamType}; use gstreamer_rtsp::RTSPUrl; use gstreamer_rtsp_server::prelude::*; @@ -23,14 +23,12 @@ use log::*; use neolink_core::bcmedia::model::*; use std::{ collections::HashSet, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, + sync::{atomic::Ordering, Arc}, }; use tokio::{ sync::mpsc::{channel, Sender}, task::JoinSet, + time::Duration, }; use tokio_stream::wrappers::ReceiverStream; @@ -113,7 +111,7 @@ unsafe impl Sync for NeoMediaFactory {} pub(crate) struct NeoMediaFactoryImpl { sender: Sender, - clientsender: Sender, + clientsender: Sender, shared: Arc, #[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj threads: JoinSet>, @@ -134,14 +132,11 @@ impl Default for NeoMediaFactoryImpl { // Prepare thread that sends data into the appsrcs let mut threads: JoinSet> = Default::default(); - let mut sender = NeoMediaSender { - data_source: ReceiverStream::new(datarx), - clientsource: ReceiverStream::new(rx_clientsender), - shared: shared.clone(), - uid: Default::default(), - clientdata: Default::default(), - waiting_for_iframe: true, - }; + let mut sender = NeoMediaSenders::new( + shared.clone(), + ReceiverStream::new(datarx), + ReceiverStream::new(rx_clientsender), + ); threads.spawn(async move { loop { tokio::task::yield_now().await; @@ -153,7 +148,6 @@ impl Default for NeoMediaFactoryImpl { break; } } - sender.waiting_for_iframe = true; } Ok(()) }); @@ -231,12 +225,7 @@ impl NeoMediaFactoryImpl { bin.remove(&element)?; } - let mut client_data = ClientPipelineData { - start_time: Arc::new(AtomicU64::new( - self.shared.microseconds.load(Ordering::Relaxed), - )), - ..Default::default() - }; + let mut client_data = NeoMediaSender::new(); // Now contruct the actual ones match *self.shared.vid_format.blocking_read() { @@ -245,38 +234,28 @@ impl NeoMediaFactoryImpl { let source = make_element("appsrc", "vidsrc")? .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_base_time(ClockTime::from_mseconds( - self.shared.microseconds.load(Ordering::Relaxed), - )); source.set_is_live(true); source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); + source.set_stream_type(AppStreamType::RandomAccess); - let enough_data_start = client_data.enough_data.clone(); - let enough_data_stop = client_data.enough_data.clone(); - let seek_shared = self.shared.clone(); - let seek_start_time = client_data.start_time.clone(); + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + let seek_command = client_data.get_commader(); source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { - if enough_data_stop.load(Ordering::Relaxed) { - log::debug!("Client is in need of data"); - } - enough_data_stop.store(false, Ordering::Relaxed); + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); }) .enough_data(move |_appsrc| { - if !enough_data_start.load(Ordering::Relaxed) { - log::debug!("Client is full of of data"); - } - enough_data_start.store(true, Ordering::Relaxed); + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); }) .seek_data(move |_appsrc, seek_pos| { - log::debug!("Seeking to {}", &seek_pos); - let current_time = seek_shared.microseconds.load(Ordering::Relaxed); - seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + debug!("Send seek H265"); + let _ = + seek_command.blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); true }) .build(), @@ -290,7 +269,6 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); - queue.set_property("min-threshold-time", 500u64 * 1000u64); let parser = make_element("h265parse", "parser")?; // parser.set_property("config-interval", 5i32); let payload = make_element("rtph265pay", "pay0")?; @@ -300,46 +278,47 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.vidsrc.replace(source); + client_data.update_vid(source); } VidFormats::H264 => { debug!("Building H264 Pipeline"); let source = make_element("appsrc", "vidsrc")? .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_base_time(ClockTime::from_mseconds( - self.shared.microseconds.load(Ordering::Relaxed), - )); + source.set_is_live(true); source.set_block(false); source.set_property("emit-signals", false); - source.set_max_bytes(52428800); + source.set_max_bytes(50000000u64); // 50MB + // source.set_property("max-buffers", 0u32); // v1.20 + // source.set_property("max-time", Duration::from_secs_f32(2.25).as_nanos() as u64); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); + source.set_stream_type(AppStreamType::RandomAccess); + let clock = source.clock(); + if let Some(clock) = clock { + debug!("Clock: {:?}", clock.time()); + } else { + debug!("No clock"); + } - let enough_data_start = client_data.enough_data.clone(); - let enough_data_stop = client_data.enough_data.clone(); - let seek_shared = self.shared.clone(); - let seek_start_time = client_data.start_time.clone(); + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + let seek_command = client_data.get_commader(); source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { - if enough_data_stop.load(Ordering::Relaxed) { - log::debug!("Client is in need of data"); - } - enough_data_stop.store(false, Ordering::Relaxed); + let _ = need_command.try_send(NeoMediaSenderCommand::Resume); }) .enough_data(move |_appsrc| { - if !enough_data_start.load(Ordering::Relaxed) { - log::debug!("Client is full of of data"); - } - enough_data_start.store(true, Ordering::Relaxed); + let _ = enough_command.try_send(NeoMediaSenderCommand::Pause); }) .seek_data(move |_appsrc, seek_pos| { - log::debug!("Seeking to {}", &seek_pos); - let current_time = seek_shared.microseconds.load(Ordering::Relaxed); - seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); - true + debug!("Send seek H264: {}", seek_pos); + let result = seek_command + .try_send(NeoMediaSenderCommand::Seek(seek_pos)) + .is_ok(); + debug!(" - Sent seek H264: {}", result); + result }) .build(), ); @@ -351,18 +330,36 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); - queue.set_property("min-threshold-time", 500u64 * 1000u64); + // queue.set_property("max-size-bytes", 0u32); + // queue.set_property("max-size-buffers", 0u32); + queue.set_property("max-size-time", Duration::from_secs(2).as_nanos() as u64); let parser = make_element("h264parse", "parser")?; // parser.set_property("update-timecode", true); let payload = make_element("rtph264pay", "pay0")?; + // payload.set_property("config-interval", 1i32); + // let storage = make_element("rtpstorage", "vidstorage")?; + // storage.set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64); + // let jitter = make_element("rtpjitterbuffer", "pay0")?; + // jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32); + // payload.set_property("config-interval", 5i32); - bin.add_many(&[&source, &queue, &parser, &payload])?; - Element::link_many(&[&source, &queue, &parser, &payload])?; + bin.add_many(&[ + &source, &queue, &parser, + &payload, + // &storage, + // &jitter, + ])?; + Element::link_many(&[ + &source, &queue, &parser, + &payload, + // &storage, + // &jitter, + ])?; let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.vidsrc.replace(source); + client_data.update_vid(source); } VidFormats::Unknown => { debug!("Building Unknown Pipeline"); @@ -371,7 +368,6 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "queue0")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); - queue.set_property("min-threshold-time", 500u64 * 1000u64 * 1000u64); let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -391,171 +387,158 @@ impl NeoMediaFactoryImpl { .build(), )?; // source.link(&queue)?; - queue.link(&overlay)?; - overlay.link(&encoder)?; - encoder.link(&payload)?; + Element::link_many(&[&queue, &overlay, &encoder, &payload])?; } } - match *self.shared.aud_format.blocking_read() { - AudFormats::Unknown => {} - AudFormats::Aac => { - debug!("Building Aac pipeline"); - let source = make_element("appsrc", "audsrc")? - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_base_time(ClockTime::from_mseconds( - self.shared.microseconds.load(Ordering::Relaxed), - )); - source.set_is_live(true); - source.set_block(false); - source.set_property("emit-signals", false); - source.set_max_bytes(52428800); - source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); + let do_aud = false; + if do_aud { + match *self.shared.aud_format.blocking_read() { + AudFormats::Unknown => {} + AudFormats::Aac => { + debug!("Building Aac pipeline"); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - let enough_data_start = client_data.enough_data.clone(); - let enough_data_stop = client_data.enough_data.clone(); - let seek_shared = self.shared.clone(); - let seek_start_time = client_data.start_time.clone(); - source.set_callbacks( - AppSrcCallbacks::builder() - .need_data(move |_appsrc, _amt| { - if enough_data_stop.load(Ordering::Relaxed) { - log::debug!("Client is in need of data"); - } - enough_data_stop.store(false, Ordering::Relaxed); - }) - .enough_data(move |_appsrc| { - if !enough_data_start.load(Ordering::Relaxed) { - log::debug!("Client is full of of data"); - } - enough_data_start.store(true, Ordering::Relaxed); - }) - .seek_data(move |_appsrc, seek_pos| { - log::debug!("Seeking to {}", &seek_pos); - let current_time = seek_shared.microseconds.load(Ordering::Relaxed); - seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); - true - }) - .build(), - ); + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Seekable); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast back"))?; + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + let seek_command = client_data.get_commader(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); + }) + .enough_data(move |_appsrc| { + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); + }) + .seek_data(move |_appsrc, seek_pos| { + debug!("Send seek AAC"); + let _ = seek_command + .blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); + true + }) + .build(), + ); - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); - queue.set_property("min-threshold-time", 500u64 * 1000u64); - let parser = make_element("aacparse", "audparser")?; - let decoder = make_element("decodebin", "auddecoder")?; - let encoder = make_element("audioconvert", "audencoder")?; - let payload = make_element("rtpL16pay", "pay1")?; + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; - bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &parser, &decoder])?; - Element::link_many(&[&encoder, &payload])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link AAC decoder to encoder"); - }); + let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); + let parser = make_element("aacparse", "audparser")?; + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "audpayload")?; + let storage = make_element("rtpstorage", "audstorage")?; + storage + .set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64); + let jitter = make_element("rtpjitterbuffer", "pay1")?; + jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.audsrc.replace(source); - } - AudFormats::Adpcm(block_size) => { - debug!("Building Adpcm pipeline"); - // Original command line - // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 - // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 - // ! adpcmdec - // ! audioconvert - // ! rtpL16pay name=pay1 + bin.add_many(&[ + &source, &queue, &parser, &decoder, &encoder, &payload, &storage, &jitter, + ])?; + Element::link_many(&[&source, &queue, &parser, &decoder])?; + Element::link_many(&[&encoder, &payload, &storage, &jitter])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link AAC decoder to encoder"); + }); - let source = make_element("appsrc", "audsrc")? - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_base_time(ClockTime::from_mseconds( - self.shared.microseconds.load(Ordering::Relaxed), - )); - source.set_is_live(true); - source.set_block(false); - source.set_property("emit-signals", false); - source.set_max_bytes(52428800); - source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); - source.set_caps(Some( - &Caps::builder("audio/x-adpcm") - .field("layout", "div") - .field("block_align", block_size as i32) - .field("channels", 1i32) - .field("rate", 8000i32) - .build(), - )); + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + client_data.update_aud(source); + } + AudFormats::Adpcm(block_size) => { + debug!("Building Adpcm pipeline"); + // Original command line + // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 + // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 + // ! adpcmdec + // ! audioconvert + // ! rtpL16pay name=pay1 - let enough_data_start = client_data.enough_data.clone(); - let enough_data_stop = client_data.enough_data.clone(); - let seek_shared = self.shared.clone(); - let seek_start_time = client_data.start_time.clone(); - source.set_callbacks( - AppSrcCallbacks::builder() - .need_data(move |_appsrc, _amt| { - if enough_data_stop.load(Ordering::Relaxed) { - log::debug!("Client is in need of data"); - } - enough_data_stop.store(false, Ordering::Relaxed); - }) - .enough_data(move |_appsrc| { - if !enough_data_start.load(Ordering::Relaxed) { - log::debug!("Client is full of of data"); - } - enough_data_start.store(true, Ordering::Relaxed); - }) - .seek_data(move |_appsrc, seek_pos| { - log::debug!("Seeking to {}", &seek_pos); - let current_time = seek_shared.microseconds.load(Ordering::Relaxed); - seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); - true - }) - .build(), - ); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::RandomAccess); + source.set_caps(Some( + &Caps::builder("audio/x-adpcm") + .field("layout", "div") + .field("block_align", block_size as i32) + .field("channels", 1i32) + .field("rate", 8000i32) + .build(), + )); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast back"))?; + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + let seek_command = client_data.get_commader(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); + }) + .enough_data(move |_appsrc| { + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); + }) + .seek_data(move |_appsrc, seek_pos| { + debug!("Send seek Adpcm"); + let _ = seek_command + .blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); + true + }) + .build(), + ); - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); - queue.set_property("min-threshold-time", 500u64 * 1000u64); - let decoder = make_element("decodebin", "auddecoder")?; - let encoder = make_element("audioconvert", "audencoder")?; - let payload = make_element("rtpL16pay", "pay1")?; + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; - bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &decoder])?; - Element::link_many(&[&encoder, &payload])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link ADPCM decoder to encoder"); - }); + let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.audsrc.replace(source); + bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &decoder])?; + Element::link_many(&[&encoder, &payload])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link ADPCM decoder to encoder"); + }); + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + client_data.update_aud(source); + } } } diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index ac6b6f0..13568df 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -2,363 +2,546 @@ //! gstreamer media stream use anyhow::anyhow; use futures::stream::StreamExt; -use gstreamer::ClockTime; +use gstreamer::{prelude::*, ClockTime}; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use log::*; use neolink_core::bcmedia::model::*; -use std::collections::{hash_map::Entry, HashMap}; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, +use std::collections::{ + VecDeque, + {hash_map::Entry, HashMap}, +}; +use std::{ + iter::Iterator, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; // use tokio::time::{interval, Duration, MissedTickBehavior}; +use tokio::{ + sync::mpsc::{channel, Receiver, Sender}, + time::Duration, +}; use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; -pub(super) struct NeoMediaSender { - pub(super) data_source: ReceiverStream, - pub(super) clientsource: ReceiverStream, - pub(super) shared: Arc, - // Used to generate the key for the hashmaps - pub(super) uid: AtomicU64, - // Hashmap so we can alter the data in ClientData while also allowing for remove at any point - // If you know a better collections pm me - pub(super) clientdata: HashMap, - pub(super) waiting_for_iframe: bool, +#[derive(Default)] +struct NeoBuffer { + buf: VecDeque>, } -#[derive(Clone)] -struct StampedData { - ms: u64, - data: Vec, +impl NeoBuffer { + fn push(&mut self, item: BcMedia) { + match item { + iframe @ BcMedia::Iframe(_) => self.buf.push_back(vec![iframe]), + pframe @ BcMedia::Pframe(_) => { + if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + last.push(pframe); + } + } + aac @ BcMedia::Aac(_) => { + if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + last.push(aac); + } + } + adpcm @ BcMedia::Adpcm(_) => { + if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + last.push(adpcm); + } + } + BcMedia::InfoV1(_) | BcMedia::InfoV2(_) => {} + } + while self.buf.len() > 25 { + // 25 iframes + let _ = self.buf.pop_front(); + } + } + + fn start_time(&self) -> Option { + if let Some(BcMedia::Iframe(data)) = + self.buf.front().and_then(|inner_buf| inner_buf.first()) + { + Some(data.microseconds as u64) + } else { + None + } + } + + fn end_time(&self) -> Option { + if let Some(innerbuffer) = self.buf.back() { + let mut last_ms = None; + for frame in innerbuffer.iter() { + match frame { + BcMedia::Iframe(frame) => { + let v = last_ms.get_or_insert(frame.microseconds as u64); + if *v < frame.microseconds as u64 { + *v = frame.microseconds as u64; + } + } + BcMedia::Pframe(frame) => { + let v = last_ms.get_or_insert(frame.microseconds as u64); + if *v < frame.microseconds as u64 { + *v = frame.microseconds as u64; + } + } + _ => {} + } + } + last_ms + } else { + None + } + } +} + +pub(super) struct NeoMediaSenders { + data_source: ReceiverStream, + client_source: ReceiverStream, + shared: Arc, + uid: AtomicU64, + client_data: HashMap, + buffer: NeoBuffer, +} + +impl NeoMediaSenders { + pub(super) fn new( + shared: Arc, + data_source: ReceiverStream, + client_source: ReceiverStream, + ) -> Self { + Self { + data_source, + client_source, + shared, + uid: Default::default(), + client_data: Default::default(), + buffer: Default::default(), + } + } + + async fn handle_new_client(&mut self, mut client: NeoMediaSender) -> AnyResult<()> { + client.initialise(&self.buffer).await?; + self.client_data + .insert(self.uid.fetch_add(1, Ordering::Relaxed), client); + Ok(()) + } + + async fn update_mediatypes(&self, data: &BcMedia) { + match data { + BcMedia::Iframe(BcMediaIframe { + video_type: VideoType::H264, + .. + }) => { + if !matches!(*self.shared.vid_format.read().await, VidFormats::H264) { + *self.shared.vid_format.write().await = VidFormats::H264; + } + } + BcMedia::Pframe(BcMediaPframe { + video_type: VideoType::H264, + .. + }) => { + if !matches!(*self.shared.vid_format.read().await, VidFormats::H264) { + *self.shared.vid_format.write().await = VidFormats::H264; + } + } + BcMedia::Iframe(BcMediaIframe { + video_type: VideoType::H265, + .. + }) => { + if !matches!(*self.shared.vid_format.read().await, VidFormats::H265) { + *self.shared.vid_format.write().await = VidFormats::H265; + } + } + BcMedia::Pframe(BcMediaPframe { + video_type: VideoType::H265, + .. + }) => { + if !matches!(*self.shared.vid_format.read().await, VidFormats::H265) { + *self.shared.vid_format.write().await = VidFormats::H265; + } + } + BcMedia::Aac(_) => { + if !matches!(*self.shared.aud_format.read().await, AudFormats::Aac) { + *self.shared.aud_format.write().await = AudFormats::Aac; + } + } + BcMedia::Adpcm(data) => { + if !matches!(*self.shared.aud_format.read().await, AudFormats::Adpcm(_)) { + *self.shared.aud_format.write().await = + AudFormats::Adpcm(data.data.len() as u16); + } + } + _ => {} + } + } + + async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> { + // debug!("Handle new data"); + self.update_mediatypes(&data).await; + for key in self.client_data.keys().copied().collect::>() { + // debug!(" - Client: {}", key); + match self.client_data.entry(key) { + Entry::Occupied(mut occ) => { + if let Err(e) = occ.get_mut().initialise(&self.buffer).await { + debug!("Could not init client: {:?}", e); + occ.remove(); + continue; + } + if let Err(e) = occ.get_mut().process_commands(&self.buffer).await { + debug!("Could not process client command: {:?}", e); + occ.remove(); + continue; + } + if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await { + debug!("Could not handle jump to live: {:?}", e); + occ.remove(); + continue; + } + if let Err(e) = occ.get_mut().process_buffer(&self.buffer).await { + debug!("Could not send client data: {:?}", e); + occ.remove(); + continue; + } + } + Entry::Vacant(_) => {} + } + } + self.buffer.push(data); + + Ok(()) + } + + pub(super) async fn run(&mut self) -> AnyResult<()> { + loop { + tokio::select! { + Some(v) = self.data_source.next() => { + self.handle_new_data(v).await + }, + Some(v) = self.client_source.next() => { + self.handle_new_client(v).await + }, + else => { + Err(anyhow!("Sender data source closed")) + } + }?; + } + } +} + +pub(super) enum NeoMediaSenderCommand { + Seek(u64), + Pause, + Resume, +} +#[derive(Debug)] +pub(super) struct NeoMediaSender { + needle: u64, + start_time: u64, + vid: Option, + aud: Option, + command_reciever: Receiver, + command_sender: Sender, + inited: bool, + playing: bool, } impl NeoMediaSender { - pub(super) async fn run(&mut self) -> AnyResult<()> { - // let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32)); - // resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip); - let mut buffer_a: Vec = Default::default(); - let mut buffer_b: Vec = Default::default(); - let buffer_read = &mut buffer_a; - let buffer_write = &mut buffer_b; + pub(super) fn new() -> Self { + let (tx, rx) = channel(3); + Self { + needle: 0, + start_time: 0, + vid: None, + aud: None, + command_reciever: rx, + command_sender: tx, + inited: false, + playing: true, + } + } - let mut audbuffer_a: Vec = Default::default(); - let mut audbuffer_b: Vec = Default::default(); - let audbuffer_read = &mut audbuffer_a; - let audbuffer_write = &mut audbuffer_b; - loop { - self.shared - .number_of_clients - .store(self.clientdata.len(), Ordering::Relaxed); - self.shared - .buffer_ready - .store(!buffer_read.is_empty(), Ordering::Relaxed); - // debug!("Sender: Get"); - tokio::select! { - v = self.data_source.next() => { - // debug!("Sender: Got Data"); - // debug!("data_source recieved"); - if let Some(bc_media) = v { - if ! self.skip_bcmedia(&bc_media) { - // debug!("Not skipped"); - // resend_pause.reset(); - self.inspect_bcmedia(&bc_media).await; - match bc_media { - BcMedia::Iframe(frame) => { - if buffer_write.len() > 25 { - std::mem::swap(buffer_read, buffer_write); - buffer_write.clear(); - std::mem::swap(audbuffer_read, audbuffer_write); - audbuffer_write.clear(); - } - let new_data = StampedData { - ms: frame.microseconds as u64, - data: frame.data, - }; - self.process_vidbuffer(&new_data).await; - buffer_write.push(new_data); - } - BcMedia::Pframe(frame) => { - let new_data = StampedData { - ms: frame.microseconds as u64, - data: frame.data, - }; - self.process_vidbuffer(&new_data).await; - buffer_write.push(new_data); - } - BcMedia::Aac(aac) => { - if let Some(last) = buffer_write.last().as_ref() { - let new_data = StampedData { - ms: last.ms, - data: aac.data, - }; - self.process_audbuffer(&new_data).await; - audbuffer_write.push(new_data); - } + pub(super) fn update_vid(&mut self, source: AppSrc) { + self.vid.replace(source); + } - } - BcMedia::Adpcm(adpcm) => { - if let Some(last) = buffer_write.last().as_ref() { - let new_data = StampedData { - ms: last.ms, - data: adpcm.data, - }; - self.process_audbuffer(&new_data).await; - audbuffer_write.push(new_data); - } + pub(super) fn update_aud(&mut self, source: AppSrc) { + self.aud.replace(source); + } - } - _ => {} - } + pub(super) fn get_commader(&self) -> Sender { + self.command_sender.clone() + } + + async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if self.needle > self.start_time { + self.seek(self.needle - self.start_time, buffer).await + } else { + self.seek(0, buffer).await + } + } + + async fn seek(&mut self, dest: u64, buffer: &NeoBuffer) -> AnyResult<()> { + if let (Some(first_ms), Some(last_ms)) = (buffer.start_time(), buffer.end_time()) { + let target_seek = self.start_time + dest; + if target_seek >= first_ms && target_seek <= last_ms { + debug!("Seeking inside buffer: {}", target_seek); + self.needle = target_seek; + self.process_buffer(buffer).await?; + debug!("Seeked inside buffer"); + } else { + // Fake seek + debug!( + "Seeking outside buffer: {} - {} = {}", + self.needle, + target_seek, + self.needle - dest + ); + self.start_time = self.needle - dest; + } + } else { + // Buffers arent valid yet + debug!("Seek buffers are not ready: {}", dest); + debug!(" - {:?}", (buffer.start_time(), buffer.end_time())); + } + + Ok(()) + } + + async fn process_commands(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if self.inited { + if let Ok(command) = self.command_reciever.try_recv() { + match command { + NeoMediaSenderCommand::Seek(dest) => { + debug!("Got Seek Request: {}", dest); + self.seek(dest, buffer).await?; + } + NeoMediaSenderCommand::Pause => { + if self.playing { + debug!("Pausing"); + self.playing = false; + } + } + NeoMediaSenderCommand::Resume => { + if !self.playing { + debug!("Resuming"); + self.playing = true; + self.jump_to_live(buffer).await?; } - } else { - break; } } - v = self.clientsource.next() => { - // debug!("Sender: Got Client"); - if let Some(mut clientdata) = v { - // Resend the video and audio buffers - if !clientdata.inited { - clientdata.inited = true; - if let Some(last) = buffer_read.first() { - debug!("Start PTS SET: {:?}", tokio::time::Duration::from_micros(last.ms)); - clientdata.set_start_time(last.ms); - } - let mut vid_resend = buffer_read.iter().chain(buffer_write.iter()); - let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter()); - loop { - let next_vid = vid_resend.next(); - if let Some(next_vid) = next_vid.as_ref() { - if let Some(src) = clientdata.vidsrc.as_ref() { - if Self::send_buffer( - src, - next_vid.data.as_slice(), - next_vid.ms, - clientdata.get_start_time(), - ).await.is_err() { - break; - }; - } - } - let next_aud = aud_resend.next(); - if let Some(next_aud) = next_aud.as_ref() { - if next_aud.ms >= clientdata.get_start_time() { - if let Some(src) = clientdata.audsrc.as_ref() { - if Self::send_buffer( - src, - next_aud.data.as_slice(), - next_aud.ms, - clientdata.get_start_time(), - ).await.is_err() { - break; - }; - } - } - } - if next_vid.is_none() && next_aud.is_none() { - break; - } - } - debug!("Advance PTS Pushed"); - } - // Save the client for future sends - self.clientdata.insert(self.uid.fetch_add(1, Ordering::Relaxed) , clientdata); - } else { - break; - } - }, - // _ = resend_pause.tick() => { - // if let Some(live) = buffer.last().cloned() { - // self.process_vidbuffer(&ProcessData { - // live, - // resend: buffer.as_slice() - // })?; - // } - // } } } Ok(()) } - async fn process_vidbuffer(&mut self, stamped_data: &StampedData) { - for key in self - .clientdata - .keys() - .cloned() - .collect::>() - .drain(..) - { - match self.clientdata.entry(key) { - Entry::Occupied(data) => { - if let Some(vidsrc) = data.get().vidsrc.as_ref() { - if !data.get().enough_data.load(Ordering::Relaxed) - && Self::send_buffer( - vidsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.get().get_start_time(), - ) - .await - .is_err() - { - data.remove(); + async fn initialise(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if !self.inited { + if let (Some(first_ms), Some(end_ms)) = (buffer.start_time(), buffer.end_time()) { + // if let Some(source) = self.vid.as_ref() { + // source.set_base_time(ClockTime::from_mseconds(0)); + // } + // if let Some(source) = self.aud.as_ref() { + // source.set_base_time(ClockTime::from_mseconds(0)); + // } + + self.start_time = (first_ms + end_ms) / 2; + self.needle = self.start_time; + + self.inited = true; + + self.process_buffer(buffer).await?; + } + } + Ok(()) + } + + async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { + if self.inited && self.playing { + let mut buf_it = buf.buf.iter().peekable(); + while let Some(buffer) = buf_it.next() { + tokio::task::yield_now().await; + let next = buf_it.peek(); + // Look at the next frame set and see if it is after the needle (or if empty) + let deep = next + .map(|buffer| { + buffer + .first() + .map(|frame| { + if let BcMedia::Iframe(data) = frame { + data.microseconds as u64 >= self.needle + } else { + unreachable!() + } + }) + .unwrap_or(true) // No first data + }) + .unwrap_or(true); // Next buffer is empty + // This frame set contains our data process it + if deep { + let mut send_it = false; + for frame in buffer.iter() { + if !send_it { + let microseconds = match frame { + BcMedia::Iframe(data) => Some(data.microseconds), + BcMedia::Pframe(data) => Some(data.microseconds), + _ => None, + }; + if let Some(microseconds) = microseconds { + if microseconds as u64 > self.needle { + send_it = true; + } + } } - } else { - data.remove(); - } - } - Entry::Vacant(_) => {} - } - } - } - async fn process_audbuffer(&mut self, stamped_data: &StampedData) { - for key in self - .clientdata - .keys() - .cloned() - .collect::>() - .drain(..) - { - match self.clientdata.entry(key) { - Entry::Occupied(data) => { - if let Some(audsrc) = data.get().audsrc.as_ref() { - if !data.get().enough_data.load(Ordering::Relaxed) - && Self::send_buffer( - audsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.get().get_start_time(), - ) - .await - .is_err() - { - data.remove(); + if send_it && !self.send_buffer(frame).await? { + // Buffer too far is ahead of needle early exit + break; } + } + } + } + } + + Ok(()) + } + + async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if let Some(first_buftime) = buffer.start_time() { + if first_buftime > self.needle { + debug!("Jump to live first_buftime > self.needle"); + self.jump_to_live(buffer).await?; + } else if let Some(runtime) = self.get_runtime() { + if first_buftime > self.start_time { + let buftime = Duration::from_micros(first_buftime - self.start_time); + if buftime > runtime { + debug!("Jump to live buftime > runtime"); + self.jump_to_live(buffer).await?; + } + } else { + debug!("Jump to live first_buftime > self.start_time "); + self.jump_to_live(buffer).await?; + } + } + } + Ok(()) + } + + fn get_runtime(&self) -> Option { + if let Some(appsrc) = self.vid.as_ref() { + if let Some(clock) = appsrc.clock() { + if let Some(time) = clock.time() { + if let Some(base_time) = appsrc.base_time() { + let runtime = time - base_time; + return Some(Duration::from_nanos(runtime.nseconds())); + } + } + } + } + None + } + + async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { + if self.inited && self.playing { + let microseconds = match media { + BcMedia::Iframe(data) => { + if data.microseconds as u64 > self.start_time { + data.microseconds as u64 - self.start_time } else { - data.remove(); + 0 } } - Entry::Vacant(_) => {} - } - } - } - - async fn send_buffer( - appsrc: &AppSrc, - buf: &[u8], - frame_ms: u64, - start_time: u64, - ) -> AnyResult<()> { - let micros = if frame_ms >= start_time { - frame_ms - start_time - } else { - 0 - }; - debug!("PTS: {:?}", tokio::time::Duration::from_micros(micros)); - - let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); - { - let gst_buf_mut = gst_buf.get_mut().unwrap(); - - let time = ClockTime::from_useconds(micros); - gst_buf_mut.set_pts(time); - // debug!( - // "PTS set to: {:?} ({}-{}={:?})", - // time, frame_ms, start_time, micros - // ); - let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); - gst_buf_data.copy_from_slice(buf); - } - // debug!("Buffer pushed"); - let thread_appsrc = appsrc.clone(); // GObjects are refcounted - let res = tokio::task::spawn_blocking(move || { - thread_appsrc - .push_buffer(gst_buf.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) - }) - .await; - match &res { - Err(e) => { - debug!("Failed to send buffer: {:?}", e); - } - Ok(Err(e)) => { - debug!("Failed to send buffer: {:?}", e); - } - Ok(Ok(_)) => {} - }; - res? - } - - fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> bool { - if self.waiting_for_iframe { - if let BcMedia::Iframe(_) = bc_media { - self.waiting_for_iframe = false; - } else { - log::debug!("Skipping bcmedia"); - return true; - } - } - false - } - - async fn inspect_bcmedia(&mut self, bc_media: &BcMedia) { - let old_vid = *self.shared.vid_format.read().await; - let old_aud = *self.shared.aud_format.read().await; - match bc_media { - BcMedia::Iframe(frame) => { - match frame.video_type { - VideoType::H264 => { - (*self.shared.vid_format.write().await) = VidFormats::H264; - } - VideoType::H265 => { - (*self.shared.vid_format.write().await) = VidFormats::H265; + BcMedia::Pframe(data) => { + if data.microseconds as u64 > self.start_time { + data.microseconds as u64 - self.start_time + } else { + 0 } } - self.shared - .microseconds - .store(frame.microseconds as u64, Ordering::Relaxed); - // log::debug!( - // "Time set to {}", - // self.shared.microseconds.load(Ordering::Relaxed) - // ); - } - BcMedia::Pframe(frame) => { - match frame.video_type { - VideoType::H264 => { - (*self.shared.vid_format.write().await) = VidFormats::H264; - } - VideoType::H265 => { - (*self.shared.vid_format.write().await) = VidFormats::H265; + _ => { + if self.needle > self.start_time { + self.needle - self.start_time + } else { + 0 } } - self.shared - .microseconds - .store(frame.microseconds as u64, Ordering::Relaxed); - // log::debug!( - // "Time set to {}", - // self.shared.microseconds.load(Ordering::Relaxed) - // ); + }; + let buf = match media { + BcMedia::Iframe(data) => Some(&data.data), + BcMedia::Pframe(data) => Some(&data.data), + BcMedia::Aac(data) => Some(&data.data), + BcMedia::Adpcm(data) => Some(&data.data), + _ => None, + }; + let appsrc = match media { + BcMedia::Iframe(_) | BcMedia::Pframe(_) => self.vid.as_ref(), + BcMedia::Aac(_) | BcMedia::Adpcm(_) => self.aud.as_ref(), + _ => None, + }; + + if let (Some(buf), Some(appsrc)) = (buf, appsrc) { + if let Some(runtime) = self.get_runtime() { + let buftimr = Duration::from_micros(microseconds); + if buftimr > runtime + Duration::from_micros(2000) { + // Stop advancing the buffer + return Ok(false); + } else { + debug!(" - runtime: {:?}", runtime); + debug!(" - buftime: {:?}", buftimr); + } + } + + debug!("PTS: {:?}", Duration::from_micros(microseconds)); + + let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); + + let time = ClockTime::from_useconds(microseconds); + gst_buf_mut.set_pts(time); + let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(buf); + } + // debug!("Buffer pushed"); + let thread_appsrc = appsrc.clone(); // GObjects are refcounted + let res = tokio::task::spawn_blocking(move || { + thread_appsrc + .push_buffer(gst_buf.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await; + match &res { + Err(e) => { + debug!("Failed to send buffer: {:?}", e); + } + Ok(Err(e)) => { + debug!("Failed to send buffer: {:?}", e); + } + Ok(Ok(_)) => {} + }; + res??; + // Buffer sent sucessfully + // + // Update the needle position + match media { + BcMedia::Iframe(data) => { + if data.microseconds as u64 > self.needle { + // debug!("Updating needle to: {}", data.microseconds); + self.needle = data.microseconds as u64; + } + } + BcMedia::Pframe(data) => { + if data.microseconds as u64 > self.needle { + // debug!("Updating needle to: {}", data.microseconds); + self.needle = data.microseconds as u64; + } + } + _ => {} + } } - BcMedia::Aac(_aac) => { - (*self.shared.aud_format.write().await) = AudFormats::Aac; - } - BcMedia::Adpcm(adpcm) => { - (*self.shared.aud_format.write().await) = AudFormats::Adpcm(adpcm.data.len() as u16) - } - _ => {} - } - let new_vid = *self.shared.vid_format.read().await; - if new_vid != old_vid { - log::debug!("Video format set to: {:?} from {:?}", new_vid, old_vid); - } - let new_aud = *self.shared.aud_format.read().await; - if old_aud != new_aud { - log::debug!("Audio format set to: {:?} from {:?}", new_aud, old_aud); } + Ok(true) } } diff --git a/src/rtsp/gst/shared.rs b/src/rtsp/gst/shared.rs index e3a2e24..685695f 100644 --- a/src/rtsp/gst/shared.rs +++ b/src/rtsp/gst/shared.rs @@ -1,9 +1,7 @@ //! Data shared between the various //! components that manage a media stream -use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize}; use tokio::sync::RwLock; #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -23,7 +21,6 @@ pub(super) enum AudFormats { pub(super) struct NeoMediaShared { pub(super) vid_format: RwLock, pub(super) aud_format: RwLock, - pub(super) microseconds: AtomicU64, pub(super) number_of_clients: AtomicUsize, pub(super) buffer_ready: AtomicBool, } @@ -33,27 +30,8 @@ impl Default for NeoMediaShared { Self { vid_format: RwLock::new(VidFormats::Unknown), aud_format: RwLock::new(AudFormats::Unknown), - microseconds: AtomicU64::new(0), number_of_clients: AtomicUsize::new(0), buffer_ready: AtomicBool::new(false), } } } - -#[derive(Default, Debug)] -pub(super) struct ClientPipelineData { - pub(super) vidsrc: Option, - pub(super) audsrc: Option, - pub(super) start_time: Arc, - pub(super) inited: bool, - pub(super) enough_data: Arc, -} - -impl ClientPipelineData { - pub(super) fn get_start_time(&self) -> u64 { - self.start_time.load(Ordering::Relaxed) - } - pub(super) fn set_start_time(&self, time: u64) { - self.start_time.store(time, Ordering::Relaxed) - } -}