diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index e01b5c6..dfe2531 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -23,7 +23,10 @@ use log::*; use neolink_core::bcmedia::model::*; use std::{ collections::HashSet, - sync::{atomic::Ordering, Arc}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; use tokio::{ sync::mpsc::{channel, Sender}, @@ -118,7 +121,7 @@ pub(crate) struct NeoMediaFactoryImpl { impl Drop for NeoMediaFactoryImpl { fn drop(&mut self) { - log::info!("Dopping NeoMediaFactoryImpl"); + log::debug!("Dopping NeoMediaFactoryImpl"); } } @@ -223,15 +226,15 @@ impl NeoMediaFactoryImpl { let bin = media .dynamic_cast::() .map_err(|_| anyhow!("Media source's element should be a bin"))?; - let clock = bin.clock(); // Clear the autogenerated ones for element in bin.iterate_elements().into_iter().flatten() { bin.remove(&element)?; } let mut client_data = ClientPipelineData { - start_time: self.shared.microseconds.load(Ordering::Relaxed), - clock, + start_time: Arc::new(AtomicU64::new( + self.shared.microseconds.load(Ordering::Relaxed), + )), ..Default::default() }; @@ -250,10 +253,12 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Stream); + source.set_stream_type(AppStreamType::Seekable); let enough_data_start = client_data.enough_data.clone(); let enough_data_stop = client_data.enough_data.clone(); + let seek_shared = self.shared.clone(); + let seek_start_time = client_data.start_time.clone(); source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { @@ -268,6 +273,12 @@ impl NeoMediaFactoryImpl { } enough_data_start.store(true, Ordering::Relaxed); }) + .seek_data(move |_appsrc, seek_pos| { + log::debug!("Seeking to {}", &seek_pos); + let current_time = seek_shared.microseconds.load(Ordering::Relaxed); + seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + true + }) .build(), ); // source.set_caps(Some( @@ -279,7 +290,9 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); + queue.set_property("min-threshold-time", 500u64 * 1000u64); let parser = make_element("h265parse", "parser")?; + // parser.set_property("config-interval", 5i32); let payload = make_element("rtph265pay", "pay0")?; bin.add_many(&[&source, &queue, &parser, &payload])?; Element::link_many(&[&source, &queue, &parser, &payload])?; @@ -302,10 +315,12 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Stream); + source.set_stream_type(AppStreamType::Seekable); let enough_data_start = client_data.enough_data.clone(); let enough_data_stop = client_data.enough_data.clone(); + let seek_shared = self.shared.clone(); + let seek_start_time = client_data.start_time.clone(); source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { @@ -320,6 +335,12 @@ impl NeoMediaFactoryImpl { } enough_data_start.store(true, Ordering::Relaxed); }) + .seek_data(move |_appsrc, seek_pos| { + log::debug!("Seeking to {}", &seek_pos); + let current_time = seek_shared.microseconds.load(Ordering::Relaxed); + seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + true + }) .build(), ); // source.set_caps(Some( @@ -330,9 +351,11 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); - + queue.set_property("min-threshold-time", 500u64 * 1000u64); let parser = make_element("h264parse", "parser")?; + // parser.set_property("update-timecode", true); let payload = make_element("rtph264pay", "pay0")?; + // payload.set_property("config-interval", 5i32); bin.add_many(&[&source, &queue, &parser, &payload])?; Element::link_many(&[&source, &queue, &parser, &payload])?; @@ -348,6 +371,7 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "queue0")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); + queue.set_property("min-threshold-time", 500u64 * 1000u64 * 1000u64); let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -388,7 +412,35 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Stream); + source.set_stream_type(AppStreamType::Seekable); + + let enough_data_start = client_data.enough_data.clone(); + let enough_data_stop = client_data.enough_data.clone(); + let seek_shared = self.shared.clone(); + let seek_start_time = client_data.start_time.clone(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + if enough_data_stop.load(Ordering::Relaxed) { + log::debug!("Client is in need of data"); + } + enough_data_stop.store(false, Ordering::Relaxed); + }) + .enough_data(move |_appsrc| { + if !enough_data_start.load(Ordering::Relaxed) { + log::debug!("Client is full of of data"); + } + enough_data_start.store(true, Ordering::Relaxed); + }) + .seek_data(move |_appsrc, seek_pos| { + log::debug!("Seeking to {}", &seek_pos); + let current_time = seek_shared.microseconds.load(Ordering::Relaxed); + seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + true + }) + .build(), + ); + let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; @@ -396,6 +448,7 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "audqueue")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); + queue.set_property("min-threshold-time", 500u64 * 1000u64); let parser = make_element("aacparse", "audparser")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; @@ -438,7 +491,7 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Stream); + source.set_stream_type(AppStreamType::Seekable); source.set_caps(Some( &Caps::builder("audio/x-adpcm") .field("layout", "div") @@ -447,6 +500,34 @@ impl NeoMediaFactoryImpl { .field("rate", 8000i32) .build(), )); + + let enough_data_start = client_data.enough_data.clone(); + let enough_data_stop = client_data.enough_data.clone(); + let seek_shared = self.shared.clone(); + let seek_start_time = client_data.start_time.clone(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + if enough_data_stop.load(Ordering::Relaxed) { + log::debug!("Client is in need of data"); + } + enough_data_stop.store(false, Ordering::Relaxed); + }) + .enough_data(move |_appsrc| { + if !enough_data_start.load(Ordering::Relaxed) { + log::debug!("Client is full of of data"); + } + enough_data_start.store(true, Ordering::Relaxed); + }) + .seek_data(move |_appsrc, seek_pos| { + log::debug!("Seeking to {}", &seek_pos); + let current_time = seek_shared.microseconds.load(Ordering::Relaxed); + seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + true + }) + .build(), + ); + let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; @@ -454,6 +535,7 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "audqueue")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); + queue.set_property("min-threshold-time", 500u64 * 1000u64); let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index a5a2973..ac6b6f0 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -2,7 +2,7 @@ //! gstreamer media stream use anyhow::anyhow; use futures::stream::StreamExt; -use gstreamer::{prelude::*, ClockTime}; +use gstreamer::ClockTime; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use log::*; @@ -67,7 +67,7 @@ impl NeoMediaSender { self.inspect_bcmedia(&bc_media).await; match bc_media { BcMedia::Iframe(frame) => { - if buffer_write.len() > 100 { + if buffer_write.len() > 25 { std::mem::swap(buffer_read, buffer_write); buffer_write.clear(); std::mem::swap(audbuffer_read, audbuffer_write); @@ -123,39 +123,46 @@ impl NeoMediaSender { // Resend the video and audio buffers if !clientdata.inited { clientdata.inited = true; - let mut vid_resend = buffer_read.iter().chain(buffer_write.iter()).enumerate(); - let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter()).enumerate(); + if let Some(last) = buffer_read.first() { + debug!("Start PTS SET: {:?}", tokio::time::Duration::from_micros(last.ms)); + clientdata.set_start_time(last.ms); + } + let mut vid_resend = buffer_read.iter().chain(buffer_write.iter()); + let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter()); loop { let next_vid = vid_resend.next(); - if let Some((idx, next_vid)) = next_vid.as_ref() { + if let Some(next_vid) = next_vid.as_ref() { if let Some(src) = clientdata.vidsrc.as_ref() { if Self::send_buffer( src, next_vid.data.as_slice(), - *idx as u64, - 0, + next_vid.ms, + clientdata.get_start_time(), ).await.is_err() { break; }; } } let next_aud = aud_resend.next(); - if let Some((idx, next_aud)) = next_aud.as_ref() { - if let Some(src) = clientdata.audsrc.as_ref() { - if Self::send_buffer( - src, - next_aud.data.as_slice(), - *idx as u64, - 0, - ).await.is_err() { - break; - }; + if let Some(next_aud) = next_aud.as_ref() { + if next_aud.ms >= clientdata.get_start_time() { + if let Some(src) = clientdata.audsrc.as_ref() { + if Self::send_buffer( + src, + next_aud.data.as_slice(), + next_aud.ms, + clientdata.get_start_time(), + ).await.is_err() { + break; + }; + } } } if next_vid.is_none() && next_aud.is_none() { break; } } + debug!("Advance PTS Pushed"); } // Save the client for future sends self.clientdata.insert(self.uid.fetch_add(1, Ordering::Relaxed) , clientdata); @@ -187,35 +194,17 @@ impl NeoMediaSender { match self.clientdata.entry(key) { Entry::Occupied(data) => { if let Some(vidsrc) = data.get().vidsrc.as_ref() { - if !data.get().enough_data.load(Ordering::Relaxed) { - let ms = stamped_data.ms; - let st = data.get().start_time; - let micros = ms - st; - if let Some(clock) = data.get().clock.as_ref() { - if let Some(time) = clock.time() { - let buffer_time = ClockTime::from_useconds(micros); - log::debug!( - "Buffer time: {:?}. Clock time: {:?}", - buffer_time, - time - ); - } - } - debug!( - "Pushed Video Buffer: ms: {}, st: {}, delta: {}", - ms, st, micros - ); - if Self::send_buffer( + if !data.get().enough_data.load(Ordering::Relaxed) + && Self::send_buffer( vidsrc, stamped_data.data.as_slice(), stamped_data.ms, - data.get().start_time, + data.get().get_start_time(), ) .await .is_err() - { - data.remove(); - } + { + data.remove(); } } else { data.remove(); @@ -237,19 +226,17 @@ impl NeoMediaSender { match self.clientdata.entry(key) { Entry::Occupied(data) => { if let Some(audsrc) = data.get().audsrc.as_ref() { - if !data.get().enough_data.load(Ordering::Relaxed) { - debug!("Pushed Audio Buffer"); - if Self::send_buffer( + if !data.get().enough_data.load(Ordering::Relaxed) + && Self::send_buffer( audsrc, stamped_data.data.as_slice(), stamped_data.ms, - data.get().start_time, + data.get().get_start_time(), ) .await .is_err() - { - data.remove(); - } + { + data.remove(); } } else { data.remove(); @@ -266,11 +253,12 @@ impl NeoMediaSender { frame_ms: u64, start_time: u64, ) -> AnyResult<()> { - let micros = if frame_ms > start_time { + let micros = if frame_ms >= start_time { frame_ms - start_time } else { - start_time + 0 }; + debug!("PTS: {:?}", tokio::time::Duration::from_micros(micros)); let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { diff --git a/src/rtsp/gst/shared.rs b/src/rtsp/gst/shared.rs index 847119d..e3a2e24 100644 --- a/src/rtsp/gst/shared.rs +++ b/src/rtsp/gst/shared.rs @@ -1,9 +1,8 @@ //! Data shared between the various //! components that manage a media stream -use gstreamer::Clock; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use tokio::sync::RwLock; @@ -45,8 +44,16 @@ impl Default for NeoMediaShared { pub(super) struct ClientPipelineData { pub(super) vidsrc: Option, pub(super) audsrc: Option, - pub(super) start_time: u64, + pub(super) start_time: Arc, pub(super) inited: bool, - pub(super) clock: Option, pub(super) enough_data: Arc, } + +impl ClientPipelineData { + pub(super) fn get_start_time(&self) -> u64 { + self.start_time.load(Ordering::Relaxed) + } + pub(super) fn set_start_time(&self, time: u64) { + self.start_time.store(time, Ordering::Relaxed) + } +}