From 6daf1f5b3da1cb5b140b627f8762ac317829030b Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Thu, 5 Oct 2023 11:21:30 +0700 Subject: [PATCH] More adjustments to timestamping --- Cargo.toml | 6 +- src/common/streamthread.rs | 9 ++- src/rtsp/gst/factory.rs | 1 + src/rtsp/mod.rs | 161 ++++++++++++++++++++++--------------- 4 files changed, 104 insertions(+), 73 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 480dfb1..d682141 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,9 +23,9 @@ env_logger = "0.10.0" fcm-push-listener = "2.0.1" futures = "0.3.28" gstreamer = "0.21.0" -gstreamer-app = { version = "0.21.0", features = ["v1_16"] } -gstreamer-rtsp = { version = "0.21.0", features = ["v1_16"] } -gstreamer-rtsp-server = { version = "0.21.0", features = ["v1_16"] } +gstreamer-app = { version = "0.21.0", features = ["v1_18"] } +gstreamer-rtsp = { version = "0.21.0", features = ["v1_18"] } +gstreamer-rtsp-server = { version = "0.21.0", features = ["v1_18"] } heck = "0.4.1" lazy_static = "1.4.0" log = { version = "0.4.17", features = [ "release_max_level_debug" ] } diff --git a/src/common/streamthread.rs b/src/common/streamthread.rs index a0f5fd4..0abdd33 100644 --- a/src/common/streamthread.rs +++ b/src/common/streamthread.rs @@ -517,8 +517,9 @@ impl StreamData { }; let _ = vid_tx.send(d.clone()); vid_history.send_modify(|history| { + let drop_time = d.ts.saturating_sub(Duration::from_secs(5)); history.push_back(d); - while history.len() > 100 { + while history.front().is_some_and(|di| di.ts < drop_time) { history.pop_front(); } }); @@ -535,8 +536,9 @@ impl StreamData { }; let _ = vid_tx.send(d.clone()); vid_history.send_modify(|history| { + let drop_time = d.ts.saturating_sub(Duration::from_secs(5)); history.push_back(d); - while history.len() > 100 { + while history.front().is_some_and(|di| di.ts < drop_time) { history.pop_front(); } }); @@ -551,8 +553,9 @@ impl StreamData { aud_keyframe = false; let _ = aud_tx.send(d.clone())?; aud_history.send_modify(|history| { + let drop_time = d.ts.saturating_sub(Duration::from_secs(5)); history.push_back(d); - while history.len() > 100 { + while history.front().is_some_and(|di| di.ts < drop_time) { history.pop_front(); } }); diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 5ae819a..4b377a4 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -39,6 +39,7 @@ impl NeoMediaFactory { factory.set_shared(false); factory.set_eos_shutdown(true); factory.set_stop_on_disconnect(true); + // factory.set_publish_clock_mode(gstreamer_rtsp_server::RTSPPublishClockMode::Clock); factory.set_suspend_mode(gstreamer_rtsp_server::RTSPSuspendMode::Reset); factory.set_launch("videotestsrc pattern=\"snow\" ! video/x-raw,width=896,height=512,framerate=25/1 ! textoverlay name=\"inittextoverlay\" text=\"Stream not Ready\" valignment=top halignment=left font-desc=\"Sans, 32\" ! jpegenc ! rtpjpegpay name=pay0"); factory.set_suspend_mode(RTSPSuspendMode::None); diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index d95a6ea..873f570 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -784,15 +784,19 @@ async fn stream_run( v = async { // Send Initial let mut found_key_frame = false; - for data in thread_vid_history.borrow().iter() { - if data.keyframe || found_key_frame { - found_key_frame = true; - thread_vid_data_tx.send( - StreamData::Media { - data: data.data.clone(), - ts: Duration::ZERO, - } - )?; + { + let borrowed = thread_vid_history.borrow(); + let last_ts = borrowed.back().map(|lf| lf.ts); + for data in borrowed.iter() { + if data.keyframe || found_key_frame { + found_key_frame = true; + thread_vid_data_tx.send( + StreamData::Media { + data: data.data.clone(), + ts: last_ts.unwrap() + } + )?; + } } } @@ -825,16 +829,20 @@ async fn stream_run( _ = thread_stream_cancel.cancelled() => AnyResult::Ok(()), v = async { // Send Initial - let mut found_key_frame = false; - for data in thread_aud_history.borrow().iter() { - if data.keyframe || found_key_frame { - thread_aud_data_tx.send( - StreamData::Media { - data: data.data.clone(), - ts: Duration::ZERO, - } - )?; - found_key_frame = true; + { + let mut found_key_frame = false; + let borrowed = thread_aud_history.borrow(); + let last_ts = borrowed.back().map(|lf| lf.ts); + for data in borrowed.iter() { + if data.keyframe || found_key_frame { + found_key_frame = true; + thread_aud_data_tx.send( + StreamData::Media { + data: data.data.clone(), + ts: last_ts.unwrap() + } + )?; + } } } // Send new @@ -873,18 +881,22 @@ async fn stream_run( // Send seek vid_data_tx.send(data)?; // Send initial buffer - if let Some(seek_ts) = seek_ts { + if let Some(_seek_ts) = seek_ts { // Send Initial - let mut found_key_frame = false; - for data in thread_vid_history.borrow().iter() { - if data.keyframe || found_key_frame { - vid_data_tx.send( - StreamData::Media { - data: data.data.clone(), - ts: seek_ts, - } - )?; - found_key_frame = true; + { + let mut found_key_frame = false; + let borrowed = thread_vid_history.borrow(); + let last_ts = borrowed.back().map(|lf| lf.ts); + for data in borrowed.iter() { + if data.keyframe || found_key_frame { + found_key_frame = true; + vid_data_tx.send( + StreamData::Media { + data: data.data.clone(), + ts: last_ts.unwrap() + } + )?; + } } } } @@ -913,18 +925,22 @@ async fn stream_run( }; aud_data_tx.send(data)?; // Send initial buffer - if let Some(seek_ts) = seek_ts { + if let Some(_seek_ts) = seek_ts { // Send Initial - let mut found_key_frame = false; - for data in thread_aud_history.borrow().iter() { - if data.keyframe || found_key_frame { - aud_data_tx.send( - StreamData::Media { - data: data.data.clone(), - ts: seek_ts, - } - )?; - found_key_frame = true; + { + let mut found_key_frame = false; + let borrowed = thread_aud_history.borrow(); + let last_ts = borrowed.back().map(|lf| lf.ts); + for data in borrowed.iter() { + if data.keyframe || found_key_frame { + found_key_frame = true; + aud_data_tx.send( + StreamData::Media { + data: data.data.clone(), + ts: last_ts.unwrap() + } + )?; + } } } } @@ -1144,10 +1160,9 @@ async fn handle_data> + Unpin, E>( mut data_rx: T, ) -> Result<()> { if let Some(app) = app { - let mut last_ft: Option = None; - let mut ft = Duration::ZERO; - let mut last_rt: Option = None; - let mut rt = Duration::ZERO; + let mut start_rt: Option = None; + let mut start_ft: Option = None; + let mut rt: Duration = Duration::ZERO; let appsrc = app.clone(); let (tx, mut rx) = mpsc(150); @@ -1186,42 +1201,47 @@ async fn handle_data> + Unpin, E>( } } match data { - StreamData::Seek { ts, reply } => { - rt = ts; - last_rt = None; + StreamData::Seek { ts: _, reply } => { let _ = reply.send(()).await; } StreamData::Media { data, ts: ft_i } => { log::trace!("Frame recieved with ts: {ft_i:?}"); // Update rt if let Some(rt_i) = get_runtime(app) { - if let Some(last_rt) = last_rt { - let delta_rt = rt_i.saturating_sub(last_rt); - rt += delta_rt; + log::debug!("Appsrc runtime: {:?}", rt_i); + rt = rt_i; + } + + if start_rt.is_none() { + start_rt = Some(rt); + } + if let Some(i_start_ft) = start_ft { + // Have we gone back in time? + if ft_i < i_start_ft { + start_rt = Some(rt); + start_ft = Some(ft_i); } - last_rt = Some(rt_i); + } else { + start_ft = Some(ft_i); } - // Update ft - if let Some(last_ft) = last_ft { - let delta_ft = ft_i.saturating_sub(last_ft); - ft += delta_ft; - } - last_ft = Some(ft_i); + let mut ts = (ft_i - start_ft.unwrap()) + start_rt.unwrap(); // Sync ft to rt if > 1500ms difference const MAX_DELTA_T: Duration = Duration::from_millis(200); - let delta_t = if rt > ft { rt - ft } else { ft - rt }; + let delta_t = if rt > ts { rt - ts } else { ts - rt }; if delta_t > MAX_DELTA_T { - log::debug!("delta_t > MAX_DELTA_T = {delta_t:?}, {MAX_DELTA_T:?}"); - ft = rt; + log::debug!("delta_t > MAX_DELTA_T = {delta_t:?} > {MAX_DELTA_T:?}"); + ts = rt; + start_rt = Some(rt); + start_ft = Some(ft_i); } let buf = { let mut gst_buf = gstreamer::Buffer::with_size(data.len()).unwrap(); { let gst_buf_mut = gst_buf.get_mut().unwrap(); - log::debug!("Setting PTS: {ft:?}, Runtime: {rt:?}"); - let time = ClockTime::from_useconds(ft.as_micros() as u64); + log::debug!("Setting PTS: {ts:?}, Runtime: {ts:?}"); + let time = ClockTime::from_useconds(ts.as_micros() as u64); gst_buf_mut.set_dts(time); gst_buf_mut.set_pts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); @@ -1251,8 +1271,11 @@ fn check_live(app: &AppSrc) -> Result<()> { fn get_runtime(app: &AppSrc) -> Option { if let Some(clock) = app.clock() { if let Some(time) = clock.time() { + log::debug!("time: {time:?}"); if let Some(base_time) = app.base_time() { + log::debug!("base_time: {base_time:?}"); let runtime = time.saturating_sub(base_time); + log::debug!("runtime: {runtime:?}"); return Some(Duration::from_micros(runtime.useconds())); } } @@ -1320,8 +1343,9 @@ fn build_h264(bin: &Element, stream_config: &StreamConfig) -> Result { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_is_live(false); + source.set_is_live(true); source.set_block(false); + source.set_min_latency(0); source.set_property("emit-signals", false); source.set_max_bytes(buffer_size as u64); source.set_do_timestamp(false); @@ -1353,8 +1377,9 @@ fn build_h265(bin: &Element, stream_config: &StreamConfig) -> Result { let source = make_element("appsrc", "vidsrc")? .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_is_live(false); + source.set_is_live(true); source.set_block(false); + source.set_min_latency(0); source.set_property("emit-signals", false); source.set_max_bytes(buffer_size as u64); source.set_do_timestamp(false); @@ -1387,8 +1412,9 @@ fn build_aac(bin: &Element, stream_config: &StreamConfig) -> Result { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_is_live(false); + source.set_is_live(true); source.set_block(false); + source.set_min_latency(0); source.set_property("emit-signals", false); source.set_max_bytes(buffer_size as u64); source.set_do_timestamp(false); @@ -1434,8 +1460,9 @@ fn build_adpcm(bin: &Element, block_size: u32, stream_config: &StreamConfig) -> let source = make_element("appsrc", "audsrc")? .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_is_live(false); + source.set_is_live(true); source.set_block(false); + source.set_min_latency(0); source.set_property("emit-signals", false); source.set_max_bytes(buffer_size as u64); source.set_do_timestamp(false);