From f7c16bf5a3fed147263e7dd911caa6b1b3935c7b Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 5 Jun 2023 07:53:25 +0700 Subject: [PATCH 1/7] Allow zero buffer and remove buffer refill --- crates/core/src/bc_protocol/ptz.rs | 12 +- src/rtsp/gst/sender.rs | 174 ++++------------------------- 2 files changed, 27 insertions(+), 159 deletions(-) diff --git a/crates/core/src/bc_protocol/ptz.rs b/crates/core/src/bc_protocol/ptz.rs index 0e08843..a90fb6e 100644 --- a/crates/core/src/bc_protocol/ptz.rs +++ b/crates/core/src/bc_protocol/ptz.rs @@ -139,12 +139,6 @@ impl BcCamera { let mut sub_set = connection.subscribe(msg_num).await?; let command = if name.is_some() { "setPos" } else { "toPos" }; - let preset = Preset { - id: preset_id, - name, - command: Some(command.to_string()), - ..Default::default() - }; let send = Bc { meta: BcMeta { msg_id: MSG_ID_PTZ_CONTROL_PRESET, @@ -163,7 +157,11 @@ impl BcCamera { payload: Some(BcPayloads::BcXml(BcXml { ptz_preset: Some(PtzPreset { preset_list: Some(PresetList { - preset: vec![preset], + preset: vec![Preset { + id: preset_id, + name, + command: Some(command.to_string()), + }], }), ..Default::default() }), diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index ad4ccc3..0bc2f33 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -43,6 +43,12 @@ struct NeoBuffer { max_size: usize, } +/// The historical buffer is how much we keep extra after live. +/// +/// This represents the upper limit we keep in storage after live +/// while waiting for the time to play arrives +const HISTORICAL_BUFFER_SIZE: usize = 100; + impl NeoBuffer { fn new(max_size: usize) -> Self { Self { @@ -98,65 +104,21 @@ impl NeoBuffer { last.data.push(media); } - while self.buf.len() > self.max_size { + while self.buf.len() > self.max_size + HISTORICAL_BUFFER_SIZE { if self.buf.pop_front().is_none() { break; } } } - fn prep_size(&self) -> usize { - self.max_size * 2 / 3 - } - fn live_size(&self) -> usize { - self.max_size / 3 + self.max_size } pub(crate) fn ready(&self) -> bool { - self.buf.len() > self.prep_size() + self.buf.len() >= self.live_size() } - pub(crate) fn ready_play(&self) -> bool { - self.buf.len() > self.live_size() - } - - // fn last_iframe_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // backs - // .iter() - // .rev() - // .chain(&mut fronts.iter().rev()) - // .flat_map(|frame| match frame { - // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - // _ => None, - // }) - // .next() - // .map(|i| i as FrameTime) - // } - - // fn first_iframe_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // fronts - // .iter() - // .chain(&mut backs.iter()) - // .flat_map(|frame| match frame { - // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - // _ => None, - // }) - // .next() - // .map(|i| i as FrameTime) - // } - - // fn start_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // fronts - // .iter() - // .chain(&mut backs.iter()) - // .map(|frame| frame.time) - // .next() - // } - fn end_time(&self) -> Option { let (fronts, backs) = self.buf.as_slices(); backs @@ -166,35 +128,6 @@ impl NeoBuffer { .map(|frame| frame.time) .next() } - - // fn min_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // fronts - // .iter() - // .chain(&mut backs.iter()) - // .flat_map(|frame| match Arc::as_ref(frame) { - // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - // BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), - // _ => None, - // }) - // .min() - // .map(|i| i as FrameTime) - // } - - // fn max_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // backs - // .iter() - // .rev() - // .chain(&mut fronts.iter().rev()) - // .flat_map(|frame| match Arc::as_ref(frame) { - // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - // BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), - // _ => None, - // }) - // .max() - // .map(|i| i as FrameTime) - // } } pub(super) struct NeoMediaSenders { @@ -288,52 +221,9 @@ impl NeoMediaSenders { async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> { // trace!("Handle new data"); self.update_mediatypes(&data).await; - let time = match &data { - BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => *microseconds as FrameTime, - BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => *microseconds as FrameTime, - _ => self.buffer.end_time().unwrap_or(0), - }; let data = Arc::new(data); - let end_time = self.buffer.end_time(); - let frame_time = time; - // Ocassionally the camera will make a jump in timestamps of about 15s (on sub 9s on main) - // This could mean that it runs on some fixed sized buffer - if let Some(end_time) = end_time { - let delta_frame = (self.buffer.buf.back().unwrap().time - - self.buffer.buf.front().unwrap().time) - / self.buffer.buf.len() as i64; - let delta_time = frame_time - end_time - delta_frame; - let delta_duration = Duration::from_micros(delta_time.unsigned_abs()); - if delta_duration > Duration::from_secs(1) { - trace!( - "Reschedule buffer due to jump: {:?}, Prev: {}, New: {}", - delta_duration, - end_time, - frame_time - ); - trace!("Adjusting master: {}", self.buffer.buf.len()); - for frame in self.buffer.buf.iter_mut() { - let old_frame_time = frame.time; - frame.time = frame.time.saturating_add(delta_time); - trace!( - " - New frame time: {} -> {} (target {})", - old_frame_time, - frame.time, - frame_time - ); - } - - for (_, client) in self.client_data.iter_mut() { - for frame in client.buffer.buf.iter_mut() { - frame.time = frame.time.saturating_add(delta_time); - } - client.start_time.mod_value(delta_time as f64); - } - } - } - for client_data in self.client_data.values_mut() { client_data.add_data(data.clone()).await?; } @@ -465,9 +355,6 @@ impl NeoMediaSenders { }, FactoryCommand::JumpToLive => { for client in self.client_data.values_mut() { - // Set them into the non init state - // This will make them wait for the - // buffer to be enough then jump to live let _ = client.jump_to_live().await; } }, @@ -512,7 +399,6 @@ pub(super) struct NeoMediaSender { command_sender: Sender, inited: bool, playing: bool, - refilling: bool, use_smoothing: bool, } @@ -529,7 +415,6 @@ impl NeoMediaSender { command_sender: tx, inited: false, playing: true, - refilling: false, use_smoothing, } } @@ -687,7 +572,7 @@ impl NeoMediaSender { self.start_time.reset_to(target_time as f64); // Send preprocess now - self.send_buffers(preprocess.as_slice()).await?; + self.send_buffers(preprocess.as_slice(), true).await?; trace!("Preprocessed"); // Send these later for frame in buffer.drain(..) { @@ -741,18 +626,9 @@ impl NeoMediaSender { debug!("Buffer overfull"); self.jump_to_live().await?; } - if self.refilling && self.buffer.ready_play() { - self.refilling = false; - self.jump_to_live().await?; - } else if self.refilling { - trace!( - "Refilling: {}/{} ({:.2}%)", - self.buffer.buf.len(), - self.buffer.live_size(), - self.buffer.buf.len() as f32 / (self.buffer.live_size()) as f32 * 100.0 - ); - } else if !self.refilling && self.inited && self.playing { + if self.inited && self.playing { self.update_starttime().await?; + let ignore_stamps = self.buffer.live_size() == 0; // Check app src is live if !self .vid @@ -763,17 +639,6 @@ impl NeoMediaSender { return Err(anyhow!("Vid src is closed")); } - // Check if buffers are ok - if self.buffer.buf.len() <= self.buffer.max_size / 10 { - warn!( - "Buffer exhausted. Not enough data from Camera. Pausing RTSP until refilled." - ); - info!("Try reducing your Max Bitrate using the offical app"); - self.refilling = true; - } else { - trace!("Buffer size: {}", self.buffer.buf.len()); - } - // Send buffers let mut buffers = vec![]; if let Some(buftime) = self.get_buftime() { @@ -785,7 +650,6 @@ impl NeoMediaSender { .map(|data| data.time <= buftime) .unwrap_or(false) { - tokio::task::yield_now().await; match self.buffer.buf.pop_front() { Some(frame) => { buffers.push(frame); @@ -797,13 +661,13 @@ impl NeoMediaSender { tokio::task::yield_now().await; // collect certain frames - self.send_buffers(&buffers).await?; + self.send_buffers(&buffers, ignore_stamps).await?; } Ok(()) } - async fn send_buffers(&mut self, medias: &[Stamped]) -> AnyResult<()> { + async fn send_buffers(&mut self, medias: &[Stamped], ignore_stamps: bool) -> AnyResult<()> { if medias.is_empty() { return Ok(()); } @@ -852,7 +716,10 @@ impl NeoMediaSender { let buffers_ref = buffers.get_mut().unwrap(); for (time, buf) in vid_buffers.drain(..) { tokio::task::yield_now().await; - let runtime = self.buftime_to_runtime(time); + let runtime = match (self.get_runtime(), ignore_stamps) { + (None, _) | (_, false) => self.buftime_to_runtime(time), + (Some(runtime), true) => runtime, + }; trace!( " - Sending vid frame at time {} ({:?} Expect: {:?})", time, @@ -913,7 +780,10 @@ impl NeoMediaSender { let buffers_ref = buffers.get_mut().unwrap(); for (time, buf) in aud_buffers.drain(..) { tokio::task::yield_now().await; - let runtime = self.buftime_to_runtime(time); + let runtime = match (self.get_runtime(), ignore_stamps) { + (None, _) | (_, false) => self.buftime_to_runtime(time), + (Some(runtime), true) => runtime, + }; let gst_buf = { let mut gst_buf = From a165377d7aaeb71a6a21d969121b3999574ab372 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 5 Jun 2023 07:58:16 +0700 Subject: [PATCH 2/7] Not smoothing when no buffer --- src/rtsp/gst/sender.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 0bc2f33..cc913ed 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -415,7 +415,7 @@ impl NeoMediaSender { command_sender: tx, inited: false, playing: true, - use_smoothing, + use_smoothing: use_smoothing && buffer_size > 0, } } From 73e828f940099e06e1c1e5589a2bc1981705d645 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 6 Jun 2023 10:45:58 +0700 Subject: [PATCH 3/7] Restore stream timestamp stitching for the pause --- src/config.rs | 2 +- src/rtsp/gst/sender.rs | 62 +++++++++++++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/config.rs b/src/config.rs index 2cfd870..3af8eee 100644 --- a/src/config.rs +++ b/src/config.rs @@ -104,7 +104,7 @@ pub(crate) struct CameraConfig { pub(crate) update_time: bool, #[validate(range( - min = 10, + min = 0, max = 500, message = "Invalid buffer size", code = "buffer_size" diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index cc913ed..b4bc9b9 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -224,6 +224,57 @@ impl NeoMediaSenders { let data = Arc::new(data); + let end_time = self.buffer.end_time(); + let frame_time = match data.as_ref() { + BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => *microseconds as FrameTime, + BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => *microseconds as FrameTime, + _ => self.buffer.end_time().unwrap_or(0), + }; + // If there is a jump in the time stamps for any reason + // (including a resume from a pause) + // Then we stitch the old and new frames together without + // waiting for the time to elapse + if let Some(end_time) = end_time { + // An approximate fps + let delta_frame = (self.buffer.buf.back().unwrap().time + - self.buffer.buf.front().unwrap().time) + / self.buffer.buf.len() as i64; + + // Shift current frames backwards from the new start - approx fps + let delta_time = frame_time - end_time - delta_frame; + let delta_duration = Duration::from_micros(delta_time.unsigned_abs()); + + // If that shift >1s then we stitch it together + if delta_duration > Duration::from_secs(1) { + trace!( + "Reschedule buffer due to jump: {:?}, Prev: {}, New: {}", + delta_duration, + end_time, + frame_time + ); + trace!("Adjusting master: {}", self.buffer.buf.len()); + for frame in self.buffer.buf.iter_mut() { + let old_frame_time = frame.time; + frame.time = frame.time.saturating_add(delta_time); + trace!( + " - New frame time: {} -> {} (target {})", + old_frame_time, + frame.time, + frame_time + ); + } + + for (_, client) in self.client_data.iter_mut() { + for frame in client.buffer.buf.iter_mut() { + frame.time = frame.time.saturating_add(delta_time); + } + // We also move the start time up so that the new frames will play + // with the stiching + client.start_time.mod_value(delta_time as f64); + } + } + } + for client_data in self.client_data.values_mut() { client_data.add_data(data.clone()).await?; } @@ -453,12 +504,7 @@ impl NeoMediaSender { } } else { trace!("Not enough timestamps for target live: {:?}", stamps); - if let Some(st) = stamps.first() { - trace!("Setting to 1s behind first frame in buffer"); - Some(st - Duration::from_secs(1).as_micros() as FrameTime) - } else { - None - } + None } } @@ -622,10 +668,6 @@ impl NeoMediaSender { } async fn update(&mut self) -> AnyResult<()> { - if self.buffer.buf.len() >= self.buffer.max_size * 9 / 10 { - debug!("Buffer overfull"); - self.jump_to_live().await?; - } if self.inited && self.playing { self.update_starttime().await?; let ignore_stamps = self.buffer.live_size() == 0; From 9e981df990ade9a2d7bf66934a9204ac68d3f465 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 6 Jun 2023 10:52:45 +0700 Subject: [PATCH 4/7] Fixes for zero buffer config --- src/rtsp/gst/sender.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index b4bc9b9..b223868 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -489,6 +489,10 @@ impl NeoMediaSender { fn target_live_for(buffer: &NeoBuffer) -> Option { let target_idx = buffer.live_size(); + if target_idx == 0 { + // No buffer needed grab the last + return buffer.buf.back().map(|item| item.time); + } let stamps = buffer.buf.iter().map(|item| item.time).collect::>(); if stamps.len() >= target_idx { From 60638441aaa8cea09f0d4448b51d0b7134efa438 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 6 Jun 2023 10:57:39 +0700 Subject: [PATCH 5/7] Reduce default buffer size --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 3af8eee..5b23831 100644 --- a/src/config.rs +++ b/src/config.rs @@ -260,7 +260,7 @@ fn default_smoothing() -> bool { } fn default_buffer_size() -> usize { - 100 + 25 } pub(crate) static RESERVED_NAMES: &[&str] = &["anyone", "anonymous"]; From a08019bc0943a33b8a707ce93edd14ca0e7c10e2 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 6 Jun 2023 11:07:43 +0700 Subject: [PATCH 6/7] Ensure some buffer for pause before we declare ready --- src/rtsp/gst/sender.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index b223868..1fee3fb 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -116,7 +116,7 @@ impl NeoBuffer { } pub(crate) fn ready(&self) -> bool { - self.buf.len() >= self.live_size() + self.buf.len() >= self.live_size() + HISTORICAL_BUFFER_SIZE / 2 } fn end_time(&self) -> Option { @@ -623,12 +623,12 @@ impl NeoMediaSender { // Send preprocess now self.send_buffers(preprocess.as_slice(), true).await?; - trace!("Preprocessed"); + trace!("Preprocessed: {}", preprocess.len()); // Send these later for frame in buffer.drain(..) { self.buffer.buf.push_back(frame); } - trace!("Buffer filled"); + trace!("Buffer filled: {}", self.buffer.buf.len()); self.jump_to_live().await?; } else { From de0528b1b871179cbf1543a5be856f98baf1bde7 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 6 Jun 2023 11:39:03 +0700 Subject: [PATCH 7/7] Set pts and dts --- src/rtsp/gst/sender.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 1fee3fb..58af684 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -783,6 +783,7 @@ impl NeoMediaSender { runtime.try_into().unwrap(), ); gst_buf_mut.set_dts(time); + gst_buf_mut.set_pts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf.as_slice()); @@ -841,6 +842,7 @@ impl NeoMediaSender { runtime.try_into().unwrap(), ); gst_buf_mut.set_dts(time); + gst_buf_mut.set_pts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf.as_slice());