diff --git a/crates/core/src/bc_protocol/ptz.rs b/crates/core/src/bc_protocol/ptz.rs index 0e08843..a90fb6e 100644 --- a/crates/core/src/bc_protocol/ptz.rs +++ b/crates/core/src/bc_protocol/ptz.rs @@ -139,12 +139,6 @@ impl BcCamera { let mut sub_set = connection.subscribe(msg_num).await?; let command = if name.is_some() { "setPos" } else { "toPos" }; - let preset = Preset { - id: preset_id, - name, - command: Some(command.to_string()), - ..Default::default() - }; let send = Bc { meta: BcMeta { msg_id: MSG_ID_PTZ_CONTROL_PRESET, @@ -163,7 +157,11 @@ impl BcCamera { payload: Some(BcPayloads::BcXml(BcXml { ptz_preset: Some(PtzPreset { preset_list: Some(PresetList { - preset: vec![preset], + preset: vec![Preset { + id: preset_id, + name, + command: Some(command.to_string()), + }], }), ..Default::default() }), diff --git a/src/config.rs b/src/config.rs index 2cfd870..5b23831 100644 --- a/src/config.rs +++ b/src/config.rs @@ -104,7 +104,7 @@ pub(crate) struct CameraConfig { pub(crate) update_time: bool, #[validate(range( - min = 10, + min = 0, max = 500, message = "Invalid buffer size", code = "buffer_size" @@ -260,7 +260,7 @@ fn default_smoothing() -> bool { } fn default_buffer_size() -> usize { - 100 + 25 } pub(crate) static RESERVED_NAMES: &[&str] = &["anyone", "anonymous"]; diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index ad4ccc3..58af684 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -43,6 +43,12 @@ struct NeoBuffer { max_size: usize, } +/// The historical buffer is how much we keep extra after live. +/// +/// This represents the upper limit we keep in storage after live +/// while waiting for the time to play arrives +const HISTORICAL_BUFFER_SIZE: usize = 100; + impl NeoBuffer { fn new(max_size: usize) -> Self { Self { @@ -98,65 +104,21 @@ impl NeoBuffer { last.data.push(media); } - while self.buf.len() > self.max_size { + while self.buf.len() > self.max_size + HISTORICAL_BUFFER_SIZE { if self.buf.pop_front().is_none() { break; } } } - fn prep_size(&self) -> usize { - self.max_size * 2 / 3 - } - fn live_size(&self) -> usize { - self.max_size / 3 + self.max_size } pub(crate) fn ready(&self) -> bool { - self.buf.len() > self.prep_size() + self.buf.len() >= self.live_size() + HISTORICAL_BUFFER_SIZE / 2 } - pub(crate) fn ready_play(&self) -> bool { - self.buf.len() > self.live_size() - } - - // fn last_iframe_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // backs - // .iter() - // .rev() - // .chain(&mut fronts.iter().rev()) - // .flat_map(|frame| match frame { - // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - // _ => None, - // }) - // .next() - // .map(|i| i as FrameTime) - // } - - // fn first_iframe_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // fronts - // .iter() - // .chain(&mut backs.iter()) - // .flat_map(|frame| match frame { - // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - // _ => None, - // }) - // .next() - // .map(|i| i as FrameTime) - // } - - // fn start_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // fronts - // .iter() - // .chain(&mut backs.iter()) - // .map(|frame| frame.time) - // .next() - // } - fn end_time(&self) -> Option { let (fronts, backs) = self.buf.as_slices(); backs @@ -166,35 +128,6 @@ impl NeoBuffer { .map(|frame| frame.time) .next() } - - // fn min_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // fronts - // .iter() - // .chain(&mut backs.iter()) - // .flat_map(|frame| match Arc::as_ref(frame) { - // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - // BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), - // _ => None, - // }) - // .min() - // .map(|i| i as FrameTime) - // } - - // fn max_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // backs - // .iter() - // .rev() - // .chain(&mut fronts.iter().rev()) - // .flat_map(|frame| match Arc::as_ref(frame) { - // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - // BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), - // _ => None, - // }) - // .max() - // .map(|i| i as FrameTime) - // } } pub(super) struct NeoMediaSenders { @@ -288,24 +221,30 @@ impl NeoMediaSenders { async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> { // trace!("Handle new data"); self.update_mediatypes(&data).await; - let time = match &data { - BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => *microseconds as FrameTime, - BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => *microseconds as FrameTime, - _ => self.buffer.end_time().unwrap_or(0), - }; let data = Arc::new(data); let end_time = self.buffer.end_time(); - let frame_time = time; - // Ocassionally the camera will make a jump in timestamps of about 15s (on sub 9s on main) - // This could mean that it runs on some fixed sized buffer + let frame_time = match data.as_ref() { + BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => *microseconds as FrameTime, + BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => *microseconds as FrameTime, + _ => self.buffer.end_time().unwrap_or(0), + }; + // If there is a jump in the time stamps for any reason + // (including a resume from a pause) + // Then we stitch the old and new frames together without + // waiting for the time to elapse if let Some(end_time) = end_time { + // An approximate fps let delta_frame = (self.buffer.buf.back().unwrap().time - self.buffer.buf.front().unwrap().time) / self.buffer.buf.len() as i64; + + // Shift current frames backwards from the new start - approx fps let delta_time = frame_time - end_time - delta_frame; let delta_duration = Duration::from_micros(delta_time.unsigned_abs()); + + // If that shift >1s then we stitch it together if delta_duration > Duration::from_secs(1) { trace!( "Reschedule buffer due to jump: {:?}, Prev: {}, New: {}", @@ -329,6 +268,8 @@ impl NeoMediaSenders { for frame in client.buffer.buf.iter_mut() { frame.time = frame.time.saturating_add(delta_time); } + // We also move the start time up so that the new frames will play + // with the stiching client.start_time.mod_value(delta_time as f64); } } @@ -465,9 +406,6 @@ impl NeoMediaSenders { }, FactoryCommand::JumpToLive => { for client in self.client_data.values_mut() { - // Set them into the non init state - // This will make them wait for the - // buffer to be enough then jump to live let _ = client.jump_to_live().await; } }, @@ -512,7 +450,6 @@ pub(super) struct NeoMediaSender { command_sender: Sender, inited: bool, playing: bool, - refilling: bool, use_smoothing: bool, } @@ -529,8 +466,7 @@ impl NeoMediaSender { command_sender: tx, inited: false, playing: true, - refilling: false, - use_smoothing, + use_smoothing: use_smoothing && buffer_size > 0, } } @@ -553,6 +489,10 @@ impl NeoMediaSender { fn target_live_for(buffer: &NeoBuffer) -> Option { let target_idx = buffer.live_size(); + if target_idx == 0 { + // No buffer needed grab the last + return buffer.buf.back().map(|item| item.time); + } let stamps = buffer.buf.iter().map(|item| item.time).collect::>(); if stamps.len() >= target_idx { @@ -568,12 +508,7 @@ impl NeoMediaSender { } } else { trace!("Not enough timestamps for target live: {:?}", stamps); - if let Some(st) = stamps.first() { - trace!("Setting to 1s behind first frame in buffer"); - Some(st - Duration::from_secs(1).as_micros() as FrameTime) - } else { - None - } + None } } @@ -687,13 +622,13 @@ impl NeoMediaSender { self.start_time.reset_to(target_time as f64); // Send preprocess now - self.send_buffers(preprocess.as_slice()).await?; - trace!("Preprocessed"); + self.send_buffers(preprocess.as_slice(), true).await?; + trace!("Preprocessed: {}", preprocess.len()); // Send these later for frame in buffer.drain(..) { self.buffer.buf.push_back(frame); } - trace!("Buffer filled"); + trace!("Buffer filled: {}", self.buffer.buf.len()); self.jump_to_live().await?; } else { @@ -737,22 +672,9 @@ impl NeoMediaSender { } async fn update(&mut self) -> AnyResult<()> { - if self.buffer.buf.len() >= self.buffer.max_size * 9 / 10 { - debug!("Buffer overfull"); - self.jump_to_live().await?; - } - if self.refilling && self.buffer.ready_play() { - self.refilling = false; - self.jump_to_live().await?; - } else if self.refilling { - trace!( - "Refilling: {}/{} ({:.2}%)", - self.buffer.buf.len(), - self.buffer.live_size(), - self.buffer.buf.len() as f32 / (self.buffer.live_size()) as f32 * 100.0 - ); - } else if !self.refilling && self.inited && self.playing { + if self.inited && self.playing { self.update_starttime().await?; + let ignore_stamps = self.buffer.live_size() == 0; // Check app src is live if !self .vid @@ -763,17 +685,6 @@ impl NeoMediaSender { return Err(anyhow!("Vid src is closed")); } - // Check if buffers are ok - if self.buffer.buf.len() <= self.buffer.max_size / 10 { - warn!( - "Buffer exhausted. Not enough data from Camera. Pausing RTSP until refilled." - ); - info!("Try reducing your Max Bitrate using the offical app"); - self.refilling = true; - } else { - trace!("Buffer size: {}", self.buffer.buf.len()); - } - // Send buffers let mut buffers = vec![]; if let Some(buftime) = self.get_buftime() { @@ -785,7 +696,6 @@ impl NeoMediaSender { .map(|data| data.time <= buftime) .unwrap_or(false) { - tokio::task::yield_now().await; match self.buffer.buf.pop_front() { Some(frame) => { buffers.push(frame); @@ -797,13 +707,13 @@ impl NeoMediaSender { tokio::task::yield_now().await; // collect certain frames - self.send_buffers(&buffers).await?; + self.send_buffers(&buffers, ignore_stamps).await?; } Ok(()) } - async fn send_buffers(&mut self, medias: &[Stamped]) -> AnyResult<()> { + async fn send_buffers(&mut self, medias: &[Stamped], ignore_stamps: bool) -> AnyResult<()> { if medias.is_empty() { return Ok(()); } @@ -852,7 +762,10 @@ impl NeoMediaSender { let buffers_ref = buffers.get_mut().unwrap(); for (time, buf) in vid_buffers.drain(..) { tokio::task::yield_now().await; - let runtime = self.buftime_to_runtime(time); + let runtime = match (self.get_runtime(), ignore_stamps) { + (None, _) | (_, false) => self.buftime_to_runtime(time), + (Some(runtime), true) => runtime, + }; trace!( " - Sending vid frame at time {} ({:?} Expect: {:?})", time, @@ -870,6 +783,7 @@ impl NeoMediaSender { runtime.try_into().unwrap(), ); gst_buf_mut.set_dts(time); + gst_buf_mut.set_pts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf.as_slice()); @@ -913,7 +827,10 @@ impl NeoMediaSender { let buffers_ref = buffers.get_mut().unwrap(); for (time, buf) in aud_buffers.drain(..) { tokio::task::yield_now().await; - let runtime = self.buftime_to_runtime(time); + let runtime = match (self.get_runtime(), ignore_stamps) { + (None, _) | (_, false) => self.buftime_to_runtime(time), + (Some(runtime), true) => runtime, + }; let gst_buf = { let mut gst_buf = @@ -925,6 +842,7 @@ impl NeoMediaSender { runtime.try_into().unwrap(), ); gst_buf_mut.set_dts(time); + gst_buf_mut.set_pts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf.as_slice());