More adjustments to how the pre-buffer is sent

This commit is contained in:
QuantumEntangledAndy committed 2023-05-01 12:26:00 +07:00
1 parent 8cea9e79e5
commit d24a4d12e2
5 files changed
+232 -190

No files matched your search

+13 -8
View File
@@ -12,11 +12,15 @@ pub struct BcMediaCodex {
/// If true we will not search for the start of the next packet
/// in the event that the stream appears to be corrupted
strict: bool,
amount_skipped: usize,
}
impl BcMediaCodex {
pub(crate) fn new(strict: bool) -> Self {
Self { strict }
Self {
strict,
amount_skipped: 0,
}
}
}
@@ -46,18 +50,19 @@ impl Decoder for BcMediaCodex {
}
fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>> {
let mut amount_skipped: usize = 0;
loop {
match { BcMedia::deserialize(src) } {
Ok(bc) => {
if amount_skipped > 0 {
debug!("Amount skipped to restore stream: {}", amount_skipped);
if self.amount_skipped > 0 {
debug!("Amount skipped to restore stream: {}", self.amount_skipped);
self.amount_skipped = 0;
}
return Ok(Some(bc));
}
Err(Error::NomIncomplete(_)) => {
if amount_skipped > 0 {
debug!("Amount skipped to restore stream: {}", amount_skipped);
if self.amount_skipped > 0 {
debug!("Amount skipped to restore stream: {}", self.amount_skipped);
self.amount_skipped = 0;
}
return Ok(None);
}
@@ -67,12 +72,12 @@ impl Decoder for BcMediaCodex {
} else if src.is_empty() {
return Ok(None);
} else {
if amount_skipped == 0 {
if self.amount_skipped == 0 {
debug!("Error in stream attempting to restore");
trace!(" Stream Error: {:?}", e);
}
// Drop the whole packet and wait for a packet that starts with magic
amount_skipped += src.len();
self.amount_skipped += src.len();
src.clear();
continue;
}
+138 -146
View File
@@ -26,7 +26,10 @@ use std::{
sync::{atomic::Ordering, Arc},
};
use tokio::{
sync::mpsc::{channel, Sender},
sync::{
mpsc::{channel, Sender},
RwLock,
},
task::JoinSet,
time::Duration,
};
@@ -104,6 +107,10 @@ impl NeoMediaFactory {
pub(crate) fn buffer_ready(&self) -> bool {
self.imp().buffer_ready()
}
pub(crate) async fn join(&self) -> AnyResult<()> {
self.imp().join().await
}
}
unsafe impl Send for NeoMediaFactory {}
@@ -114,7 +121,7 @@ pub(crate) struct NeoMediaFactoryImpl {
clientsender: Sender<NeoMediaSender>,
shared: Arc<NeoMediaShared>,
#[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj
threads: JoinSet<AnyResult<()>>,
threads: RwLock<JoinSet<AnyResult<()>>>,
}
impl Drop for NeoMediaFactoryImpl {
@@ -149,14 +156,15 @@ impl Default for NeoMediaFactoryImpl {
}
}
}
Ok(())
unreachable!();
// Ok(())
});
Self {
sender: datasender,
clientsender,
shared,
threads,
threads: RwLock::new(threads),
}
}
}
@@ -294,12 +302,6 @@ impl NeoMediaFactoryImpl {
// source.set_property("max-time", Duration::from_secs_f32(2.25).as_nanos() as u64);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::RandomAccess);
let clock = source.clock();
if let Some(clock) = clock {
debug!("Clock: {:?}", clock.time());
} else {
debug!("No clock");
}
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
@@ -391,154 +393,136 @@ impl NeoMediaFactoryImpl {
}
}
let do_aud = false;
if do_aud {
match *self.shared.aud_format.blocking_read() {
AudFormats::Unknown => {}
AudFormats::Aac => {
debug!("Building Aac pipeline");
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
// let do_aud = false;
// if do_aud {
match *self.shared.aud_format.blocking_read() {
AudFormats::Unknown => {}
AudFormats::Aac => {
debug!("Building Aac pipeline");
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
let seek_command = client_data.get_commader();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
})
.enough_data(move |_appsrc| {
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
})
.seek_data(move |_appsrc, seek_pos| {
debug!("Send seek AAC");
let _ = seek_command
.blocking_send(NeoMediaSenderCommand::Seek(seek_pos));
true
})
.build(),
);
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
// let seek_command = client_data.get_commader();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
})
.enough_data(move |_appsrc| {
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
})
.seek_data(move |_appsrc, _seek_pos| true)
.build(),
);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
let parser = make_element("aacparse", "audparser")?;
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "audpayload")?;
let storage = make_element("rtpstorage", "audstorage")?;
storage
.set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64);
let jitter = make_element("rtpjitterbuffer", "pay1")?;
jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32);
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
let parser = make_element("aacparse", "audparser")?;
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
bin.add_many(&[
&source, &queue, &parser, &decoder, &encoder, &payload, &storage, &jitter,
])?;
Element::link_many(&[&source, &queue, &parser, &decoder])?;
Element::link_many(&[&encoder, &payload, &storage, &jitter])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link AAC decoder to encoder");
});
bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
Element::link_many(&[&source, &queue, &parser, &decoder])?;
Element::link_many(&[&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link AAC decoder to encoder");
});
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.update_aud(source);
}
AudFormats::Adpcm(block_size) => {
debug!("Building Adpcm pipeline");
// Original command line
// caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000
// ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024
// ! adpcmdec
// ! audioconvert
// ! rtpL16pay name=pay1
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.update_aud(source);
}
AudFormats::Adpcm(block_size) => {
debug!("Building Adpcm pipeline");
// Original command line
// caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000
// ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024
// ! adpcmdec
// ! audioconvert
// ! rtpL16pay name=pay1
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::RandomAccess);
source.set_caps(Some(
&Caps::builder("audio/x-adpcm")
.field("layout", "div")
.field("block_align", block_size as i32)
.field("channels", 1i32)
.field("rate", 8000i32)
.build(),
));
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::RandomAccess);
source.set_caps(Some(
&Caps::builder("audio/x-adpcm")
.field("layout", "div")
.field("block_align", block_size as i32)
.field("channels", 1i32)
.field("rate", 8000i32)
.build(),
));
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
let seek_command = client_data.get_commader();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
})
.enough_data(move |_appsrc| {
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
})
.seek_data(move |_appsrc, seek_pos| {
debug!("Send seek Adpcm");
let _ = seek_command
.blocking_send(NeoMediaSenderCommand::Seek(seek_pos));
true
})
.build(),
);
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
// let seek_command = client_data.get_commader();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
})
.enough_data(move |_appsrc| {
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
})
.seek_data(move |_appsrc, _seek_pos| true)
.build(),
);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?;
Element::link_many(&[&source, &queue, &decoder])?;
Element::link_many(&[&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link ADPCM decoder to encoder");
});
bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?;
Element::link_many(&[&source, &queue, &decoder])?;
Element::link_many(&[&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link ADPCM decoder to encoder");
});
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.update_aud(source);
}
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.update_aud(source);
}
}
@@ -547,4 +531,12 @@ impl NeoMediaFactoryImpl {
bin.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))
}
async fn join(&self) -> AnyResult<()> {
let mut threads = self.threads.write().await;
while let Some(thread) = threads.join_next().await {
thread??;
}
Ok(())
}
}
+42 -21
View File
@@ -27,8 +27,9 @@ use tokio_stream::wrappers::ReceiverStream;
use super::{shared::*, AnyResult};
const LATENCY: Duration = Duration::from_millis(200);
const LATENCY: Duration = Duration::from_millis(20);
const BUFFER_TIME: Duration = Duration::from_secs(10);
const BUFFER_SIZE: usize = 2;
#[derive(Default)]
struct NeoBuffer {
@@ -68,7 +69,7 @@ impl NeoBuffer {
if let Some(last_frame_time) = self.last_iframe_time() {
let mut time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff);
while let Some(time) = time_delta {
if time > BUFFER_TIME && self.buf.len() > 2 {
if time > BUFFER_TIME && self.buf.len() > BUFFER_SIZE {
let _ = self.buf.pop_front();
debug!("Popping frame with time difference of {:?}", time);
time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff);
@@ -242,6 +243,9 @@ impl NeoMediaSenders {
_ => {}
}
self.buffer.push(data);
if self.buffer.buf.len() >= BUFFER_SIZE {
self.shared.buffer_ready.store(true, Ordering::Relaxed);
}
Ok(())
}
@@ -320,7 +324,7 @@ pub(super) struct NeoMediaSender {
impl NeoMediaSender {
pub(super) fn new() -> Self {
let (tx, rx) = channel(3);
let (tx, rx) = channel(30);
Self {
start_time: Duration::ZERO,
last_sent_time: Duration::ZERO,
@@ -347,25 +351,24 @@ impl NeoMediaSender {
async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
if self.inited {
if let Some(runtime) = self.get_runtime() {
let (fronts, backs) = buffer.buf.as_slices();
let frames = fronts.iter().chain(backs.iter()).collect::<Vec<_>>();
let idx = frames.len().saturating_div(2);
let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| {
if let BcMedia::Iframe(data) = f {
Some(Duration::from_micros(data.microseconds as u64))
} else {
None
}
});
if let Some(target_time) = target_time {
self.start_time = target_time - runtime;
self.last_sent_time = target_time;
debug!(
"Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}",
target_time, self.start_time, target_time - self.start_time, runtime
);
let runtime = self.get_runtime().unwrap_or(Duration::ZERO);
let (fronts, backs) = buffer.buf.as_slices();
let frames = fronts.iter().chain(backs.iter()).collect::<Vec<_>>();
let idx = frames.len().saturating_div(2);
let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| {
if let BcMedia::Iframe(data) = f {
Some(Duration::from_micros(data.microseconds as u64))
} else {
None
}
});
if let Some(target_time) = target_time {
self.start_time = target_time - runtime;
self.last_sent_time = target_time;
debug!(
"Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}",
target_time, self.start_time, target_time - self.start_time, runtime
);
}
}
@@ -484,7 +487,25 @@ impl NeoMediaSender {
}
} else {
// Not live. Send the WHOLE buffer for analysis
// Up to the last sent frame
debug!("Sending pre buffer");
for frame in buf.iter() {
let frame_time = match frame {
BcMedia::Iframe(data) => {
Some(Duration::from_micros(data.microseconds as u64))
}
BcMedia::Pframe(data) => {
Some(Duration::from_micros(data.microseconds as u64))
}
_ => None,
};
debug!(" - Maybe: {:?} > {:?}", frame_time, self.last_sent_time);
if let Some(frame_time) = frame_time {
if frame_time > self.last_sent_time {
return Ok(());
}
}
debug!(" - Send: {:?}", frame_time);
self.send_buffer(frame).await?;
}
}
+34 -9
View File
@@ -24,7 +24,7 @@ use std::{
};
use tokio::{
sync::{mpsc::Sender, RwLock},
task::JoinHandle,
task::JoinSet,
};
glib::wrapper! {
@@ -83,11 +83,7 @@ impl NeoRtspServer {
self.imp().add_permitted_roles(tag, permitted_users).await
}
pub(crate) async fn run(
&self,
bind_addr: &str,
bind_port: u16,
) -> AnyResult<(JoinHandle<AnyResult<()>>, Arc<MainLoop>)> {
pub(crate) async fn run(&self, bind_addr: &str, bind_port: u16) -> AnyResult<()> {
let server = self;
server.set_address(bind_addr);
server.set_service(&format!("{}", bind_port));
@@ -96,10 +92,32 @@ impl NeoRtspServer {
let main_loop = Arc::new(MainLoop::new(None, false));
// Run the Glib main loop.
let main_loop_thread = main_loop.clone();
#[allow(clippy::unit_arg)]
let handle = tokio::task::spawn_blocking(move || Ok(main_loop_thread.run()));
let handle = tokio::task::spawn_blocking(move || {
main_loop_thread.run();
AnyResult::Ok(())
});
self.imp()
.threads
.write()
.await
.spawn(async move { handle.await? });
self.imp().main_loop.write().await.replace(main_loop);
Ok(())
}
Ok((handle, main_loop))
pub(crate) async fn quit(&self) -> AnyResult<()> {
if let Some(main_loop) = self.imp().main_loop.read().await.as_ref() {
main_loop.quit();
}
Ok(())
}
pub(crate) async fn join(&self) -> AnyResult<()> {
let mut threads = self.imp().threads.write().await;
while let Some(thread) = threads.join_next().await {
thread??;
}
Ok(())
}
pub(crate) fn set_up_tls(&self, config: &Config) {
@@ -163,6 +181,8 @@ struct FactoryData {
#[derive(Default)]
pub(crate) struct NeoRtspServerImpl {
medias: RwLock<HashMap<String, FactoryData>>,
threads: RwLock<JoinSet<AnyResult<()>>>,
main_loop: RwLock<Option<Arc<MainLoop>>>,
}
impl ObjectImpl for NeoRtspServerImpl {}
@@ -182,6 +202,11 @@ impl NeoRtspServerImpl {
Entry::Occupied(_occ) => {}
Entry::Vacant(vac) => {
let media = NeoMediaFactory::new();
let thread_media = media.clone();
self.threads
.write()
.await
.spawn(async move { thread_media.join().await });
vac.insert(FactoryData {
factory: media,
paths: Default::default(),
+5 -6
View File
@@ -131,14 +131,15 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> {
let bind_addr = config.bind_addr.clone();
let bind_port = config.bind_port;
let (handle, main_loop) = rtsp.run(&bind_addr, bind_port).await?;
set.spawn(async move { handle.await? });
rtsp.run(&bind_addr, bind_port).await?;
let thread_rtsp = rtsp.clone();
set.spawn(async move { thread_rtsp.join().await });
while let Some(joined) = set.join_next().await {
match &joined {
Err(_) | Ok(Err(_)) => {
// Panicked or error in task
main_loop.quit();
rtsp.quit().await?;
}
Ok(Ok(_)) => {
// All good
@@ -200,9 +201,7 @@ async fn camera_main(camera: Camera<Disconnected>) -> Result<(), CameraFailureKi
break;
}
}
if streaming.get_config().pause.on_motion || streaming.get_config().pause.on_disconnect {
log::info!("Pause buffer prepared");
}
log::info!("{}: Buffers prepared", name);
// tokio::task::spawn(async move {
// let mut inter = tokio::time::interval(tokio::time::Duration::from_secs_f32(0.01));