From c4c35d1bdec94d58a3e313beaaeaf6b881577114 Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 20 Apr 2023 15:26:21 +0700 Subject: [PATCH 01/64] Implement fake seek for VLC --- src/rtsp/gst/factory.rs | 102 ++++++++++++++++++++++++++++++++++++---- src/rtsp/gst/sender.rs | 86 +++++++++++++++------------------ src/rtsp/gst/shared.rs | 15 ++++-- 3 files changed, 140 insertions(+), 63 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index e01b5c6..dfe2531 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -23,7 +23,10 @@ use log::*; use neolink_core::bcmedia::model::*; use std::{ collections::HashSet, - sync::{atomic::Ordering, Arc}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; use tokio::{ sync::mpsc::{channel, Sender}, @@ -118,7 +121,7 @@ pub(crate) struct NeoMediaFactoryImpl { impl Drop for NeoMediaFactoryImpl { fn drop(&mut self) { - log::info!("Dopping NeoMediaFactoryImpl"); + log::debug!("Dopping NeoMediaFactoryImpl"); } } @@ -223,15 +226,15 @@ impl NeoMediaFactoryImpl { let bin = media .dynamic_cast::() .map_err(|_| anyhow!("Media source's element should be a bin"))?; - let clock = bin.clock(); // Clear the autogenerated ones for element in bin.iterate_elements().into_iter().flatten() { bin.remove(&element)?; } let mut client_data = ClientPipelineData { - start_time: self.shared.microseconds.load(Ordering::Relaxed), - clock, + start_time: Arc::new(AtomicU64::new( + self.shared.microseconds.load(Ordering::Relaxed), + )), ..Default::default() }; @@ -250,10 +253,12 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Stream); + source.set_stream_type(AppStreamType::Seekable); let enough_data_start = client_data.enough_data.clone(); let enough_data_stop = client_data.enough_data.clone(); + let seek_shared = self.shared.clone(); + let seek_start_time = client_data.start_time.clone(); source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { @@ -268,6 +273,12 @@ impl NeoMediaFactoryImpl { } enough_data_start.store(true, Ordering::Relaxed); }) + .seek_data(move |_appsrc, seek_pos| { + log::debug!("Seeking to {}", &seek_pos); + let current_time = seek_shared.microseconds.load(Ordering::Relaxed); + seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + true + }) .build(), ); // source.set_caps(Some( @@ -279,7 +290,9 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); + queue.set_property("min-threshold-time", 500u64 * 1000u64); let parser = make_element("h265parse", "parser")?; + // parser.set_property("config-interval", 5i32); let payload = make_element("rtph265pay", "pay0")?; bin.add_many(&[&source, &queue, &parser, &payload])?; Element::link_many(&[&source, &queue, &parser, &payload])?; @@ -302,10 +315,12 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Stream); + source.set_stream_type(AppStreamType::Seekable); let enough_data_start = client_data.enough_data.clone(); let enough_data_stop = client_data.enough_data.clone(); + let seek_shared = self.shared.clone(); + let seek_start_time = client_data.start_time.clone(); source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { @@ -320,6 +335,12 @@ impl NeoMediaFactoryImpl { } enough_data_start.store(true, Ordering::Relaxed); }) + .seek_data(move |_appsrc, seek_pos| { + log::debug!("Seeking to {}", &seek_pos); + let current_time = seek_shared.microseconds.load(Ordering::Relaxed); + seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + true + }) .build(), ); // source.set_caps(Some( @@ -330,9 +351,11 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); - + queue.set_property("min-threshold-time", 500u64 * 1000u64); let parser = make_element("h264parse", "parser")?; + // parser.set_property("update-timecode", true); let payload = make_element("rtph264pay", "pay0")?; + // payload.set_property("config-interval", 5i32); bin.add_many(&[&source, &queue, &parser, &payload])?; Element::link_many(&[&source, &queue, &parser, &payload])?; @@ -348,6 +371,7 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "queue0")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); + queue.set_property("min-threshold-time", 500u64 * 1000u64 * 1000u64); let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -388,7 +412,35 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Stream); + source.set_stream_type(AppStreamType::Seekable); + + let enough_data_start = client_data.enough_data.clone(); + let enough_data_stop = client_data.enough_data.clone(); + let seek_shared = self.shared.clone(); + let seek_start_time = client_data.start_time.clone(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + if enough_data_stop.load(Ordering::Relaxed) { + log::debug!("Client is in need of data"); + } + enough_data_stop.store(false, Ordering::Relaxed); + }) + .enough_data(move |_appsrc| { + if !enough_data_start.load(Ordering::Relaxed) { + log::debug!("Client is full of of data"); + } + enough_data_start.store(true, Ordering::Relaxed); + }) + .seek_data(move |_appsrc, seek_pos| { + log::debug!("Seeking to {}", &seek_pos); + let current_time = seek_shared.microseconds.load(Ordering::Relaxed); + seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + true + }) + .build(), + ); + let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; @@ -396,6 +448,7 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "audqueue")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); + queue.set_property("min-threshold-time", 500u64 * 1000u64); let parser = make_element("aacparse", "audparser")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; @@ -438,7 +491,7 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Stream); + source.set_stream_type(AppStreamType::Seekable); source.set_caps(Some( &Caps::builder("audio/x-adpcm") .field("layout", "div") @@ -447,6 +500,34 @@ impl NeoMediaFactoryImpl { .field("rate", 8000i32) .build(), )); + + let enough_data_start = client_data.enough_data.clone(); + let enough_data_stop = client_data.enough_data.clone(); + let seek_shared = self.shared.clone(); + let seek_start_time = client_data.start_time.clone(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + if enough_data_stop.load(Ordering::Relaxed) { + log::debug!("Client is in need of data"); + } + enough_data_stop.store(false, Ordering::Relaxed); + }) + .enough_data(move |_appsrc| { + if !enough_data_start.load(Ordering::Relaxed) { + log::debug!("Client is full of of data"); + } + enough_data_start.store(true, Ordering::Relaxed); + }) + .seek_data(move |_appsrc, seek_pos| { + log::debug!("Seeking to {}", &seek_pos); + let current_time = seek_shared.microseconds.load(Ordering::Relaxed); + seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + true + }) + .build(), + ); + let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; @@ -454,6 +535,7 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "audqueue")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); + queue.set_property("min-threshold-time", 500u64 * 1000u64); let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index a5a2973..ac6b6f0 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -2,7 +2,7 @@ //! gstreamer media stream use anyhow::anyhow; use futures::stream::StreamExt; -use gstreamer::{prelude::*, ClockTime}; +use gstreamer::ClockTime; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use log::*; @@ -67,7 +67,7 @@ impl NeoMediaSender { self.inspect_bcmedia(&bc_media).await; match bc_media { BcMedia::Iframe(frame) => { - if buffer_write.len() > 100 { + if buffer_write.len() > 25 { std::mem::swap(buffer_read, buffer_write); buffer_write.clear(); std::mem::swap(audbuffer_read, audbuffer_write); @@ -123,39 +123,46 @@ impl NeoMediaSender { // Resend the video and audio buffers if !clientdata.inited { clientdata.inited = true; - let mut vid_resend = buffer_read.iter().chain(buffer_write.iter()).enumerate(); - let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter()).enumerate(); + if let Some(last) = buffer_read.first() { + debug!("Start PTS SET: {:?}", tokio::time::Duration::from_micros(last.ms)); + clientdata.set_start_time(last.ms); + } + let mut vid_resend = buffer_read.iter().chain(buffer_write.iter()); + let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter()); loop { let next_vid = vid_resend.next(); - if let Some((idx, next_vid)) = next_vid.as_ref() { + if let Some(next_vid) = next_vid.as_ref() { if let Some(src) = clientdata.vidsrc.as_ref() { if Self::send_buffer( src, next_vid.data.as_slice(), - *idx as u64, - 0, + next_vid.ms, + clientdata.get_start_time(), ).await.is_err() { break; }; } } let next_aud = aud_resend.next(); - if let Some((idx, next_aud)) = next_aud.as_ref() { - if let Some(src) = clientdata.audsrc.as_ref() { - if Self::send_buffer( - src, - next_aud.data.as_slice(), - *idx as u64, - 0, - ).await.is_err() { - break; - }; + if let Some(next_aud) = next_aud.as_ref() { + if next_aud.ms >= clientdata.get_start_time() { + if let Some(src) = clientdata.audsrc.as_ref() { + if Self::send_buffer( + src, + next_aud.data.as_slice(), + next_aud.ms, + clientdata.get_start_time(), + ).await.is_err() { + break; + }; + } } } if next_vid.is_none() && next_aud.is_none() { break; } } + debug!("Advance PTS Pushed"); } // Save the client for future sends self.clientdata.insert(self.uid.fetch_add(1, Ordering::Relaxed) , clientdata); @@ -187,35 +194,17 @@ impl NeoMediaSender { match self.clientdata.entry(key) { Entry::Occupied(data) => { if let Some(vidsrc) = data.get().vidsrc.as_ref() { - if !data.get().enough_data.load(Ordering::Relaxed) { - let ms = stamped_data.ms; - let st = data.get().start_time; - let micros = ms - st; - if let Some(clock) = data.get().clock.as_ref() { - if let Some(time) = clock.time() { - let buffer_time = ClockTime::from_useconds(micros); - log::debug!( - "Buffer time: {:?}. Clock time: {:?}", - buffer_time, - time - ); - } - } - debug!( - "Pushed Video Buffer: ms: {}, st: {}, delta: {}", - ms, st, micros - ); - if Self::send_buffer( + if !data.get().enough_data.load(Ordering::Relaxed) + && Self::send_buffer( vidsrc, stamped_data.data.as_slice(), stamped_data.ms, - data.get().start_time, + data.get().get_start_time(), ) .await .is_err() - { - data.remove(); - } + { + data.remove(); } } else { data.remove(); @@ -237,19 +226,17 @@ impl NeoMediaSender { match self.clientdata.entry(key) { Entry::Occupied(data) => { if let Some(audsrc) = data.get().audsrc.as_ref() { - if !data.get().enough_data.load(Ordering::Relaxed) { - debug!("Pushed Audio Buffer"); - if Self::send_buffer( + if !data.get().enough_data.load(Ordering::Relaxed) + && Self::send_buffer( audsrc, stamped_data.data.as_slice(), stamped_data.ms, - data.get().start_time, + data.get().get_start_time(), ) .await .is_err() - { - data.remove(); - } + { + data.remove(); } } else { data.remove(); @@ -266,11 +253,12 @@ impl NeoMediaSender { frame_ms: u64, start_time: u64, ) -> AnyResult<()> { - let micros = if frame_ms > start_time { + let micros = if frame_ms >= start_time { frame_ms - start_time } else { - start_time + 0 }; + debug!("PTS: {:?}", tokio::time::Duration::from_micros(micros)); let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { diff --git a/src/rtsp/gst/shared.rs b/src/rtsp/gst/shared.rs index 847119d..e3a2e24 100644 --- a/src/rtsp/gst/shared.rs +++ b/src/rtsp/gst/shared.rs @@ -1,9 +1,8 @@ //! Data shared between the various //! components that manage a media stream -use gstreamer::Clock; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use tokio::sync::RwLock; @@ -45,8 +44,16 @@ impl Default for NeoMediaShared { pub(super) struct ClientPipelineData { pub(super) vidsrc: Option, pub(super) audsrc: Option, - pub(super) start_time: u64, + pub(super) start_time: Arc, pub(super) inited: bool, - pub(super) clock: Option, pub(super) enough_data: Arc, } + +impl ClientPipelineData { + pub(super) fn get_start_time(&self) -> u64 { + self.start_time.load(Ordering::Relaxed) + } + pub(super) fn set_start_time(&self, time: u64) { + self.start_time.store(time, Ordering::Relaxed) + } +} From 107902ef3acbee68b35fcb4fd7d9d2d430fb5194 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 25 Apr 2023 14:28:35 +0700 Subject: [PATCH 02/64] Complete rework of buffer time --- src/rtsp/gst/factory.rs | 421 ++++++++++---------- src/rtsp/gst/sender.rs | 823 ++++++++++++++++++++++++---------------- src/rtsp/gst/shared.rs | 24 +- 3 files changed, 706 insertions(+), 562 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index dfe2531..34823ab 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -11,7 +11,7 @@ use gstreamer::{ glib::{self, Object}, Structure, }; -use gstreamer::{Bin, Caps, ClockTime, Element, ElementFactory}; +use gstreamer::{Bin, Caps, Element, ElementFactory}; use gstreamer_app::{AppSrc, AppSrcCallbacks, AppStreamType}; use gstreamer_rtsp::RTSPUrl; use gstreamer_rtsp_server::prelude::*; @@ -23,14 +23,12 @@ use log::*; use neolink_core::bcmedia::model::*; use std::{ collections::HashSet, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, + sync::{atomic::Ordering, Arc}, }; use tokio::{ sync::mpsc::{channel, Sender}, task::JoinSet, + time::Duration, }; use tokio_stream::wrappers::ReceiverStream; @@ -113,7 +111,7 @@ unsafe impl Sync for NeoMediaFactory {} pub(crate) struct NeoMediaFactoryImpl { sender: Sender, - clientsender: Sender, + clientsender: Sender, shared: Arc, #[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj threads: JoinSet>, @@ -134,14 +132,11 @@ impl Default for NeoMediaFactoryImpl { // Prepare thread that sends data into the appsrcs let mut threads: JoinSet> = Default::default(); - let mut sender = NeoMediaSender { - data_source: ReceiverStream::new(datarx), - clientsource: ReceiverStream::new(rx_clientsender), - shared: shared.clone(), - uid: Default::default(), - clientdata: Default::default(), - waiting_for_iframe: true, - }; + let mut sender = NeoMediaSenders::new( + shared.clone(), + ReceiverStream::new(datarx), + ReceiverStream::new(rx_clientsender), + ); threads.spawn(async move { loop { tokio::task::yield_now().await; @@ -153,7 +148,6 @@ impl Default for NeoMediaFactoryImpl { break; } } - sender.waiting_for_iframe = true; } Ok(()) }); @@ -231,12 +225,7 @@ impl NeoMediaFactoryImpl { bin.remove(&element)?; } - let mut client_data = ClientPipelineData { - start_time: Arc::new(AtomicU64::new( - self.shared.microseconds.load(Ordering::Relaxed), - )), - ..Default::default() - }; + let mut client_data = NeoMediaSender::new(); // Now contruct the actual ones match *self.shared.vid_format.blocking_read() { @@ -245,38 +234,28 @@ impl NeoMediaFactoryImpl { let source = make_element("appsrc", "vidsrc")? .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_base_time(ClockTime::from_mseconds( - self.shared.microseconds.load(Ordering::Relaxed), - )); source.set_is_live(true); source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); + source.set_stream_type(AppStreamType::RandomAccess); - let enough_data_start = client_data.enough_data.clone(); - let enough_data_stop = client_data.enough_data.clone(); - let seek_shared = self.shared.clone(); - let seek_start_time = client_data.start_time.clone(); + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + let seek_command = client_data.get_commader(); source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { - if enough_data_stop.load(Ordering::Relaxed) { - log::debug!("Client is in need of data"); - } - enough_data_stop.store(false, Ordering::Relaxed); + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); }) .enough_data(move |_appsrc| { - if !enough_data_start.load(Ordering::Relaxed) { - log::debug!("Client is full of of data"); - } - enough_data_start.store(true, Ordering::Relaxed); + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); }) .seek_data(move |_appsrc, seek_pos| { - log::debug!("Seeking to {}", &seek_pos); - let current_time = seek_shared.microseconds.load(Ordering::Relaxed); - seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); + debug!("Send seek H265"); + let _ = + seek_command.blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); true }) .build(), @@ -290,7 +269,6 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); - queue.set_property("min-threshold-time", 500u64 * 1000u64); let parser = make_element("h265parse", "parser")?; // parser.set_property("config-interval", 5i32); let payload = make_element("rtph265pay", "pay0")?; @@ -300,46 +278,47 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.vidsrc.replace(source); + client_data.update_vid(source); } VidFormats::H264 => { debug!("Building H264 Pipeline"); let source = make_element("appsrc", "vidsrc")? .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_base_time(ClockTime::from_mseconds( - self.shared.microseconds.load(Ordering::Relaxed), - )); + source.set_is_live(true); source.set_block(false); source.set_property("emit-signals", false); - source.set_max_bytes(52428800); + source.set_max_bytes(50000000u64); // 50MB + // source.set_property("max-buffers", 0u32); // v1.20 + // source.set_property("max-time", Duration::from_secs_f32(2.25).as_nanos() as u64); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); + source.set_stream_type(AppStreamType::RandomAccess); + let clock = source.clock(); + if let Some(clock) = clock { + debug!("Clock: {:?}", clock.time()); + } else { + debug!("No clock"); + } - let enough_data_start = client_data.enough_data.clone(); - let enough_data_stop = client_data.enough_data.clone(); - let seek_shared = self.shared.clone(); - let seek_start_time = client_data.start_time.clone(); + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + let seek_command = client_data.get_commader(); source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { - if enough_data_stop.load(Ordering::Relaxed) { - log::debug!("Client is in need of data"); - } - enough_data_stop.store(false, Ordering::Relaxed); + let _ = need_command.try_send(NeoMediaSenderCommand::Resume); }) .enough_data(move |_appsrc| { - if !enough_data_start.load(Ordering::Relaxed) { - log::debug!("Client is full of of data"); - } - enough_data_start.store(true, Ordering::Relaxed); + let _ = enough_command.try_send(NeoMediaSenderCommand::Pause); }) .seek_data(move |_appsrc, seek_pos| { - log::debug!("Seeking to {}", &seek_pos); - let current_time = seek_shared.microseconds.load(Ordering::Relaxed); - seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); - true + debug!("Send seek H264: {}", seek_pos); + let result = seek_command + .try_send(NeoMediaSenderCommand::Seek(seek_pos)) + .is_ok(); + debug!(" - Sent seek H264: {}", result); + result }) .build(), ); @@ -351,18 +330,36 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); - queue.set_property("min-threshold-time", 500u64 * 1000u64); + // queue.set_property("max-size-bytes", 0u32); + // queue.set_property("max-size-buffers", 0u32); + queue.set_property("max-size-time", Duration::from_secs(2).as_nanos() as u64); let parser = make_element("h264parse", "parser")?; // parser.set_property("update-timecode", true); let payload = make_element("rtph264pay", "pay0")?; + // payload.set_property("config-interval", 1i32); + // let storage = make_element("rtpstorage", "vidstorage")?; + // storage.set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64); + // let jitter = make_element("rtpjitterbuffer", "pay0")?; + // jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32); + // payload.set_property("config-interval", 5i32); - bin.add_many(&[&source, &queue, &parser, &payload])?; - Element::link_many(&[&source, &queue, &parser, &payload])?; + bin.add_many(&[ + &source, &queue, &parser, + &payload, + // &storage, + // &jitter, + ])?; + Element::link_many(&[ + &source, &queue, &parser, + &payload, + // &storage, + // &jitter, + ])?; let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.vidsrc.replace(source); + client_data.update_vid(source); } VidFormats::Unknown => { debug!("Building Unknown Pipeline"); @@ -371,7 +368,6 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "queue0")?; queue.set_property_from_str("leaky", "downstream"); queue.set_property("max-size-bytes", 104857600u32); - queue.set_property("min-threshold-time", 500u64 * 1000u64 * 1000u64); let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -391,171 +387,158 @@ impl NeoMediaFactoryImpl { .build(), )?; // source.link(&queue)?; - queue.link(&overlay)?; - overlay.link(&encoder)?; - encoder.link(&payload)?; + Element::link_many(&[&queue, &overlay, &encoder, &payload])?; } } - match *self.shared.aud_format.blocking_read() { - AudFormats::Unknown => {} - AudFormats::Aac => { - debug!("Building Aac pipeline"); - let source = make_element("appsrc", "audsrc")? - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_base_time(ClockTime::from_mseconds( - self.shared.microseconds.load(Ordering::Relaxed), - )); - source.set_is_live(true); - source.set_block(false); - source.set_property("emit-signals", false); - source.set_max_bytes(52428800); - source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); + let do_aud = false; + if do_aud { + match *self.shared.aud_format.blocking_read() { + AudFormats::Unknown => {} + AudFormats::Aac => { + debug!("Building Aac pipeline"); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - let enough_data_start = client_data.enough_data.clone(); - let enough_data_stop = client_data.enough_data.clone(); - let seek_shared = self.shared.clone(); - let seek_start_time = client_data.start_time.clone(); - source.set_callbacks( - AppSrcCallbacks::builder() - .need_data(move |_appsrc, _amt| { - if enough_data_stop.load(Ordering::Relaxed) { - log::debug!("Client is in need of data"); - } - enough_data_stop.store(false, Ordering::Relaxed); - }) - .enough_data(move |_appsrc| { - if !enough_data_start.load(Ordering::Relaxed) { - log::debug!("Client is full of of data"); - } - enough_data_start.store(true, Ordering::Relaxed); - }) - .seek_data(move |_appsrc, seek_pos| { - log::debug!("Seeking to {}", &seek_pos); - let current_time = seek_shared.microseconds.load(Ordering::Relaxed); - seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); - true - }) - .build(), - ); + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Seekable); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast back"))?; + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + let seek_command = client_data.get_commader(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); + }) + .enough_data(move |_appsrc| { + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); + }) + .seek_data(move |_appsrc, seek_pos| { + debug!("Send seek AAC"); + let _ = seek_command + .blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); + true + }) + .build(), + ); - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); - queue.set_property("min-threshold-time", 500u64 * 1000u64); - let parser = make_element("aacparse", "audparser")?; - let decoder = make_element("decodebin", "auddecoder")?; - let encoder = make_element("audioconvert", "audencoder")?; - let payload = make_element("rtpL16pay", "pay1")?; + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; - bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &parser, &decoder])?; - Element::link_many(&[&encoder, &payload])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link AAC decoder to encoder"); - }); + let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); + let parser = make_element("aacparse", "audparser")?; + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "audpayload")?; + let storage = make_element("rtpstorage", "audstorage")?; + storage + .set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64); + let jitter = make_element("rtpjitterbuffer", "pay1")?; + jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.audsrc.replace(source); - } - AudFormats::Adpcm(block_size) => { - debug!("Building Adpcm pipeline"); - // Original command line - // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 - // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 - // ! adpcmdec - // ! audioconvert - // ! rtpL16pay name=pay1 + bin.add_many(&[ + &source, &queue, &parser, &decoder, &encoder, &payload, &storage, &jitter, + ])?; + Element::link_many(&[&source, &queue, &parser, &decoder])?; + Element::link_many(&[&encoder, &payload, &storage, &jitter])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link AAC decoder to encoder"); + }); - let source = make_element("appsrc", "audsrc")? - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_base_time(ClockTime::from_mseconds( - self.shared.microseconds.load(Ordering::Relaxed), - )); - source.set_is_live(true); - source.set_block(false); - source.set_property("emit-signals", false); - source.set_max_bytes(52428800); - source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); - source.set_caps(Some( - &Caps::builder("audio/x-adpcm") - .field("layout", "div") - .field("block_align", block_size as i32) - .field("channels", 1i32) - .field("rate", 8000i32) - .build(), - )); + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + client_data.update_aud(source); + } + AudFormats::Adpcm(block_size) => { + debug!("Building Adpcm pipeline"); + // Original command line + // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 + // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 + // ! adpcmdec + // ! audioconvert + // ! rtpL16pay name=pay1 - let enough_data_start = client_data.enough_data.clone(); - let enough_data_stop = client_data.enough_data.clone(); - let seek_shared = self.shared.clone(); - let seek_start_time = client_data.start_time.clone(); - source.set_callbacks( - AppSrcCallbacks::builder() - .need_data(move |_appsrc, _amt| { - if enough_data_stop.load(Ordering::Relaxed) { - log::debug!("Client is in need of data"); - } - enough_data_stop.store(false, Ordering::Relaxed); - }) - .enough_data(move |_appsrc| { - if !enough_data_start.load(Ordering::Relaxed) { - log::debug!("Client is full of of data"); - } - enough_data_start.store(true, Ordering::Relaxed); - }) - .seek_data(move |_appsrc, seek_pos| { - log::debug!("Seeking to {}", &seek_pos); - let current_time = seek_shared.microseconds.load(Ordering::Relaxed); - seek_start_time.store(current_time - seek_pos, Ordering::Relaxed); - true - }) - .build(), - ); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::RandomAccess); + source.set_caps(Some( + &Caps::builder("audio/x-adpcm") + .field("layout", "div") + .field("block_align", block_size as i32) + .field("channels", 1i32) + .field("rate", 8000i32) + .build(), + )); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast back"))?; + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + let seek_command = client_data.get_commader(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); + }) + .enough_data(move |_appsrc| { + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); + }) + .seek_data(move |_appsrc, seek_pos| { + debug!("Send seek Adpcm"); + let _ = seek_command + .blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); + true + }) + .build(), + ); - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); - queue.set_property("min-threshold-time", 500u64 * 1000u64); - let decoder = make_element("decodebin", "auddecoder")?; - let encoder = make_element("audioconvert", "audencoder")?; - let payload = make_element("rtpL16pay", "pay1")?; + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; - bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &decoder])?; - Element::link_many(&[&encoder, &payload])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link ADPCM decoder to encoder"); - }); + let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.audsrc.replace(source); + bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &decoder])?; + Element::link_many(&[&encoder, &payload])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link ADPCM decoder to encoder"); + }); + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + client_data.update_aud(source); + } } } diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index ac6b6f0..13568df 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -2,363 +2,546 @@ //! gstreamer media stream use anyhow::anyhow; use futures::stream::StreamExt; -use gstreamer::ClockTime; +use gstreamer::{prelude::*, ClockTime}; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use log::*; use neolink_core::bcmedia::model::*; -use std::collections::{hash_map::Entry, HashMap}; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, +use std::collections::{ + VecDeque, + {hash_map::Entry, HashMap}, +}; +use std::{ + iter::Iterator, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; // use tokio::time::{interval, Duration, MissedTickBehavior}; +use tokio::{ + sync::mpsc::{channel, Receiver, Sender}, + time::Duration, +}; use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; -pub(super) struct NeoMediaSender { - pub(super) data_source: ReceiverStream, - pub(super) clientsource: ReceiverStream, - pub(super) shared: Arc, - // Used to generate the key for the hashmaps - pub(super) uid: AtomicU64, - // Hashmap so we can alter the data in ClientData while also allowing for remove at any point - // If you know a better collections pm me - pub(super) clientdata: HashMap, - pub(super) waiting_for_iframe: bool, +#[derive(Default)] +struct NeoBuffer { + buf: VecDeque>, } -#[derive(Clone)] -struct StampedData { - ms: u64, - data: Vec, +impl NeoBuffer { + fn push(&mut self, item: BcMedia) { + match item { + iframe @ BcMedia::Iframe(_) => self.buf.push_back(vec![iframe]), + pframe @ BcMedia::Pframe(_) => { + if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + last.push(pframe); + } + } + aac @ BcMedia::Aac(_) => { + if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + last.push(aac); + } + } + adpcm @ BcMedia::Adpcm(_) => { + if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + last.push(adpcm); + } + } + BcMedia::InfoV1(_) | BcMedia::InfoV2(_) => {} + } + while self.buf.len() > 25 { + // 25 iframes + let _ = self.buf.pop_front(); + } + } + + fn start_time(&self) -> Option { + if let Some(BcMedia::Iframe(data)) = + self.buf.front().and_then(|inner_buf| inner_buf.first()) + { + Some(data.microseconds as u64) + } else { + None + } + } + + fn end_time(&self) -> Option { + if let Some(innerbuffer) = self.buf.back() { + let mut last_ms = None; + for frame in innerbuffer.iter() { + match frame { + BcMedia::Iframe(frame) => { + let v = last_ms.get_or_insert(frame.microseconds as u64); + if *v < frame.microseconds as u64 { + *v = frame.microseconds as u64; + } + } + BcMedia::Pframe(frame) => { + let v = last_ms.get_or_insert(frame.microseconds as u64); + if *v < frame.microseconds as u64 { + *v = frame.microseconds as u64; + } + } + _ => {} + } + } + last_ms + } else { + None + } + } +} + +pub(super) struct NeoMediaSenders { + data_source: ReceiverStream, + client_source: ReceiverStream, + shared: Arc, + uid: AtomicU64, + client_data: HashMap, + buffer: NeoBuffer, +} + +impl NeoMediaSenders { + pub(super) fn new( + shared: Arc, + data_source: ReceiverStream, + client_source: ReceiverStream, + ) -> Self { + Self { + data_source, + client_source, + shared, + uid: Default::default(), + client_data: Default::default(), + buffer: Default::default(), + } + } + + async fn handle_new_client(&mut self, mut client: NeoMediaSender) -> AnyResult<()> { + client.initialise(&self.buffer).await?; + self.client_data + .insert(self.uid.fetch_add(1, Ordering::Relaxed), client); + Ok(()) + } + + async fn update_mediatypes(&self, data: &BcMedia) { + match data { + BcMedia::Iframe(BcMediaIframe { + video_type: VideoType::H264, + .. + }) => { + if !matches!(*self.shared.vid_format.read().await, VidFormats::H264) { + *self.shared.vid_format.write().await = VidFormats::H264; + } + } + BcMedia::Pframe(BcMediaPframe { + video_type: VideoType::H264, + .. + }) => { + if !matches!(*self.shared.vid_format.read().await, VidFormats::H264) { + *self.shared.vid_format.write().await = VidFormats::H264; + } + } + BcMedia::Iframe(BcMediaIframe { + video_type: VideoType::H265, + .. + }) => { + if !matches!(*self.shared.vid_format.read().await, VidFormats::H265) { + *self.shared.vid_format.write().await = VidFormats::H265; + } + } + BcMedia::Pframe(BcMediaPframe { + video_type: VideoType::H265, + .. + }) => { + if !matches!(*self.shared.vid_format.read().await, VidFormats::H265) { + *self.shared.vid_format.write().await = VidFormats::H265; + } + } + BcMedia::Aac(_) => { + if !matches!(*self.shared.aud_format.read().await, AudFormats::Aac) { + *self.shared.aud_format.write().await = AudFormats::Aac; + } + } + BcMedia::Adpcm(data) => { + if !matches!(*self.shared.aud_format.read().await, AudFormats::Adpcm(_)) { + *self.shared.aud_format.write().await = + AudFormats::Adpcm(data.data.len() as u16); + } + } + _ => {} + } + } + + async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> { + // debug!("Handle new data"); + self.update_mediatypes(&data).await; + for key in self.client_data.keys().copied().collect::>() { + // debug!(" - Client: {}", key); + match self.client_data.entry(key) { + Entry::Occupied(mut occ) => { + if let Err(e) = occ.get_mut().initialise(&self.buffer).await { + debug!("Could not init client: {:?}", e); + occ.remove(); + continue; + } + if let Err(e) = occ.get_mut().process_commands(&self.buffer).await { + debug!("Could not process client command: {:?}", e); + occ.remove(); + continue; + } + if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await { + debug!("Could not handle jump to live: {:?}", e); + occ.remove(); + continue; + } + if let Err(e) = occ.get_mut().process_buffer(&self.buffer).await { + debug!("Could not send client data: {:?}", e); + occ.remove(); + continue; + } + } + Entry::Vacant(_) => {} + } + } + self.buffer.push(data); + + Ok(()) + } + + pub(super) async fn run(&mut self) -> AnyResult<()> { + loop { + tokio::select! { + Some(v) = self.data_source.next() => { + self.handle_new_data(v).await + }, + Some(v) = self.client_source.next() => { + self.handle_new_client(v).await + }, + else => { + Err(anyhow!("Sender data source closed")) + } + }?; + } + } +} + +pub(super) enum NeoMediaSenderCommand { + Seek(u64), + Pause, + Resume, +} +#[derive(Debug)] +pub(super) struct NeoMediaSender { + needle: u64, + start_time: u64, + vid: Option, + aud: Option, + command_reciever: Receiver, + command_sender: Sender, + inited: bool, + playing: bool, } impl NeoMediaSender { - pub(super) async fn run(&mut self) -> AnyResult<()> { - // let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32)); - // resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip); - let mut buffer_a: Vec = Default::default(); - let mut buffer_b: Vec = Default::default(); - let buffer_read = &mut buffer_a; - let buffer_write = &mut buffer_b; + pub(super) fn new() -> Self { + let (tx, rx) = channel(3); + Self { + needle: 0, + start_time: 0, + vid: None, + aud: None, + command_reciever: rx, + command_sender: tx, + inited: false, + playing: true, + } + } - let mut audbuffer_a: Vec = Default::default(); - let mut audbuffer_b: Vec = Default::default(); - let audbuffer_read = &mut audbuffer_a; - let audbuffer_write = &mut audbuffer_b; - loop { - self.shared - .number_of_clients - .store(self.clientdata.len(), Ordering::Relaxed); - self.shared - .buffer_ready - .store(!buffer_read.is_empty(), Ordering::Relaxed); - // debug!("Sender: Get"); - tokio::select! { - v = self.data_source.next() => { - // debug!("Sender: Got Data"); - // debug!("data_source recieved"); - if let Some(bc_media) = v { - if ! self.skip_bcmedia(&bc_media) { - // debug!("Not skipped"); - // resend_pause.reset(); - self.inspect_bcmedia(&bc_media).await; - match bc_media { - BcMedia::Iframe(frame) => { - if buffer_write.len() > 25 { - std::mem::swap(buffer_read, buffer_write); - buffer_write.clear(); - std::mem::swap(audbuffer_read, audbuffer_write); - audbuffer_write.clear(); - } - let new_data = StampedData { - ms: frame.microseconds as u64, - data: frame.data, - }; - self.process_vidbuffer(&new_data).await; - buffer_write.push(new_data); - } - BcMedia::Pframe(frame) => { - let new_data = StampedData { - ms: frame.microseconds as u64, - data: frame.data, - }; - self.process_vidbuffer(&new_data).await; - buffer_write.push(new_data); - } - BcMedia::Aac(aac) => { - if let Some(last) = buffer_write.last().as_ref() { - let new_data = StampedData { - ms: last.ms, - data: aac.data, - }; - self.process_audbuffer(&new_data).await; - audbuffer_write.push(new_data); - } + pub(super) fn update_vid(&mut self, source: AppSrc) { + self.vid.replace(source); + } - } - BcMedia::Adpcm(adpcm) => { - if let Some(last) = buffer_write.last().as_ref() { - let new_data = StampedData { - ms: last.ms, - data: adpcm.data, - }; - self.process_audbuffer(&new_data).await; - audbuffer_write.push(new_data); - } + pub(super) fn update_aud(&mut self, source: AppSrc) { + self.aud.replace(source); + } - } - _ => {} - } + pub(super) fn get_commader(&self) -> Sender { + self.command_sender.clone() + } + + async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if self.needle > self.start_time { + self.seek(self.needle - self.start_time, buffer).await + } else { + self.seek(0, buffer).await + } + } + + async fn seek(&mut self, dest: u64, buffer: &NeoBuffer) -> AnyResult<()> { + if let (Some(first_ms), Some(last_ms)) = (buffer.start_time(), buffer.end_time()) { + let target_seek = self.start_time + dest; + if target_seek >= first_ms && target_seek <= last_ms { + debug!("Seeking inside buffer: {}", target_seek); + self.needle = target_seek; + self.process_buffer(buffer).await?; + debug!("Seeked inside buffer"); + } else { + // Fake seek + debug!( + "Seeking outside buffer: {} - {} = {}", + self.needle, + target_seek, + self.needle - dest + ); + self.start_time = self.needle - dest; + } + } else { + // Buffers arent valid yet + debug!("Seek buffers are not ready: {}", dest); + debug!(" - {:?}", (buffer.start_time(), buffer.end_time())); + } + + Ok(()) + } + + async fn process_commands(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if self.inited { + if let Ok(command) = self.command_reciever.try_recv() { + match command { + NeoMediaSenderCommand::Seek(dest) => { + debug!("Got Seek Request: {}", dest); + self.seek(dest, buffer).await?; + } + NeoMediaSenderCommand::Pause => { + if self.playing { + debug!("Pausing"); + self.playing = false; + } + } + NeoMediaSenderCommand::Resume => { + if !self.playing { + debug!("Resuming"); + self.playing = true; + self.jump_to_live(buffer).await?; } - } else { - break; } } - v = self.clientsource.next() => { - // debug!("Sender: Got Client"); - if let Some(mut clientdata) = v { - // Resend the video and audio buffers - if !clientdata.inited { - clientdata.inited = true; - if let Some(last) = buffer_read.first() { - debug!("Start PTS SET: {:?}", tokio::time::Duration::from_micros(last.ms)); - clientdata.set_start_time(last.ms); - } - let mut vid_resend = buffer_read.iter().chain(buffer_write.iter()); - let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter()); - loop { - let next_vid = vid_resend.next(); - if let Some(next_vid) = next_vid.as_ref() { - if let Some(src) = clientdata.vidsrc.as_ref() { - if Self::send_buffer( - src, - next_vid.data.as_slice(), - next_vid.ms, - clientdata.get_start_time(), - ).await.is_err() { - break; - }; - } - } - let next_aud = aud_resend.next(); - if let Some(next_aud) = next_aud.as_ref() { - if next_aud.ms >= clientdata.get_start_time() { - if let Some(src) = clientdata.audsrc.as_ref() { - if Self::send_buffer( - src, - next_aud.data.as_slice(), - next_aud.ms, - clientdata.get_start_time(), - ).await.is_err() { - break; - }; - } - } - } - if next_vid.is_none() && next_aud.is_none() { - break; - } - } - debug!("Advance PTS Pushed"); - } - // Save the client for future sends - self.clientdata.insert(self.uid.fetch_add(1, Ordering::Relaxed) , clientdata); - } else { - break; - } - }, - // _ = resend_pause.tick() => { - // if let Some(live) = buffer.last().cloned() { - // self.process_vidbuffer(&ProcessData { - // live, - // resend: buffer.as_slice() - // })?; - // } - // } } } Ok(()) } - async fn process_vidbuffer(&mut self, stamped_data: &StampedData) { - for key in self - .clientdata - .keys() - .cloned() - .collect::>() - .drain(..) - { - match self.clientdata.entry(key) { - Entry::Occupied(data) => { - if let Some(vidsrc) = data.get().vidsrc.as_ref() { - if !data.get().enough_data.load(Ordering::Relaxed) - && Self::send_buffer( - vidsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.get().get_start_time(), - ) - .await - .is_err() - { - data.remove(); + async fn initialise(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if !self.inited { + if let (Some(first_ms), Some(end_ms)) = (buffer.start_time(), buffer.end_time()) { + // if let Some(source) = self.vid.as_ref() { + // source.set_base_time(ClockTime::from_mseconds(0)); + // } + // if let Some(source) = self.aud.as_ref() { + // source.set_base_time(ClockTime::from_mseconds(0)); + // } + + self.start_time = (first_ms + end_ms) / 2; + self.needle = self.start_time; + + self.inited = true; + + self.process_buffer(buffer).await?; + } + } + Ok(()) + } + + async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { + if self.inited && self.playing { + let mut buf_it = buf.buf.iter().peekable(); + while let Some(buffer) = buf_it.next() { + tokio::task::yield_now().await; + let next = buf_it.peek(); + // Look at the next frame set and see if it is after the needle (or if empty) + let deep = next + .map(|buffer| { + buffer + .first() + .map(|frame| { + if let BcMedia::Iframe(data) = frame { + data.microseconds as u64 >= self.needle + } else { + unreachable!() + } + }) + .unwrap_or(true) // No first data + }) + .unwrap_or(true); // Next buffer is empty + // This frame set contains our data process it + if deep { + let mut send_it = false; + for frame in buffer.iter() { + if !send_it { + let microseconds = match frame { + BcMedia::Iframe(data) => Some(data.microseconds), + BcMedia::Pframe(data) => Some(data.microseconds), + _ => None, + }; + if let Some(microseconds) = microseconds { + if microseconds as u64 > self.needle { + send_it = true; + } + } } - } else { - data.remove(); - } - } - Entry::Vacant(_) => {} - } - } - } - async fn process_audbuffer(&mut self, stamped_data: &StampedData) { - for key in self - .clientdata - .keys() - .cloned() - .collect::>() - .drain(..) - { - match self.clientdata.entry(key) { - Entry::Occupied(data) => { - if let Some(audsrc) = data.get().audsrc.as_ref() { - if !data.get().enough_data.load(Ordering::Relaxed) - && Self::send_buffer( - audsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.get().get_start_time(), - ) - .await - .is_err() - { - data.remove(); + if send_it && !self.send_buffer(frame).await? { + // Buffer too far is ahead of needle early exit + break; } + } + } + } + } + + Ok(()) + } + + async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if let Some(first_buftime) = buffer.start_time() { + if first_buftime > self.needle { + debug!("Jump to live first_buftime > self.needle"); + self.jump_to_live(buffer).await?; + } else if let Some(runtime) = self.get_runtime() { + if first_buftime > self.start_time { + let buftime = Duration::from_micros(first_buftime - self.start_time); + if buftime > runtime { + debug!("Jump to live buftime > runtime"); + self.jump_to_live(buffer).await?; + } + } else { + debug!("Jump to live first_buftime > self.start_time "); + self.jump_to_live(buffer).await?; + } + } + } + Ok(()) + } + + fn get_runtime(&self) -> Option { + if let Some(appsrc) = self.vid.as_ref() { + if let Some(clock) = appsrc.clock() { + if let Some(time) = clock.time() { + if let Some(base_time) = appsrc.base_time() { + let runtime = time - base_time; + return Some(Duration::from_nanos(runtime.nseconds())); + } + } + } + } + None + } + + async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { + if self.inited && self.playing { + let microseconds = match media { + BcMedia::Iframe(data) => { + if data.microseconds as u64 > self.start_time { + data.microseconds as u64 - self.start_time } else { - data.remove(); + 0 } } - Entry::Vacant(_) => {} - } - } - } - - async fn send_buffer( - appsrc: &AppSrc, - buf: &[u8], - frame_ms: u64, - start_time: u64, - ) -> AnyResult<()> { - let micros = if frame_ms >= start_time { - frame_ms - start_time - } else { - 0 - }; - debug!("PTS: {:?}", tokio::time::Duration::from_micros(micros)); - - let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); - { - let gst_buf_mut = gst_buf.get_mut().unwrap(); - - let time = ClockTime::from_useconds(micros); - gst_buf_mut.set_pts(time); - // debug!( - // "PTS set to: {:?} ({}-{}={:?})", - // time, frame_ms, start_time, micros - // ); - let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); - gst_buf_data.copy_from_slice(buf); - } - // debug!("Buffer pushed"); - let thread_appsrc = appsrc.clone(); // GObjects are refcounted - let res = tokio::task::spawn_blocking(move || { - thread_appsrc - .push_buffer(gst_buf.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) - }) - .await; - match &res { - Err(e) => { - debug!("Failed to send buffer: {:?}", e); - } - Ok(Err(e)) => { - debug!("Failed to send buffer: {:?}", e); - } - Ok(Ok(_)) => {} - }; - res? - } - - fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> bool { - if self.waiting_for_iframe { - if let BcMedia::Iframe(_) = bc_media { - self.waiting_for_iframe = false; - } else { - log::debug!("Skipping bcmedia"); - return true; - } - } - false - } - - async fn inspect_bcmedia(&mut self, bc_media: &BcMedia) { - let old_vid = *self.shared.vid_format.read().await; - let old_aud = *self.shared.aud_format.read().await; - match bc_media { - BcMedia::Iframe(frame) => { - match frame.video_type { - VideoType::H264 => { - (*self.shared.vid_format.write().await) = VidFormats::H264; - } - VideoType::H265 => { - (*self.shared.vid_format.write().await) = VidFormats::H265; + BcMedia::Pframe(data) => { + if data.microseconds as u64 > self.start_time { + data.microseconds as u64 - self.start_time + } else { + 0 } } - self.shared - .microseconds - .store(frame.microseconds as u64, Ordering::Relaxed); - // log::debug!( - // "Time set to {}", - // self.shared.microseconds.load(Ordering::Relaxed) - // ); - } - BcMedia::Pframe(frame) => { - match frame.video_type { - VideoType::H264 => { - (*self.shared.vid_format.write().await) = VidFormats::H264; - } - VideoType::H265 => { - (*self.shared.vid_format.write().await) = VidFormats::H265; + _ => { + if self.needle > self.start_time { + self.needle - self.start_time + } else { + 0 } } - self.shared - .microseconds - .store(frame.microseconds as u64, Ordering::Relaxed); - // log::debug!( - // "Time set to {}", - // self.shared.microseconds.load(Ordering::Relaxed) - // ); + }; + let buf = match media { + BcMedia::Iframe(data) => Some(&data.data), + BcMedia::Pframe(data) => Some(&data.data), + BcMedia::Aac(data) => Some(&data.data), + BcMedia::Adpcm(data) => Some(&data.data), + _ => None, + }; + let appsrc = match media { + BcMedia::Iframe(_) | BcMedia::Pframe(_) => self.vid.as_ref(), + BcMedia::Aac(_) | BcMedia::Adpcm(_) => self.aud.as_ref(), + _ => None, + }; + + if let (Some(buf), Some(appsrc)) = (buf, appsrc) { + if let Some(runtime) = self.get_runtime() { + let buftimr = Duration::from_micros(microseconds); + if buftimr > runtime + Duration::from_micros(2000) { + // Stop advancing the buffer + return Ok(false); + } else { + debug!(" - runtime: {:?}", runtime); + debug!(" - buftime: {:?}", buftimr); + } + } + + debug!("PTS: {:?}", Duration::from_micros(microseconds)); + + let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); + + let time = ClockTime::from_useconds(microseconds); + gst_buf_mut.set_pts(time); + let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(buf); + } + // debug!("Buffer pushed"); + let thread_appsrc = appsrc.clone(); // GObjects are refcounted + let res = tokio::task::spawn_blocking(move || { + thread_appsrc + .push_buffer(gst_buf.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await; + match &res { + Err(e) => { + debug!("Failed to send buffer: {:?}", e); + } + Ok(Err(e)) => { + debug!("Failed to send buffer: {:?}", e); + } + Ok(Ok(_)) => {} + }; + res??; + // Buffer sent sucessfully + // + // Update the needle position + match media { + BcMedia::Iframe(data) => { + if data.microseconds as u64 > self.needle { + // debug!("Updating needle to: {}", data.microseconds); + self.needle = data.microseconds as u64; + } + } + BcMedia::Pframe(data) => { + if data.microseconds as u64 > self.needle { + // debug!("Updating needle to: {}", data.microseconds); + self.needle = data.microseconds as u64; + } + } + _ => {} + } } - BcMedia::Aac(_aac) => { - (*self.shared.aud_format.write().await) = AudFormats::Aac; - } - BcMedia::Adpcm(adpcm) => { - (*self.shared.aud_format.write().await) = AudFormats::Adpcm(adpcm.data.len() as u16) - } - _ => {} - } - let new_vid = *self.shared.vid_format.read().await; - if new_vid != old_vid { - log::debug!("Video format set to: {:?} from {:?}", new_vid, old_vid); - } - let new_aud = *self.shared.aud_format.read().await; - if old_aud != new_aud { - log::debug!("Audio format set to: {:?} from {:?}", new_aud, old_aud); } + Ok(true) } } diff --git a/src/rtsp/gst/shared.rs b/src/rtsp/gst/shared.rs index e3a2e24..685695f 100644 --- a/src/rtsp/gst/shared.rs +++ b/src/rtsp/gst/shared.rs @@ -1,9 +1,7 @@ //! Data shared between the various //! components that manage a media stream -use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize}; use tokio::sync::RwLock; #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -23,7 +21,6 @@ pub(super) enum AudFormats { pub(super) struct NeoMediaShared { pub(super) vid_format: RwLock, pub(super) aud_format: RwLock, - pub(super) microseconds: AtomicU64, pub(super) number_of_clients: AtomicUsize, pub(super) buffer_ready: AtomicBool, } @@ -33,27 +30,8 @@ impl Default for NeoMediaShared { Self { vid_format: RwLock::new(VidFormats::Unknown), aud_format: RwLock::new(AudFormats::Unknown), - microseconds: AtomicU64::new(0), number_of_clients: AtomicUsize::new(0), buffer_ready: AtomicBool::new(false), } } } - -#[derive(Default, Debug)] -pub(super) struct ClientPipelineData { - pub(super) vidsrc: Option, - pub(super) audsrc: Option, - pub(super) start_time: Arc, - pub(super) inited: bool, - pub(super) enough_data: Arc, -} - -impl ClientPipelineData { - pub(super) fn get_start_time(&self) -> u64 { - self.start_time.load(Ordering::Relaxed) - } - pub(super) fn set_start_time(&self, time: u64) { - self.start_time.store(time, Ordering::Relaxed) - } -} From c30e6293f68d1b314766e46f324dcc38fb8f209e Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Fri, 28 Apr 2023 19:59:06 +0700 Subject: [PATCH 03/64] Rework how buffers are sent to always be --- src/rtsp/gst/sender.rs | 247 ++++++++++++++++------------------------- 1 file changed, 98 insertions(+), 149 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 13568df..208ed60 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -27,6 +27,8 @@ use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; +const LATENCY: Duration = Duration::from_micros(200); + #[derive(Default)] struct NeoBuffer { buf: VecDeque>, @@ -59,31 +61,35 @@ impl NeoBuffer { } } - fn start_time(&self) -> Option { + fn start_time(&self) -> Option { if let Some(BcMedia::Iframe(data)) = self.buf.front().and_then(|inner_buf| inner_buf.first()) { - Some(data.microseconds as u64) + Some(Duration::from_micros(data.microseconds)) } else { None } } - fn end_time(&self) -> Option { + fn end_time(&self) -> Option { if let Some(innerbuffer) = self.buf.back() { let mut last_ms = None; for frame in innerbuffer.iter() { match frame { BcMedia::Iframe(frame) => { - let v = last_ms.get_or_insert(frame.microseconds as u64); - if *v < frame.microseconds as u64 { - *v = frame.microseconds as u64; + let frame_ms = Duration::from_micros(frame.microseconds); + let v = last_ms.get_or_insert(frame_ms); + + if *v < frame.microseconds { + *v = frame.microseconds; } } BcMedia::Pframe(frame) => { - let v = last_ms.get_or_insert(frame.microseconds as u64); - if *v < frame.microseconds as u64 { - *v = frame.microseconds as u64; + let frame_ms = Duration::from_micros(frame.microseconds); + let v = last_ms.get_or_insert(frame_ms); + + if *v < frame_ms { + *v = frame_ms; } } _ => {} @@ -237,8 +243,8 @@ pub(super) enum NeoMediaSenderCommand { } #[derive(Debug)] pub(super) struct NeoMediaSender { - needle: u64, - start_time: u64, + start_time: Duration, + last_sent_time: Durataion, vid: Option, aud: Option, command_reciever: Receiver, @@ -251,8 +257,8 @@ impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(3); Self { - needle: 0, - start_time: 0, + start_time: Duration::ZERO, + last_sent_time: Duration:ZERO, vid: None, aud: None, command_reciever: rx, @@ -275,47 +281,30 @@ impl NeoMediaSender { } async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - if self.needle > self.start_time { - self.seek(self.needle - self.start_time, buffer).await - } else { - self.seek(0, buffer).await + if self.inited { + if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = (self.get_runtime(), buffer.start_time(), buffer.end_time()) { + // Jump to the mid point + let target_time = (buffer_start + buffer_end) / 2; + self.start_time = target_time - runtime; + self.last_sent_time = target_time; + } } } - - async fn seek(&mut self, dest: u64, buffer: &NeoBuffer) -> AnyResult<()> { - if let (Some(first_ms), Some(last_ms)) = (buffer.start_time(), buffer.end_time()) { - let target_seek = self.start_time + dest; - if target_seek >= first_ms && target_seek <= last_ms { - debug!("Seeking inside buffer: {}", target_seek); - self.needle = target_seek; - self.process_buffer(buffer).await?; - debug!("Seeked inside buffer"); - } else { - // Fake seek - debug!( - "Seeking outside buffer: {} - {} = {}", - self.needle, - target_seek, - self.needle - dest - ); - self.start_time = self.needle - dest; - } - } else { - // Buffers arent valid yet - debug!("Seek buffers are not ready: {}", dest); - debug!(" - {:?}", (buffer.start_time(), buffer.end_time())); - } - + + async fn seek(&mut self, target_time: Duration) -> AnyResult<()> { + self.last_sent_time = runtime_to_buftime(target_time); Ok(()) } + + async fn process_commands(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { if let Ok(command) = self.command_reciever.try_recv() { match command { NeoMediaSenderCommand::Seek(dest) => { debug!("Got Seek Request: {}", dest); - self.seek(dest, buffer).await?; + self.seek(Duration::from_micros(dest)).await?; } NeoMediaSenderCommand::Pause => { if self.playing { @@ -347,7 +336,6 @@ impl NeoMediaSender { // } self.start_time = (first_ms + end_ms) / 2; - self.needle = self.start_time; self.inited = true; @@ -356,72 +344,78 @@ impl NeoMediaSender { } Ok(()) } + + fn runtime_to_buftime(&self, runtime: Duration) -> Duration { + runtime + self.start_time + } + + fn buftime_to_runtime(&self, buftime: Duration) -> Duration { + if buftime > self.start_time { + buftime - self.start_time + } else { + 0 + } + } + + async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { if self.inited && self.playing { - let mut buf_it = buf.buf.iter().peekable(); - while let Some(buffer) = buf_it.next() { - tokio::task::yield_now().await; - let next = buf_it.peek(); - // Look at the next frame set and see if it is after the needle (or if empty) - let deep = next - .map(|buffer| { - buffer - .first() - .map(|frame| { - if let BcMedia::Iframe(data) = frame { - data.microseconds as u64 >= self.needle - } else { - unreachable!() - } - }) - .unwrap_or(true) // No first data - }) - .unwrap_or(true); // Next buffer is empty - // This frame set contains our data process it - if deep { - let mut send_it = false; - for frame in buffer.iter() { - if !send_it { - let microseconds = match frame { - BcMedia::Iframe(data) => Some(data.microseconds), - BcMedia::Pframe(data) => Some(data.microseconds), - _ => None, - }; - if let Some(microseconds) = microseconds { - if microseconds as u64 > self.needle { - send_it = true; - } + let runtime = self.get_runtime(); + if let Some(runtime) = runtime { + // We are live only send the buffer up to the runtime + let min_time = self.last_sent_time; + let max_time = self.runtime_to_buftime(runtime) + LATENCY; + + let mut found_start = false; + let mut buf_it = buf.buf.iter().peekable(); + while let Some(frames) = buf_it.next() { + if !found_start { + let next_frames = buf_it.peek(); + if let Some(BcMedia::Iframe(frame)) = next_frames.first() { + // Get time of next IFrame + // if it is after min_time, then the + // start shuld happen between frames and next_frames + if Duration::from_micros(frame.microseconds) > min_time { + found_start = true; } } - - if send_it && !self.send_buffer(frame).await? { - // Buffer too far is ahead of needle early exit - break; + } + + if found_start { + // We have found the start send eveythin until we get passed the + // max time + for frame in frames { + let frame_time = match frame { + BcMedia::Iframe(data) => Duration::from_micros(data.microseconds), + BcMedia::Pframe(data) => Duration::from_micros(data.microseconds), + _ => self.last_sent_time, + }; + if frame_time > min_time && frame_time <= max_time { + self.send_buffer(frame).await?; + self.last_sent_time = frame_time; + } else if frame_time > max_time { + return Ok(()); + } } } } + + } else { + // Not live. Send the WHOLE buffer for analysis + for frame in buf.iter() { + self.send_buffer(frame).await?; + } } - } Ok(()) } async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - if let Some(first_buftime) = buffer.start_time() { - if first_buftime > self.needle { - debug!("Jump to live first_buftime > self.needle"); - self.jump_to_live(buffer).await?; - } else if let Some(runtime) = self.get_runtime() { - if first_buftime > self.start_time { - let buftime = Duration::from_micros(first_buftime - self.start_time); - if buftime > runtime { - debug!("Jump to live buftime > runtime"); - self.jump_to_live(buffer).await?; - } - } else { - debug!("Jump to live first_buftime > self.start_time "); - self.jump_to_live(buffer).await?; + if self.inited { + if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = (self.get_runtime(), buffer.start_time(), buffer.end_time()) { + if runtime < (buffer_start - LATENCY) || runtime > (buffer_end + LATENCY) { + self.jump_to_live(buffer)?; } } } @@ -444,29 +438,13 @@ impl NeoMediaSender { async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { if self.inited && self.playing { - let microseconds = match media { - BcMedia::Iframe(data) => { - if data.microseconds as u64 > self.start_time { - data.microseconds as u64 - self.start_time - } else { - 0 - } - } - BcMedia::Pframe(data) => { - if data.microseconds as u64 > self.start_time { - data.microseconds as u64 - self.start_time - } else { - 0 - } - } - _ => { - if self.needle > self.start_time { - self.needle - self.start_time - } else { - 0 - } - } + let buftime = match media { + BcMedia::Iframe(data) => Duration::from_micros(data.microseconds), + BcMedia::Pframe(data) => Duration::from_micros(data.microseconds), + _ => self.last_sent_time, }; + let runtime = buftime_to_runtime(buftime); + let buf = match media { BcMedia::Iframe(data) => Some(&data.data), BcMedia::Pframe(data) => Some(&data.data), @@ -481,24 +459,13 @@ impl NeoMediaSender { }; if let (Some(buf), Some(appsrc)) = (buf, appsrc) { - if let Some(runtime) = self.get_runtime() { - let buftimr = Duration::from_micros(microseconds); - if buftimr > runtime + Duration::from_micros(2000) { - // Stop advancing the buffer - return Ok(false); - } else { - debug!(" - runtime: {:?}", runtime); - debug!(" - buftime: {:?}", buftimr); - } - } - - debug!("PTS: {:?}", Duration::from_micros(microseconds)); + debug!("PTS: {:?}", runtime); let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { let gst_buf_mut = gst_buf.get_mut().unwrap(); - let time = ClockTime::from_useconds(microseconds); + let time = ClockTime::from_useconds(runtime.as_microseconds()); gst_buf_mut.set_pts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf); @@ -522,24 +489,6 @@ impl NeoMediaSender { Ok(Ok(_)) => {} }; res??; - // Buffer sent sucessfully - // - // Update the needle position - match media { - BcMedia::Iframe(data) => { - if data.microseconds as u64 > self.needle { - // debug!("Updating needle to: {}", data.microseconds); - self.needle = data.microseconds as u64; - } - } - BcMedia::Pframe(data) => { - if data.microseconds as u64 > self.needle { - // debug!("Updating needle to: {}", data.microseconds); - self.needle = data.microseconds as u64; - } - } - _ => {} - } } } Ok(true) From 093a1b1af9c8a6302a8dcd79e6d8bcd84f40d83c Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Fri, 28 Apr 2023 21:15:22 +0700 Subject: [PATCH 04/64] Close brace --- src/rtsp/gst/sender.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 208ed60..c17853c 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -407,6 +407,7 @@ impl NeoMediaSender { self.send_buffer(frame).await?; } } + } Ok(()) } From a222a2d053e6395900858a57a40c8883762346d6 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 30 Apr 2023 09:31:03 +0700 Subject: [PATCH 05/64] New live buffer method --- src/rtsp/gst/sender.rs | 104 ++++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index c17853c..95a50be 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -55,17 +55,21 @@ impl NeoBuffer { } BcMedia::InfoV1(_) | BcMedia::InfoV2(_) => {} } - while self.buf.len() > 25 { - // 25 iframes + while self.buf.len() > 10 { + // Remove excess let _ = self.buf.pop_front(); } } + fn iter(&self) -> impl Iterator { + self.buf.iter().flatten() + } + fn start_time(&self) -> Option { if let Some(BcMedia::Iframe(data)) = self.buf.front().and_then(|inner_buf| inner_buf.first()) { - Some(Duration::from_micros(data.microseconds)) + Some(Duration::from_micros(data.microseconds as u64)) } else { None } @@ -77,17 +81,17 @@ impl NeoBuffer { for frame in innerbuffer.iter() { match frame { BcMedia::Iframe(frame) => { - let frame_ms = Duration::from_micros(frame.microseconds); + let frame_ms = Duration::from_micros(frame.microseconds as u64); let v = last_ms.get_or_insert(frame_ms); - - if *v < frame.microseconds { - *v = frame.microseconds; + + if *v < frame_ms { + *v = frame_ms; } } BcMedia::Pframe(frame) => { - let frame_ms = Duration::from_micros(frame.microseconds); + let frame_ms = Duration::from_micros(frame.microseconds as u64); let v = last_ms.get_or_insert(frame_ms); - + if *v < frame_ms { *v = frame_ms; } @@ -244,7 +248,7 @@ pub(super) enum NeoMediaSenderCommand { #[derive(Debug)] pub(super) struct NeoMediaSender { start_time: Duration, - last_sent_time: Durataion, + last_sent_time: Duration, vid: Option, aud: Option, command_reciever: Receiver, @@ -258,7 +262,7 @@ impl NeoMediaSender { let (tx, rx) = channel(3); Self { start_time: Duration::ZERO, - last_sent_time: Duration:ZERO, + last_sent_time: Duration::ZERO, vid: None, aud: None, command_reciever: rx, @@ -282,21 +286,27 @@ impl NeoMediaSender { async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { - if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = (self.get_runtime(), buffer.start_time(), buffer.end_time()) { + if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = + (self.get_runtime(), buffer.start_time(), buffer.end_time()) + { // Jump to the mid point let target_time = (buffer_start + buffer_end) / 2; self.start_time = target_time - runtime; self.last_sent_time = target_time; } } - } - - async fn seek(&mut self, target_time: Duration) -> AnyResult<()> { - self.last_sent_time = runtime_to_buftime(target_time); + Ok(()) } - + async fn seek(&mut self, target_time: Duration) -> AnyResult<()> { + self.last_sent_time = self.runtime_to_buftime(target_time); + debug!( + "Seeked last_sent_time to {:?} ({:?})", + self.last_sent_time, target_time + ); + Ok(()) + } async fn process_commands(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { @@ -344,20 +354,18 @@ impl NeoMediaSender { } Ok(()) } - + fn runtime_to_buftime(&self, runtime: Duration) -> Duration { runtime + self.start_time } - + fn buftime_to_runtime(&self, buftime: Duration) -> Duration { if buftime > self.start_time { buftime - self.start_time } else { - 0 + Duration::ZERO } } - - async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { if self.inited && self.playing { @@ -366,29 +374,33 @@ impl NeoMediaSender { // We are live only send the buffer up to the runtime let min_time = self.last_sent_time; let max_time = self.runtime_to_buftime(runtime) + LATENCY; - + let mut found_start = false; let mut buf_it = buf.buf.iter().peekable(); while let Some(frames) = buf_it.next() { if !found_start { let next_frames = buf_it.peek(); - if let Some(BcMedia::Iframe(frame)) = next_frames.first() { + if let Some(BcMedia::Iframe(frame)) = next_frames.and_then(|b| b.first()) { // Get time of next IFrame // if it is after min_time, then the // start shuld happen between frames and next_frames - if Duration::from_micros(frame.microseconds) > min_time { + if Duration::from_micros(frame.microseconds as u64) > min_time { found_start = true; } } } - + if found_start { // We have found the start send eveythin until we get passed the // max time for frame in frames { let frame_time = match frame { - BcMedia::Iframe(data) => Duration::from_micros(data.microseconds), - BcMedia::Pframe(data) => Duration::from_micros(data.microseconds), + BcMedia::Iframe(data) => { + Duration::from_micros(data.microseconds as u64) + } + BcMedia::Pframe(data) => { + Duration::from_micros(data.microseconds as u64) + } _ => self.last_sent_time, }; if frame_time > min_time && frame_time <= max_time { @@ -400,7 +412,6 @@ impl NeoMediaSender { } } } - } else { // Not live. Send the WHOLE buffer for analysis for frame in buf.iter() { @@ -414,9 +425,12 @@ impl NeoMediaSender { async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { - if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = (self.get_runtime(), buffer.start_time(), buffer.end_time()) { + if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = + (self.get_buftime(), buffer.start_time(), buffer.end_time()) + { if runtime < (buffer_start - LATENCY) || runtime > (buffer_end + LATENCY) { - self.jump_to_live(buffer)?; + debug!("Outside buffer jumping to live"); + self.jump_to_live(buffer).await?; } } } @@ -437,15 +451,31 @@ impl NeoMediaSender { None } + fn get_buftime(&self) -> Option { + if let Some(appsrc) = self.vid.as_ref() { + if let Some(clock) = appsrc.clock() { + if let Some(time) = clock.time() { + if let Some(base_time) = appsrc.base_time() { + let runtime = time - base_time; + return Some( + self.runtime_to_buftime(Duration::from_nanos(runtime.nseconds())), + ); + } + } + } + } + None + } + async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { if self.inited && self.playing { let buftime = match media { - BcMedia::Iframe(data) => Duration::from_micros(data.microseconds), - BcMedia::Pframe(data) => Duration::from_micros(data.microseconds), + BcMedia::Iframe(data) => Duration::from_micros(data.microseconds as u64), + BcMedia::Pframe(data) => Duration::from_micros(data.microseconds as u64), _ => self.last_sent_time, }; - let runtime = buftime_to_runtime(buftime); - + let runtime = self.buftime_to_runtime(buftime); + let buf = match media { BcMedia::Iframe(data) => Some(&data.data), BcMedia::Pframe(data) => Some(&data.data), @@ -460,13 +490,13 @@ impl NeoMediaSender { }; if let (Some(buf), Some(appsrc)) = (buf, appsrc) { - debug!("PTS: {:?}", runtime); + debug!("PTS: {:?}, Expected: {:?}", runtime, self.get_runtime()); let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { let gst_buf_mut = gst_buf.get_mut().unwrap(); - let time = ClockTime::from_useconds(runtime.as_microseconds()); + let time = ClockTime::from_useconds(runtime.as_micros() as u64); gst_buf_mut.set_pts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf); From 8cea9e79e58d59a294c738b981abb3f562f1bcb1 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 30 Apr 2023 13:56:03 +0700 Subject: [PATCH 06/64] Use time to trim the buffer --- src/rtsp/gst/sender.rs | 181 +++++++++++++++++++++++++++++------------ 1 file changed, 127 insertions(+), 54 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 95a50be..4bbee00 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -27,7 +27,8 @@ use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; -const LATENCY: Duration = Duration::from_micros(200); +const LATENCY: Duration = Duration::from_millis(200); +const BUFFER_TIME: Duration = Duration::from_secs(10); #[derive(Default)] struct NeoBuffer { @@ -36,31 +37,78 @@ struct NeoBuffer { impl NeoBuffer { fn push(&mut self, item: BcMedia) { + let mut was_iframe = false; match item { - iframe @ BcMedia::Iframe(_) => self.buf.push_back(vec![iframe]), + BcMedia::Iframe(data) => { + debug!( + "Pushed iFrame to buffer at {:?}", + Duration::from_micros(data.microseconds as u64) + ); + was_iframe = true; + self.buf.push_back(vec![BcMedia::Iframe(data)]) + } pframe @ BcMedia::Pframe(_) => { - if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + if let Some(last) = self.buf.back_mut().as_mut() { last.push(pframe); } } aac @ BcMedia::Aac(_) => { - if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + if let Some(last) = self.buf.back_mut().as_mut() { last.push(aac); } } adpcm @ BcMedia::Adpcm(_) => { - if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() { + if let Some(last) = self.buf.back_mut().as_mut() { last.push(adpcm); } } BcMedia::InfoV1(_) | BcMedia::InfoV2(_) => {} } - while self.buf.len() > 10 { - // Remove excess - let _ = self.buf.pop_front(); + if was_iframe { + if let Some(last_frame_time) = self.last_iframe_time() { + let mut time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff); + while let Some(time) = time_delta { + if time > BUFFER_TIME && self.buf.len() > 2 { + let _ = self.buf.pop_front(); + debug!("Popping frame with time difference of {:?}", time); + time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff); + } else { + debug!("Iframes left in the buffer: {}", self.buf.len()); + break; + } + } + } } } + fn last_iframe_time(&self) -> Option { + self.buf + .back() + .as_ref() + .and_then(|b| b.first()) + .and_then(|f| { + if let BcMedia::Iframe(frame) = f { + Some(Duration::from_micros(frame.microseconds as u64)) + } else { + None + } + }) + } + + fn first_iframe_time(&self) -> Option { + self.buf + .front() + .as_ref() + .and_then(|b| b.first()) + .and_then(|f| { + if let BcMedia::Iframe(frame) = f { + Some(Duration::from_micros(frame.microseconds as u64)) + } else { + None + } + }) + } + fn iter(&self) -> impl Iterator { self.buf.iter().flatten() } @@ -77,29 +125,14 @@ impl NeoBuffer { fn end_time(&self) -> Option { if let Some(innerbuffer) = self.buf.back() { - let mut last_ms = None; - for frame in innerbuffer.iter() { - match frame { - BcMedia::Iframe(frame) => { - let frame_ms = Duration::from_micros(frame.microseconds as u64); - let v = last_ms.get_or_insert(frame_ms); - - if *v < frame_ms { - *v = frame_ms; - } - } - BcMedia::Pframe(frame) => { - let frame_ms = Duration::from_micros(frame.microseconds as u64); - let v = last_ms.get_or_insert(frame_ms); - - if *v < frame_ms { - *v = frame_ms; - } - } - _ => {} - } - } - last_ms + innerbuffer + .iter() + .flat_map(|frame| match frame { + BcMedia::Iframe(data) => Some(Duration::from_micros(data.microseconds as u64)), + BcMedia::Pframe(data) => Some(Duration::from_micros(data.microseconds as u64)), + _ => None, + }) + .max() } else { None } @@ -190,7 +223,32 @@ impl NeoMediaSenders { async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> { // debug!("Handle new data"); self.update_mediatypes(&data).await; + self.handle_buffer().await?; + match &data { + BcMedia::Iframe(data) => { + let frame_ms = Duration::from_micros(data.microseconds as u64); + debug!("New IFrame for buffer @ {:?}", frame_ms); + for client in self.client_data.values() { + debug!(" - {:?}", client.buftime_to_runtime(frame_ms)); + } + } + BcMedia::Pframe(data) => { + let frame_ms = Duration::from_micros(data.microseconds as u64); + debug!("New PFrame for buffer @ {:?}", frame_ms); + for client in self.client_data.values() { + debug!(" - {:?}", client.buftime_to_runtime(frame_ms)); + } + } + _ => {} + } + self.buffer.push(data); + + Ok(()) + } + + async fn handle_buffer(&mut self) -> AnyResult<()> { for key in self.client_data.keys().copied().collect::>() { + tokio::task::yield_now().await; // debug!(" - Client: {}", key); match self.client_data.entry(key) { Entry::Occupied(mut occ) => { @@ -218,14 +276,17 @@ impl NeoMediaSenders { Entry::Vacant(_) => {} } } - self.buffer.push(data); - Ok(()) } pub(super) async fn run(&mut self) -> AnyResult<()> { + let mut interval = tokio::time::interval(LATENCY / 3); loop { + tokio::task::yield_now().await; tokio::select! { + _ = interval.tick() => { + self.handle_buffer().await + }, Some(v) = self.data_source.next() => { self.handle_new_data(v).await }, @@ -286,13 +347,25 @@ impl NeoMediaSender { async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { - if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = - (self.get_runtime(), buffer.start_time(), buffer.end_time()) - { - // Jump to the mid point - let target_time = (buffer_start + buffer_end) / 2; - self.start_time = target_time - runtime; - self.last_sent_time = target_time; + if let Some(runtime) = self.get_runtime() { + let (fronts, backs) = buffer.buf.as_slices(); + let frames = fronts.iter().chain(backs.iter()).collect::>(); + let idx = frames.len().saturating_div(2); + let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| { + if let BcMedia::Iframe(data) = f { + Some(Duration::from_micros(data.microseconds as u64)) + } else { + None + } + }); + if let Some(target_time) = target_time { + self.start_time = target_time - runtime; + self.last_sent_time = target_time; + debug!( + "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", + target_time, self.start_time, target_time - self.start_time, runtime + ); + } } } @@ -338,18 +411,14 @@ impl NeoMediaSender { async fn initialise(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if !self.inited { if let (Some(first_ms), Some(end_ms)) = (buffer.start_time(), buffer.end_time()) { - // if let Some(source) = self.vid.as_ref() { - // source.set_base_time(ClockTime::from_mseconds(0)); - // } - // if let Some(source) = self.aud.as_ref() { - // source.set_base_time(ClockTime::from_mseconds(0)); - // } + if end_ms - first_ms > LATENCY * 5 { + // Minimum buffer + self.inited = true; - self.start_time = (first_ms + end_ms) / 2; + self.jump_to_live(buffer).await?; - self.inited = true; - - self.process_buffer(buffer).await?; + self.process_buffer(buffer).await?; + } } } Ok(()) @@ -378,6 +447,7 @@ impl NeoMediaSender { let mut found_start = false; let mut buf_it = buf.buf.iter().peekable(); while let Some(frames) = buf_it.next() { + tokio::task::yield_now().await; if !found_start { let next_frames = buf_it.peek(); if let Some(BcMedia::Iframe(frame)) = next_frames.and_then(|b| b.first()) { @@ -428,8 +498,11 @@ impl NeoMediaSender { if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = (self.get_buftime(), buffer.start_time(), buffer.end_time()) { - if runtime < (buffer_start - LATENCY) || runtime > (buffer_end + LATENCY) { - debug!("Outside buffer jumping to live"); + if runtime < (buffer_start - LATENCY * 2) || runtime > (buffer_end + LATENCY * 2) { + debug!( + "Outside buffer jumping to live: {:?} < {:?} < {:?}", + buffer_start, runtime, buffer_end + ); self.jump_to_live(buffer).await?; } } @@ -490,14 +563,14 @@ impl NeoMediaSender { }; if let (Some(buf), Some(appsrc)) = (buf, appsrc) { - debug!("PTS: {:?}, Expected: {:?}", runtime, self.get_runtime()); + debug!("DTS: {:?}, Expected: {:?}", runtime, self.get_runtime()); let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { let gst_buf_mut = gst_buf.get_mut().unwrap(); let time = ClockTime::from_useconds(runtime.as_micros() as u64); - gst_buf_mut.set_pts(time); + gst_buf_mut.set_dts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf); } From d24a4d12e213062bc7958239d7e65f5602be85e2 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 1 May 2023 12:26:00 +0700 Subject: [PATCH 07/64] More adjustments to how the pre-buffer is sent --- crates/core/src/bcmedia/codex.rs | 21 ++- src/rtsp/gst/factory.rs | 284 +++++++++++++++---------------- src/rtsp/gst/sender.rs | 63 ++++--- src/rtsp/gst/server.rs | 43 ++++- src/rtsp/mod.rs | 11 +- 5 files changed, 232 insertions(+), 190 deletions(-) diff --git a/crates/core/src/bcmedia/codex.rs b/crates/core/src/bcmedia/codex.rs index 4fc40bc..a8aa28e 100644 --- a/crates/core/src/bcmedia/codex.rs +++ b/crates/core/src/bcmedia/codex.rs @@ -12,11 +12,15 @@ pub struct BcMediaCodex { /// If true we will not search for the start of the next packet /// in the event that the stream appears to be corrupted strict: bool, + amount_skipped: usize, } impl BcMediaCodex { pub(crate) fn new(strict: bool) -> Self { - Self { strict } + Self { + strict, + amount_skipped: 0, + } } } @@ -46,18 +50,19 @@ impl Decoder for BcMediaCodex { } fn decode(&mut self, src: &mut BytesMut) -> Result> { - let mut amount_skipped: usize = 0; loop { match { BcMedia::deserialize(src) } { Ok(bc) => { - if amount_skipped > 0 { - debug!("Amount skipped to restore stream: {}", amount_skipped); + if self.amount_skipped > 0 { + debug!("Amount skipped to restore stream: {}", self.amount_skipped); + self.amount_skipped = 0; } return Ok(Some(bc)); } Err(Error::NomIncomplete(_)) => { - if amount_skipped > 0 { - debug!("Amount skipped to restore stream: {}", amount_skipped); + if self.amount_skipped > 0 { + debug!("Amount skipped to restore stream: {}", self.amount_skipped); + self.amount_skipped = 0; } return Ok(None); } @@ -67,12 +72,12 @@ impl Decoder for BcMediaCodex { } else if src.is_empty() { return Ok(None); } else { - if amount_skipped == 0 { + if self.amount_skipped == 0 { debug!("Error in stream attempting to restore"); trace!(" Stream Error: {:?}", e); } // Drop the whole packet and wait for a packet that starts with magic - amount_skipped += src.len(); + self.amount_skipped += src.len(); src.clear(); continue; } diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 34823ab..0d0f94f 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -26,7 +26,10 @@ use std::{ sync::{atomic::Ordering, Arc}, }; use tokio::{ - sync::mpsc::{channel, Sender}, + sync::{ + mpsc::{channel, Sender}, + RwLock, + }, task::JoinSet, time::Duration, }; @@ -104,6 +107,10 @@ impl NeoMediaFactory { pub(crate) fn buffer_ready(&self) -> bool { self.imp().buffer_ready() } + + pub(crate) async fn join(&self) -> AnyResult<()> { + self.imp().join().await + } } unsafe impl Send for NeoMediaFactory {} @@ -114,7 +121,7 @@ pub(crate) struct NeoMediaFactoryImpl { clientsender: Sender, shared: Arc, #[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj - threads: JoinSet>, + threads: RwLock>>, } impl Drop for NeoMediaFactoryImpl { @@ -149,14 +156,15 @@ impl Default for NeoMediaFactoryImpl { } } } - Ok(()) + unreachable!(); + // Ok(()) }); Self { sender: datasender, clientsender, shared, - threads, + threads: RwLock::new(threads), } } } @@ -294,12 +302,6 @@ impl NeoMediaFactoryImpl { // source.set_property("max-time", Duration::from_secs_f32(2.25).as_nanos() as u64); source.set_do_timestamp(false); source.set_stream_type(AppStreamType::RandomAccess); - let clock = source.clock(); - if let Some(clock) = clock { - debug!("Clock: {:?}", clock.time()); - } else { - debug!("No clock"); - } let need_command = client_data.get_commader(); let enough_command = client_data.get_commader(); @@ -391,154 +393,136 @@ impl NeoMediaFactoryImpl { } } - let do_aud = false; - if do_aud { - match *self.shared.aud_format.blocking_read() { - AudFormats::Unknown => {} - AudFormats::Aac => { - debug!("Building Aac pipeline"); - let source = make_element("appsrc", "audsrc")? - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + // let do_aud = false; + // if do_aud { + match *self.shared.aud_format.blocking_read() { + AudFormats::Unknown => {} + AudFormats::Aac => { + debug!("Building Aac pipeline"); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_is_live(true); - source.set_block(false); - source.set_property("emit-signals", false); - source.set_max_bytes(52428800); - source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::Seekable); + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Seekable); - let need_command = client_data.get_commader(); - let enough_command = client_data.get_commader(); - let seek_command = client_data.get_commader(); - source.set_callbacks( - AppSrcCallbacks::builder() - .need_data(move |_appsrc, _amt| { - let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); - }) - .enough_data(move |_appsrc| { - let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); - }) - .seek_data(move |_appsrc, seek_pos| { - debug!("Send seek AAC"); - let _ = seek_command - .blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); - true - }) - .build(), - ); + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + // let seek_command = client_data.get_commader(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); + }) + .enough_data(move |_appsrc| { + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); + }) + .seek_data(move |_appsrc, _seek_pos| true) + .build(), + ); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast back"))?; + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); - let parser = make_element("aacparse", "audparser")?; - let decoder = make_element("decodebin", "auddecoder")?; - let encoder = make_element("audioconvert", "audencoder")?; - let payload = make_element("rtpL16pay", "audpayload")?; - let storage = make_element("rtpstorage", "audstorage")?; - storage - .set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64); - let jitter = make_element("rtpjitterbuffer", "pay1")?; - jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32); + let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); + let parser = make_element("aacparse", "audparser")?; + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; - bin.add_many(&[ - &source, &queue, &parser, &decoder, &encoder, &payload, &storage, &jitter, - ])?; - Element::link_many(&[&source, &queue, &parser, &decoder])?; - Element::link_many(&[&encoder, &payload, &storage, &jitter])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link AAC decoder to encoder"); - }); + bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &parser, &decoder])?; + Element::link_many(&[&encoder, &payload])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link AAC decoder to encoder"); + }); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.update_aud(source); - } - AudFormats::Adpcm(block_size) => { - debug!("Building Adpcm pipeline"); - // Original command line - // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 - // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 - // ! adpcmdec - // ! audioconvert - // ! rtpL16pay name=pay1 + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + client_data.update_aud(source); + } + AudFormats::Adpcm(block_size) => { + debug!("Building Adpcm pipeline"); + // Original command line + // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 + // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 + // ! adpcmdec + // ! audioconvert + // ! rtpL16pay name=pay1 - let source = make_element("appsrc", "audsrc")? - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast to appsrc."))?; - source.set_is_live(true); - source.set_block(false); - source.set_property("emit-signals", false); - source.set_max_bytes(52428800); - source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::RandomAccess); - source.set_caps(Some( - &Caps::builder("audio/x-adpcm") - .field("layout", "div") - .field("block_align", block_size as i32) - .field("channels", 1i32) - .field("rate", 8000i32) - .build(), - )); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::RandomAccess); + source.set_caps(Some( + &Caps::builder("audio/x-adpcm") + .field("layout", "div") + .field("block_align", block_size as i32) + .field("channels", 1i32) + .field("rate", 8000i32) + .build(), + )); - let need_command = client_data.get_commader(); - let enough_command = client_data.get_commader(); - let seek_command = client_data.get_commader(); - source.set_callbacks( - AppSrcCallbacks::builder() - .need_data(move |_appsrc, _amt| { - let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); - }) - .enough_data(move |_appsrc| { - let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); - }) - .seek_data(move |_appsrc, seek_pos| { - debug!("Send seek Adpcm"); - let _ = seek_command - .blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); - true - }) - .build(), - ); + let need_command = client_data.get_commader(); + let enough_command = client_data.get_commader(); + // let seek_command = client_data.get_commader(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); + }) + .enough_data(move |_appsrc| { + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); + }) + .seek_data(move |_appsrc, _seek_pos| true) + .build(), + ); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot cast back"))?; + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); - let decoder = make_element("decodebin", "auddecoder")?; - let encoder = make_element("audioconvert", "audencoder")?; - let payload = make_element("rtpL16pay", "pay1")?; + let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; - bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &decoder])?; - Element::link_many(&[&encoder, &payload])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link ADPCM decoder to encoder"); - }); + bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &decoder])?; + Element::link_many(&[&encoder, &payload])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link ADPCM decoder to encoder"); + }); - let source = source - .dynamic_cast::() - .map_err(|_| anyhow!("Cannot convert appsrc"))?; - client_data.update_aud(source); - } + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + client_data.update_aud(source); } } @@ -547,4 +531,12 @@ impl NeoMediaFactoryImpl { bin.dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back")) } + + async fn join(&self) -> AnyResult<()> { + let mut threads = self.threads.write().await; + while let Some(thread) = threads.join_next().await { + thread??; + } + Ok(()) + } } diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 4bbee00..0737ba2 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -27,8 +27,9 @@ use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; -const LATENCY: Duration = Duration::from_millis(200); +const LATENCY: Duration = Duration::from_millis(20); const BUFFER_TIME: Duration = Duration::from_secs(10); +const BUFFER_SIZE: usize = 2; #[derive(Default)] struct NeoBuffer { @@ -68,7 +69,7 @@ impl NeoBuffer { if let Some(last_frame_time) = self.last_iframe_time() { let mut time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff); while let Some(time) = time_delta { - if time > BUFFER_TIME && self.buf.len() > 2 { + if time > BUFFER_TIME && self.buf.len() > BUFFER_SIZE { let _ = self.buf.pop_front(); debug!("Popping frame with time difference of {:?}", time); time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff); @@ -242,6 +243,9 @@ impl NeoMediaSenders { _ => {} } self.buffer.push(data); + if self.buffer.buf.len() >= BUFFER_SIZE { + self.shared.buffer_ready.store(true, Ordering::Relaxed); + } Ok(()) } @@ -320,7 +324,7 @@ pub(super) struct NeoMediaSender { impl NeoMediaSender { pub(super) fn new() -> Self { - let (tx, rx) = channel(3); + let (tx, rx) = channel(30); Self { start_time: Duration::ZERO, last_sent_time: Duration::ZERO, @@ -347,25 +351,24 @@ impl NeoMediaSender { async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { - if let Some(runtime) = self.get_runtime() { - let (fronts, backs) = buffer.buf.as_slices(); - let frames = fronts.iter().chain(backs.iter()).collect::>(); - let idx = frames.len().saturating_div(2); - let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| { - if let BcMedia::Iframe(data) = f { - Some(Duration::from_micros(data.microseconds as u64)) - } else { - None - } - }); - if let Some(target_time) = target_time { - self.start_time = target_time - runtime; - self.last_sent_time = target_time; - debug!( - "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", - target_time, self.start_time, target_time - self.start_time, runtime - ); + let runtime = self.get_runtime().unwrap_or(Duration::ZERO); + let (fronts, backs) = buffer.buf.as_slices(); + let frames = fronts.iter().chain(backs.iter()).collect::>(); + let idx = frames.len().saturating_div(2); + let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| { + if let BcMedia::Iframe(data) = f { + Some(Duration::from_micros(data.microseconds as u64)) + } else { + None } + }); + if let Some(target_time) = target_time { + self.start_time = target_time - runtime; + self.last_sent_time = target_time; + debug!( + "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", + target_time, self.start_time, target_time - self.start_time, runtime + ); } } @@ -484,7 +487,25 @@ impl NeoMediaSender { } } else { // Not live. Send the WHOLE buffer for analysis + // Up to the last sent frame + debug!("Sending pre buffer"); for frame in buf.iter() { + let frame_time = match frame { + BcMedia::Iframe(data) => { + Some(Duration::from_micros(data.microseconds as u64)) + } + BcMedia::Pframe(data) => { + Some(Duration::from_micros(data.microseconds as u64)) + } + _ => None, + }; + debug!(" - Maybe: {:?} > {:?}", frame_time, self.last_sent_time); + if let Some(frame_time) = frame_time { + if frame_time > self.last_sent_time { + return Ok(()); + } + } + debug!(" - Send: {:?}", frame_time); self.send_buffer(frame).await?; } } diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 3a85da0..15e2bad 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -24,7 +24,7 @@ use std::{ }; use tokio::{ sync::{mpsc::Sender, RwLock}, - task::JoinHandle, + task::JoinSet, }; glib::wrapper! { @@ -83,11 +83,7 @@ impl NeoRtspServer { self.imp().add_permitted_roles(tag, permitted_users).await } - pub(crate) async fn run( - &self, - bind_addr: &str, - bind_port: u16, - ) -> AnyResult<(JoinHandle>, Arc)> { + pub(crate) async fn run(&self, bind_addr: &str, bind_port: u16) -> AnyResult<()> { let server = self; server.set_address(bind_addr); server.set_service(&format!("{}", bind_port)); @@ -96,10 +92,32 @@ impl NeoRtspServer { let main_loop = Arc::new(MainLoop::new(None, false)); // Run the Glib main loop. let main_loop_thread = main_loop.clone(); - #[allow(clippy::unit_arg)] - let handle = tokio::task::spawn_blocking(move || Ok(main_loop_thread.run())); + let handle = tokio::task::spawn_blocking(move || { + main_loop_thread.run(); + AnyResult::Ok(()) + }); + self.imp() + .threads + .write() + .await + .spawn(async move { handle.await? }); + self.imp().main_loop.write().await.replace(main_loop); + Ok(()) + } - Ok((handle, main_loop)) + pub(crate) async fn quit(&self) -> AnyResult<()> { + if let Some(main_loop) = self.imp().main_loop.read().await.as_ref() { + main_loop.quit(); + } + Ok(()) + } + + pub(crate) async fn join(&self) -> AnyResult<()> { + let mut threads = self.imp().threads.write().await; + while let Some(thread) = threads.join_next().await { + thread??; + } + Ok(()) } pub(crate) fn set_up_tls(&self, config: &Config) { @@ -163,6 +181,8 @@ struct FactoryData { #[derive(Default)] pub(crate) struct NeoRtspServerImpl { medias: RwLock>, + threads: RwLock>>, + main_loop: RwLock>>, } impl ObjectImpl for NeoRtspServerImpl {} @@ -182,6 +202,11 @@ impl NeoRtspServerImpl { Entry::Occupied(_occ) => {} Entry::Vacant(vac) => { let media = NeoMediaFactory::new(); + let thread_media = media.clone(); + self.threads + .write() + .await + .spawn(async move { thread_media.join().await }); vac.insert(FactoryData { factory: media, paths: Default::default(), diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index e07d11d..bf4bc70 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -131,14 +131,15 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { let bind_addr = config.bind_addr.clone(); let bind_port = config.bind_port; - let (handle, main_loop) = rtsp.run(&bind_addr, bind_port).await?; - set.spawn(async move { handle.await? }); + rtsp.run(&bind_addr, bind_port).await?; + let thread_rtsp = rtsp.clone(); + set.spawn(async move { thread_rtsp.join().await }); while let Some(joined) = set.join_next().await { match &joined { Err(_) | Ok(Err(_)) => { // Panicked or error in task - main_loop.quit(); + rtsp.quit().await?; } Ok(Ok(_)) => { // All good @@ -200,9 +201,7 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi break; } } - if streaming.get_config().pause.on_motion || streaming.get_config().pause.on_disconnect { - log::info!("Pause buffer prepared"); - } + log::info!("{}: Buffers prepared", name); // tokio::task::spawn(async move { // let mut inter = tokio::time::interval(tokio::time::Duration::from_secs_f32(0.01)); From cce519de6d30037c29a24d6c3c8eb3330728338d Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 1 May 2023 12:34:31 +0700 Subject: [PATCH 08/64] Use queue2 --- src/rtsp/gst/factory.rs | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 0d0f94f..b3cbe6a 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -31,7 +31,6 @@ use tokio::{ RwLock, }, task::JoinSet, - time::Duration, }; use tokio_stream::wrappers::ReceiverStream; @@ -215,6 +214,12 @@ fn make_element(kind: &str, name: &str) -> AnyResult { }) } +fn make_queue(name: &str) -> AnyResult { + let queue = make_element("queue2", name)?; + queue.set_property("use-buffering", true); + Ok(queue) +} + impl NeoMediaFactoryImpl { pub(crate) fn buffer_ready(&self) -> bool { self.shared.buffer_ready.load(Ordering::Relaxed) @@ -274,9 +279,7 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; - let queue = make_element("queue", "source_queue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); + let queue = make_queue("source_queue")?; let parser = make_element("h265parse", "parser")?; // parser.set_property("config-interval", 5i32); let payload = make_element("rtph265pay", "pay0")?; @@ -330,11 +333,7 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; - let queue = make_element("queue", "source_queue")?; - queue.set_property_from_str("leaky", "downstream"); - // queue.set_property("max-size-bytes", 0u32); - // queue.set_property("max-size-buffers", 0u32); - queue.set_property("max-size-time", Duration::from_secs(2).as_nanos() as u64); + let queue = make_queue("source_queue")?; let parser = make_element("h264parse", "parser")?; // parser.set_property("update-timecode", true); let payload = make_element("rtph264pay", "pay0")?; @@ -367,9 +366,7 @@ impl NeoMediaFactoryImpl { debug!("Building Unknown Pipeline"); let source = make_element("videotestsrc", "vidsrc")?; source.set_property_from_str("pattern", "snow"); - let queue = make_element("queue", "queue0")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); + let queue = make_queue("queue0")?; let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -429,9 +426,7 @@ impl NeoMediaFactoryImpl { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); + let queue = make_queue("audqueue")?; let parser = make_element("aacparse", "audparser")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; @@ -500,9 +495,7 @@ impl NeoMediaFactoryImpl { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; - let queue = make_element("queue", "audqueue")?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 104857600u32); + let queue = make_queue("audqueue")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; From 4a08b8148ba493f531ee72590ef1e35839ea95e9 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 1 May 2023 14:19:32 +0700 Subject: [PATCH 09/64] Use the bin (faster) method for sending the prebuffer --- src/rtsp/gst/sender.rs | 120 ++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 69 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 0737ba2..8a9a0a8 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -11,12 +11,9 @@ use std::collections::{ VecDeque, {hash_map::Entry, HashMap}, }; -use std::{ - iter::Iterator, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, }; // use tokio::time::{interval, Duration, MissedTickBehavior}; use tokio::{ @@ -110,10 +107,6 @@ impl NeoBuffer { }) } - fn iter(&self) -> impl Iterator { - self.buf.iter().flatten() - } - fn start_time(&self) -> Option { if let Some(BcMedia::Iframe(data)) = self.buf.front().and_then(|inner_buf| inner_buf.first()) @@ -320,6 +313,7 @@ pub(super) struct NeoMediaSender { command_sender: Sender, inited: bool, playing: bool, + prebuffered: Duration, } impl NeoMediaSender { @@ -334,6 +328,7 @@ impl NeoMediaSender { command_sender: tx, inited: false, playing: true, + prebuffered: Duration::ZERO, } } @@ -446,79 +441,66 @@ impl NeoMediaSender { // We are live only send the buffer up to the runtime let min_time = self.last_sent_time; let max_time = self.runtime_to_buftime(runtime) + LATENCY; + self.last_sent_time = self.send_buffer_between(buf, min_time, max_time).await?; + } else { + // We are not playing send pre buffers so that the elements can init themselves + let min_time = self.prebuffered; + let max_time = self.last_sent_time; + self.prebuffered = self.send_buffer_between(buf, min_time, max_time).await?; + } + } + Ok(()) + } - let mut found_start = false; - let mut buf_it = buf.buf.iter().peekable(); - while let Some(frames) = buf_it.next() { - tokio::task::yield_now().await; - if !found_start { - let next_frames = buf_it.peek(); - if let Some(BcMedia::Iframe(frame)) = next_frames.and_then(|b| b.first()) { - // Get time of next IFrame - // if it is after min_time, then the - // start shuld happen between frames and next_frames - if Duration::from_micros(frame.microseconds as u64) > min_time { - found_start = true; - } - } - } - - if found_start { - // We have found the start send eveythin until we get passed the - // max time - for frame in frames { - let frame_time = match frame { - BcMedia::Iframe(data) => { - Duration::from_micros(data.microseconds as u64) - } - BcMedia::Pframe(data) => { - Duration::from_micros(data.microseconds as u64) - } - _ => self.last_sent_time, - }; - if frame_time > min_time && frame_time <= max_time { - self.send_buffer(frame).await?; - self.last_sent_time = frame_time; - } else if frame_time > max_time { - return Ok(()); - } - } + async fn send_buffer_between( + &mut self, + buf: &NeoBuffer, + min_time: Duration, + max_time: Duration, + ) -> AnyResult { + let mut last_sent_time = min_time; + let mut found_start = false; + let mut buf_it = buf.buf.iter().peekable(); + while let Some(frames) = buf_it.next() { + tokio::task::yield_now().await; + if !found_start { + let next_frames = buf_it.peek(); + if let Some(BcMedia::Iframe(frame)) = next_frames.and_then(|b| b.first()) { + // Get time of next IFrame + // if it is after min_time, then the + // start shuld happen between frames and next_frames + if Duration::from_micros(frame.microseconds as u64) > min_time { + found_start = true; } } - } else { - // Not live. Send the WHOLE buffer for analysis - // Up to the last sent frame - debug!("Sending pre buffer"); - for frame in buf.iter() { + } + + if found_start { + // We have found the start send eveythin until we get passed the + // max time + for frame in frames { let frame_time = match frame { - BcMedia::Iframe(data) => { - Some(Duration::from_micros(data.microseconds as u64)) - } - BcMedia::Pframe(data) => { - Some(Duration::from_micros(data.microseconds as u64)) - } - _ => None, + BcMedia::Iframe(data) => Duration::from_micros(data.microseconds as u64), + BcMedia::Pframe(data) => Duration::from_micros(data.microseconds as u64), + _ => last_sent_time, }; - debug!(" - Maybe: {:?} > {:?}", frame_time, self.last_sent_time); - if let Some(frame_time) = frame_time { - if frame_time > self.last_sent_time { - return Ok(()); - } + if frame_time > min_time && frame_time <= max_time { + self.send_buffer(frame).await?; + last_sent_time = frame_time; + } else if frame_time > max_time { + return Ok(last_sent_time); } - debug!(" - Send: {:?}", frame_time); - self.send_buffer(frame).await?; } } } - - Ok(()) + Ok(last_sent_time) } async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { - if let (Some(runtime), Some(buffer_start), Some(buffer_end)) = - (self.get_buftime(), buffer.start_time(), buffer.end_time()) + if let (Some(buffer_start), Some(buffer_end)) = (buffer.start_time(), buffer.end_time()) { + let runtime = self.get_buftime().unwrap_or(Duration::ZERO); if runtime < (buffer_start - LATENCY * 2) || runtime > (buffer_end + LATENCY * 2) { debug!( "Outside buffer jumping to live: {:?} < {:?} < {:?}", From 048af9280878a954a63e9c28c8cda92bff880748 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 1 May 2023 15:40:14 +0700 Subject: [PATCH 10/64] Saturating subs all around --- src/rtsp/gst/sender.rs | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 8a9a0a8..40b08d1 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -64,12 +64,16 @@ impl NeoBuffer { } if was_iframe { if let Some(last_frame_time) = self.last_iframe_time() { - let mut time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff); + let mut time_delta = self + .first_iframe_time() + .map(|ff| last_frame_time.saturating_sub(ff)); while let Some(time) = time_delta { if time > BUFFER_TIME && self.buf.len() > BUFFER_SIZE { let _ = self.buf.pop_front(); debug!("Popping frame with time difference of {:?}", time); - time_delta = self.first_iframe_time().map(|ff| last_frame_time - ff); + time_delta = self + .first_iframe_time() + .map(|ff| last_frame_time.saturating_sub(ff)); } else { debug!("Iframes left in the buffer: {}", self.buf.len()); break; @@ -358,11 +362,11 @@ impl NeoMediaSender { } }); if let Some(target_time) = target_time { - self.start_time = target_time - runtime; + self.start_time = target_time.saturating_sub(runtime); self.last_sent_time = target_time; debug!( "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", - target_time, self.start_time, target_time - self.start_time, runtime + target_time, self.start_time, target_time.saturating_sub(self.start_time), runtime ); } } @@ -409,7 +413,7 @@ impl NeoMediaSender { async fn initialise(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if !self.inited { if let (Some(first_ms), Some(end_ms)) = (buffer.start_time(), buffer.end_time()) { - if end_ms - first_ms > LATENCY * 5 { + if end_ms.saturating_sub(first_ms) > LATENCY * 5 { // Minimum buffer self.inited = true; @@ -423,15 +427,11 @@ impl NeoMediaSender { } fn runtime_to_buftime(&self, runtime: Duration) -> Duration { - runtime + self.start_time + runtime.saturating_add(self.start_time) } fn buftime_to_runtime(&self, buftime: Duration) -> Duration { - if buftime > self.start_time { - buftime - self.start_time - } else { - Duration::ZERO - } + buftime.saturating_sub(self.start_time) } async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { @@ -440,7 +440,7 @@ impl NeoMediaSender { if let Some(runtime) = runtime { // We are live only send the buffer up to the runtime let min_time = self.last_sent_time; - let max_time = self.runtime_to_buftime(runtime) + LATENCY; + let max_time = self.runtime_to_buftime(runtime).saturating_add(LATENCY); self.last_sent_time = self.send_buffer_between(buf, min_time, max_time).await?; } else { // We are not playing send pre buffers so that the elements can init themselves @@ -501,7 +501,9 @@ impl NeoMediaSender { if let (Some(buffer_start), Some(buffer_end)) = (buffer.start_time(), buffer.end_time()) { let runtime = self.get_buftime().unwrap_or(Duration::ZERO); - if runtime < (buffer_start - LATENCY * 2) || runtime > (buffer_end + LATENCY * 2) { + if runtime < buffer_start.saturating_sub(LATENCY * 2) + || runtime > buffer_end.saturating_add(LATENCY * 2) + { debug!( "Outside buffer jumping to live: {:?} < {:?} < {:?}", buffer_start, runtime, buffer_end @@ -518,7 +520,7 @@ impl NeoMediaSender { if let Some(clock) = appsrc.clock() { if let Some(time) = clock.time() { if let Some(base_time) = appsrc.base_time() { - let runtime = time - base_time; + let runtime = time.saturating_sub(base_time); return Some(Duration::from_nanos(runtime.nseconds())); } } @@ -532,7 +534,7 @@ impl NeoMediaSender { if let Some(clock) = appsrc.clock() { if let Some(time) = clock.time() { if let Some(base_time) = appsrc.base_time() { - let runtime = time - base_time; + let runtime = time.saturating_sub(base_time); return Some( self.runtime_to_buftime(Duration::from_nanos(runtime.nseconds())), ); From 31ac6e4707d5829be286f050d60df9b4dc5c26d5 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 1 May 2023 18:37:57 +0700 Subject: [PATCH 11/64] Reconnect streams when media type changes --- src/rtsp/gst/server.rs | 29 ++++++++++++++++++++++++++++- src/rtsp/mod.rs | 7 +++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 15e2bad..04ff087 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -13,7 +13,7 @@ use gstreamer_rtsp_server::{ gio::{TlsAuthenticationMode, TlsCertificate}, prelude::*, subclass::prelude::*, - RTSPAuth, RTSPServer, RTSPToken, RTSP_TOKEN_MEDIA_FACTORY_ROLE, + RTSPAuth, RTSPFilterResult, RTSPServer, RTSPToken, RTSP_TOKEN_MEDIA_FACTORY_ROLE, }; use log::*; use neolink_core::bcmedia::model::*; @@ -168,6 +168,33 @@ impl NeoRtspServer { pub(crate) async fn buffer_ready>(&self, tag: T) -> Option { self.imp().buffer_ready(tag).await } + + pub(crate) async fn clear_session>(&self, tag: T) { + let tag: String = tag.into(); + if let Some(media) = self.imp().medias.read().await.get(&tag) { + self.clear_session_paths(media.paths.iter()); + } + } + + fn clear_session_paths>(&self, paths: impl Iterator) { + let paths: Vec<_> = paths.collect(); + for client in self.client_filter(None) { + for session in client.session_filter(None) { + session.filter(Some(&mut |_session, session_media| { + if paths.iter().any(|p| { + session_media + .matches(p.as_ref()) + .map(|amt| amt > 0) + .unwrap_or(false) + }) { + RTSPFilterResult::Remove + } else { + RTSPFilterResult::Keep + } + })); + } + } + } } unsafe impl Send for NeoRtspServer {} diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index bf4bc70..c3ee6cd 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -201,6 +201,13 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi break; } } + // Clear all current media and force a reconnect + // This shoudld stop them from watching the "Stream Not Ready Thing" + tags.iter() + .map(|tag| rtsp_thread.clear_session(tag)) + .collect::>() + .collect::>() + .await; log::info!("{}: Buffers prepared", name); // tokio::task::spawn(async move { From 84bf5ef05c0243f411e4d3bd38845b8b01bd9d3b Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 2 May 2023 10:20:24 +0700 Subject: [PATCH 12/64] Fix the jump to live process --- src/rtsp/gst/sender.rs | 2 +- src/rtsp/gst/server.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 40b08d1..2bb557a 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -500,7 +500,7 @@ impl NeoMediaSender { if self.inited { if let (Some(buffer_start), Some(buffer_end)) = (buffer.start_time(), buffer.end_time()) { - let runtime = self.get_buftime().unwrap_or(Duration::ZERO); + let runtime = self.get_buftime().unwrap_or(self.last_sent_time); if runtime < buffer_start.saturating_sub(LATENCY * 2) || runtime > buffer_end.saturating_add(LATENCY * 2) { diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 04ff087..50ffdc4 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -187,6 +187,7 @@ impl NeoRtspServer { .map(|amt| amt > 0) .unwrap_or(false) }) { + info!("Removing Session Media"); RTSPFilterResult::Remove } else { RTSPFilterResult::Keep From dc6860378d5e83c0774d275af6e8282b6db62bcc Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 2 May 2023 10:57:41 +0700 Subject: [PATCH 13/64] Drop whole session --- src/rtsp/gst/server.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 50ffdc4..b20bfc4 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -179,21 +179,30 @@ impl NeoRtspServer { fn clear_session_paths>(&self, paths: impl Iterator) { let paths: Vec<_> = paths.collect(); for client in self.client_filter(None) { - for session in client.session_filter(None) { + client.session_filter(Some(&mut |_client, session| { + let mut clean_up = false; session.filter(Some(&mut |_session, session_media| { if paths.iter().any(|p| { + let s = p.as_ref(); session_media - .matches(p.as_ref()) - .map(|amt| amt > 0) + .matches(s) + .map(|amt| amt == s.len() as i32) .unwrap_or(false) }) { info!("Removing Session Media"); + clean_up = true; RTSPFilterResult::Remove } else { RTSPFilterResult::Keep } })); - } + + if clean_up { + RTSPFilterResult::Remove + } else { + RTSPFilterResult::Keep + } + })); } } } From c53b0955f820929fab6bd02f0182117c8f191d87 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 2 May 2023 13:15:22 +0700 Subject: [PATCH 14/64] Bullseye docker --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fc7acda..fd7c920 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # Miroslav Å edivý # SPDX-License-Identifier: AGPL-3.0-only -FROM docker.io/rust:slim-buster AS build +FROM docker.io/rust:slim-bullseye AS build ARG TARGETPLATFORM ENV DEBIAN_FRONTEND=noninteractive @@ -34,7 +34,7 @@ RUN echo "TARGETPLATFORM: ${TARGETPLATFORM}"; \ fi # Create the release container. Match the base OS used to build -FROM debian:buster-slim +FROM debian:bullseye-slim ARG TARGETPLATFORM ARG REPO ARG VERSION From f2b609f12c4448588f7d2d77a7c99f760f197177 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 2 May 2023 13:53:20 +0700 Subject: [PATCH 15/64] Allow negative start times --- src/rtsp/gst/sender.rs | 86 +++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 2bb557a..c6d2a1e 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -7,14 +7,17 @@ use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use log::*; use neolink_core::bcmedia::model::*; -use std::collections::{ - VecDeque, - {hash_map::Entry, HashMap}, -}; use std::sync::{ atomic::{AtomicU64, Ordering}, Arc, }; +use std::{ + collections::{ + VecDeque, + {hash_map::Entry, HashMap}, + }, + convert::TryInto, +}; // use tokio::time::{interval, Duration, MissedTickBehavior}; use tokio::{ sync::mpsc::{channel, Receiver, Sender}, @@ -40,7 +43,7 @@ impl NeoBuffer { BcMedia::Iframe(data) => { debug!( "Pushed iFrame to buffer at {:?}", - Duration::from_micros(data.microseconds as u64) + Duration::from_micros(data.microseconds.try_into().unwrap()) ); was_iframe = true; self.buf.push_back(vec![BcMedia::Iframe(data)]) @@ -90,7 +93,9 @@ impl NeoBuffer { .and_then(|b| b.first()) .and_then(|f| { if let BcMedia::Iframe(frame) = f { - Some(Duration::from_micros(frame.microseconds as u64)) + Some(Duration::from_micros( + frame.microseconds.try_into().unwrap(), + )) } else { None } @@ -104,7 +109,9 @@ impl NeoBuffer { .and_then(|b| b.first()) .and_then(|f| { if let BcMedia::Iframe(frame) = f { - Some(Duration::from_micros(frame.microseconds as u64)) + Some(Duration::from_micros( + frame.microseconds.try_into().unwrap(), + )) } else { None } @@ -115,7 +122,7 @@ impl NeoBuffer { if let Some(BcMedia::Iframe(data)) = self.buf.front().and_then(|inner_buf| inner_buf.first()) { - Some(Duration::from_micros(data.microseconds as u64)) + Some(Duration::from_micros(data.microseconds.try_into().unwrap())) } else { None } @@ -126,8 +133,12 @@ impl NeoBuffer { innerbuffer .iter() .flat_map(|frame| match frame { - BcMedia::Iframe(data) => Some(Duration::from_micros(data.microseconds as u64)), - BcMedia::Pframe(data) => Some(Duration::from_micros(data.microseconds as u64)), + BcMedia::Iframe(data) => { + Some(Duration::from_micros(data.microseconds.try_into().unwrap())) + } + BcMedia::Pframe(data) => { + Some(Duration::from_micros(data.microseconds.try_into().unwrap())) + } _ => None, }) .max() @@ -211,7 +222,7 @@ impl NeoMediaSenders { BcMedia::Adpcm(data) => { if !matches!(*self.shared.aud_format.read().await, AudFormats::Adpcm(_)) { *self.shared.aud_format.write().await = - AudFormats::Adpcm(data.data.len() as u16); + AudFormats::Adpcm(data.data.len().try_into().unwrap()); } } _ => {} @@ -224,14 +235,14 @@ impl NeoMediaSenders { self.handle_buffer().await?; match &data { BcMedia::Iframe(data) => { - let frame_ms = Duration::from_micros(data.microseconds as u64); + let frame_ms = Duration::from_micros(data.microseconds.try_into().unwrap()); debug!("New IFrame for buffer @ {:?}", frame_ms); for client in self.client_data.values() { debug!(" - {:?}", client.buftime_to_runtime(frame_ms)); } } BcMedia::Pframe(data) => { - let frame_ms = Duration::from_micros(data.microseconds as u64); + let frame_ms = Duration::from_micros(data.microseconds.try_into().unwrap()); debug!("New PFrame for buffer @ {:?}", frame_ms); for client in self.client_data.values() { debug!(" - {:?}", client.buftime_to_runtime(frame_ms)); @@ -309,7 +320,7 @@ pub(super) enum NeoMediaSenderCommand { } #[derive(Debug)] pub(super) struct NeoMediaSender { - start_time: Duration, + start_time: i128, last_sent_time: Duration, vid: Option, aud: Option, @@ -324,7 +335,7 @@ impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(30); Self { - start_time: Duration::ZERO, + start_time: 0, last_sent_time: Duration::ZERO, vid: None, aud: None, @@ -356,17 +367,18 @@ impl NeoMediaSender { let idx = frames.len().saturating_div(2); let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| { if let BcMedia::Iframe(data) = f { - Some(Duration::from_micros(data.microseconds as u64)) + Some(Duration::from_micros(data.microseconds.try_into().unwrap())) } else { None } }); if let Some(target_time) = target_time { - self.start_time = target_time.saturating_sub(runtime); + self.start_time = TryInto::::try_into(target_time.as_micros()).unwrap() + - TryInto::::try_into(runtime.as_micros()).unwrap(); self.last_sent_time = target_time; debug!( "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", - target_time, self.start_time, target_time.saturating_sub(self.start_time), runtime + target_time, self.start_time, self.buftime_to_runtime(target_time), runtime ); } } @@ -427,11 +439,25 @@ impl NeoMediaSender { } fn runtime_to_buftime(&self, runtime: Duration) -> Duration { - runtime.saturating_add(self.start_time) + Duration::from_micros( + TryInto::::try_into( + TryInto::::try_into(runtime.as_micros()) + .unwrap() + .saturating_add(self.start_time), + ) + .unwrap_or(0), + ) } fn buftime_to_runtime(&self, buftime: Duration) -> Duration { - buftime.saturating_sub(self.start_time) + Duration::from_micros( + TryInto::::try_into( + TryInto::::try_into(buftime.as_micros()) + .unwrap() + .saturating_sub(self.start_time), + ) + .unwrap_or(0), + ) } async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { @@ -469,7 +495,7 @@ impl NeoMediaSender { // Get time of next IFrame // if it is after min_time, then the // start shuld happen between frames and next_frames - if Duration::from_micros(frame.microseconds as u64) > min_time { + if Duration::from_micros(frame.microseconds.try_into().unwrap()) > min_time { found_start = true; } } @@ -480,8 +506,12 @@ impl NeoMediaSender { // max time for frame in frames { let frame_time = match frame { - BcMedia::Iframe(data) => Duration::from_micros(data.microseconds as u64), - BcMedia::Pframe(data) => Duration::from_micros(data.microseconds as u64), + BcMedia::Iframe(data) => { + Duration::from_micros(data.microseconds.try_into().unwrap()) + } + BcMedia::Pframe(data) => { + Duration::from_micros(data.microseconds.try_into().unwrap()) + } _ => last_sent_time, }; if frame_time > min_time && frame_time <= max_time { @@ -548,8 +578,12 @@ impl NeoMediaSender { async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { if self.inited && self.playing { let buftime = match media { - BcMedia::Iframe(data) => Duration::from_micros(data.microseconds as u64), - BcMedia::Pframe(data) => Duration::from_micros(data.microseconds as u64), + BcMedia::Iframe(data) => { + Duration::from_micros(data.microseconds.try_into().unwrap()) + } + BcMedia::Pframe(data) => { + Duration::from_micros(data.microseconds.try_into().unwrap()) + } _ => self.last_sent_time, }; let runtime = self.buftime_to_runtime(buftime); @@ -574,7 +608,7 @@ impl NeoMediaSender { { let gst_buf_mut = gst_buf.get_mut().unwrap(); - let time = ClockTime::from_useconds(runtime.as_micros() as u64); + let time = ClockTime::from_useconds(runtime.as_micros().try_into().unwrap()); gst_buf_mut.set_dts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf); From d50fa87baf5147e0e0e019ab91042461c76352b6 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 2 May 2023 14:40:56 +0700 Subject: [PATCH 16/64] Rework the type of the time --- src/rtsp/gst/sender.rs | 179 +++++++++++++++++++---------------------- 1 file changed, 84 insertions(+), 95 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index c6d2a1e..011ffd1 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -18,7 +18,6 @@ use std::{ }, convert::TryInto, }; -// use tokio::time::{interval, Duration, MissedTickBehavior}; use tokio::{ sync::mpsc::{channel, Receiver, Sender}, time::Duration, @@ -27,8 +26,14 @@ use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; -const LATENCY: Duration = Duration::from_millis(20); -const BUFFER_TIME: Duration = Duration::from_secs(10); +type FrameTime = i64; + +const SECS: FrameTime = 1000000; +const MILLIS: FrameTime = 1000; +// const MICROS: FrameTime = 1; + +const LATENCY: FrameTime = 20 * MILLIS; +const BUFFER_TIME: FrameTime = 10 * SECS; const BUFFER_SIZE: usize = 2; #[derive(Default)] @@ -73,7 +78,10 @@ impl NeoBuffer { while let Some(time) = time_delta { if time > BUFFER_TIME && self.buf.len() > BUFFER_SIZE { let _ = self.buf.pop_front(); - debug!("Popping frame with time difference of {:?}", time); + debug!( + "Popping frame with time difference of {:?}", + Duration::from_micros(time.try_into().unwrap_or(0)) + ); time_delta = self .first_iframe_time() .map(|ff| last_frame_time.saturating_sub(ff)); @@ -86,59 +94,51 @@ impl NeoBuffer { } } - fn last_iframe_time(&self) -> Option { + fn last_iframe_time(&self) -> Option { self.buf .back() .as_ref() .and_then(|b| b.first()) .and_then(|f| { if let BcMedia::Iframe(frame) = f { - Some(Duration::from_micros( - frame.microseconds.try_into().unwrap(), - )) + Some(frame.microseconds as FrameTime) } else { None } }) } - fn first_iframe_time(&self) -> Option { + fn first_iframe_time(&self) -> Option { self.buf .front() .as_ref() .and_then(|b| b.first()) .and_then(|f| { if let BcMedia::Iframe(frame) = f { - Some(Duration::from_micros( - frame.microseconds.try_into().unwrap(), - )) + Some(frame.microseconds as FrameTime) } else { None } }) } - fn start_time(&self) -> Option { + fn start_time(&self) -> Option { if let Some(BcMedia::Iframe(data)) = self.buf.front().and_then(|inner_buf| inner_buf.first()) { - Some(Duration::from_micros(data.microseconds.try_into().unwrap())) + Some(data.microseconds as FrameTime) } else { None } } - fn end_time(&self) -> Option { + fn end_time(&self) -> Option { if let Some(innerbuffer) = self.buf.back() { innerbuffer .iter() .flat_map(|frame| match frame { - BcMedia::Iframe(data) => { - Some(Duration::from_micros(data.microseconds.try_into().unwrap())) - } - BcMedia::Pframe(data) => { - Some(Duration::from_micros(data.microseconds.try_into().unwrap())) - } + BcMedia::Iframe(data) => Some(data.microseconds as FrameTime), + BcMedia::Pframe(data) => Some(data.microseconds as FrameTime), _ => None, }) .max() @@ -235,17 +235,33 @@ impl NeoMediaSenders { self.handle_buffer().await?; match &data { BcMedia::Iframe(data) => { - let frame_ms = Duration::from_micros(data.microseconds.try_into().unwrap()); - debug!("New IFrame for buffer @ {:?}", frame_ms); + let frame_ms = data.microseconds as FrameTime; + debug!( + "New IFrame for buffer @ {:?}", + Duration::from_micros(frame_ms.try_into().unwrap_or(0)) + ); for client in self.client_data.values() { - debug!(" - {:?}", client.buftime_to_runtime(frame_ms)); + debug!( + " - {:?}", + Duration::from_micros( + client.buftime_to_runtime(frame_ms).try_into().unwrap_or(0) + ) + ); } } BcMedia::Pframe(data) => { - let frame_ms = Duration::from_micros(data.microseconds.try_into().unwrap()); - debug!("New PFrame for buffer @ {:?}", frame_ms); + let frame_ms = data.microseconds as FrameTime; + debug!( + "New PFrame for buffer @ {:?}", + Duration::from_micros(frame_ms.try_into().unwrap_or(0)) + ); for client in self.client_data.values() { - debug!(" - {:?}", client.buftime_to_runtime(frame_ms)); + debug!( + " - {:?}", + Duration::from_micros( + client.buftime_to_runtime(frame_ms).try_into().unwrap_or(0) + ) + ); } } _ => {} @@ -292,7 +308,7 @@ impl NeoMediaSenders { } pub(super) async fn run(&mut self) -> AnyResult<()> { - let mut interval = tokio::time::interval(LATENCY / 3); + let mut interval = tokio::time::interval(Duration::from_micros(LATENCY as u64 / 3)); loop { tokio::task::yield_now().await; tokio::select! { @@ -320,15 +336,15 @@ pub(super) enum NeoMediaSenderCommand { } #[derive(Debug)] pub(super) struct NeoMediaSender { - start_time: i128, - last_sent_time: Duration, + start_time: FrameTime, + last_sent_time: FrameTime, vid: Option, aud: Option, command_reciever: Receiver, command_sender: Sender, inited: bool, playing: bool, - prebuffered: Duration, + prebuffered: FrameTime, } impl NeoMediaSender { @@ -336,14 +352,14 @@ impl NeoMediaSender { let (tx, rx) = channel(30); Self { start_time: 0, - last_sent_time: Duration::ZERO, + last_sent_time: 0, vid: None, aud: None, command_reciever: rx, command_sender: tx, inited: false, playing: true, - prebuffered: Duration::ZERO, + prebuffered: 0, } } @@ -361,24 +377,23 @@ impl NeoMediaSender { async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { - let runtime = self.get_runtime().unwrap_or(Duration::ZERO); + let runtime = self.get_runtime().unwrap_or(0); let (fronts, backs) = buffer.buf.as_slices(); let frames = fronts.iter().chain(backs.iter()).collect::>(); let idx = frames.len().saturating_div(2); let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| { if let BcMedia::Iframe(data) = f { - Some(Duration::from_micros(data.microseconds.try_into().unwrap())) + Some(data.microseconds as FrameTime) } else { None } }); if let Some(target_time) = target_time { - self.start_time = TryInto::::try_into(target_time.as_micros()).unwrap() - - TryInto::::try_into(runtime.as_micros()).unwrap(); + self.start_time = target_time - runtime; self.last_sent_time = target_time; debug!( "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", - target_time, self.start_time, self.buftime_to_runtime(target_time), runtime + Duration::from_micros(target_time.try_into().unwrap_or(0)), Duration::from_micros(self.start_time.try_into().unwrap_or(0)), Duration::from_micros(self.buftime_to_runtime(target_time).try_into().unwrap_or(0)), Duration::from_micros(runtime.try_into().unwrap_or(0)) ); } } @@ -386,11 +401,12 @@ impl NeoMediaSender { Ok(()) } - async fn seek(&mut self, target_time: Duration) -> AnyResult<()> { + async fn seek(&mut self, target_time: FrameTime) -> AnyResult<()> { self.last_sent_time = self.runtime_to_buftime(target_time); debug!( "Seeked last_sent_time to {:?} ({:?})", - self.last_sent_time, target_time + Duration::from_micros(self.last_sent_time.try_into().unwrap_or(0)), + Duration::from_micros(target_time.try_into().unwrap_or(0)) ); Ok(()) } @@ -400,8 +416,8 @@ impl NeoMediaSender { if let Ok(command) = self.command_reciever.try_recv() { match command { NeoMediaSenderCommand::Seek(dest) => { - debug!("Got Seek Request: {}", dest); - self.seek(Duration::from_micros(dest)).await?; + debug!("Got Seek Request: {:?}", Duration::from_micros(dest)); + self.seek(dest as FrameTime).await?; } NeoMediaSenderCommand::Pause => { if self.playing { @@ -438,26 +454,12 @@ impl NeoMediaSender { Ok(()) } - fn runtime_to_buftime(&self, runtime: Duration) -> Duration { - Duration::from_micros( - TryInto::::try_into( - TryInto::::try_into(runtime.as_micros()) - .unwrap() - .saturating_add(self.start_time), - ) - .unwrap_or(0), - ) + fn runtime_to_buftime(&self, runtime: FrameTime) -> FrameTime { + runtime.saturating_add(self.start_time) } - fn buftime_to_runtime(&self, buftime: Duration) -> Duration { - Duration::from_micros( - TryInto::::try_into( - TryInto::::try_into(buftime.as_micros()) - .unwrap() - .saturating_sub(self.start_time), - ) - .unwrap_or(0), - ) + fn buftime_to_runtime(&self, buftime: FrameTime) -> FrameTime { + buftime.saturating_sub(self.start_time).max(0) } async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { @@ -481,9 +483,9 @@ impl NeoMediaSender { async fn send_buffer_between( &mut self, buf: &NeoBuffer, - min_time: Duration, - max_time: Duration, - ) -> AnyResult { + min_time: FrameTime, + max_time: FrameTime, + ) -> AnyResult { let mut last_sent_time = min_time; let mut found_start = false; let mut buf_it = buf.buf.iter().peekable(); @@ -495,7 +497,7 @@ impl NeoMediaSender { // Get time of next IFrame // if it is after min_time, then the // start shuld happen between frames and next_frames - if Duration::from_micros(frame.microseconds.try_into().unwrap()) > min_time { + if frame.microseconds as FrameTime > min_time { found_start = true; } } @@ -506,12 +508,8 @@ impl NeoMediaSender { // max time for frame in frames { let frame_time = match frame { - BcMedia::Iframe(data) => { - Duration::from_micros(data.microseconds.try_into().unwrap()) - } - BcMedia::Pframe(data) => { - Duration::from_micros(data.microseconds.try_into().unwrap()) - } + BcMedia::Iframe(data) => data.microseconds as FrameTime, + BcMedia::Pframe(data) => data.microseconds as FrameTime, _ => last_sent_time, }; if frame_time > min_time && frame_time <= max_time { @@ -536,7 +534,9 @@ impl NeoMediaSender { { debug!( "Outside buffer jumping to live: {:?} < {:?} < {:?}", - buffer_start, runtime, buffer_end + Duration::from_micros(buffer_start.try_into().unwrap_or(0)), + Duration::from_micros(runtime.try_into().unwrap_or(0)), + Duration::from_micros(buffer_end.try_into().unwrap_or(0)) ); self.jump_to_live(buffer).await?; } @@ -545,13 +545,13 @@ impl NeoMediaSender { Ok(()) } - fn get_runtime(&self) -> Option { + fn get_runtime(&self) -> Option { if let Some(appsrc) = self.vid.as_ref() { if let Some(clock) = appsrc.clock() { if let Some(time) = clock.time() { if let Some(base_time) = appsrc.base_time() { let runtime = time.saturating_sub(base_time); - return Some(Duration::from_nanos(runtime.nseconds())); + return Some(runtime.useconds() as FrameTime); } } } @@ -559,31 +559,15 @@ impl NeoMediaSender { None } - fn get_buftime(&self) -> Option { - if let Some(appsrc) = self.vid.as_ref() { - if let Some(clock) = appsrc.clock() { - if let Some(time) = clock.time() { - if let Some(base_time) = appsrc.base_time() { - let runtime = time.saturating_sub(base_time); - return Some( - self.runtime_to_buftime(Duration::from_nanos(runtime.nseconds())), - ); - } - } - } - } - None + fn get_buftime(&self) -> Option { + self.get_runtime().map(|time| self.runtime_to_buftime(time)) } async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { if self.inited && self.playing { let buftime = match media { - BcMedia::Iframe(data) => { - Duration::from_micros(data.microseconds.try_into().unwrap()) - } - BcMedia::Pframe(data) => { - Duration::from_micros(data.microseconds.try_into().unwrap()) - } + BcMedia::Iframe(data) => data.microseconds as FrameTime, + BcMedia::Pframe(data) => data.microseconds as FrameTime, _ => self.last_sent_time, }; let runtime = self.buftime_to_runtime(buftime); @@ -602,13 +586,18 @@ impl NeoMediaSender { }; if let (Some(buf), Some(appsrc)) = (buf, appsrc) { - debug!("DTS: {:?}, Expected: {:?}", runtime, self.get_runtime()); + debug!( + "DTS: {:?}, Expected: {:?}", + Duration::from_micros(runtime.try_into().unwrap_or(0)), + self.get_runtime() + .map(|t| Duration::from_micros(t.try_into().unwrap_or(0))) + ); let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { let gst_buf_mut = gst_buf.get_mut().unwrap(); - let time = ClockTime::from_useconds(runtime.as_micros().try_into().unwrap()); + let time = ClockTime::from_useconds(runtime.try_into().unwrap()); gst_buf_mut.set_dts(time); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf); From af9f181ff5dca70290ddd10c4f9b91df9e6424e5 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 2 May 2023 15:19:01 +0700 Subject: [PATCH 17/64] Increase minimum buffer amount --- src/rtsp/gst/sender.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 011ffd1..c0f5399 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -34,7 +34,7 @@ const MILLIS: FrameTime = 1000; const LATENCY: FrameTime = 20 * MILLIS; const BUFFER_TIME: FrameTime = 10 * SECS; -const BUFFER_SIZE: usize = 2; +const BUFFER_SIZE: usize = 3; #[derive(Default)] struct NeoBuffer { From ccd5778f576aabf1f9e43a33450a264a0bd44665 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 2 May 2023 15:25:40 +0700 Subject: [PATCH 18/64] Display stream not ready if connecing before buffers prepared --- src/rtsp/gst/factory.rs | 72 ++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index b3cbe6a..ee7c32d 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -241,8 +241,37 @@ impl NeoMediaFactoryImpl { let mut client_data = NeoMediaSender::new(); // Now contruct the actual ones - match *self.shared.vid_format.blocking_read() { - VidFormats::H265 => { + match ( + *self.shared.vid_format.blocking_read(), + self.shared.buffer_ready.load(Ordering::Relaxed), + ) { + (VidFormats::Unknown, true) | (_, false) => { + debug!("Building Unknown Pipeline"); + let source = make_element("videotestsrc", "vidsrc")?; + source.set_property_from_str("pattern", "snow"); + let queue = make_queue("queue0")?; + let overlay = make_element("textoverlay", "overlay")?; + overlay.set_property("text", "Stream not Ready"); + overlay.set_property_from_str("valignment", "top"); + overlay.set_property_from_str("halignment", "left"); + overlay.set_property("font-desc", "Sans, 16"); + let encoder = make_element("jpegenc", "encoder")?; + let payload = make_element("rtpjpegpay", "pay0")?; + + bin.add_many(&[&source, &queue, &overlay, &encoder, &payload])?; + source.link_filtered( + &queue, + &Caps::builder("video/x-raw") + .field("format", "YUY2") + .field("width", 896i32) + .field("height", 512i32) + .field("framerate", gstreamer::Fraction::new(25, 1)) + .build(), + )?; + // source.link(&queue)?; + Element::link_many(&[&queue, &overlay, &encoder, &payload])?; + } + (VidFormats::H265, true) => { debug!("Building H265 Pipeline"); let source = make_element("appsrc", "vidsrc")? .dynamic_cast::() @@ -291,7 +320,7 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot convert appsrc"))?; client_data.update_vid(source); } - VidFormats::H264 => { + (VidFormats::H264, true) => { debug!("Building H264 Pipeline"); let source = make_element("appsrc", "vidsrc")? .dynamic_cast::() @@ -362,39 +391,16 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot convert appsrc"))?; client_data.update_vid(source); } - VidFormats::Unknown => { - debug!("Building Unknown Pipeline"); - let source = make_element("videotestsrc", "vidsrc")?; - source.set_property_from_str("pattern", "snow"); - let queue = make_queue("queue0")?; - let overlay = make_element("textoverlay", "overlay")?; - overlay.set_property("text", "Stream not Ready"); - overlay.set_property_from_str("valignment", "top"); - overlay.set_property_from_str("halignment", "left"); - overlay.set_property("font-desc", "Sans, 16"); - let encoder = make_element("jpegenc", "encoder")?; - let payload = make_element("rtpjpegpay", "pay0")?; - - bin.add_many(&[&source, &queue, &overlay, &encoder, &payload])?; - source.link_filtered( - &queue, - &Caps::builder("video/x-raw") - .field("format", "YUY2") - .field("width", 896i32) - .field("height", 512i32) - .field("framerate", gstreamer::Fraction::new(25, 1)) - .build(), - )?; - // source.link(&queue)?; - Element::link_many(&[&queue, &overlay, &encoder, &payload])?; - } } // let do_aud = false; // if do_aud { - match *self.shared.aud_format.blocking_read() { - AudFormats::Unknown => {} - AudFormats::Aac => { + match ( + *self.shared.aud_format.blocking_read(), + self.shared.buffer_ready.load(Ordering::Relaxed), + ) { + (AudFormats::Unknown, true) | (_, false) => {} + (AudFormats::Aac, true) => { debug!("Building Aac pipeline"); let source = make_element("appsrc", "audsrc")? .dynamic_cast::() @@ -449,7 +455,7 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot convert appsrc"))?; client_data.update_aud(source); } - AudFormats::Adpcm(block_size) => { + (AudFormats::Adpcm(block_size), true) => { debug!("Building Adpcm pipeline"); // Original command line // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 From 2ff1ee474ec37e930d9fb553eafa5962a90a5d92 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 2 May 2023 15:49:10 +0700 Subject: [PATCH 19/64] Handle error in appsrc push during init --- src/rtsp/gst/sender.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index c0f5399..b52b817 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -380,14 +380,17 @@ impl NeoMediaSender { let runtime = self.get_runtime().unwrap_or(0); let (fronts, backs) = buffer.buf.as_slices(); let frames = fronts.iter().chain(backs.iter()).collect::>(); - let idx = frames.len().saturating_div(2); - let target_time = frames.get(idx).and_then(|f| f.first()).and_then(|f| { - if let BcMedia::Iframe(data) = f { - Some(data.microseconds as FrameTime) - } else { - None - } - }); + let target_time = { + // Middle iframe + let idx = frames.len().saturating_div(2); + frames.get(idx).and_then(|f| f.first()).and_then(|f| { + if let BcMedia::Iframe(data) = f { + Some(data.microseconds as FrameTime) + } else { + None + } + }) + }; if let Some(target_time) = target_time { self.start_time = target_time - runtime; self.last_sent_time = target_time; @@ -447,7 +450,7 @@ impl NeoMediaSender { self.jump_to_live(buffer).await?; - self.process_buffer(buffer).await?; + let _ = self.process_buffer(buffer).await; // Ignore errors let them be handled later in the loop } } } From 86f6ea55ec681b34d241ab25368ecc01e0d8ab57 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 08:15:40 +0700 Subject: [PATCH 20/64] Without jump to lives --- src/rtsp/gst/sender.rs | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index b52b817..a56cb10 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -290,11 +290,11 @@ impl NeoMediaSenders { occ.remove(); continue; } - if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await { - debug!("Could not handle jump to live: {:?}", e); - occ.remove(); - continue; - } + // if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await { + // debug!("Could not handle jump to live: {:?}", e); + // occ.remove(); + // continue; + // } if let Err(e) = occ.get_mut().process_buffer(&self.buffer).await { debug!("Could not send client data: {:?}", e); occ.remove(); @@ -527,26 +527,26 @@ impl NeoMediaSender { Ok(last_sent_time) } - async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - if self.inited { - if let (Some(buffer_start), Some(buffer_end)) = (buffer.start_time(), buffer.end_time()) - { - let runtime = self.get_buftime().unwrap_or(self.last_sent_time); - if runtime < buffer_start.saturating_sub(LATENCY * 2) - || runtime > buffer_end.saturating_add(LATENCY * 2) - { - debug!( - "Outside buffer jumping to live: {:?} < {:?} < {:?}", - Duration::from_micros(buffer_start.try_into().unwrap_or(0)), - Duration::from_micros(runtime.try_into().unwrap_or(0)), - Duration::from_micros(buffer_end.try_into().unwrap_or(0)) - ); - self.jump_to_live(buffer).await?; - } - } - } - Ok(()) - } + // async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + // if self.inited { + // if let (Some(buffer_start), Some(buffer_end)) = (buffer.start_time(), buffer.end_time()) + // { + // let runtime = self.get_buftime().unwrap_or(self.last_sent_time); + // if runtime < buffer_start.saturating_sub(LATENCY * 2) + // || runtime > buffer_end.saturating_add(LATENCY * 2) + // { + // debug!( + // "Outside buffer jumping to live: {:?} < {:?} < {:?}", + // Duration::from_micros(buffer_start.try_into().unwrap_or(0)), + // Duration::from_micros(runtime.try_into().unwrap_or(0)), + // Duration::from_micros(buffer_end.try_into().unwrap_or(0)) + // ); + // self.jump_to_live(buffer).await?; + // } + // } + // } + // Ok(()) + // } fn get_runtime(&self) -> Option { if let Some(appsrc) = self.vid.as_ref() { From 9d57b2dafd25c1d01b9a24591c6b4d6d8ea01bf1 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 08:46:27 +0700 Subject: [PATCH 21/64] Change how queues work --- src/rtsp/gst/factory.rs | 40 ++++++++++++++++++++++++++++++---------- src/rtsp/gst/sender.rs | 6 +++--- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index ee7c32d..3da9c6d 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -215,6 +215,19 @@ fn make_element(kind: &str, name: &str) -> AnyResult { } fn make_queue(name: &str) -> AnyResult { + let queue = make_element("queue", name)?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 0u32); + queue.set_property("max-size-buffers", 0u32); + queue.set_property( + "max-size-time", + std::convert::TryInto::::try_into(tokio::time::Duration::from_secs(5).as_nanos()) + .unwrap_or(0), + ); + Ok(queue) +} + +fn make_queue2(name: &str) -> AnyResult { let queue = make_element("queue2", name)?; queue.set_property("use-buffering", true); Ok(queue) @@ -250,6 +263,7 @@ impl NeoMediaFactoryImpl { let source = make_element("videotestsrc", "vidsrc")?; source.set_property_from_str("pattern", "snow"); let queue = make_queue("queue0")?; + let queue2 = make_queue2("queue2")?; let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -258,7 +272,7 @@ impl NeoMediaFactoryImpl { let encoder = make_element("jpegenc", "encoder")?; let payload = make_element("rtpjpegpay", "pay0")?; - bin.add_many(&[&source, &queue, &overlay, &encoder, &payload])?; + bin.add_many(&[&source, &queue, &queue2, &overlay, &encoder, &payload])?; source.link_filtered( &queue, &Caps::builder("video/x-raw") @@ -269,7 +283,7 @@ impl NeoMediaFactoryImpl { .build(), )?; // source.link(&queue)?; - Element::link_many(&[&queue, &overlay, &encoder, &payload])?; + Element::link_many(&[&queue, &queue2, &overlay, &encoder, &payload])?; } (VidFormats::H265, true) => { debug!("Building H265 Pipeline"); @@ -309,11 +323,12 @@ impl NeoMediaFactoryImpl { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_queue("source_queue")?; + let queue2 = make_queue2("queue2")?; let parser = make_element("h265parse", "parser")?; // parser.set_property("config-interval", 5i32); let payload = make_element("rtph265pay", "pay0")?; - bin.add_many(&[&source, &queue, &parser, &payload])?; - Element::link_many(&[&source, &queue, &parser, &payload])?; + bin.add_many(&[&source, &queue, &queue2, &parser, &payload])?; + Element::link_many(&[&source, &queue, &queue2, &parser, &payload])?; let source = source .dynamic_cast::() @@ -363,6 +378,7 @@ impl NeoMediaFactoryImpl { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_queue("source_queue")?; + let queue2 = make_queue2("queue2")?; let parser = make_element("h264parse", "parser")?; // parser.set_property("update-timecode", true); let payload = make_element("rtph264pay", "pay0")?; @@ -374,13 +390,13 @@ impl NeoMediaFactoryImpl { // payload.set_property("config-interval", 5i32); bin.add_many(&[ - &source, &queue, &parser, + &source, &queue, &queue2, &parser, &payload, // &storage, // &jitter, ])?; Element::link_many(&[ - &source, &queue, &parser, + &source, &queue, &queue2, &parser, &payload, // &storage, // &jitter, @@ -433,13 +449,16 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_queue("audqueue")?; + let queue2 = make_queue2("queue2")?; let parser = make_element("aacparse", "audparser")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; - bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &parser, &decoder])?; + bin.add_many(&[ + &source, &queue, &queue2, &parser, &decoder, &encoder, &payload, + ])?; + Element::link_many(&[&source, &queue, &queue2, &parser, &decoder])?; Element::link_many(&[&encoder, &payload])?; decoder.connect_pad_added(move |_element, pad| { debug!("Linking encoder to decoder: {:?}", pad.caps()); @@ -502,12 +521,13 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_queue("audqueue")?; + let queue2 = make_queue2("audqueue2")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; - bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &decoder])?; + bin.add_many(&[&source, &queue, &queue2, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &queue2, &decoder])?; Element::link_many(&[&encoder, &payload])?; decoder.connect_pad_added(move |_element, pad| { debug!("Linking encoder to decoder: {:?}", pad.caps()); diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index a56cb10..9a5ec62 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -562,9 +562,9 @@ impl NeoMediaSender { None } - fn get_buftime(&self) -> Option { - self.get_runtime().map(|time| self.runtime_to_buftime(time)) - } + // fn get_buftime(&self) -> Option { + // self.get_runtime().map(|time| self.runtime_to_buftime(time)) + // } async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { if self.inited && self.playing { From 2403ee2b2797810976f4dad30a4165b50b302024 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 10:15:35 +0700 Subject: [PATCH 22/64] Close client on buffers prepared Don't prepare buffers more than once across restarts Use time limited queues Don't cache dummy sender --- src/rtsp/gst/factory.rs | 9 +++++- src/rtsp/gst/sender.rs | 58 +++++++++++++++++++++++++++------- src/rtsp/gst/server.rs | 12 +++++-- src/rtsp/mod.rs | 70 +++++++++++++++++++++-------------------- 4 files changed, 101 insertions(+), 48 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 3da9c6d..88b7af9 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -230,6 +230,13 @@ fn make_queue(name: &str) -> AnyResult { fn make_queue2(name: &str) -> AnyResult { let queue = make_element("queue2", name)?; queue.set_property("use-buffering", true); + queue.set_property("max-size-bytes", 0u32); + queue.set_property("max-size-buffers", 0u32); + queue.set_property( + "max-size-time", + std::convert::TryInto::::try_into(tokio::time::Duration::from_secs(5).as_nanos()) + .unwrap_or(0), + ); Ok(queue) } @@ -449,7 +456,7 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_queue("audqueue")?; - let queue2 = make_queue2("queue2")?; + let queue2 = make_queue2("audqueue2")?; let parser = make_element("aacparse", "audparser")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 9a5ec62..fb98e89 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -174,9 +174,15 @@ impl NeoMediaSenders { } async fn handle_new_client(&mut self, mut client: NeoMediaSender) -> AnyResult<()> { - client.initialise(&self.buffer).await?; - self.client_data - .insert(self.uid.fetch_add(1, Ordering::Relaxed), client); + if client.vid.is_some() || client.aud.is_some() { + // Must have at least one type of source + // + // If not this is the dummy stream + // we don't keep a reference to that + client.initialise(&self.buffer).await?; + self.client_data + .insert(self.uid.fetch_add(1, Ordering::Relaxed), client); + } Ok(()) } @@ -347,6 +353,12 @@ pub(super) struct NeoMediaSender { prebuffered: FrameTime, } +#[allow(dead_code)] +enum JumpMethod { + MiddleIFrame, + BufferPerunit(f32), +} + impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(30); @@ -380,18 +392,33 @@ impl NeoMediaSender { let runtime = self.get_runtime().unwrap_or(0); let (fronts, backs) = buffer.buf.as_slices(); let frames = fronts.iter().chain(backs.iter()).collect::>(); - let target_time = { - // Middle iframe - let idx = frames.len().saturating_div(2); - frames.get(idx).and_then(|f| f.first()).and_then(|f| { - if let BcMedia::Iframe(data) = f { - Some(data.microseconds as FrameTime) + let jump_method = JumpMethod::BufferPerunit(0.75); + let target_time = match jump_method { + JumpMethod::MiddleIFrame => { + let idx = frames.len().saturating_div(2); + frames.get(idx).and_then(|f| f.first()).and_then(|f| { + if let BcMedia::Iframe(data) = f { + Some(data.microseconds as FrameTime) + } else { + None + } + }) + } + JumpMethod::BufferPerunit(perunit) => { + if let (Some(st), Some(et)) = (buffer.start_time(), buffer.end_time()) { + Some(((et - st) as f32 * perunit) as FrameTime + st) } else { None } - }) + } }; if let Some(target_time) = target_time { + if let Some(et) = buffer.end_time() { + if let Ok(delta) = TryInto::::try_into(et - target_time) { + debug!("Expected latency: {:?}", Duration::from_micros(delta)); + } + } + self.start_time = target_time - runtime; self.last_sent_time = target_time; debug!( @@ -516,6 +543,15 @@ impl NeoMediaSender { _ => last_sent_time, }; if frame_time > min_time && frame_time <= max_time { + if let (Some(start_time), Some(end_time)) = + (buf.start_time(), buf.end_time()) + { + debug!( + "Buffer Run Position: {}", + (frame_time - start_time) as f32 / (end_time - start_time) as f32 + ); + } + self.send_buffer(frame).await?; last_sent_time = frame_time; } else if frame_time > max_time { @@ -590,7 +626,7 @@ impl NeoMediaSender { if let (Some(buf), Some(appsrc)) = (buf, appsrc) { debug!( - "DTS: {:?}, Expected: {:?}", + "DTS: {:?}, Expected: {:?}, Position in Buffer", Duration::from_micros(runtime.try_into().unwrap_or(0)), self.get_runtime() .map(|t| Duration::from_micros(t.try_into().unwrap_or(0))) diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index b20bfc4..e9b849d 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -178,7 +178,8 @@ impl NeoRtspServer { fn clear_session_paths>(&self, paths: impl Iterator) { let paths: Vec<_> = paths.collect(); - for client in self.client_filter(None) { + self.client_filter(Some(&mut |_server, client| { + let mut close_client = false; client.session_filter(Some(&mut |_client, session| { let mut clean_up = false; session.filter(Some(&mut |_session, session_media| { @@ -198,12 +199,19 @@ impl NeoRtspServer { })); if clean_up { + close_client = true; RTSPFilterResult::Remove } else { RTSPFilterResult::Keep } })); - } + if close_client { + client.close(); + RTSPFilterResult::Remove + } else { + RTSPFilterResult::Keep + } + })); } } diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index c3ee6cd..dbd7018 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -179,48 +179,50 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi let _ = loggedin.manage().await; + let tags = loggedin.shared.get_tags(); + let rtsp_thread = loggedin.get_rtsp(); + + // Check if buffers are already perpared (from say previous connection) + let buffers_prepared = tags + .iter() + .map(|tag| rtsp_thread.buffer_ready(tag)) + .collect::>() + .filter_map(|a| a) + .all(|a| a) + .await; + + // Start pulling data from the camera let mut streaming = loggedin .stream() .await .with_context(|| format!("{}: Could not start stream", name)) .map_err(CameraFailureKind::Retry)?; - let tags = streaming.shared.get_tags(); - let rtsp_thread = streaming.get_rtsp(); - - let mut waiter = tokio::time::interval(Duration::from_micros(500)); - loop { - waiter.tick().await; - if tags - .iter() - .map(|tag| rtsp_thread.buffer_ready(tag)) - .collect::>() - .all(|f| f.unwrap_or(false)) - .await - { - break; + if !buffers_prepared { + // Wait for buffers to be prepared + let mut waiter = tokio::time::interval(Duration::from_micros(500)); + loop { + waiter.tick().await; + if tags + .iter() + .map(|tag| rtsp_thread.buffer_ready(tag)) + .collect::>() + .all(|f| f.unwrap_or(false)) + .await + { + break; + } } - } - // Clear all current media and force a reconnect - // This shoudld stop them from watching the "Stream Not Ready Thing" - tags.iter() - .map(|tag| rtsp_thread.clear_session(tag)) - .collect::>() - .collect::>() - .await; - log::info!("{}: Buffers prepared", name); - // tokio::task::spawn(async move { - // let mut inter = tokio::time::interval(tokio::time::Duration::from_secs_f32(0.01)); - // loop { - // inter.tick().await; - // log::info!( - // "Users: {:?}, {}", - // rtsp_thread.get_number_of_clients("Cammy01::Main").await, - // rtsp_thread.num_path_users(&["/Cammy01/mainStream"]) - // ); - // } - // }); + // Clear all current media and force a reconnect + // This shoudld stop them from watching the "Stream Not Ready Thing" + tags.iter() + .map(|tag| rtsp_thread.clear_session(tag)) + .collect::>() + .collect::>() + .await; + log::info!("{}: Buffers prepared", name); + } loop { // Wait for error or reason to pause From a6e11af08425d8ae9780d0687bcbc6c3933c1676 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 10:37:18 +0700 Subject: [PATCH 23/64] Use mt tokio --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 32edeb3..1fde31a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,20 +16,20 @@ anyhow = "1.0.70" byte-slice-cast = "1.2.2" clap = { version = "4.2.2", features = ["derive", "cargo"] } crossbeam-channel = "0.5.8" -env_logger = "*" +env_logger = "0.10.0" futures = "0.3.28" gstreamer = "0.20.3" gstreamer-app = "0.20.0" gstreamer-rtsp = "0.20.0" gstreamer-rtsp-server = { version = "0.20.3", features = ["v1_16"] } lazy_static = "1.4.0" -log = { version = "0.4.17" } +log = "0.4.17" neolink_core = { path = "crates/core", version = "0.5.8" } regex = "1.7.3" rumqttc = "0.20.0" serde = { version = "1.0.160", features = ["derive"] } time = "0.3.20" -tokio = { version = "1.27.0", features = ["rt", "macros", "io-util"] } +tokio = { version = "1.27.0", features = ["rt-multi-thread", "macros", "io-util"] } tokio-stream = "0.1.12" toml = "0.7.3" validator = "0.16.0" From 8a0d4a8f0367dbba3c63254aa5c6911b007cc795 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 11:24:34 +0700 Subject: [PATCH 24/64] Changes to github action to reduce reruns --- .github/workflows/build.yml | 55 +++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a5da993..9e22599 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,23 @@ name: CI on: [push, pull_request, workflow_dispatch] jobs: + pre_job: + # continue-on-error: true # Uncomment once integration is finished + runs-on: ubuntu-latest + # Map a step output to a job output + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@v5 + with: + # All of these options are optional, so you can remove them if you are happy with the defaults + concurrent_skipping: 'same_content_newer' + skip_after_successful_duplicate: 'false' + native: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' name: native runs-on: ${{ matrix.os }} strategy: @@ -69,6 +85,8 @@ jobs: name: release-${{ matrix.os }} path: "target/release/neolink*" cross: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' name: cross runs-on: ubuntu-latest container: "node:current-bullseye-slim" @@ -134,16 +152,15 @@ jobs: TARGET: ${{ matrix.target }} - uses: actions/upload-artifact@v3 with: - name: release-${{ matrix.arch }}-buster + name: release-${{ matrix.arch }}-bullseye path: "target/${{ matrix.target }}/release/neolink*" push_to_registry: name: Build Docker image - if: github.event_name == 'push' + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name runs-on: ubuntu-latest needs: - "cross" - - "native" steps: - name: Check token is set id: vars @@ -186,11 +203,13 @@ jobs: shell: bash run: | tags="" - if [[ "${GITHUB_REF}" =~ ^'refs/tags/'(.*) ]]; then - gittag="$(echo "${BASH_REMATCH[1]}" | sed -E 's|[/]|-|g')" + gittag="$(git tag --points-at HEAD)" + branch="$)git rev-parse --abbrev-ref HEAD)" + + if [[ -n "${gittag}" ]]; then tags="${REPO_NAME}:${gittag}" - elif [[ "${GITHUB_REF}" =~ ^'refs/heads/'(.*) ]]; then - branch="$(echo "${BASH_REMATCH[1]}" | sed -E 's|[/]|-|g')" + elif [[ -n "${branch}" ]]; then + branch="$(echo "${branch}" | sed -E 's|[/]|-|g')" tags="${REPO_NAME}:${branch}" if [ "${branch}" == "master" ]; then tags="${tags},${REPO_NAME}:latest" @@ -218,22 +237,22 @@ jobs: - name: Download Linux x86_64 uses: actions/download-artifact@v3 with: - name: release-amd64-buster + name: release-amd64-bullseye path: linux/amd64 - name: Download Linux armv7 uses: actions/download-artifact@v3 with: - name: release-armhf-buster + name: release-armhf-bullseye path: linux/arm/v7 - name: Download Linux arm64 uses: actions/download-artifact@v3 with: - name: release-arm64-buster + name: release-arm64-bullseye path: linux/arm64 - name: Download Linux i386 uses: actions/download-artifact@v3 with: - name: release-i386-buster + name: release-i386-bullseye path: linux/386 - name: Push to Docker Hub if: ${{ steps.vars.outputs.HAS_SECRET_TOKEN }} @@ -287,22 +306,22 @@ jobs: - name: Download Linux x86_64 uses: actions/download-artifact@v3 with: - name: release-amd64-buster - path: neolink_linux_x86_64_buster + name: release-amd64-bullseye + path: neolink_linux_x86_64_bullseye - name: Download Linux armhf uses: actions/download-artifact@v3 with: - name: release-armhf-buster + name: release-armhf-bbullseye path: neolink_linux_armhf - name: Download Linux arm64 uses: actions/download-artifact@v3 with: - name: release-arm64-buster + name: release-arm64-bullseye path: neolink_linux_arm64 - name: Download Linux i386 uses: actions/download-artifact@v3 with: - name: release-i386-buster + name: release-i386-bullseye path: neolink_linux_i386 - name: Prepare Releases run: | @@ -310,7 +329,7 @@ jobs: neolink_windows neolink_macos neolink_linux_x86_64_ubuntu - neolink_linux_x86_64_buster + neolink_linux_x86_64_bullseye neolink_linux_armhf neolink_linux_arm64 neolink_linux_i386 @@ -327,7 +346,7 @@ jobs: neolink_windows.zip neolink_macos.zip neolink_linux_x86_64_ubuntu.zip - neolink_linux_x86_64_buster.zip + neolink_linux_x86_64_bullseye.zip neolink_linux_armhf.zip neolink_linux_arm64.zip neolink_linux_i386.zip From e4bcf9bd239f998951e1e54d536444aa1c4b65e5 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 11:27:36 +0700 Subject: [PATCH 25/64] Fixes to docker build tagging --- .github/workflows/build.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e22599..8846169 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -204,13 +204,18 @@ jobs: run: | tags="" gittag="$(git tag --points-at HEAD)" - branch="$)git rev-parse --abbrev-ref HEAD)" + branch="$(git rev-parse --abbrev-ref HEAD)" if [[ -n "${gittag}" ]]; then tags="${REPO_NAME}:${gittag}" - elif [[ -n "${branch}" ]]; then + fi + if [[ -n "${branch}" ]]; then branch="$(echo "${branch}" | sed -E 's|[/]|-|g')" - tags="${REPO_NAME}:${branch}" + if [[ -z "${tags}" ]]; then + tags="${REPO_NAME}:${branch}" + else + tags="${tags},${REPO_NAME}:${branch}" + fi if [ "${branch}" == "master" ]; then tags="${tags},${REPO_NAME}:latest" fi From c666a4c3bdd99105e5c201acada48a8dd03d5aad Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 11:30:20 +0700 Subject: [PATCH 26/64] Add skip build to style checks too --- .github/workflows/dockerprune.yml | 16 ++++++++++++++++ .github/workflows/style_checks.yml | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.github/workflows/dockerprune.yml b/.github/workflows/dockerprune.yml index 0ede5a8..d932958 100644 --- a/.github/workflows/dockerprune.yml +++ b/.github/workflows/dockerprune.yml @@ -3,7 +3,23 @@ name: DockerHub on: [push, pull_request, workflow_dispatch] jobs: + pre_job: + # continue-on-error: true # Uncomment once integration is finished + runs-on: ubuntu-latest + # Map a step output to a job output + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@v5 + with: + # All of these options are optional, so you can remove them if you are happy with the defaults + concurrent_skipping: 'same_content_newer' + skip_after_successful_duplicate: 'false' + native: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' name: native runs-on: ubuntu-latest steps: diff --git a/.github/workflows/style_checks.yml b/.github/workflows/style_checks.yml index 300e3bb..d56488c 100644 --- a/.github/workflows/style_checks.yml +++ b/.github/workflows/style_checks.yml @@ -3,7 +3,23 @@ name: Style on: [push, pull_request] jobs: + pre_job: + # continue-on-error: true # Uncomment once integration is finished + runs-on: ubuntu-latest + # Map a step output to a job output + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@v5 + with: + # All of these options are optional, so you can remove them if you are happy with the defaults + concurrent_skipping: 'same_content_newer' + skip_after_successful_duplicate: 'false' + check_clippy: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' name: Clippy runs-on: ubuntu-22.04 steps: @@ -21,6 +37,8 @@ jobs: run: cargo +nightly clippy --workspace --all-targets --all-features check_fmt: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' name: Rust-fmt runs-on: ubuntu-latest steps: @@ -34,6 +52,8 @@ jobs: cargo +nightly fmt --all -- --check check_lua: + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' name: Luacheck runs-on: ubuntu-latest steps: From ae3f9365b16dcdbd7f841e76a26d2a1ad1a1cd2c Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 11:51:05 +0700 Subject: [PATCH 27/64] Skip printing of pkg_config join tags in array --- .github/workflows/build.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8846169..65b693b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -78,7 +78,7 @@ jobs: echo "PATH=${PATH}" echo "GSTREAMER_1_0_ROOT_MSVC_X86_64=${GSTREAMER_1_0_ROOT_MSVC_X86_64}" echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" - pkg-config --variable pc_path pkg-config + # pkg-config --variable pc_path pkg-config cargo build --release --all-features - uses: actions/upload-artifact@v3 with: @@ -202,25 +202,29 @@ jobs: id: tags shell: bash run: | - tags="" + tags=( ) gittag="$(git tag --points-at HEAD)" branch="$(git rev-parse --abbrev-ref HEAD)" if [[ -n "${gittag}" ]]; then - tags="${REPO_NAME}:${gittag}" + tags=( "${tags[@]}" "${REPO_NAME}:${gittag}" ) fi if [[ -n "${branch}" ]]; then branch="$(echo "${branch}" | sed -E 's|[/]|-|g')" - if [[ -z "${tags}" ]]; then - tags="${REPO_NAME}:${branch}" - else - tags="${tags},${REPO_NAME}:${branch}" - fi + tags=( "${tags[@]}" "${REPO_NAME}:${branch}" ) if [ "${branch}" == "master" ]; then - tags="${tags},${REPO_NAME}:latest" + tags=( "${tags[@]}" "${tags},${REPO_NAME}:latest" ) fi fi - echo "TAGS=${tags}" >> "${GITHUB_OUTPUT}" + function join_by { + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi + } + tagstr="$(join_by "," "${tags[@]}")" + echo "TAGS=${tagstr}" + echo "TAGS=${tagstr}" >> "${GITHUB_OUTPUT}" env: REPO_NAME: ${{ steps.string_repo.outputs.lowercase }} - name: Install latest rust From 29b6ff8c24e1dac023ce1a30357e2cd0ecc80b24 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 13:19:59 +0700 Subject: [PATCH 28/64] Timed polling --- Cargo.toml | 2 +- src/main.rs | 2 ++ src/rtsp/gst/factory.rs | 4 ++-- src/rtsp/gst/sender.rs | 23 +++++++++++++---------- src/rtsp/gst/server.rs | 24 ++++++++++++++++-------- src/rtsp/mod.rs | 8 +++++--- src/timed_poll.rs | 37 +++++++++++++++++++++++++++++++++++++ 7 files changed, 76 insertions(+), 24 deletions(-) create mode 100644 src/timed_poll.rs diff --git a/Cargo.toml b/Cargo.toml index 1fde31a..45938a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ regex = "1.7.3" rumqttc = "0.20.0" serde = { version = "1.0.160", features = ["derive"] } time = "0.3.20" -tokio = { version = "1.27.0", features = ["rt-multi-thread", "macros", "io-util"] } +tokio = { version = "1.27.0", features = ["rt", "macros", "io-util"] } # rt-multi-thread tokio-stream = "0.1.12" toml = "0.7.3" validator = "0.16.0" diff --git a/src/main.rs b/src/main.rs index 9a2b01d..a3a1db9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,10 +38,12 @@ mod reboot; mod rtsp; mod statusled; mod talk; +mod timed_poll; mod utils; use cmdline::{Command, Opt}; use config::Config; +pub(crate) use timed_poll::TimedPoll; #[tokio::main] async fn main() -> Result<()> { diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 88b7af9..513b056 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -143,7 +143,7 @@ impl Default for NeoMediaFactoryImpl { ReceiverStream::new(datarx), ReceiverStream::new(rx_clientsender), ); - threads.spawn(async move { + threads.spawn(crate::TimedPoll::new("MediaSend", async move { loop { tokio::task::yield_now().await; match sender.run().await { @@ -157,7 +157,7 @@ impl Default for NeoMediaFactoryImpl { } unreachable!(); // Ok(()) - }); + })); Self { sender: datasender, diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index fb98e89..d17b5b6 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -314,13 +314,13 @@ impl NeoMediaSenders { } pub(super) async fn run(&mut self) -> AnyResult<()> { - let mut interval = tokio::time::interval(Duration::from_micros(LATENCY as u64 / 3)); + // let mut interval = tokio::time::interval(Duration::from_micros(LATENCY as u64 / 3)); loop { tokio::task::yield_now().await; tokio::select! { - _ = interval.tick() => { - self.handle_buffer().await - }, + // _ = interval.tick() => { + // self.handle_buffer().await + // }, Some(v) = self.data_source.next() => { self.handle_new_data(v).await }, @@ -643,12 +643,15 @@ impl NeoMediaSender { } // debug!("Buffer pushed"); let thread_appsrc = appsrc.clone(); // GObjects are refcounted - let res = tokio::task::spawn_blocking(move || { - thread_appsrc - .push_buffer(gst_buf.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) - }) + let res = crate::TimedPoll::new( + "SendBuffer", + tokio::task::spawn_blocking(move || { + thread_appsrc + .push_buffer(gst_buf.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }), + ) .await; match &res { Err(e) => { diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index e9b849d..37d135c 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -96,24 +96,30 @@ impl NeoRtspServer { main_loop_thread.run(); AnyResult::Ok(()) }); - self.imp() - .threads - .write() + crate::TimedPoll::new("ServerThreadsLock", self.imp().threads.write()) .await - .spawn(async move { handle.await? }); - self.imp().main_loop.write().await.replace(main_loop); + .spawn(crate::TimedPoll::new( + "ServerRun", + async move { handle.await? }, + )); + crate::TimedPoll::new("ServerMainLoop", self.imp().main_loop.write()) + .await + .replace(main_loop); Ok(()) } pub(crate) async fn quit(&self) -> AnyResult<()> { - if let Some(main_loop) = self.imp().main_loop.read().await.as_ref() { + if let Some(main_loop) = crate::TimedPoll::new("MainLoopQuit", self.imp().main_loop.read()) + .await + .as_ref() + { main_loop.quit(); } Ok(()) } pub(crate) async fn join(&self) -> AnyResult<()> { - let mut threads = self.imp().threads.write().await; + let mut threads = crate::TimedPoll::new("MainLoopJoin", self.imp().threads.write()).await; while let Some(thread) = threads.join_next().await { thread??; } @@ -251,7 +257,9 @@ impl NeoRtspServerImpl { self.threads .write() .await - .spawn(async move { thread_media.join().await }); + .spawn(crate::TimedPoll::new("RtspJoin", async move { + thread_media.join().await + })); vac.insert(FactoryData { factory: media, paths: Default::default(), diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index dbd7018..198a8b3 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -92,7 +92,7 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { let mut set = tokio::task::JoinSet::new(); for mut camera in cameras.drain(..) { // Spawn each camera controller in it's own thread - set.spawn(async move { + set.spawn(crate::TimedPoll::new("CameraMain", async move { let shared = camera.shared.clone(); let name = camera.get_name(); let mut backoff = Duration::from_micros(125); @@ -122,7 +122,7 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { } } Ok(()) - }); + })); } info!( "Starting RTSP Server at {}:{}", @@ -133,7 +133,9 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { let bind_port = config.bind_port; rtsp.run(&bind_addr, bind_port).await?; let thread_rtsp = rtsp.clone(); - set.spawn(async move { thread_rtsp.join().await }); + set.spawn(crate::TimedPoll::new("RTSP Join", async move { + thread_rtsp.join().await + })); while let Some(joined) = set.join_next().await { match &joined { diff --git a/src/timed_poll.rs b/src/timed_poll.rs new file mode 100644 index 0000000..9d82edc --- /dev/null +++ b/src/timed_poll.rs @@ -0,0 +1,37 @@ +//! For debugging polls +//! +//! https://users.rust-lang.org/t/how-do-i-debug-a-tokio-hang-tokio-0-3-3/51235/3 +//! +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +pub(crate) struct TimedPoll { + name: String, + future: F, +} + +impl TimedPoll { + pub fn new>(name: T, future: F) -> Self { + Self { + name: name.into(), + future, + } + } +} + +impl Future for TimedPoll { + type Output = F::Output; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let name = self.name.clone(); + // log::debug!("{}: Start poll", name); + let before = std::time::Instant::now(); + let future = unsafe { Pin::map_unchecked_mut(self, |me| &mut me.future) }; + let res = future.poll(cx); + let elapsed = before.elapsed(); + if elapsed > std::time::Duration::from_micros(500) { + log::debug!("{}: Stop poll: {:?}", name, before.elapsed()); + } + res + } +} From 0b922aeeb5fbd353e68ad64ca5d51fa57807d0a8 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 13:29:47 +0700 Subject: [PATCH 29/64] Always timed polls --- src/timed_poll.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/timed_poll.rs b/src/timed_poll.rs index 9d82edc..afb42f3 100644 --- a/src/timed_poll.rs +++ b/src/timed_poll.rs @@ -24,14 +24,14 @@ impl Future for TimedPoll { type Output = F::Output; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let name = self.name.clone(); - // log::debug!("{}: Start poll", name); + log::debug!("{}: Start poll", name); let before = std::time::Instant::now(); let future = unsafe { Pin::map_unchecked_mut(self, |me| &mut me.future) }; let res = future.poll(cx); let elapsed = before.elapsed(); - if elapsed > std::time::Duration::from_micros(500) { - log::debug!("{}: Stop poll: {:?}", name, before.elapsed()); - } + // if elapsed > std::time::Duration::from_micros(500) { + log::debug!("{}: Stop poll: {:?}", name, elapsed); + // } res } } From 359597131ecd9da79558e041e2d055f87cc7be61 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 14:43:26 +0700 Subject: [PATCH 30/64] Tokio console --- .github/workflows/build.yml | 1 + Cargo.lock | 493 +++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + src/main.rs | 13 +- src/rtsp/gst/factory.rs | 4 +- src/rtsp/gst/sender.rs | 15 +- src/rtsp/gst/server.rs | 24 +- src/rtsp/mod.rs | 8 +- src/timed_poll.rs | 37 --- 9 files changed, 524 insertions(+), 72 deletions(-) delete mode 100644 src/timed_poll.rs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65b693b..0c595c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -150,6 +150,7 @@ jobs: PKG_CONFIG_ALLOW_CROSS: 1 PKG_CONFIG_LIBDIR: /usr/lib/${{ matrix.pkgconfig }}/pkgconfig TARGET: ${{ matrix.target }} + RUSTFLAGS: "--cfg tokio_unstable" - uses: actions/upload-artifact@v3 with: name: release-${{ matrix.arch }}-bullseye diff --git a/Cargo.lock b/Cargo.lock index 970afd5..ef39f9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aes" version = "0.8.2" @@ -94,6 +100,39 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "atomic_refcell" version = "0.1.9" @@ -106,6 +145,57 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.0" @@ -237,6 +327,42 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "console-api" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57ff02e8ad8e06ab9731d5dc72dc23bef9200778eae1a89d555d8c42e5d4a86" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a3a81dfaf6b66bce5d159eddae701e3a002f194d378cbf7be5f053c281d9be" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "cookie-factory" version = "0.3.2" @@ -342,6 +468,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + [[package]] name = "env_logger" version = "0.10.0" @@ -399,6 +531,16 @@ dependencies = [ "instant", ] +[[package]] +name = "flate2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.10.14" @@ -412,6 +554,12 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -860,6 +1008,25 @@ dependencies = [ "system-deps", ] +[[package]] +name = "h2" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -869,6 +1036,19 @@ dependencies = [ "ahash", ] +[[package]] +name = "hdrhistogram" +version = "7.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +dependencies = [ + "base64 0.13.1", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.4.1" @@ -896,12 +1076,82 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "848ec2dd093df965a34b434580d94852197fc83feac5b2c1962399bbf2cb4f0b" +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "idna" version = "0.2.3" @@ -986,6 +1236,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -1050,12 +1309,27 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + [[package]] name = "md5" version = "0.7.0" @@ -1068,12 +1342,27 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.6" @@ -1118,6 +1407,7 @@ dependencies = [ "anyhow", "byte-slice-cast", "clap", + "console-subscriber", "crossbeam-channel", "env_logger", "futures", @@ -1401,6 +1691,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.26" @@ -1469,6 +1791,15 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -1578,7 +1909,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] @@ -1681,6 +2012,15 @@ dependencies = [ "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "shell-words" version = "1.1.0" @@ -1791,6 +2131,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synstructure" version = "0.12.6" @@ -1869,6 +2215,16 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "time" version = "0.3.20" @@ -1928,9 +2284,20 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.45.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.0.0" @@ -2017,6 +2384,70 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.13.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -2025,9 +2456,21 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "tracing-core" version = "0.1.30" @@ -2035,8 +2478,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", + "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "matchers", + "once_cell", + "regex", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.16.0" @@ -2156,6 +2631,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "version-compare" version = "0.1.1" @@ -2168,6 +2649,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 45938a9..39e70d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ anyhow = "1.0.70" byte-slice-cast = "1.2.2" clap = { version = "4.2.2", features = ["derive", "cargo"] } +console-subscriber = "0.1.8" crossbeam-channel = "0.5.8" env_logger = "0.10.0" futures = "0.3.28" diff --git a/src/main.rs b/src/main.rs index a3a1db9..47c02aa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,15 +38,24 @@ mod reboot; mod rtsp; mod statusled; mod talk; -mod timed_poll; mod utils; use cmdline::{Command, Opt}; use config::Config; -pub(crate) use timed_poll::TimedPoll; +use console_subscriber as _; + +#[cfg(tokio_unstable)] +fn tokio_console_enable() { + info!("Tokio Console Enabled"); + console_subscriber::init(); +} + +#[cfg(not(tokio_unstable))] +fn tokio_console_enable() {} #[tokio::main] async fn main() -> Result<()> { + tokio_console_enable(); env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); info!( diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 513b056..88b7af9 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -143,7 +143,7 @@ impl Default for NeoMediaFactoryImpl { ReceiverStream::new(datarx), ReceiverStream::new(rx_clientsender), ); - threads.spawn(crate::TimedPoll::new("MediaSend", async move { + threads.spawn(async move { loop { tokio::task::yield_now().await; match sender.run().await { @@ -157,7 +157,7 @@ impl Default for NeoMediaFactoryImpl { } unreachable!(); // Ok(()) - })); + }); Self { sender: datasender, diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index d17b5b6..38833cb 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -643,15 +643,12 @@ impl NeoMediaSender { } // debug!("Buffer pushed"); let thread_appsrc = appsrc.clone(); // GObjects are refcounted - let res = crate::TimedPoll::new( - "SendBuffer", - tokio::task::spawn_blocking(move || { - thread_appsrc - .push_buffer(gst_buf.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) - }), - ) + let res = tokio::task::spawn_blocking(move || { + thread_appsrc + .push_buffer(gst_buf.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) .await; match &res { Err(e) => { diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 37d135c..e9b849d 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -96,30 +96,24 @@ impl NeoRtspServer { main_loop_thread.run(); AnyResult::Ok(()) }); - crate::TimedPoll::new("ServerThreadsLock", self.imp().threads.write()) + self.imp() + .threads + .write() .await - .spawn(crate::TimedPoll::new( - "ServerRun", - async move { handle.await? }, - )); - crate::TimedPoll::new("ServerMainLoop", self.imp().main_loop.write()) - .await - .replace(main_loop); + .spawn(async move { handle.await? }); + self.imp().main_loop.write().await.replace(main_loop); Ok(()) } pub(crate) async fn quit(&self) -> AnyResult<()> { - if let Some(main_loop) = crate::TimedPoll::new("MainLoopQuit", self.imp().main_loop.read()) - .await - .as_ref() - { + if let Some(main_loop) = self.imp().main_loop.read().await.as_ref() { main_loop.quit(); } Ok(()) } pub(crate) async fn join(&self) -> AnyResult<()> { - let mut threads = crate::TimedPoll::new("MainLoopJoin", self.imp().threads.write()).await; + let mut threads = self.imp().threads.write().await; while let Some(thread) = threads.join_next().await { thread??; } @@ -257,9 +251,7 @@ impl NeoRtspServerImpl { self.threads .write() .await - .spawn(crate::TimedPoll::new("RtspJoin", async move { - thread_media.join().await - })); + .spawn(async move { thread_media.join().await }); vac.insert(FactoryData { factory: media, paths: Default::default(), diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 198a8b3..dbd7018 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -92,7 +92,7 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { let mut set = tokio::task::JoinSet::new(); for mut camera in cameras.drain(..) { // Spawn each camera controller in it's own thread - set.spawn(crate::TimedPoll::new("CameraMain", async move { + set.spawn(async move { let shared = camera.shared.clone(); let name = camera.get_name(); let mut backoff = Duration::from_micros(125); @@ -122,7 +122,7 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { } } Ok(()) - })); + }); } info!( "Starting RTSP Server at {}:{}", @@ -133,9 +133,7 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { let bind_port = config.bind_port; rtsp.run(&bind_addr, bind_port).await?; let thread_rtsp = rtsp.clone(); - set.spawn(crate::TimedPoll::new("RTSP Join", async move { - thread_rtsp.join().await - })); + set.spawn(async move { thread_rtsp.join().await }); while let Some(joined) = set.join_next().await { match &joined { diff --git a/src/timed_poll.rs b/src/timed_poll.rs deleted file mode 100644 index afb42f3..0000000 --- a/src/timed_poll.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! For debugging polls -//! -//! https://users.rust-lang.org/t/how-do-i-debug-a-tokio-hang-tokio-0-3-3/51235/3 -//! -use std::future::Future; -use std::pin::Pin; -use std::task::{Context, Poll}; - -pub(crate) struct TimedPoll { - name: String, - future: F, -} - -impl TimedPoll { - pub fn new>(name: T, future: F) -> Self { - Self { - name: name.into(), - future, - } - } -} - -impl Future for TimedPoll { - type Output = F::Output; - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let name = self.name.clone(); - log::debug!("{}: Start poll", name); - let before = std::time::Instant::now(); - let future = unsafe { Pin::map_unchecked_mut(self, |me| &mut me.future) }; - let res = future.poll(cx); - let elapsed = before.elapsed(); - // if elapsed > std::time::Duration::from_micros(500) { - log::debug!("{}: Stop poll: {:?}", name, elapsed); - // } - res - } -} From fd3282efe2deea947793b3cf22e3b547af0f072f Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 15:15:37 +0700 Subject: [PATCH 31/64] Add rustflag directly --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c595c3..30a68ff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,13 +144,12 @@ jobs: TARGET: ${{ matrix.target }} - name: Build run: | - cargo build --release --all-features --target=${TARGET} + RUSTFLAGS="--cfg tokio_unstable" cargo build --release --all-features --target=${TARGET} env: # Retarget pkg-config as described in https://www.freedesktop.org/wiki/Software/pkg-config/CrossCompileProposal/ PKG_CONFIG_ALLOW_CROSS: 1 PKG_CONFIG_LIBDIR: /usr/lib/${{ matrix.pkgconfig }}/pkgconfig TARGET: ${{ matrix.target }} - RUSTFLAGS: "--cfg tokio_unstable" - uses: actions/upload-artifact@v3 with: name: release-${{ matrix.arch }}-bullseye From 929dac374bcb84a5477e034f79ecd09f556187bd Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 15:21:49 +0700 Subject: [PATCH 32/64] Nightly --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30a68ff..25e3a9c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,6 +136,7 @@ jobs: - name: Install rustup run: | curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --quiet -y + rustup toolchain install nightly echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" - name: Install ${{ matrix.arch }} Rust toolchain run: | @@ -144,12 +145,13 @@ jobs: TARGET: ${{ matrix.target }} - name: Build run: | - RUSTFLAGS="--cfg tokio_unstable" cargo build --release --all-features --target=${TARGET} + cargo +nightly build --verbose --release --all-features --target=${TARGET} env: # Retarget pkg-config as described in https://www.freedesktop.org/wiki/Software/pkg-config/CrossCompileProposal/ PKG_CONFIG_ALLOW_CROSS: 1 PKG_CONFIG_LIBDIR: /usr/lib/${{ matrix.pkgconfig }}/pkgconfig TARGET: ${{ matrix.target }} + RUSTFLAGS: "--cfg tokio_unstable" - uses: actions/upload-artifact@v3 with: name: release-${{ matrix.arch }}-bullseye From f60de36bae12221808f92e1d8665da03f2500589 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 15:27:57 +0700 Subject: [PATCH 33/64] Source /.cargo/env --- .cargo/config | 5 ++++- .github/workflows/build.yml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.cargo/config b/.cargo/config index 2282ebe..e83f9f7 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,5 +1,5 @@ [target.x86_64-pc-windows-msvc] -rustflags = ["-C", "target-feature=+crt-static"] +rustflags = ["-C", "target-feature=+crt-static", "--cfg", "tokio_unstable"] # TODO not good as a default [target.armv7-unknown-linux-gnueabihf] @@ -10,3 +10,6 @@ linker = "aarch64-linux-gnu-gcc" [target.i686-unknown-linux-gnu] linker = "i686-linux-gnu-gcc" + +[build] +rustflags = ["--cfg", "tokio_unstable"] \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25e3a9c..61f0ef9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,6 +136,7 @@ jobs: - name: Install rustup run: | curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --quiet -y + source "$HOME/.cargo/env" rustup toolchain install nightly echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" - name: Install ${{ matrix.arch }} Rust toolchain From c20b944411704901c0e5e136818f6128c2fd9bb9 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 15:31:28 +0700 Subject: [PATCH 34/64] Sh it's sh --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61f0ef9..7b7728c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,7 +136,7 @@ jobs: - name: Install rustup run: | curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --quiet -y - source "$HOME/.cargo/env" + . "$HOME/.cargo/env" rustup toolchain install nightly echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" - name: Install ${{ matrix.arch }} Rust toolchain From 26d894222a1af122cdad31f1ced81993b87bbbe2 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 16:00:25 +0700 Subject: [PATCH 35/64] Build this way --- .github/workflows/build.yml | 5 +---- Cargo.toml | 2 +- crates/core/Cargo.toml | 2 +- src/main.rs | 6 ++++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b7728c..a842065 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,8 +136,6 @@ jobs: - name: Install rustup run: | curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --quiet -y - . "$HOME/.cargo/env" - rustup toolchain install nightly echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" - name: Install ${{ matrix.arch }} Rust toolchain run: | @@ -146,13 +144,12 @@ jobs: TARGET: ${{ matrix.target }} - name: Build run: | - cargo +nightly build --verbose --release --all-features --target=${TARGET} + cargo build --verbose --release --all-features --target=${TARGET} env: # Retarget pkg-config as described in https://www.freedesktop.org/wiki/Software/pkg-config/CrossCompileProposal/ PKG_CONFIG_ALLOW_CROSS: 1 PKG_CONFIG_LIBDIR: /usr/lib/${{ matrix.pkgconfig }}/pkgconfig TARGET: ${{ matrix.target }} - RUSTFLAGS: "--cfg tokio_unstable" - uses: actions/upload-artifact@v3 with: name: release-${{ matrix.arch }}-bullseye diff --git a/Cargo.toml b/Cargo.toml index 39e70d4..aa751d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ regex = "1.7.3" rumqttc = "0.20.0" serde = { version = "1.0.160", features = ["derive"] } time = "0.3.20" -tokio = { version = "1.27.0", features = ["rt", "macros", "io-util"] } # rt-multi-thread +tokio = { version = "1.27.0", features = ["rt-multi-thread", "macros", "io-util", "tracing"] } tokio-stream = "0.1.12" toml = "0.7.3" validator = "0.16.0" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 9a6ecb4..079047c 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -28,7 +28,7 @@ serde = { version = "1.0.106", features = ["derive"] } time = { version = "0.3.20" , features = [ "macros", "parsing", "local-offset" ]} tokio = { version = "1.27.0", features = ["full"] } tokio-stream = { version = "0.1.12", features = ["sync", "time", "net"] } -tokio-util = { version = "0.7.7", features = ["full"] } +tokio-util = { version = "0.7.7", features = ["full", "tracing"] } yaserde = "0.8.0" yaserde_derive = "0.8.0" diff --git a/src/main.rs b/src/main.rs index 47c02aa..ae2c2ec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,12 +51,14 @@ fn tokio_console_enable() { } #[cfg(not(tokio_unstable))] -fn tokio_console_enable() {} +fn tokio_console_enable() { + debug!("Tokio Console Disabled"); +} #[tokio::main] async fn main() -> Result<()> { - tokio_console_enable(); env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + tokio_console_enable(); info!( "Neolink {} {}", From e2602b5c1774e56e0a776d3c7add9ac5c87b3688 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 18:17:05 +0700 Subject: [PATCH 36/64] Add tokio console to the config --- src/config.rs | 7 +++++++ src/main.rs | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 6298bc3..d2f8f69 100644 --- a/src/config.rs +++ b/src/config.rs @@ -27,6 +27,9 @@ pub(crate) struct Config { #[serde(default = "default_bind_port")] pub(crate) bind_port: u16, + #[serde(default = "default_tokio_console")] + pub(crate) tokio_console: bool, + #[serde(default = "default_certificate")] pub(crate) certificate: Option, @@ -192,6 +195,10 @@ fn default_tls_client_auth() -> String { "none".to_string() } +fn default_tokio_console() -> bool { + false +} + fn default_channel_id() -> u8 { 0 } diff --git a/src/main.rs b/src/main.rs index ae2c2ec..efa9a3c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,7 +58,6 @@ fn tokio_console_enable() { #[tokio::main] async fn main() -> Result<()> { env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); - tokio_console_enable(); info!( "Neolink {} {}", @@ -79,6 +78,10 @@ async fn main() -> Result<()> { .validate() .with_context(|| format!("Failed to validate the {:?} config file", conf_path))?; + if config.tokio_console { + tokio_console_enable(); + } + match opt.cmd { None => { warn!( From 386fec422073f08b08ef567c9695e84204fd00cf Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 3 May 2023 20:26:13 +0700 Subject: [PATCH 37/64] Allow decode Bc frame over packet boundary --- crates/core/src/bc/codex.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/core/src/bc/codex.rs b/crates/core/src/bc/codex.rs index 203a143..6ad3525 100644 --- a/crates/core/src/bc/codex.rs +++ b/crates/core/src/bc/codex.rs @@ -48,19 +48,23 @@ impl Decoder for BcCodex { type Error = Error; fn decode_eof(&mut self, buf: &mut BytesMut) -> Result> { + // match self.decode(buf)? { + // Some(frame) => Ok(Some(frame)), + // None => { + // if buf.is_empty() { + // Ok(None) + // } else { + // Err(io::Error::new( + // io::ErrorKind::Other, + // format!("bytes remaining on BC stream: {:X?}", buf), + // ) + // .into()) + // } + // } + // } match self.decode(buf)? { Some(frame) => Ok(Some(frame)), - None => { - if buf.is_empty() { - Ok(None) - } else { - Err(io::Error::new( - io::ErrorKind::Other, - format!("bytes remaining on BC stream: {:X?}", buf), - ) - .into()) - } - } + None => Ok(None), } } From b07aa2ddfe1cc02f27933bd2c640eb7d0ae4f46d Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Thu, 4 May 2023 09:10:52 +0700 Subject: [PATCH 38/64] Jump to live if more than 15s difference --- crates/core/src/bc/codex.rs | 32 ++++++++++----------- src/rtsp/gst/sender.rs | 56 ++++++++++++++++++------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/crates/core/src/bc/codex.rs b/crates/core/src/bc/codex.rs index 6ad3525..efba7e6 100644 --- a/crates/core/src/bc/codex.rs +++ b/crates/core/src/bc/codex.rs @@ -7,7 +7,7 @@ use crate::bc::model::*; use crate::bc::xml::*; use crate::{Credentials, Error, Result}; use bytes::BytesMut; -use std::io; +use nom::AsBytes; use tokio_util::codec::{Decoder, Encoder}; pub(crate) struct BcCodex { @@ -48,24 +48,24 @@ impl Decoder for BcCodex { type Error = Error; fn decode_eof(&mut self, buf: &mut BytesMut) -> Result> { - // match self.decode(buf)? { - // Some(frame) => Ok(Some(frame)), - // None => { - // if buf.is_empty() { - // Ok(None) - // } else { - // Err(io::Error::new( - // io::ErrorKind::Other, - // format!("bytes remaining on BC stream: {:X?}", buf), - // ) - // .into()) - // } - // } - // } match self.decode(buf)? { Some(frame) => Ok(Some(frame)), - None => Ok(None), + None => { + if buf.is_empty() { + Ok(None) + } else { + log::debug!("bytes remaining on BC stream: {:X?}", buf.as_bytes()); + // Right after this we seem to get an issue with the camera dropping us + // Needs probing + // \xf0\xde\xbc\n\x03\0\0\0@\x9c\0\0\0\x01\x04\0\xc8\0\0\0\0\0\0\0\t\x11\xc6\xa0\x12\xdb\xf7:E\x96* + Ok(None) + } + } } + // match self.decode(buf)? { + // Some(frame) => Ok(Some(frame)), + // None => Ok(None), + // } } fn decode(&mut self, src: &mut BytesMut) -> Result> { diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 38833cb..eae7765 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -296,11 +296,11 @@ impl NeoMediaSenders { occ.remove(); continue; } - // if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await { - // debug!("Could not handle jump to live: {:?}", e); - // occ.remove(); - // continue; - // } + if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await { + debug!("Could not handle jump to live: {:?}", e); + occ.remove(); + continue; + } if let Err(e) = occ.get_mut().process_buffer(&self.buffer).await { debug!("Could not send client data: {:?}", e); occ.remove(); @@ -563,26 +563,26 @@ impl NeoMediaSender { Ok(last_sent_time) } - // async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - // if self.inited { - // if let (Some(buffer_start), Some(buffer_end)) = (buffer.start_time(), buffer.end_time()) - // { - // let runtime = self.get_buftime().unwrap_or(self.last_sent_time); - // if runtime < buffer_start.saturating_sub(LATENCY * 2) - // || runtime > buffer_end.saturating_add(LATENCY * 2) - // { - // debug!( - // "Outside buffer jumping to live: {:?} < {:?} < {:?}", - // Duration::from_micros(buffer_start.try_into().unwrap_or(0)), - // Duration::from_micros(runtime.try_into().unwrap_or(0)), - // Duration::from_micros(buffer_end.try_into().unwrap_or(0)) - // ); - // self.jump_to_live(buffer).await?; - // } - // } - // } - // Ok(()) - // } + async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if self.inited { + if let (Some(buffer_start), Some(buffer_end)) = (buffer.start_time(), buffer.end_time()) + { + let runtime = self.get_buftime().unwrap_or(self.last_sent_time); + if runtime < buffer_start.saturating_sub(15 * SECS) + || runtime > buffer_end.saturating_add(15 * SECS) + { + debug!( + "Outside buffer jumping to live: {:?} < {:?} < {:?}", + Duration::from_micros(buffer_start.try_into().unwrap_or(0)), + Duration::from_micros(runtime.try_into().unwrap_or(0)), + Duration::from_micros(buffer_end.try_into().unwrap_or(0)) + ); + self.jump_to_live(buffer).await?; + } + } + } + Ok(()) + } fn get_runtime(&self) -> Option { if let Some(appsrc) = self.vid.as_ref() { @@ -598,9 +598,9 @@ impl NeoMediaSender { None } - // fn get_buftime(&self) -> Option { - // self.get_runtime().map(|time| self.runtime_to_buftime(time)) - // } + fn get_buftime(&self) -> Option { + self.get_runtime().map(|time| self.runtime_to_buftime(time)) + } async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { if self.inited && self.playing { From 262d2b2ed49824a839fac7b3e1ab104d84895e72 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Fri, 5 May 2023 08:09:35 +0700 Subject: [PATCH 39/64] Lockless subscription --- crates/core/src/bc/codex.rs | 11 +- .../core/src/bc_protocol/connection/bcconn.rs | 258 ++++++++++-------- crates/core/src/bc_protocol/errors.rs | 6 + src/rtsp/gst/sender.rs | 114 ++++++-- 4 files changed, 255 insertions(+), 134 deletions(-) diff --git a/crates/core/src/bc/codex.rs b/crates/core/src/bc/codex.rs index efba7e6..9a44438 100644 --- a/crates/core/src/bc/codex.rs +++ b/crates/core/src/bc/codex.rs @@ -54,10 +54,17 @@ impl Decoder for BcCodex { if buf.is_empty() { Ok(None) } else { - log::debug!("bytes remaining on BC stream: {:X?}", buf.as_bytes()); + log::debug!( + "bytes remaining on BC stream: {:X?}", + buf.as_bytes().chunks(25).next() + ); // Right after this we seem to get an issue with the camera dropping us // Needs probing - // \xf0\xde\xbc\n\x03\0\0\0@\x9c\0\0\0\x01\x04\0\xc8\0\0\0\0\0\0\0\t\x11\xc6\xa0\x12\xdb\xf7:E\x96* + // F0, DE, BC, A, 3, 0, 0, 0, 88, 6, 0, 0, 0, 1, 4, 0, C8, 0, 0, 0, 0, 0, 0, 0, 30, 31, 64, 63, 48, + // 32, 36, 34, 6A, 6, 0, 0, 0, 0, 0, 0, D8, F5, C7, 86, 56, 0, 0, 0, 0, 0, 0, 1, 21, 9A, FC, 22, 7F, 6, AE, F6, 15, FF, E5, 71, 4, 2F, 24, 61, 15, 96, F0, BF, 83, DE, 10, BE, B4, 2E, 3 + // 9, 76, 56, 92, 7E, 48, 79, 20, 9A, DC, 1B, BB, AC, 22, 60, 5C, 72, B5, 3D, 8, E0, 34, 43, 3F, 2E, A7, 81, A8, 11, 75, 7F, 58, 3E, 8, 54, 91, 43, 21, EC, 6B, D6, 1A, D5, CB, D5, 6C, + // 8C, 2E, 6E, A3, 51, C3, A4, F0, CF, 2B, 61, 81, D0, 1C, A1, 76, EE, BF, 7A, D5, D8, D1, C4, D, B0, 45, EE, 3E, 93, 9A, CE, 5F, AB, 75, 55, AC, 9D, 66, DE, 23, 6D, 5F, 25, 57, DA, F5 + //, E, 7F, 8D, 30, A7, 66, C4, 60, 76, 41, D0, 6A, 23, E, A9, C5, 51, EE, F6, DD, 19, E7, A8, 96, 9F, 2B, AF, 31, 90, 9D, FC, BE Ok(None) } } diff --git a/crates/core/src/bc_protocol/connection/bcconn.rs b/crates/core/src/bc_protocol/connection/bcconn.rs index 770a093..2512770 100644 --- a/crates/core/src/bc_protocol/connection/bcconn.rs +++ b/crates/core/src/bc_protocol/connection/bcconn.rs @@ -5,10 +5,10 @@ use futures::stream::{Stream, StreamExt}; use log::*; use std::collections::btree_map::Entry; use std::collections::BTreeMap; -use std::sync::Arc; use tokio::sync::mpsc::{channel, Sender}; use tokio::task::yield_now; use tokio_stream::wrappers::ReceiverStream; +use tokio_util::sync::PollSender; use tokio::{sync::RwLock, task::JoinSet}; @@ -16,9 +16,9 @@ type MsgHandler = Box Option + Send + Sync>; #[derive(Default)] struct Subscriber { /// Subscribers based on their Num - num: RwLock>>>, + num: BTreeMap>>, /// Subscribers based on their ID - id: RwLock>, + id: BTreeMap, } pub(crate) type BcConnSink = Box + Send + Sync + Unpin>; @@ -31,35 +31,41 @@ pub(crate) type BcConnSource = Box> + Send + Sync + /// There can be only one subscriber per kind of message at a time. pub struct BcConnection { sink: Sender>, - subscribers: Arc, - #[allow(dead_code)] // Not dead we just need to hold a reference to keep it alive + poll_commander: Sender, rx_thread: RwLock>>, } impl BcConnection { - pub async fn new(mut sink: BcConnSink, mut source: BcConnSource) -> Result { - let subscribers: Arc = Default::default(); - let subs = subscribers.clone(); + pub async fn new(mut sink: BcConnSink, source: BcConnSource) -> Result { let (sinker, sinker_rx) = channel::>(100); - let mut rx_thread = JoinSet::new(); - let sink_thread = sinker.clone(); + + let (poll_commander, poll_commanded) = channel(200); + let mut poller = Poller { + subscribers: Default::default(), + sink: sinker.clone(), + reciever: ReceiverStream::new(poll_commanded), + }; + + let mut rx_thread = JoinSet::>::new(); + let thread_poll_commander = poll_commander.clone(); rx_thread.spawn(async move { - loop { - yield_now().await; // Give other tasks a chance to run before pulling next packet - let packet = source.next().await; - let bc = match packet { - Some(res) => res, - None => continue, - }; - Self::poll(bc, &subs, &sink_thread).await? - } + PollSender::new(thread_poll_commander) + .send_all(&mut source.map(|bc| Ok(PollCommand::Bc(Box::new(bc))))) + .await?; + Ok(()) }); rx_thread.spawn(async move { sink.send_all(&mut ReceiverStream::new(sinker_rx)).await }); + rx_thread.spawn(async move { + loop { + poller.run().await?; + } + }); + Ok(BcConnection { sink: sinker, - subscribers, + poll_commander, rx_thread: RwLock::new(rx_thread), }) } @@ -71,18 +77,9 @@ impl BcConnection { pub async fn subscribe(&self, msg_num: u16) -> Result { let (tx, rx) = channel(100); - match self.subscribers.num.write().await.entry(msg_num) { - Entry::Vacant(vac_entry) => { - vac_entry.insert(tx); - } - Entry::Occupied(mut occ_entry) => { - if occ_entry.get().is_closed() { - occ_entry.insert(tx); - } else { - return Err(Error::SimultaneousSubscription { msg_num }); - } - } - }; + self.poll_commander + .send(PollCommand::AddSubscriber(msg_num, tx)) + .await?; Ok(BcSubscription::new(rx, msg_num as u32, self)) } @@ -93,90 +90,18 @@ impl BcConnection { where T: Fn(&Bc) -> Option + Send + Sync + 'static, { - match self.subscribers.id.write().await.entry(msg_id) { - Entry::Vacant(vac_entry) => { - vac_entry.insert(Box::new(handler)); - } - Entry::Occupied(_) => { - return Err(Error::SimultaneousSubscriptionId { msg_id }); - } - }; + self.poll_commander + .send(PollCommand::AddHandler(msg_id, Box::new(handler))) + .await?; Ok(()) } /// Stop a message handler created using [`handle_msg`] #[allow(dead_code)] // Currently unused but added for future use pub async fn unhandle_msg(&self, msg_id: u32) -> Result<()> { - self.subscribers.id.write().await.remove(&msg_id); - Ok(()) - } - - async fn poll( - response: Result, - subscribers: &Subscriber, - sink: &Sender>, - ) -> Result<()> { - match response { - Ok(response) => { - let msg_num = response.meta.msg_num; - let msg_id = response.meta.msg_id; - - let mut remove_it_num = false; - match ( - subscribers.id.read().await.get(&msg_id), - subscribers.num.read().await.get(&msg_num), - ) { - (Some(occ), _) => { - if let Some(reply) = occ(&response) { - assert!(reply.meta.msg_num == response.meta.msg_num); - sink.send(Ok(reply)).await?; - } - } - (None, Some(occ)) => { - if occ.capacity() == 0 { - warn!("Reaching limit of channel"); - warn!( - "Remaining: {} of {} message space for {} (ID: {})", - occ.capacity(), - occ.max_capacity(), - &msg_num, - &msg_id - ); - } else { - trace!( - "Remaining: {} of {} message space for {} (ID: {})", - occ.capacity(), - occ.max_capacity(), - &msg_num, - &msg_id - ); - } - if occ.send(Ok(response)).await.is_err() { - remove_it_num = true; - } - } - (None, None) => { - debug!( - "Ignoring uninteresting message id {} (number: {})", - msg_id, msg_num - ); - trace!("Contents: {:?}", response); - } - } - if remove_it_num { - subscribers.num.write().await.remove(&msg_num); - } - } - Err(e) => { - for sub in subscribers.num.read().await.values() { - let _ = sub.send(Err(e.clone())).await; - } - subscribers.num.write().await.clear(); - subscribers.id.write().await.clear(); - return Err(e); - } - } - + self.poll_commander + .send(PollCommand::RemoveHandler(msg_id)) + .await?; Ok(()) } @@ -198,3 +123,116 @@ impl BcConnection { Ok(()) } } + +enum PollCommand { + Bc(Box>), + AddHandler(u32, MsgHandler), + RemoveHandler(u32), + AddSubscriber(u16, Sender>), +} + +struct Poller { + subscribers: Subscriber, + sink: Sender>, + reciever: ReceiverStream, +} + +impl Poller { + async fn run(&mut self) -> Result<()> { + while let Some(command) = self.reciever.next().await { + yield_now().await; + match command { + PollCommand::Bc(boxed_response) => match *boxed_response { + Ok(response) => { + let msg_num = response.meta.msg_num; + let msg_id = response.meta.msg_id; + + let mut remove_it_num = false; + match ( + self.subscribers.id.get(&msg_id), + self.subscribers.num.get(&msg_num), + ) { + (Some(occ), _) => { + if let Some(reply) = occ(&response) { + assert!(reply.meta.msg_num == response.meta.msg_num); + self.sink.send(Ok(reply)).await?; + } + } + (None, Some(occ)) => { + if occ.capacity() == 0 { + warn!("Reaching limit of channel"); + warn!( + "Remaining: {} of {} message space for {} (ID: {})", + occ.capacity(), + occ.max_capacity(), + &msg_num, + &msg_id + ); + } else { + trace!( + "Remaining: {} of {} message space for {} (ID: {})", + occ.capacity(), + occ.max_capacity(), + &msg_num, + &msg_id + ); + } + if occ.send(Ok(response)).await.is_err() { + remove_it_num = true; + } + } + (None, None) => { + debug!( + "Ignoring uninteresting message id {} (number: {})", + msg_id, msg_num + ); + trace!("Contents: {:?}", response); + } + } + if remove_it_num { + self.subscribers.num.remove(&msg_num); + } + } + Err(e) => { + for sub in self.subscribers.num.values() { + let _ = sub.send(Err(e.clone())).await; + } + self.subscribers.num.clear(); + self.subscribers.id.clear(); + return Err(e); + } + }, + PollCommand::AddHandler(msg_id, handler) => { + match self.subscribers.id.entry(msg_id) { + Entry::Vacant(vac_entry) => { + vac_entry.insert(handler); + } + Entry::Occupied(_) => { + return Err(Error::SimultaneousSubscriptionId { msg_id }); + } + }; + } + PollCommand::RemoveHandler(msg_id) => { + self.subscribers.id.remove(&msg_id); + } + PollCommand::AddSubscriber(msg_num, tx) => { + match self.subscribers.num.entry(msg_num) { + Entry::Vacant(vac_entry) => { + vac_entry.insert(tx); + } + Entry::Occupied(mut occ_entry) => { + if occ_entry.get().is_closed() { + occ_entry.insert(tx); + } else { + let _ = tx + .send(Err(Error::SimultaneousSubscription { msg_num })) + .await; + } + } + }; + } + } + } + Ok(()) + } +} diff --git a/crates/core/src/bc_protocol/errors.rs b/crates/core/src/bc_protocol/errors.rs index d7a5687..5346f7f 100644 --- a/crates/core/src/bc_protocol/errors.rs +++ b/crates/core/src/bc_protocol/errors.rs @@ -208,6 +208,12 @@ impl From> for Error { } } +impl From> for Error { + fn from(_: tokio_util::sync::PollSendError) -> Self { + Error::TokioBcSendError + } +} + impl From for Error { fn from(k: cookie_factory::GenError) -> Self { Error::GenError(std::sync::Arc::new(k)) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index eae7765..5c1ba75 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -20,7 +20,7 @@ use std::{ }; use tokio::{ sync::mpsc::{channel, Receiver, Sender}, - time::Duration, + time::{Duration, Instant}, }; use tokio_stream::wrappers::ReceiverStream; @@ -301,6 +301,11 @@ impl NeoMediaSenders { occ.remove(); continue; } + if let Err(e) = occ.get_mut().stretch_live(&self.buffer).await { + debug!("Could not sretch live: {:?}", e); + occ.remove(); + continue; + } if let Err(e) = occ.get_mut().process_buffer(&self.buffer).await { debug!("Could not send client data: {:?}", e); occ.remove(); @@ -343,6 +348,9 @@ pub(super) enum NeoMediaSenderCommand { #[derive(Debug)] pub(super) struct NeoMediaSender { start_time: FrameTime, + target_start_time: FrameTime, + start_time_v: f32, + start_time_update: Instant, last_sent_time: FrameTime, vid: Option, aud: Option, @@ -364,6 +372,9 @@ impl NeoMediaSender { let (tx, rx) = channel(30); Self { start_time: 0, + target_start_time: 0, + start_time_v: 0.0, + start_time_update: Instant::now(), last_sent_time: 0, vid: None, aud: None, @@ -387,31 +398,35 @@ impl NeoMediaSender { self.command_sender.clone() } - async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - if self.inited { - let runtime = self.get_runtime().unwrap_or(0); - let (fronts, backs) = buffer.buf.as_slices(); - let frames = fronts.iter().chain(backs.iter()).collect::>(); - let jump_method = JumpMethod::BufferPerunit(0.75); - let target_time = match jump_method { - JumpMethod::MiddleIFrame => { - let idx = frames.len().saturating_div(2); - frames.get(idx).and_then(|f| f.first()).and_then(|f| { - if let BcMedia::Iframe(data) = f { - Some(data.microseconds as FrameTime) - } else { - None - } - }) - } - JumpMethod::BufferPerunit(perunit) => { - if let (Some(st), Some(et)) = (buffer.start_time(), buffer.end_time()) { - Some(((et - st) as f32 * perunit) as FrameTime + st) + async fn target_live_time(&self, buffer: &NeoBuffer) -> AnyResult> { + let (fronts, backs) = buffer.buf.as_slices(); + let frames = fronts.iter().chain(backs.iter()).collect::>(); + let jump_method = JumpMethod::BufferPerunit(0.75); + Ok(match jump_method { + JumpMethod::MiddleIFrame => { + let idx = frames.len().saturating_div(2); + frames.get(idx).and_then(|f| f.first()).and_then(|f| { + if let BcMedia::Iframe(data) = f { + Some(data.microseconds as FrameTime) } else { None } + }) + } + JumpMethod::BufferPerunit(perunit) => { + if let (Some(st), Some(et)) = (buffer.start_time(), buffer.end_time()) { + Some(((et - st) as f32 * perunit) as FrameTime + st) + } else { + None } - }; + } + }) + } + + async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + if self.inited { + let runtime = self.get_runtime().unwrap_or(0); + let target_time = self.target_live_time(buffer).await?; if let Some(target_time) = target_time { if let Some(et) = buffer.end_time() { if let Ok(delta) = TryInto::::try_into(et - target_time) { @@ -420,6 +435,8 @@ impl NeoMediaSender { } self.start_time = target_time - runtime; + self.target_start_time = self.start_time; + self.start_time_v = 0.0; self.last_sent_time = target_time; debug!( "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", @@ -431,6 +448,37 @@ impl NeoMediaSender { Ok(()) } + async fn stretch_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + // Desired live time + let target_time = self.target_live_time(buffer).await?; + // Actual live time + let actual_time = self.get_buftime(); + let st = buffer.start_time(); + let et = buffer.end_time(); + if let (Some(target_time), Some(actual_time), Some(st), Some(et)) = + (target_time, actual_time, st, et) + { + if actual_time > st && actual_time < et { + // Only do this while inside the buffer + self.target_start_time += target_time - actual_time; // Adjust + + // Now for the spring + let dt = (Instant::now() - self.start_time_update).as_secs_f32(); + let mut new_start_time = self.start_time as f32; + spring_update( + &mut new_start_time, + self.target_start_time as f32, + &mut self.start_time_v, + 0.5, + dt, + ); + self.start_time = new_start_time as FrameTime; + } + } + self.start_time_update = Instant::now(); + Ok(()) + } + async fn seek(&mut self, target_time: FrameTime) -> AnyResult<()> { self.last_sent_time = self.runtime_to_buftime(target_time); debug!( @@ -665,3 +713,25 @@ impl NeoMediaSender { Ok(true) } } + +const EPS: f32 = 1e-5f32; +fn halflife_to_damping(halflife: f32) -> f32 { + (4.0f32 * std::f32::consts::LN_2) / (halflife + EPS) +} + +// fn damping_to_halflife(damping: f32) -> f32 { +// (4.0f32 * std::f32::consts::LN_2) / (damping + EPS) +// } +fn fast_negexp(x: f32) -> f32 { + 1.0f32 / (1.0f32 + x + 0.48f32 * x * x + 0.235f32 * x * x * x) +} + +fn spring_update(value: &mut f32, target: f32, velocity: &mut f32, halflife: f32, dt: f32) { + let y = halflife_to_damping(halflife) / 2.0; + let j0 = *value - target; + let j1 = *velocity + j0 * y; + let eydt = fast_negexp(y * dt); + + *value = eydt * (j0 + j1 * dt) + target; + *velocity = eydt * (*velocity - j1 * y * dt); +} From 8c061efc1eec89db3906ae2b38ca9ab3ddc673f2 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Fri, 5 May 2023 11:01:20 +0700 Subject: [PATCH 40/64] Added drifting start_time --- src/rtsp/gst/sender.rs | 39 ++++++++++++++++++++++-------------- src/rtsp/gst/server.rs | 11 ++++++---- src/rtsp/states/streaming.rs | 10 ++++++--- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 5c1ba75..60ba19e 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -349,7 +349,8 @@ pub(super) enum NeoMediaSenderCommand { pub(super) struct NeoMediaSender { start_time: FrameTime, target_start_time: FrameTime, - start_time_v: f32, + smoothed_start_time: f64, + start_time_v: f64, start_time_update: Instant, last_sent_time: FrameTime, vid: Option, @@ -373,6 +374,7 @@ impl NeoMediaSender { Self { start_time: 0, target_start_time: 0, + smoothed_start_time: 0.0, start_time_v: 0.0, start_time_update: Instant::now(), last_sent_time: 0, @@ -436,6 +438,7 @@ impl NeoMediaSender { self.start_time = target_time - runtime; self.target_start_time = self.start_time; + self.smoothed_start_time = self.start_time as f64; self.start_time_v = 0.0; self.last_sent_time = target_time; debug!( @@ -458,21 +461,27 @@ impl NeoMediaSender { if let (Some(target_time), Some(actual_time), Some(st), Some(et)) = (target_time, actual_time, st, et) { - if actual_time > st && actual_time < et { + if actual_time > st.saturating_sub(LATENCY * 2) + && actual_time < et.saturating_add(LATENCY * 2) + { // Only do this while inside the buffer - self.target_start_time += target_time - actual_time; // Adjust + self.target_start_time = self.start_time + (target_time - actual_time); // Adjust // Now for the spring - let dt = (Instant::now() - self.start_time_update).as_secs_f32(); - let mut new_start_time = self.start_time as f32; + let dt = (Instant::now() - self.start_time_update) + .min(Duration::from_millis(200)) + .as_secs_f64(); spring_update( - &mut new_start_time, - self.target_start_time as f32, + &mut self.smoothed_start_time, + self.target_start_time as f64, &mut self.start_time_v, - 0.5, + 2.5, dt, ); - self.start_time = new_start_time as FrameTime; + if let Some(new_actual_time) = self.get_buftime() { + let expected_position = (new_actual_time - st) as f32 / (et - st) as f32; + debug!("expected_position: {}", expected_position); + } } } self.start_time_update = Instant::now(); @@ -714,19 +723,19 @@ impl NeoMediaSender { } } -const EPS: f32 = 1e-5f32; -fn halflife_to_damping(halflife: f32) -> f32 { - (4.0f32 * std::f32::consts::LN_2) / (halflife + EPS) +const EPS: f64 = 1e-5; +fn halflife_to_damping(halflife: f64) -> f64 { + (4.0 * std::f64::consts::LN_2) / (halflife + EPS) } // fn damping_to_halflife(damping: f32) -> f32 { // (4.0f32 * std::f32::consts::LN_2) / (damping + EPS) // } -fn fast_negexp(x: f32) -> f32 { - 1.0f32 / (1.0f32 + x + 0.48f32 * x * x + 0.235f32 * x * x * x) +fn fast_negexp(x: f64) -> f64 { + 1.0 / (1.0 + x + 0.48 * x * x + 0.235 * x * x * x) } -fn spring_update(value: &mut f32, target: f32, velocity: &mut f32, halflife: f32, dt: f32) { +fn spring_update(value: &mut f64, target: f64, velocity: &mut f64, halflife: f64, dt: f64) { let y = halflife_to_damping(halflife) / 2.0; let j0 = *value - target; let j1 = *velocity + j0 * y; diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index e9b849d..e34a671 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -25,6 +25,7 @@ use std::{ use tokio::{ sync::{mpsc::Sender, RwLock}, task::JoinSet, + time::{timeout, Duration}, }; glib::wrapper! { @@ -96,12 +97,14 @@ impl NeoRtspServer { main_loop_thread.run(); AnyResult::Ok(()) }); - self.imp() - .threads - .write() + timeout(Duration::from_secs(5), self.imp().threads.write()) .await + .with_context(|| "Timeout waiting to lock Server threads")? .spawn(async move { handle.await? }); - self.imp().main_loop.write().await.replace(main_loop); + timeout(Duration::from_secs(5), self.imp().main_loop.write()) + .await + .with_context(|| "Timeout waiting to lock Server main_loop")? + .replace(main_loop); Ok(()) } diff --git a/src/rtsp/states/streaming.rs b/src/rtsp/states/streaming.rs index d1593e6..8ec31d4 100644 --- a/src/rtsp/states/streaming.rs +++ b/src/rtsp/states/streaming.rs @@ -2,7 +2,7 @@ // // Data is streamed into a gstreamer source -use anyhow::{anyhow, Error, Result}; +use anyhow::{anyhow, Context, Error, Result}; use log::*; use neolink_core::bcmedia::model::BcMedia; use tokio::time::{timeout, Duration}; @@ -54,7 +54,9 @@ impl Camera { loop { tokio::task::yield_now().await; // debug!("Straming: Get"); - let data = timeout(Duration::from_secs(15), stream_data.get_data()).await??; + let data = timeout(Duration::from_secs(15), stream_data.get_data()) + .await + .with_context(|| "Timed out waiting for new Media Frame")??; // debug!("Straming: Got"); match &data { Ok(BcMedia::InfoV1(_)) => trace!("{}: - InfoV1", &tag_thread), @@ -66,7 +68,9 @@ impl Camera { Err(_) => trace!(" - Error"), } // debug!("Straming: Send"); - sender.send(data?).await?; + timeout(Duration::from_secs(15), sender.send(data?)) + .await + .with_context(|| "Timed out waiting to send Media Frame")??; // debug!("Straming: Sent"); } }); From 60e4b994ab2e0dc986bba6d550e26995bb9c019b Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Fri, 5 May 2023 20:16:01 +0700 Subject: [PATCH 41/64] Some changes to how we poll buffer processing --- src/rtsp/gst/sender.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 60ba19e..582738d 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -238,7 +238,6 @@ impl NeoMediaSenders { async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> { // debug!("Handle new data"); self.update_mediatypes(&data).await; - self.handle_buffer().await?; match &data { BcMedia::Iframe(data) => { let frame_ms = data.microseconds as FrameTime; @@ -319,13 +318,13 @@ impl NeoMediaSenders { } pub(super) async fn run(&mut self) -> AnyResult<()> { - // let mut interval = tokio::time::interval(Duration::from_micros(LATENCY as u64 / 3)); + let mut interval = tokio::time::interval(Duration::from_millis(20)); // 50 FPS loop { tokio::task::yield_now().await; tokio::select! { - // _ = interval.tick() => { - // self.handle_buffer().await - // }, + _ = interval.tick() => { + self.handle_buffer().await + }, Some(v) = self.data_source.next() => { self.handle_new_data(v).await }, @@ -555,7 +554,7 @@ impl NeoMediaSender { if let Some(runtime) = runtime { // We are live only send the buffer up to the runtime let min_time = self.last_sent_time; - let max_time = self.runtime_to_buftime(runtime).saturating_add(LATENCY); + let max_time = self.runtime_to_buftime(runtime); self.last_sent_time = self.send_buffer_between(buf, min_time, max_time).await?; } else { // We are not playing send pre buffers so that the elements can init themselves @@ -573,6 +572,14 @@ impl NeoMediaSender { min_time: FrameTime, max_time: FrameTime, ) -> AnyResult { + if let (Some(buftime), Some(start_time), Some(end_time)) = + (self.get_buftime(), buf.start_time(), buf.end_time()) + { + debug!( + "Buffer Run Position: {}", + (buftime - start_time) as f32 / (end_time - start_time) as f32 + ); + } let mut last_sent_time = min_time; let mut found_start = false; let mut buf_it = buf.buf.iter().peekable(); @@ -604,11 +611,10 @@ impl NeoMediaSender { (buf.start_time(), buf.end_time()) { debug!( - "Buffer Run Position: {}", + "Frame Run Position: {}", (frame_time - start_time) as f32 / (end_time - start_time) as f32 ); } - self.send_buffer(frame).await?; last_sent_time = frame_time; } else if frame_time > max_time { From dbf21d9c909bcce106a8da2343a7a4d13b3080da Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 7 May 2023 09:29:47 +0700 Subject: [PATCH 42/64] Change how frames are popped from buffer. Change how target time is updated --- Cargo.lock | 1 + Cargo.toml | 3 +- src/rtsp/gst.rs | 2 +- src/rtsp/gst/factory.rs | 11 +- src/rtsp/gst/sender.rs | 387 +++++++++++++++++------------------ src/rtsp/gst/server.rs | 31 ++- src/rtsp/mod.rs | 53 +++-- src/rtsp/spring.rs | 88 ++++++++ src/rtsp/states/streaming.rs | 10 +- 9 files changed, 362 insertions(+), 224 deletions(-) create mode 100644 src/rtsp/spring.rs diff --git a/Cargo.lock b/Cargo.lock index ef39f9d..d17e923 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1415,6 +1415,7 @@ dependencies = [ "gstreamer-app", "gstreamer-rtsp", "gstreamer-rtsp-server", + "itertools", "lazy_static", "log", "neolink_core", diff --git a/Cargo.toml b/Cargo.toml index aa751d2..9d32046 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,8 +23,9 @@ gstreamer = "0.20.3" gstreamer-app = "0.20.0" gstreamer-rtsp = "0.20.0" gstreamer-rtsp-server = { version = "0.20.3", features = ["v1_16"] } +itertools = "0.10.5" lazy_static = "1.4.0" -log = "0.4.17" +log = { version = "0.4.17", features = [ "release_max_level_debug" ] } neolink_core = { path = "crates/core", version = "0.5.8" } regex = "1.7.3" rumqttc = "0.20.0" diff --git a/src/rtsp/gst.rs b/src/rtsp/gst.rs index f323e38..7fedf33 100644 --- a/src/rtsp/gst.rs +++ b/src/rtsp/gst.rs @@ -6,7 +6,7 @@ mod sender; mod server; mod shared; -use factory::*; +pub(crate) use factory::*; pub(crate) use self::server::NeoRtspServer; diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 88b7af9..4642afc 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -56,7 +56,7 @@ impl NeoMediaFactory { factory } - pub(crate) fn get_sender(&self) -> Sender { + pub(crate) fn get_sender(&self) -> Sender { self.imp().sender.clone() } @@ -115,8 +115,15 @@ impl NeoMediaFactory { unsafe impl Send for NeoMediaFactory {} unsafe impl Sync for NeoMediaFactory {} +#[derive(Debug)] +pub(crate) enum FactoryCommand { + BcMedia(BcMedia), + ClearBuffer, + JumpToLive, +} + pub(crate) struct NeoMediaFactoryImpl { - sender: Sender, + sender: Sender, clientsender: Sender, shared: Arc, #[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 582738d..aacf495 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -5,6 +5,7 @@ use futures::stream::StreamExt; use gstreamer::{prelude::*, ClockTime}; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; +use itertools::Itertools; use log::*; use neolink_core::bcmedia::model::*; use std::sync::{ @@ -20,11 +21,12 @@ use std::{ }; use tokio::{ sync::mpsc::{channel, Receiver, Sender}, - time::{Duration, Instant}, + time::Duration, }; use tokio_stream::wrappers::ReceiverStream; -use super::{shared::*, AnyResult}; +use super::{factory::FactoryCommand, shared::*, AnyResult}; +use crate::rtsp::Spring; type FrameTime = i64; @@ -34,122 +36,106 @@ const MILLIS: FrameTime = 1000; const LATENCY: FrameTime = 20 * MILLIS; const BUFFER_TIME: FrameTime = 10 * SECS; -const BUFFER_SIZE: usize = 3; +const BUFFER_SIZE: usize = 100; #[derive(Default)] struct NeoBuffer { - buf: VecDeque>, + buf: VecDeque, } impl NeoBuffer { fn push(&mut self, item: BcMedia) { - let mut was_iframe = false; - match item { - BcMedia::Iframe(data) => { - debug!( - "Pushed iFrame to buffer at {:?}", - Duration::from_micros(data.microseconds.try_into().unwrap()) - ); - was_iframe = true; - self.buf.push_back(vec![BcMedia::Iframe(data)]) - } - pframe @ BcMedia::Pframe(_) => { - if let Some(last) = self.buf.back_mut().as_mut() { - last.push(pframe); + self.buf.push_back(item); + let (fronts, backs) = self.buf.as_slices(); + let last_frame = backs + .iter() + .rev() + .chain(&mut fronts.iter().rev()) + .flat_map(|frame| match frame { + BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), + BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), + _ => None, + }) + .next() + .map(|i| i as FrameTime); + if let Some(last_frame) = last_frame { + while match self.buf.front() { + Some(BcMedia::Iframe(BcMediaIframe { microseconds, .. })) => { + last_frame - *microseconds as FrameTime > BUFFER_TIME } - } - aac @ BcMedia::Aac(_) => { - if let Some(last) = self.buf.back_mut().as_mut() { - last.push(aac); + Some(BcMedia::Pframe(BcMediaPframe { microseconds, .. })) => { + last_frame - *microseconds as FrameTime > BUFFER_TIME } - } - adpcm @ BcMedia::Adpcm(_) => { - if let Some(last) = self.buf.back_mut().as_mut() { - last.push(adpcm); - } - } - BcMedia::InfoV1(_) | BcMedia::InfoV2(_) => {} - } - if was_iframe { - if let Some(last_frame_time) = self.last_iframe_time() { - let mut time_delta = self - .first_iframe_time() - .map(|ff| last_frame_time.saturating_sub(ff)); - while let Some(time) = time_delta { - if time > BUFFER_TIME && self.buf.len() > BUFFER_SIZE { - let _ = self.buf.pop_front(); - debug!( - "Popping frame with time difference of {:?}", - Duration::from_micros(time.try_into().unwrap_or(0)) - ); - time_delta = self - .first_iframe_time() - .map(|ff| last_frame_time.saturating_sub(ff)); - } else { - debug!("Iframes left in the buffer: {}", self.buf.len()); - break; - } + Some(_) => true, + None => false, + } && self.buf.len() > BUFFER_SIZE + { + if self.buf.pop_front().is_none() { + break; } } } } - fn last_iframe_time(&self) -> Option { - self.buf - .back() - .as_ref() - .and_then(|b| b.first()) - .and_then(|f| { - if let BcMedia::Iframe(frame) = f { - Some(frame.microseconds as FrameTime) - } else { - None - } - }) - } + // fn last_iframe_time(&self) -> Option { + // let (fronts, backs) = self.buf.as_slices(); + // backs + // .iter() + // .rev() + // .chain(&mut fronts.iter().rev()) + // .flat_map(|frame| match frame { + // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), + // _ => None, + // }) + // .next() + // .map(|i| i as FrameTime) + // } - fn first_iframe_time(&self) -> Option { - self.buf - .front() - .as_ref() - .and_then(|b| b.first()) - .and_then(|f| { - if let BcMedia::Iframe(frame) = f { - Some(frame.microseconds as FrameTime) - } else { - None - } - }) - } + // fn first_iframe_time(&self) -> Option { + // let (fronts, backs) = self.buf.as_slices(); + // fronts + // .iter() + // .chain(&mut backs.iter()) + // .flat_map(|frame| match frame { + // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), + // _ => None, + // }) + // .next() + // .map(|i| i as FrameTime) + // } fn start_time(&self) -> Option { - if let Some(BcMedia::Iframe(data)) = - self.buf.front().and_then(|inner_buf| inner_buf.first()) - { - Some(data.microseconds as FrameTime) - } else { - None - } + let (fronts, backs) = self.buf.as_slices(); + fronts + .iter() + .chain(&mut backs.iter()) + .flat_map(|frame| match frame { + BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), + BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), + _ => None, + }) + .next() + .map(|i| i as FrameTime) } fn end_time(&self) -> Option { - if let Some(innerbuffer) = self.buf.back() { - innerbuffer - .iter() - .flat_map(|frame| match frame { - BcMedia::Iframe(data) => Some(data.microseconds as FrameTime), - BcMedia::Pframe(data) => Some(data.microseconds as FrameTime), - _ => None, - }) - .max() - } else { - None - } + let (fronts, backs) = self.buf.as_slices(); + backs + .iter() + .rev() + .chain(&mut fronts.iter().rev()) + .flat_map(|frame| match frame { + BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), + BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), + _ => None, + }) + .next() + .map(|i| i as FrameTime) } } pub(super) struct NeoMediaSenders { - data_source: ReceiverStream, + data_source: ReceiverStream, client_source: ReceiverStream, shared: Arc, uid: AtomicU64, @@ -160,7 +146,7 @@ pub(super) struct NeoMediaSenders { impl NeoMediaSenders { pub(super) fn new( shared: Arc, - data_source: ReceiverStream, + data_source: ReceiverStream, client_source: ReceiverStream, ) -> Self { Self { @@ -317,16 +303,60 @@ impl NeoMediaSenders { Ok(()) } + async fn clear_buffer(&mut self) -> AnyResult<()> { + self.buffer.buf.clear(); + self.shared.buffer_ready.store(false, Ordering::Relaxed); + for client in self.client_data.values_mut() { + // Set them into the non init state + // This will make them wait for the + // buffer to be enough then jump to live + client.inited = false; + } + Ok(()) + } + + async fn update_springs(&mut self) -> AnyResult<()> { + for client in self.client_data.values_mut() { + client.start_time.update().await; + if let Some(new_target) = client + .target_live_time(&self.buffer) + .await? + .map(|i| i as f64) + { + client.target_time.set_target(new_target); + } + client.target_time.update().await; + } + Ok(()) + } + pub(super) async fn run(&mut self) -> AnyResult<()> { let mut interval = tokio::time::interval(Duration::from_millis(20)); // 50 FPS loop { tokio::task::yield_now().await; + self.update_springs().await?; tokio::select! { _ = interval.tick() => { self.handle_buffer().await }, Some(v) = self.data_source.next() => { - self.handle_new_data(v).await + match v { + FactoryCommand::BcMedia(media) => { + self.handle_new_data(media).await?; + }, + FactoryCommand::ClearBuffer => { + self.clear_buffer().await?; + }, + FactoryCommand::JumpToLive => { + for client in self.client_data.values_mut() { + // Set them into the non init state + // This will make them wait for the + // buffer to be enough then jump to live + client.inited = false; + } + }, + } + Ok(()) }, Some(v) = self.client_source.next() => { self.handle_new_client(v).await @@ -346,11 +376,8 @@ pub(super) enum NeoMediaSenderCommand { } #[derive(Debug)] pub(super) struct NeoMediaSender { - start_time: FrameTime, - target_start_time: FrameTime, - smoothed_start_time: f64, - start_time_v: f64, - start_time_update: Instant, + start_time: Spring, + target_time: Spring, last_sent_time: FrameTime, vid: Option, aud: Option, @@ -371,11 +398,8 @@ impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(30); Self { - start_time: 0, - target_start_time: 0, - smoothed_start_time: 0.0, - start_time_v: 0.0, - start_time_update: Instant::now(), + start_time: Spring::new(0.0, 0.0, 2.5), + target_time: Spring::new(0.0, 0.0, 0.25), last_sent_time: 0, vid: None, aud: None, @@ -405,8 +429,12 @@ impl NeoMediaSender { let jump_method = JumpMethod::BufferPerunit(0.75); Ok(match jump_method { JumpMethod::MiddleIFrame => { - let idx = frames.len().saturating_div(2); - frames.get(idx).and_then(|f| f.first()).and_then(|f| { + let iframes = frames + .iter() + .filter(|f| matches!(f, BcMedia::Iframe(_))) + .collect::>(); + let idx = iframes.len().saturating_div(2); + iframes.get(idx).and_then(|f| { if let BcMedia::Iframe(data) = f { Some(data.microseconds as FrameTime) } else { @@ -426,23 +454,25 @@ impl NeoMediaSender { async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { - let runtime = self.get_runtime().unwrap_or(0); - let target_time = self.target_live_time(buffer).await?; - if let Some(target_time) = target_time { + if let Some(new_target_time) = self.target_live_time(buffer).await? { + self.target_time.reset_to(new_target_time as f64); + + let runtime = self.get_runtime().unwrap_or(0); if let Some(et) = buffer.end_time() { - if let Ok(delta) = TryInto::::try_into(et - target_time) { + if let Ok(delta) = TryInto::::try_into(et - self.target_time.value_i64()) { debug!("Expected latency: {:?}", Duration::from_micros(delta)); } } - self.start_time = target_time - runtime; - self.target_start_time = self.start_time; - self.smoothed_start_time = self.start_time as f64; - self.start_time_v = 0.0; - self.last_sent_time = target_time; + self.start_time + .reset_to(self.target_time.value() - (runtime as f64)); + self.last_sent_time = self.target_time.value_i64(); debug!( "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", - Duration::from_micros(target_time.try_into().unwrap_or(0)), Duration::from_micros(self.start_time.try_into().unwrap_or(0)), Duration::from_micros(self.buftime_to_runtime(target_time).try_into().unwrap_or(0)), Duration::from_micros(runtime.try_into().unwrap_or(0)) + Duration::from_micros(self.target_time.value_u64()), + Duration::from_micros(self.start_time.value_u64()), + Duration::from_micros(self.target_time.value_u64()), + Duration::from_micros(runtime.try_into().unwrap_or(0)) ); } } @@ -451,39 +481,26 @@ impl NeoMediaSender { } async fn stretch_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - // Desired live time - let target_time = self.target_live_time(buffer).await?; // Actual live time let actual_time = self.get_buftime(); let st = buffer.start_time(); let et = buffer.end_time(); - if let (Some(target_time), Some(actual_time), Some(st), Some(et)) = - (target_time, actual_time, st, et) - { + if let (Some(actual_time), Some(st), Some(et)) = (actual_time, st, et) { if actual_time > st.saturating_sub(LATENCY * 2) && actual_time < et.saturating_add(LATENCY * 2) { // Only do this while inside the buffer - self.target_start_time = self.start_time + (target_time - actual_time); // Adjust + self.start_time.set_target( + self.start_time.value() + self.target_time.value() - (actual_time as f64), + ); // Adjust - // Now for the spring - let dt = (Instant::now() - self.start_time_update) - .min(Duration::from_millis(200)) - .as_secs_f64(); - spring_update( - &mut self.smoothed_start_time, - self.target_start_time as f64, - &mut self.start_time_v, - 2.5, - dt, - ); if let Some(new_actual_time) = self.get_buftime() { let expected_position = (new_actual_time - st) as f32 / (et - st) as f32; debug!("expected_position: {}", expected_position); } } } - self.start_time_update = Instant::now(); + self.start_time.update().await; Ok(()) } @@ -541,11 +558,11 @@ impl NeoMediaSender { } fn runtime_to_buftime(&self, runtime: FrameTime) -> FrameTime { - runtime.saturating_add(self.start_time) + runtime.saturating_add(self.start_time.value_i64()) } fn buftime_to_runtime(&self, buftime: FrameTime) -> FrameTime { - buftime.saturating_sub(self.start_time).max(0) + buftime.saturating_sub(self.start_time.value_i64()).max(0) } async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { @@ -581,47 +598,43 @@ impl NeoMediaSender { ); } let mut last_sent_time = min_time; - let mut found_start = false; - let mut buf_it = buf.buf.iter().peekable(); - while let Some(frames) = buf_it.next() { - tokio::task::yield_now().await; - if !found_start { - let next_frames = buf_it.peek(); - if let Some(BcMedia::Iframe(frame)) = next_frames.and_then(|b| b.first()) { - // Get time of next IFrame - // if it is after min_time, then the - // start shuld happen between frames and next_frames - if frame.microseconds as FrameTime > min_time { - found_start = true; - } - } - } + let mut buf_it = buf.buf.iter(); - if found_start { - // We have found the start send eveythin until we get passed the - // max time - for frame in frames { - let frame_time = match frame { - BcMedia::Iframe(data) => data.microseconds as FrameTime, - BcMedia::Pframe(data) => data.microseconds as FrameTime, - _ => last_sent_time, - }; - if frame_time > min_time && frame_time <= max_time { - if let (Some(start_time), Some(end_time)) = - (buf.start_time(), buf.end_time()) - { - debug!( - "Frame Run Position: {}", - (frame_time - start_time) as f32 / (end_time - start_time) as f32 - ); - } - self.send_buffer(frame).await?; - last_sent_time = frame_time; - } else if frame_time > max_time { - return Ok(last_sent_time); - } + // Remove before start frame + buf_it + .take_while_ref(|f| match f { + BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => { + *microseconds as FrameTime <= min_time } + BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => { + *microseconds as FrameTime <= min_time + } + _ => true, + }) + .for_each(drop); + + // Send frames afterwards + let frames_to_send = buf_it.take_while(|f| match f { + BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => { + *microseconds as FrameTime <= max_time } + BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => { + *microseconds as FrameTime <= max_time + } + _ => true, + }); + + for frame in frames_to_send { + match frame { + BcMedia::Iframe(data) => { + last_sent_time = data.microseconds as FrameTime; + } + BcMedia::Pframe(data) => { + last_sent_time = data.microseconds as FrameTime; + } + _ => {} + }; + self.send_buffer(frame).await?; } Ok(last_sent_time) } @@ -728,25 +741,3 @@ impl NeoMediaSender { Ok(true) } } - -const EPS: f64 = 1e-5; -fn halflife_to_damping(halflife: f64) -> f64 { - (4.0 * std::f64::consts::LN_2) / (halflife + EPS) -} - -// fn damping_to_halflife(damping: f32) -> f32 { -// (4.0f32 * std::f32::consts::LN_2) / (damping + EPS) -// } -fn fast_negexp(x: f64) -> f64 { - 1.0 / (1.0 + x + 0.48 * x * x + 0.235 * x * x * x) -} - -fn spring_update(value: &mut f64, target: f64, velocity: &mut f64, halflife: f64, dt: f64) { - let y = halflife_to_damping(halflife) / 2.0; - let j0 = *value - target; - let j1 = *velocity + j0 * y; - let eydt = fast_negexp(y * dt); - - *value = eydt * (j0 + j1 * dt) + target; - *velocity = eydt * (*velocity - j1 * y * dt); -} diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index e34a671..727b2fb 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -16,7 +16,6 @@ use gstreamer_rtsp_server::{ RTSPAuth, RTSPFilterResult, RTSPServer, RTSPToken, RTSP_TOKEN_MEDIA_FACTORY_ROLE, }; use log::*; -use neolink_core::bcmedia::model::*; use std::{ collections::{hash_map::Entry, HashMap, HashSet}, fs, @@ -46,7 +45,10 @@ impl NeoRtspServer { Ok(factory) } - pub(crate) async fn get_sender>(&self, tag: T) -> Option> { + pub(crate) async fn get_sender>( + &self, + tag: T, + ) -> Option> { self.imp().get_sender(tag).await } @@ -216,6 +218,26 @@ impl NeoRtspServer { } })); } + + // Clear buffers on all senders of a tag + pub(crate) async fn clear_buffer>(&self, tag: T) -> AnyResult<()> { + if let Some(sender) = self.imp().get_sender(tag).await { + sender.send(FactoryCommand::ClearBuffer).await?; + Ok(()) + } else { + Err(anyhow!("No such tag")) + } + } + + // Jump to live on all senders of a tag + pub(crate) async fn jump_to_live>(&self, tag: T) -> AnyResult<()> { + if let Some(sender) = self.imp().get_sender(tag).await { + sender.send(FactoryCommand::JumpToLive).await?; + Ok(()) + } else { + Err(anyhow!("No such tag")) + } + } } unsafe impl Send for NeoRtspServer {} @@ -264,7 +286,10 @@ impl NeoRtspServerImpl { Ok(()) } - pub(crate) async fn get_sender>(&self, tag: T) -> Option> { + pub(crate) async fn get_sender>( + &self, + tag: T, + ) -> Option> { let key = tag.into(); self.medias .read() diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index dbd7018..8db2491 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -61,11 +61,13 @@ use tokio_stream::StreamExt; mod cmdline; mod gst; +mod spring; mod states; use super::config::Config; pub(crate) use cmdline::Opt; use gst::NeoRtspServer; +pub(crate) use spring::*; use states::*; /// Entry point for the rtsp subcommand @@ -182,7 +184,7 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi let tags = loggedin.shared.get_tags(); let rtsp_thread = loggedin.get_rtsp(); - // Check if buffers are already perpared (from say previous connection) + // Check if buffers were prepared before we clear let buffers_prepared = tags .iter() .map(|tag| rtsp_thread.buffer_ready(tag)) @@ -191,6 +193,13 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi .all(|a| a) .await; + // Clear all buffers present + tags.iter() + .map(|tag| rtsp_thread.clear_buffer(tag)) + .collect::>() + .collect::>() + .await; + // Start pulling data from the camera let mut streaming = loggedin .stream() @@ -198,22 +207,28 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi .with_context(|| format!("{}: Could not start stream", name)) .map_err(CameraFailureKind::Retry)?; - if !buffers_prepared { - // Wait for buffers to be prepared - let mut waiter = tokio::time::interval(Duration::from_micros(500)); - loop { - waiter.tick().await; - if tags - .iter() - .map(|tag| rtsp_thread.buffer_ready(tag)) - .collect::>() - .all(|f| f.unwrap_or(false)) - .await - { - break; - } + // Wait for buffers to be prepared + let mut waiter = tokio::time::interval(Duration::from_micros(500)); + loop { + waiter.tick().await; + if tags + .iter() + .map(|tag| rtsp_thread.buffer_ready(tag)) + .collect::>() + .all(|f| f.unwrap_or(false)) + .await + { + break; } + } + tags.iter() + .map(|tag| rtsp_thread.jump_to_live(tag)) + .collect::>() + .collect::>() + .await; + + if !buffers_prepared { // Clear all current media and force a reconnect // This shoudld stop them from watching the "Stream Not Ready Thing" tags.iter() @@ -221,8 +236,8 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi .collect::>() .collect::>() .await; - log::info!("{}: Buffers prepared", name); } + log::info!("{}: Buffers prepared", name); loop { // Wait for error or reason to pause @@ -302,6 +317,12 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi .with_context(|| format!("{}: Error while paused", name)) .map_err(CameraFailureKind::Retry)?; + tags.iter() + .map(|tag| rtsp_thread.jump_to_live(tag)) + .collect::>() + .collect::>() + .await; + streaming = paused .stream() .await diff --git a/src/rtsp/spring.rs b/src/rtsp/spring.rs new file mode 100644 index 0000000..3725cef --- /dev/null +++ b/src/rtsp/spring.rs @@ -0,0 +1,88 @@ +//! A critically damped spring +use tokio::time::Instant; + +#[derive(Debug)] +pub(crate) struct Spring { + last_update: Instant, + target: f64, + value: f64, + velocity: f64, + halflife: f64, +} + +impl Spring { + pub(crate) fn new(value: f64, target: f64, halflife: f64) -> Self { + Self { + value, + target, + halflife, + last_update: Instant::now(), + velocity: 0.0f64, + } + } + + pub(crate) async fn update(&mut self) { + let now = Instant::now(); + let dt = now - self.last_update; + self.last_update = now; + spring_update( + &mut self.value, + self.target, + &mut self.velocity, + self.halflife, + dt.as_secs_f64(), + ); + } + + pub(crate) fn value(&self) -> f64 { + self.value + } + + // pub(crate) fn value_u32(&self) -> u32 { + // self.value.max(std::u32::MIN as f64).floor().min(std::u32::MAX as f64) as u32 + // } + pub(crate) fn value_i64(&self) -> i64 { + self.value + .max(std::i64::MIN as f64) + .floor() + .min(std::i64::MAX as f64) as i64 + } + pub(crate) fn value_u64(&self) -> u64 { + self.value + .max(std::u64::MIN as f64) + .floor() + .min(std::u64::MAX as f64) as u64 + } + + pub(crate) fn set_target(&mut self, target: f64) { + self.target = target; + } + + pub(crate) fn reset_to(&mut self, target: f64) { + self.target = target; + self.value = target; + self.velocity = 0.0; + } +} + +const EPS: f64 = 1e-5; +fn halflife_to_damping(halflife: f64) -> f64 { + (4.0 * std::f64::consts::LN_2) / (halflife + EPS) +} + +// fn damping_to_halflife(damping: f32) -> f32 { +// (4.0f32 * std::f32::consts::LN_2) / (damping + EPS) +// } +fn fast_negexp(x: f64) -> f64 { + 1.0 / (1.0 + x + 0.48 * x * x + 0.235 * x * x * x) +} + +fn spring_update(value: &mut f64, target: f64, velocity: &mut f64, halflife: f64, dt: f64) { + let y = halflife_to_damping(halflife) / 2.0; + let j0 = *value - target; + let j1 = *velocity + j0 * y; + let eydt = fast_negexp(y * dt); + + *value = eydt * (j0 + j1 * dt) + target; + *velocity = eydt * (*velocity - j1 * y * dt); +} diff --git a/src/rtsp/states/streaming.rs b/src/rtsp/states/streaming.rs index 8ec31d4..1560a41 100644 --- a/src/rtsp/states/streaming.rs +++ b/src/rtsp/states/streaming.rs @@ -11,6 +11,7 @@ use tokio::{sync::RwLock, task::JoinSet}; use neolink_core::bc_protocol::{BcCamera, StreamKind as Stream}; use super::{camera::Camera, LoggedIn}; +use crate::rtsp::gst::FactoryCommand; pub(crate) struct Streaming { pub(crate) camera: BcCamera, @@ -68,9 +69,12 @@ impl Camera { Err(_) => trace!(" - Error"), } // debug!("Straming: Send"); - timeout(Duration::from_secs(15), sender.send(data?)) - .await - .with_context(|| "Timed out waiting to send Media Frame")??; + timeout( + Duration::from_secs(15), + sender.send(FactoryCommand::BcMedia(data?)), + ) + .await + .with_context(|| "Timed out waiting to send Media Frame")??; // debug!("Straming: Sent"); } }); From 587c8bf3348f531eb0b083f372bc5f7458a65f06 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 7 May 2023 15:57:13 +0700 Subject: [PATCH 43/64] Fix client pause --- src/rtsp/gst/sender.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index aacf495..731d2c3 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -331,10 +331,14 @@ impl NeoMediaSenders { } pub(super) async fn run(&mut self) -> AnyResult<()> { - let mut interval = tokio::time::interval(Duration::from_millis(20)); // 50 FPS + let mut interval = tokio::time::interval(Duration::from_millis(40)); // 25 FPS + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tokio::task::yield_now().await; self.update_springs().await?; + self.shared + .number_of_clients + .store(self.client_data.len(), Ordering::Relaxed); tokio::select! { _ = interval.tick() => { self.handle_buffer().await @@ -571,7 +575,7 @@ impl NeoMediaSender { if let Some(runtime) = runtime { // We are live only send the buffer up to the runtime let min_time = self.last_sent_time; - let max_time = self.runtime_to_buftime(runtime); + let max_time = self.runtime_to_buftime(runtime).saturating_add(LATENCY); self.last_sent_time = self.send_buffer_between(buf, min_time, max_time).await?; } else { // We are not playing send pre buffers so that the elements can init themselves From 7a50d3e42f2196ecc77a671dffd11c7f76b29edd Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 8 May 2023 13:59:02 +0700 Subject: [PATCH 44/64] Try another method --- Cargo.lock | 1 - Cargo.toml | 1 - src/rtsp/gst/sender.rs | 535 ++++++++++++++++------------------------- src/rtsp/spring.rs | 5 + 4 files changed, 212 insertions(+), 330 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d17e923..ef39f9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1415,7 +1415,6 @@ dependencies = [ "gstreamer-app", "gstreamer-rtsp", "gstreamer-rtsp-server", - "itertools", "lazy_static", "log", "neolink_core", diff --git a/Cargo.toml b/Cargo.toml index 9d32046..c069078 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ gstreamer = "0.20.3" gstreamer-app = "0.20.0" gstreamer-rtsp = "0.20.0" gstreamer-rtsp-server = { version = "0.20.3", features = ["v1_16"] } -itertools = "0.10.5" lazy_static = "1.4.0" log = { version = "0.4.17", features = [ "release_max_level_debug" ] } neolink_core = { path = "crates/core", version = "0.5.8" } diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 731d2c3..2e660f6 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -5,19 +5,18 @@ use futures::stream::StreamExt; use gstreamer::{prelude::*, ClockTime}; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; -use itertools::Itertools; use log::*; use neolink_core::bcmedia::model::*; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, -}; use std::{ collections::{ VecDeque, {hash_map::Entry, HashMap}, }, convert::TryInto, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; use tokio::{ sync::mpsc::{channel, Receiver, Sender}, @@ -30,51 +29,42 @@ use crate::rtsp::Spring; type FrameTime = i64; -const SECS: FrameTime = 1000000; -const MILLIS: FrameTime = 1000; -// const MICROS: FrameTime = 1; - -const LATENCY: FrameTime = 20 * MILLIS; -const BUFFER_TIME: FrameTime = 10 * SECS; const BUFFER_SIZE: usize = 100; -#[derive(Default)] +#[derive(Debug)] +struct Stamped { + time: FrameTime, + data: BcMedia, +} + +#[derive(Default, Debug)] struct NeoBuffer { - buf: VecDeque, + buf: VecDeque>, } impl NeoBuffer { - fn push(&mut self, item: BcMedia) { - self.buf.push_back(item); - let (fronts, backs) = self.buf.as_slices(); - let last_frame = backs - .iter() - .rev() - .chain(&mut fronts.iter().rev()) - .flat_map(|frame| match frame { - BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), - _ => None, - }) - .next() - .map(|i| i as FrameTime); - if let Some(last_frame) = last_frame { - while match self.buf.front() { - Some(BcMedia::Iframe(BcMediaIframe { microseconds, .. })) => { - last_frame - *microseconds as FrameTime > BUFFER_TIME - } - Some(BcMedia::Pframe(BcMediaPframe { microseconds, .. })) => { - last_frame - *microseconds as FrameTime > BUFFER_TIME - } - Some(_) => true, - None => false, - } && self.buf.len() > BUFFER_SIZE - { - if self.buf.pop_front().is_none() { - break; - } + fn push(&mut self, item: Arc) { + let end_time = self.end_time(); + let frame_time = item.time; + if let Some(end_time) = end_time { + let delta_time = end_time - frame_time; + let delta_duration = Duration::from_micros(delta_time.unsigned_abs()); + if delta_duration > Duration::from_secs(1) { + debug!("Clearing buffer due to jump"); + self.buf.clear(); } } + + self.buf.push_back(item); + while self.buf.len() > BUFFER_SIZE { + if self.buf.pop_front().is_none() { + break; + } + } + } + + pub(crate) fn ready(&self) -> bool { + self.buf.len() > BUFFER_SIZE * 2 / 3 } // fn last_iframe_time(&self) -> Option { @@ -109,13 +99,8 @@ impl NeoBuffer { fronts .iter() .chain(&mut backs.iter()) - .flat_map(|frame| match frame { - BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), - _ => None, - }) + .map(|frame| frame.time) .next() - .map(|i| i as FrameTime) } fn end_time(&self) -> Option { @@ -124,14 +109,38 @@ impl NeoBuffer { .iter() .rev() .chain(&mut fronts.iter().rev()) - .flat_map(|frame| match frame { - BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), - BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), - _ => None, - }) + .map(|frame| frame.time) .next() - .map(|i| i as FrameTime) } + + // fn min_time(&self) -> Option { + // let (fronts, backs) = self.buf.as_slices(); + // fronts + // .iter() + // .chain(&mut backs.iter()) + // .flat_map(|frame| match Arc::as_ref(frame) { + // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), + // BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), + // _ => None, + // }) + // .min() + // .map(|i| i as FrameTime) + // } + + // fn max_time(&self) -> Option { + // let (fronts, backs) = self.buf.as_slices(); + // backs + // .iter() + // .rev() + // .chain(&mut fronts.iter().rev()) + // .flat_map(|frame| match Arc::as_ref(frame) { + // BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => Some(*microseconds), + // BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => Some(*microseconds), + // _ => None, + // }) + // .max() + // .map(|i| i as FrameTime) + // } } pub(super) struct NeoMediaSenders { @@ -224,51 +233,28 @@ impl NeoMediaSenders { async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> { // debug!("Handle new data"); self.update_mediatypes(&data).await; - match &data { - BcMedia::Iframe(data) => { - let frame_ms = data.microseconds as FrameTime; - debug!( - "New IFrame for buffer @ {:?}", - Duration::from_micros(frame_ms.try_into().unwrap_or(0)) - ); - for client in self.client_data.values() { - debug!( - " - {:?}", - Duration::from_micros( - client.buftime_to_runtime(frame_ms).try_into().unwrap_or(0) - ) - ); - } - } - BcMedia::Pframe(data) => { - let frame_ms = data.microseconds as FrameTime; - debug!( - "New PFrame for buffer @ {:?}", - Duration::from_micros(frame_ms.try_into().unwrap_or(0)) - ); - for client in self.client_data.values() { - debug!( - " - {:?}", - Duration::from_micros( - client.buftime_to_runtime(frame_ms).try_into().unwrap_or(0) - ) - ); - } - } - _ => {} + let time = match &data { + BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => *microseconds as FrameTime, + BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => *microseconds as FrameTime, + _ => self.buffer.end_time().unwrap_or(0), + }; + + let data = Arc::new(Stamped { time, data }); + for client_data in self.client_data.values_mut() { + client_data.add_data(data.clone()).await?; } self.buffer.push(data); - if self.buffer.buf.len() >= BUFFER_SIZE { - self.shared.buffer_ready.store(true, Ordering::Relaxed); - } + + self.shared + .buffer_ready + .store(self.buffer.ready(), Ordering::Relaxed); Ok(()) } - async fn handle_buffer(&mut self) -> AnyResult<()> { + async fn init_clients(&mut self) -> AnyResult<()> { for key in self.client_data.keys().copied().collect::>() { tokio::task::yield_now().await; - // debug!(" - Client: {}", key); match self.client_data.entry(key) { Entry::Occupied(mut occ) => { if let Err(e) = occ.get_mut().initialise(&self.buffer).await { @@ -276,23 +262,37 @@ impl NeoMediaSenders { occ.remove(); continue; } - if let Err(e) = occ.get_mut().process_commands(&self.buffer).await { + } + Entry::Vacant(_) => {} + } + } + Ok(()) + } + + async fn process_client_commands(&mut self) -> AnyResult<()> { + for key in self.client_data.keys().copied().collect::>() { + tokio::task::yield_now().await; + match self.client_data.entry(key) { + Entry::Occupied(mut occ) => { + if let Err(e) = occ.get_mut().process_commands().await { debug!("Could not process client command: {:?}", e); occ.remove(); continue; } - if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await { - debug!("Could not handle jump to live: {:?}", e); - occ.remove(); - continue; - } - if let Err(e) = occ.get_mut().stretch_live(&self.buffer).await { - debug!("Could not sretch live: {:?}", e); - occ.remove(); - continue; - } - if let Err(e) = occ.get_mut().process_buffer(&self.buffer).await { - debug!("Could not send client data: {:?}", e); + } + Entry::Vacant(_) => {} + } + } + Ok(()) + } + + async fn process_client_update(&mut self) -> AnyResult<()> { + for key in self.client_data.keys().copied().collect::>() { + tokio::task::yield_now().await; + match self.client_data.entry(key) { + Entry::Occupied(mut occ) => { + if let Err(e) = occ.get_mut().update().await { + debug!("Could not update client: {:?}", e); occ.remove(); continue; } @@ -315,18 +315,11 @@ impl NeoMediaSenders { Ok(()) } - async fn update_springs(&mut self) -> AnyResult<()> { - for client in self.client_data.values_mut() { - client.start_time.update().await; - if let Some(new_target) = client - .target_live_time(&self.buffer) - .await? - .map(|i| i as f64) - { - client.target_time.set_target(new_target); - } - client.target_time.update().await; - } + async fn update(&mut self) -> AnyResult<()> { + self.init_clients().await?; + self.process_client_commands().await?; + self.process_client_update().await?; + Ok(()) } @@ -335,13 +328,12 @@ impl NeoMediaSenders { interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tokio::task::yield_now().await; - self.update_springs().await?; self.shared .number_of_clients .store(self.client_data.len(), Ordering::Relaxed); tokio::select! { _ = interval.tick() => { - self.handle_buffer().await + self.update().await }, Some(v) = self.data_source.next() => { match v { @@ -381,15 +373,13 @@ pub(super) enum NeoMediaSenderCommand { #[derive(Debug)] pub(super) struct NeoMediaSender { start_time: Spring, - target_time: Spring, - last_sent_time: FrameTime, + buffer: NeoBuffer, vid: Option, aud: Option, command_reciever: Receiver, command_sender: Sender, inited: bool, playing: bool, - prebuffered: FrameTime, } #[allow(dead_code)] @@ -403,18 +393,22 @@ impl NeoMediaSender { let (tx, rx) = channel(30); Self { start_time: Spring::new(0.0, 0.0, 2.5), - target_time: Spring::new(0.0, 0.0, 0.25), - last_sent_time: 0, + buffer: NeoBuffer::default(), + vid: None, aud: None, command_reciever: rx, command_sender: tx, inited: false, playing: true, - prebuffered: 0, } } + async fn add_data(&mut self, data: Arc) -> AnyResult<()> { + self.buffer.push(data); + Ok(()) + } + pub(super) fn update_vid(&mut self, source: AppSrc) { self.vid.replace(source); } @@ -427,56 +421,15 @@ impl NeoMediaSender { self.command_sender.clone() } - async fn target_live_time(&self, buffer: &NeoBuffer) -> AnyResult> { - let (fronts, backs) = buffer.buf.as_slices(); - let frames = fronts.iter().chain(backs.iter()).collect::>(); - let jump_method = JumpMethod::BufferPerunit(0.75); - Ok(match jump_method { - JumpMethod::MiddleIFrame => { - let iframes = frames - .iter() - .filter(|f| matches!(f, BcMedia::Iframe(_))) - .collect::>(); - let idx = iframes.len().saturating_div(2); - iframes.get(idx).and_then(|f| { - if let BcMedia::Iframe(data) = f { - Some(data.microseconds as FrameTime) - } else { - None - } - }) - } - JumpMethod::BufferPerunit(perunit) => { - if let (Some(st), Some(et)) = (buffer.start_time(), buffer.end_time()) { - Some(((et - st) as f32 * perunit) as FrameTime + st) - } else { - None - } - } - }) - } - - async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + async fn jump_to_live(&mut self) -> AnyResult<()> { if self.inited { - if let Some(new_target_time) = self.target_live_time(buffer).await? { - self.target_time.reset_to(new_target_time as f64); - + if let Some(buffer_start) = self.buffer.start_time() { let runtime = self.get_runtime().unwrap_or(0); - if let Some(et) = buffer.end_time() { - if let Ok(delta) = TryInto::::try_into(et - self.target_time.value_i64()) { - debug!("Expected latency: {:?}", Duration::from_micros(delta)); - } - } - self.start_time - .reset_to(self.target_time.value() - (runtime as f64)); - self.last_sent_time = self.target_time.value_i64(); + self.start_time.reset_to((buffer_start - runtime) as f64); debug!( - "Target time: {:?}, New start time: {:?}, New Runtime: {:?}, Actual Runtime: {:?}", - Duration::from_micros(self.target_time.value_u64()), + "Jumped to live: New start time: {:?}", Duration::from_micros(self.start_time.value_u64()), - Duration::from_micros(self.target_time.value_u64()), - Duration::from_micros(runtime.try_into().unwrap_or(0)) ); } } @@ -484,41 +437,12 @@ impl NeoMediaSender { Ok(()) } - async fn stretch_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - // Actual live time - let actual_time = self.get_buftime(); - let st = buffer.start_time(); - let et = buffer.end_time(); - if let (Some(actual_time), Some(st), Some(et)) = (actual_time, st, et) { - if actual_time > st.saturating_sub(LATENCY * 2) - && actual_time < et.saturating_add(LATENCY * 2) - { - // Only do this while inside the buffer - self.start_time.set_target( - self.start_time.value() + self.target_time.value() - (actual_time as f64), - ); // Adjust - - if let Some(new_actual_time) = self.get_buftime() { - let expected_position = (new_actual_time - st) as f32 / (et - st) as f32; - debug!("expected_position: {}", expected_position); - } - } - } - self.start_time.update().await; + async fn seek(&mut self, _target_time: FrameTime) -> AnyResult<()> { + self.jump_to_live().await?; Ok(()) } - async fn seek(&mut self, target_time: FrameTime) -> AnyResult<()> { - self.last_sent_time = self.runtime_to_buftime(target_time); - debug!( - "Seeked last_sent_time to {:?} ({:?})", - Duration::from_micros(self.last_sent_time.try_into().unwrap_or(0)), - Duration::from_micros(target_time.try_into().unwrap_or(0)) - ); - Ok(()) - } - - async fn process_commands(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { + async fn process_commands(&mut self) -> AnyResult<()> { if self.inited { if let Ok(command) = self.command_reciever.try_recv() { match command { @@ -536,7 +460,7 @@ impl NeoMediaSender { if !self.playing { debug!("Resuming"); self.playing = true; - self.jump_to_live(buffer).await?; + self.jump_to_live().await?; } } } @@ -546,120 +470,31 @@ impl NeoMediaSender { } async fn initialise(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - if !self.inited { - if let (Some(first_ms), Some(end_ms)) = (buffer.start_time(), buffer.end_time()) { - if end_ms.saturating_sub(first_ms) > LATENCY * 5 { - // Minimum buffer - self.inited = true; + if !self.inited && buffer.ready() { + // Minimum buffer + self.inited = true; - self.jump_to_live(buffer).await?; + // Split buffer into 2/3 of preprocess + // and one third for the future buffer + let split_idx = buffer.buf.len() * 2 / 3; + let cloned = buffer.buf.iter().cloned().collect::>(); + let (preprocess, buffer) = cloned.split_at(split_idx); + let start_ms = buffer + .first() + .map(|data| data.time as f64) + .expect("Buffer should have a start time"); + self.start_time.reset_to(start_ms); - let _ = self.process_buffer(buffer).await; // Ignore errors let them be handled later in the loop - } + // Send preprocess now + for frame in preprocess.iter() { + self.send_buffer(frame).await?; } - } - Ok(()) - } - - fn runtime_to_buftime(&self, runtime: FrameTime) -> FrameTime { - runtime.saturating_add(self.start_time.value_i64()) - } - - fn buftime_to_runtime(&self, buftime: FrameTime) -> FrameTime { - buftime.saturating_sub(self.start_time.value_i64()).max(0) - } - - async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> { - if self.inited && self.playing { - let runtime = self.get_runtime(); - if let Some(runtime) = runtime { - // We are live only send the buffer up to the runtime - let min_time = self.last_sent_time; - let max_time = self.runtime_to_buftime(runtime).saturating_add(LATENCY); - self.last_sent_time = self.send_buffer_between(buf, min_time, max_time).await?; - } else { - // We are not playing send pre buffers so that the elements can init themselves - let min_time = self.prebuffered; - let max_time = self.last_sent_time; - self.prebuffered = self.send_buffer_between(buf, min_time, max_time).await?; - } - } - Ok(()) - } - - async fn send_buffer_between( - &mut self, - buf: &NeoBuffer, - min_time: FrameTime, - max_time: FrameTime, - ) -> AnyResult { - if let (Some(buftime), Some(start_time), Some(end_time)) = - (self.get_buftime(), buf.start_time(), buf.end_time()) - { - debug!( - "Buffer Run Position: {}", - (buftime - start_time) as f32 / (end_time - start_time) as f32 - ); - } - let mut last_sent_time = min_time; - let mut buf_it = buf.buf.iter(); - - // Remove before start frame - buf_it - .take_while_ref(|f| match f { - BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => { - *microseconds as FrameTime <= min_time - } - BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => { - *microseconds as FrameTime <= min_time - } - _ => true, - }) - .for_each(drop); - - // Send frames afterwards - let frames_to_send = buf_it.take_while(|f| match f { - BcMedia::Iframe(BcMediaIframe { microseconds, .. }) => { - *microseconds as FrameTime <= max_time - } - BcMedia::Pframe(BcMediaPframe { microseconds, .. }) => { - *microseconds as FrameTime <= max_time - } - _ => true, - }); - - for frame in frames_to_send { - match frame { - BcMedia::Iframe(data) => { - last_sent_time = data.microseconds as FrameTime; - } - BcMedia::Pframe(data) => { - last_sent_time = data.microseconds as FrameTime; - } - _ => {} - }; - self.send_buffer(frame).await?; - } - Ok(last_sent_time) - } - - async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> { - if self.inited { - if let (Some(buffer_start), Some(buffer_end)) = (buffer.start_time(), buffer.end_time()) - { - let runtime = self.get_buftime().unwrap_or(self.last_sent_time); - if runtime < buffer_start.saturating_sub(15 * SECS) - || runtime > buffer_end.saturating_add(15 * SECS) - { - debug!( - "Outside buffer jumping to live: {:?} < {:?} < {:?}", - Duration::from_micros(buffer_start.try_into().unwrap_or(0)), - Duration::from_micros(runtime.try_into().unwrap_or(0)), - Duration::from_micros(buffer_end.try_into().unwrap_or(0)) - ); - self.jump_to_live(buffer).await?; - } + debug!("Preprocessed"); + // Send these later + for frame in buffer.iter() { + self.buffer.push(frame.clone()); } + debug!("Buffer filled"); } Ok(()) } @@ -682,35 +517,79 @@ impl NeoMediaSender { self.get_runtime().map(|time| self.runtime_to_buftime(time)) } - async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult { - if self.inited && self.playing { - let buftime = match media { - BcMedia::Iframe(data) => data.microseconds as FrameTime, - BcMedia::Pframe(data) => data.microseconds as FrameTime, - _ => self.last_sent_time, - }; - let runtime = self.buftime_to_runtime(buftime); + fn runtime_to_buftime(&self, runtime: FrameTime) -> FrameTime { + runtime.saturating_add(self.start_time.value_i64()) + } - let buf = match media { + fn buftime_to_runtime(&self, buftime: FrameTime) -> FrameTime { + buftime.saturating_sub(self.start_time.value_i64()).max(0) + } + + async fn update_starttime(&mut self) -> AnyResult<()> { + self.start_time.update().await; + if let (Some(buftime), Some(buffer_start)) = (self.get_buftime(), self.buffer.start_time()) + { + self.start_time + .set_target(self.start_time.value() - (buffer_start - buftime) as f64); + } + Ok(()) + } + + async fn update(&mut self) -> AnyResult<()> { + self.update_starttime().await?; + if let Some(buftime) = self.get_buftime() { + // debug!("Update: buftime: {}", buftime); + while self + .buffer + .buf + .front() + .map(|data| data.time <= buftime) + .unwrap_or(false) + { + match self.buffer.buf.pop_front() { + Some(frame) => { + self.send_buffer(&frame).await?; + } + None => break, + } + } + } + Ok(()) + } + + async fn send_buffer(&mut self, media: &Stamped) -> AnyResult { + if self.inited && self.playing { + let buftime = media.time; + + let runtime = self.buftime_to_runtime(buftime); + // debug!("Frame: {:?}", media); + // debug!( + // "frame buftime: {}, frame runtime: {}, start_time: {}, target_start_time: {}", + // buftime, + // runtime, + // self.start_time.value_i64(), + // self.start_time.get_target(), + // ); + let buf = match &media.data { BcMedia::Iframe(data) => Some(&data.data), BcMedia::Pframe(data) => Some(&data.data), BcMedia::Aac(data) => Some(&data.data), BcMedia::Adpcm(data) => Some(&data.data), _ => None, }; - let appsrc = match media { + let appsrc = match &media.data { BcMedia::Iframe(_) | BcMedia::Pframe(_) => self.vid.as_ref(), BcMedia::Aac(_) | BcMedia::Adpcm(_) => self.aud.as_ref(), _ => None, }; if let (Some(buf), Some(appsrc)) = (buf, appsrc) { - debug!( - "DTS: {:?}, Expected: {:?}, Position in Buffer", - Duration::from_micros(runtime.try_into().unwrap_or(0)), - self.get_runtime() - .map(|t| Duration::from_micros(t.try_into().unwrap_or(0))) - ); + // debug!( + // "DTS: {:?}, Expected: {:?}", + // Duration::from_micros(runtime.try_into().unwrap_or(0)), + // self.get_runtime() + // .map(|t| Duration::from_micros(t.try_into().unwrap_or(0))) + // ); let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { diff --git a/src/rtsp/spring.rs b/src/rtsp/spring.rs index 3725cef..a610c48 100644 --- a/src/rtsp/spring.rs +++ b/src/rtsp/spring.rs @@ -58,6 +58,11 @@ impl Spring { self.target = target; } + #[allow(dead_code)] // Used in debug print + pub(crate) fn get_target(&self) -> f64 { + self.target + } + pub(crate) fn reset_to(&mut self, target: f64) { self.target = target; self.value = target; From 5e81d6a369c0907bc08b4aa5c2f9c57babf66727 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 8 May 2023 15:24:00 +0700 Subject: [PATCH 45/64] Update for pausing --- src/rtsp/gst/sender.rs | 36 +++++++++++++++++++++++++----------- src/rtsp/mod.rs | 22 +++++++++++++++++++--- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 2e660f6..b6ff7b2 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -44,17 +44,6 @@ struct NeoBuffer { impl NeoBuffer { fn push(&mut self, item: Arc) { - let end_time = self.end_time(); - let frame_time = item.time; - if let Some(end_time) = end_time { - let delta_time = end_time - frame_time; - let delta_duration = Duration::from_micros(delta_time.unsigned_abs()); - if delta_duration > Duration::from_secs(1) { - debug!("Clearing buffer due to jump"); - self.buf.clear(); - } - } - self.buf.push_back(item); while self.buf.len() > BUFFER_SIZE { if self.buf.pop_front().is_none() { @@ -243,6 +232,18 @@ impl NeoMediaSenders { for client_data in self.client_data.values_mut() { client_data.add_data(data.clone()).await?; } + + let end_time = self.buffer.end_time(); + let frame_time = data.time; + if let Some(end_time) = end_time { + let delta_time = end_time - frame_time; + let delta_duration = Duration::from_micros(delta_time.unsigned_abs()); + if delta_duration > Duration::from_secs(1) { + debug!("Clearing buffer due to jump"); + self.clear_buffer().await?; + } + } + self.buffer.push(data); self.shared @@ -310,6 +311,7 @@ impl NeoMediaSenders { // Set them into the non init state // This will make them wait for the // buffer to be enough then jump to live + client.buffer.buf.clear(); client.inited = false; } Ok(()) @@ -495,6 +497,10 @@ impl NeoMediaSender { self.buffer.push(frame.clone()); } debug!("Buffer filled"); + + self.jump_to_live().await?; + } else if !self.inited { + debug!("Buffer not ready to init: {}", buffer.buf.len()); } Ok(()) } @@ -537,6 +543,14 @@ impl NeoMediaSender { async fn update(&mut self) -> AnyResult<()> { self.update_starttime().await?; + if !self + .vid + .as_ref() + .map(|x| x.pads().iter().all(|pad| pad.is_linked())) + .unwrap_or(false) + { + return Err(anyhow!("Vid src is closed")); + } if let Some(buftime) = self.get_buftime() { // debug!("Update: buftime: {}", buftime); while self diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 8db2491..ced56da 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -284,13 +284,29 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi .map_err(CameraFailureKind::Retry)?; // Wait for reason to restart tokio::select! { + v = async { + // Wait for motion start and client for start + let mut motion = paused.get_camera().listen_on_motion().await?; + let mut inter = tokio::time::interval(tokio::time::Duration::from_secs_f32(0.01)); + loop { + motion.await_start(Duration::ZERO).await?; + let total_clients = tags.iter().map(|tag| rtsp_thread.get_number_of_clients(tag)).collect::>().fold(0usize, |acc, f| acc + f.unwrap_or(0usize)).await; + if total_clients > 0 { + return Result::<()>::Ok(()); + } + inter.tick().await; + } + }, if paused.get_config().pause.on_motion && paused.get_config().pause.on_disconnect => { + info!("Motion and Client Resume"); + v.with_context(|| "Error while processing motion/client messages") + }, v = async { // Wait for motion start let mut motion = paused.get_camera().listen_on_motion().await?; motion.await_start(Duration::ZERO).await - }, if paused.get_config().pause.on_motion => { + }, if paused.get_config().pause.on_motion && !paused.get_config().pause.on_disconnect => { info!("Motion Resume"); - v.map_err(|e| anyhow!("Error while processing motion messages: {:?}", e)) + v.with_context(|| "Error while processing motion messages") }, v = async { // Wait for client to connect @@ -303,7 +319,7 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi return Ok(()) } } - }, if paused.get_config().pause.on_disconnect => { + }, if paused.get_config().pause.on_disconnect && !paused.get_config().pause.on_motion => { info!("Client Resume"); v }, From 9f1cda3c607bdd35294a0e3ba9bd576cc7ad75f1 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 8 May 2023 19:21:00 +0700 Subject: [PATCH 46/64] Group audio playback Also use buffer lists --- src/rtsp/gst/sender.rs | 201 ++++++++++++++++++++++++++++------------- 1 file changed, 136 insertions(+), 65 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index b6ff7b2..1af8356 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -488,9 +488,7 @@ impl NeoMediaSender { self.start_time.reset_to(start_ms); // Send preprocess now - for frame in preprocess.iter() { - self.send_buffer(frame).await?; - } + self.send_buffers(preprocess).await?; debug!("Preprocessed"); // Send these later for frame in buffer.iter() { @@ -543,6 +541,7 @@ impl NeoMediaSender { async fn update(&mut self) -> AnyResult<()> { self.update_starttime().await?; + let mut buffers = vec![]; if !self .vid .as_ref() @@ -562,79 +561,151 @@ impl NeoMediaSender { { match self.buffer.buf.pop_front() { Some(frame) => { - self.send_buffer(&frame).await?; + buffers.push(frame); } None => break, } } } + + // collect certain frames + self.send_buffers(&buffers).await?; + Ok(()) } - async fn send_buffer(&mut self, media: &Stamped) -> AnyResult { + async fn send_buffers>(&mut self, medias: &[T]) -> AnyResult<()> { + if medias.is_empty() { + return Ok(()); + } if self.inited && self.playing { - let buftime = media.time; - - let runtime = self.buftime_to_runtime(buftime); - // debug!("Frame: {:?}", media); - // debug!( - // "frame buftime: {}, frame runtime: {}, start_time: {}, target_start_time: {}", - // buftime, - // runtime, - // self.start_time.value_i64(), - // self.start_time.get_target(), - // ); - let buf = match &media.data { - BcMedia::Iframe(data) => Some(&data.data), - BcMedia::Pframe(data) => Some(&data.data), - BcMedia::Aac(data) => Some(&data.data), - BcMedia::Adpcm(data) => Some(&data.data), - _ => None, - }; - let appsrc = match &media.data { - BcMedia::Iframe(_) | BcMedia::Pframe(_) => self.vid.as_ref(), - BcMedia::Aac(_) | BcMedia::Adpcm(_) => self.aud.as_ref(), - _ => None, - }; - - if let (Some(buf), Some(appsrc)) = (buf, appsrc) { - // debug!( - // "DTS: {:?}, Expected: {:?}", - // Duration::from_micros(runtime.try_into().unwrap_or(0)), - // self.get_runtime() - // .map(|t| Duration::from_micros(t.try_into().unwrap_or(0))) - // ); - - let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); - { - let gst_buf_mut = gst_buf.get_mut().unwrap(); - - let time = ClockTime::from_useconds(runtime.try_into().unwrap()); - gst_buf_mut.set_dts(time); - let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); - gst_buf_data.copy_from_slice(buf); - } - // debug!("Buffer pushed"); - let thread_appsrc = appsrc.clone(); // GObjects are refcounted - let res = tokio::task::spawn_blocking(move || { - thread_appsrc - .push_buffer(gst_buf.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) - }) - .await; - match &res { - Err(e) => { - debug!("Failed to send buffer: {:?}", e); - } - Ok(Err(e)) => { - debug!("Failed to send buffer: {:?}", e); - } - Ok(Ok(_)) => {} + let mut vid_buffers: Vec<(FrameTime, Vec)> = vec![]; + let mut aud_buffers: Vec<(FrameTime, Vec)> = vec![]; + for media in medias.iter().map(|t| t.as_ref()) { + let buffer = match &media.data { + BcMedia::Iframe(_) | BcMedia::Pframe(_) => Some(&mut vid_buffers), + BcMedia::Aac(_) | BcMedia::Adpcm(_) => Some(&mut aud_buffers), + _ => None, }; - res??; + let data = match &media.data { + BcMedia::Iframe(data) => Some(&data.data), + BcMedia::Pframe(data) => Some(&data.data), + BcMedia::Aac(data) => Some(&data.data), + BcMedia::Adpcm(data) => Some(&data.data), + _ => None, + }; + if let (Some(data), Some(buffer)) = (data, buffer) { + let next_time = media.time; + if let Some(last) = buffer.last_mut() { + let last_time = last.0; + if next_time == last_time { + last.1.extend(data.iter().copied()); + } else { + buffer.push((next_time, data.clone())) + } + } else { + buffer.push((next_time, data.clone())) + } + } + } + if !vid_buffers.is_empty() { + if let Some(appsrc) = self.vid.clone() { + let buffers = { + let mut buffers = gstreamer::BufferList::new_sized(vid_buffers.len()); + { + let buffers_ref = buffers.get_mut().unwrap(); + for (time, buf) in vid_buffers.drain(..) { + let runtime = self.buftime_to_runtime(time); + + let gst_buf = { + let mut gst_buf = + gstreamer::Buffer::with_size(buf.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); + + let time = + ClockTime::from_useconds(runtime.try_into().unwrap()); + gst_buf_mut.set_dts(time); + let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(buf.as_slice()); + } + gst_buf + }; + buffers_ref.add(gst_buf); + } + } + buffers + }; + + let res = tokio::task::spawn_blocking(move || { + appsrc + .push_buffer_list(buffers.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await; + match &res { + Err(e) => { + debug!("Paniced on send buffer list: {:?}", e); + } + Ok(Err(e)) => { + debug!("Failed to send buffer list: {:?}", e); + } + Ok(Ok(_)) => {} + }; + res??; + } + } + + if !aud_buffers.is_empty() { + if let Some(appsrc) = self.aud.clone() { + let buffers = { + let mut buffers = gstreamer::BufferList::new_sized(aud_buffers.len()); + { + let buffers_ref = buffers.get_mut().unwrap(); + for (time, buf) in aud_buffers.drain(..) { + let runtime = self.buftime_to_runtime(time); + + let gst_buf = { + let mut gst_buf = + gstreamer::Buffer::with_size(buf.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); + + let time = + ClockTime::from_useconds(runtime.try_into().unwrap()); + gst_buf_mut.set_dts(time); + let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(buf.as_slice()); + } + gst_buf + }; + buffers_ref.add(gst_buf); + } + } + buffers + }; + + let res = tokio::task::spawn_blocking(move || { + appsrc + .push_buffer_list(buffers.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await; + match &res { + Err(e) => { + debug!("Paniced on send buffer list: {:?}", e); + } + Ok(Err(e)) => { + debug!("Failed to send buffer list: {:?}", e); + } + Ok(Ok(_)) => {} + }; + res??; + } } } - Ok(true) + Ok(()) } } From d5cb12dc3c68d85ccd39da6a290b6a2d0ed48aa2 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 9 May 2023 08:56:58 +0700 Subject: [PATCH 47/64] Improve clearing of 'Stream not ready' streams --- src/rtsp/gst/factory.rs | 3 +- src/rtsp/gst/server.rs | 63 +++++++++++++++++++++++++++++++++++++++++ src/rtsp/mod.rs | 27 ++++++------------ 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 4642afc..75901a3 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -274,8 +274,9 @@ impl NeoMediaFactoryImpl { ) { (VidFormats::Unknown, true) | (_, false) => { debug!("Building Unknown Pipeline"); - let source = make_element("videotestsrc", "vidsrc")?; + let source = make_element("videotestsrc", "testvidsrc")?; source.set_property_from_str("pattern", "snow"); + source.set_property("num-buffers", 500i32); // Send buffers then EOS let queue = make_queue("queue0")?; let queue2 = make_queue2("queue2")?; let overlay = make_element("textoverlay", "overlay")?; diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 727b2fb..2a24280 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -174,6 +174,8 @@ impl NeoRtspServer { self.imp().buffer_ready(tag).await } + #[allow(dead_code)] + /// Clear all sessions on a given path pub(crate) async fn clear_session>(&self, tag: T) { let tag: String = tag.into(); if let Some(media) = self.imp().medias.read().await.get(&tag) { @@ -219,6 +221,67 @@ impl NeoRtspServer { })); } + /// Clear all sessions on the rtsp server that are using the not ready stream + pub(crate) async fn clear_session_notready>(&self, tag: T) { + let tag: String = tag.into(); + if let Some(media) = self.imp().medias.read().await.get(&tag) { + self.clear_session_paths_notready(media.paths.iter()); + } + } + + fn clear_session_paths_notready>(&self, paths: impl Iterator) { + let paths: Vec<_> = paths.collect(); + self.client_filter(Some(&mut |_server, client| { + let mut close_client = false; + client.session_filter(Some(&mut |_client, session| { + let mut clean_up = false; + session.filter(Some(&mut |_session, session_media| { + if paths.iter().any(|p| { + let s = p.as_ref(); + session_media + .matches(s) + .map(|amt| amt == s.len() as i32) + .unwrap_or(false) + }) { + // Path is correct now check for no vidsrc + let element = session_media + .media() + .expect("Media should exist") + .element() + .dynamic_cast::() + .expect("Media element should be a bin"); + // debug!("Searching for testvidsrc on {:?}", element.name()); + // for child in element.children().iter() { + // debug!(" - Child: {:?}", child.name()); + // } + if element.child_by_name("testvidsrc").is_some() { + info!("Removing Session Media"); + clean_up = true; + RTSPFilterResult::Remove + } else { + RTSPFilterResult::Keep + } + } else { + RTSPFilterResult::Keep + } + })); + + if clean_up { + close_client = true; + RTSPFilterResult::Remove + } else { + RTSPFilterResult::Keep + } + })); + if close_client { + client.close(); + RTSPFilterResult::Remove + } else { + RTSPFilterResult::Keep + } + })); + } + // Clear buffers on all senders of a tag pub(crate) async fn clear_buffer>(&self, tag: T) -> AnyResult<()> { if let Some(sender) = self.imp().get_sender(tag).await { diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index ced56da..b5da075 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -184,15 +184,6 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi let tags = loggedin.shared.get_tags(); let rtsp_thread = loggedin.get_rtsp(); - // Check if buffers were prepared before we clear - let buffers_prepared = tags - .iter() - .map(|tag| rtsp_thread.buffer_ready(tag)) - .collect::>() - .filter_map(|a| a) - .all(|a| a) - .await; - // Clear all buffers present tags.iter() .map(|tag| rtsp_thread.clear_buffer(tag)) @@ -228,15 +219,14 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi .collect::>() .await; - if !buffers_prepared { - // Clear all current media and force a reconnect - // This shoudld stop them from watching the "Stream Not Ready Thing" - tags.iter() - .map(|tag| rtsp_thread.clear_session(tag)) - .collect::>() - .collect::>() - .await; - } + // Clear "stream not ready" media to try and force a reconnect + // This shoud stop them from watching the "Stream Not Ready" thing + debug!("Clearing not ready clients"); + tags.iter() + .map(|tag| rtsp_thread.clear_session_notready(tag)) + .collect::>() + .collect::>() + .await; log::info!("{}: Buffers prepared", name); loop { @@ -261,7 +251,6 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi // Wait for client to disconnect let mut inter = tokio::time::interval(tokio::time::Duration::from_secs_f32(0.01)); - info!("Await Client Pause"); loop { inter.tick().await; let total_clients = tags.iter().map(|tag| rtsp_thread.get_number_of_clients(tag)).collect::>().fold(0usize, |acc, f| acc + f.unwrap_or(0usize)).await; From 52840348bfe2ebd698a0ba9dd341822f2abe31ac Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 9 May 2023 11:23:23 +0700 Subject: [PATCH 48/64] Send frames async --- src/config.rs | 7 + src/rtsp/gst/sender.rs | 287 ++++++++++++++++++++++-------------- src/rtsp/spring.rs | 6 +- src/rtsp/states/loggedin.rs | 18 ++- 4 files changed, 197 insertions(+), 121 deletions(-) diff --git a/src/config.rs b/src/config.rs index d2f8f69..78b3c8a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -99,6 +99,9 @@ pub(crate) struct CameraConfig { #[serde(default = "default_print", alias = "print")] pub(crate) print_format: PrintFormat, + + #[serde(default = "default_update_time", alias = "time")] + pub(crate) update_time: bool, } #[derive(Debug, Deserialize, Validate, Clone)] @@ -203,6 +206,10 @@ fn default_channel_id() -> u8 { 0 } +fn default_update_time() -> bool { + false +} + fn default_motion_timeout() -> f64 { 1. } diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 1af8356..e6c9ba0 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -44,7 +44,24 @@ struct NeoBuffer { impl NeoBuffer { fn push(&mut self, item: Arc) { - self.buf.push_back(item); + // Sort time + // debug!("sorting"); + let mut sorting_vec = vec![]; + let time = item.time; + while self + .buf + .back() + .map(|back| back.time > time) + .unwrap_or(false) + { + sorting_vec.push(self.buf.pop_back().unwrap()); + } + sorting_vec.push(item); + + for sorted_item in sorting_vec.drain(..) { + // debug!("Pushing frame with time: {}", sorted_item.time); + self.buf.push_back(sorted_item); + } while self.buf.len() > BUFFER_SIZE { if self.buf.pop_front().is_none() { break; @@ -83,14 +100,14 @@ impl NeoBuffer { // .map(|i| i as FrameTime) // } - fn start_time(&self) -> Option { - let (fronts, backs) = self.buf.as_slices(); - fronts - .iter() - .chain(&mut backs.iter()) - .map(|frame| frame.time) - .next() - } + // fn start_time(&self) -> Option { + // let (fronts, backs) = self.buf.as_slices(); + // fronts + // .iter() + // .chain(&mut backs.iter()) + // .map(|frame| frame.time) + // .next() + // } fn end_time(&self) -> Option { let (fronts, backs) = self.buf.as_slices(); @@ -275,7 +292,7 @@ impl NeoMediaSenders { tokio::task::yield_now().await; match self.client_data.entry(key) { Entry::Occupied(mut occ) => { - if let Err(e) = occ.get_mut().process_commands().await { + if let Err(e) = occ.get_mut().process_commands(&self.buffer).await { debug!("Could not process client command: {:?}", e); occ.remove(); continue; @@ -425,10 +442,13 @@ impl NeoMediaSender { async fn jump_to_live(&mut self) -> AnyResult<()> { if self.inited { - if let Some(buffer_start) = self.buffer.start_time() { + let target_frame = self.buffer.buf.len().saturating_sub(BUFFER_SIZE / 3); + let target_time = self.buffer.buf.get(target_frame).map(|frame| frame.time); + + if let Some(target_time) = target_time { let runtime = self.get_runtime().unwrap_or(0); - self.start_time.reset_to((buffer_start - runtime) as f64); + self.start_time.reset_to((target_time - runtime) as f64); debug!( "Jumped to live: New start time: {:?}", Duration::from_micros(self.start_time.value_u64()), @@ -439,18 +459,52 @@ impl NeoMediaSender { Ok(()) } - async fn seek(&mut self, _target_time: FrameTime) -> AnyResult<()> { + async fn update_starttime(&mut self) -> AnyResult<()> { + self.start_time.update().await; + let target_frame = self.buffer.buf.len().saturating_sub(BUFFER_SIZE / 3); + let target_time = self.buffer.buf.get(target_frame).map(|frame| frame.time); + if let (Some(runtime), Some(target_time)) = (self.get_runtime(), target_time) { + debug!( + "Target frame: {}, Target time: {}, Target start time: {}, Current start time: {}", + target_frame, + target_time, + target_time - runtime, + self.start_time.value_i64(), + ); + self.start_time.set_target((target_time - runtime) as f64); + } + Ok(()) + } + + async fn seek( + &mut self, + target_runtime: FrameTime, + master_buffer: &NeoBuffer, + ) -> AnyResult<()> { + let target_buftime = self.runtime_to_buftime(target_runtime); + self.buffer.buf.clear(); + + // Master buffer has historical data + // Refill our buffer with the frames it wants + for master_frame in master_buffer + .buf + .iter() + .filter(|frame| frame.time >= target_buftime) + { + self.buffer.push(master_frame.clone()); + } + // Then accelerate us to to live self.jump_to_live().await?; Ok(()) } - async fn process_commands(&mut self) -> AnyResult<()> { + async fn process_commands(&mut self, master_buffer: &NeoBuffer) -> AnyResult<()> { if self.inited { if let Ok(command) = self.command_reciever.try_recv() { match command { NeoMediaSenderCommand::Seek(dest) => { debug!("Got Seek Request: {:?}", Duration::from_micros(dest)); - self.seek(dest as FrameTime).await?; + self.seek(dest as FrameTime, master_buffer).await?; } NeoMediaSenderCommand::Pause => { if self.playing { @@ -529,16 +583,6 @@ impl NeoMediaSender { buftime.saturating_sub(self.start_time.value_i64()).max(0) } - async fn update_starttime(&mut self) -> AnyResult<()> { - self.start_time.update().await; - if let (Some(buftime), Some(buffer_start)) = (self.get_buftime(), self.buffer.start_time()) - { - self.start_time - .set_target(self.start_time.value() - (buffer_start - buftime) as f64); - } - Ok(()) - } - async fn update(&mut self) -> AnyResult<()> { self.update_starttime().await?; let mut buffers = vec![]; @@ -568,6 +612,7 @@ impl NeoMediaSender { } } + tokio::task::yield_now().await; // collect certain frames self.send_buffers(&buffers).await?; @@ -579,6 +624,7 @@ impl NeoMediaSender { return Ok(()); } if self.inited && self.playing { + tokio::task::yield_now().await; let mut vid_buffers: Vec<(FrameTime, Vec)> = vec![]; let mut aud_buffers: Vec<(FrameTime, Vec)> = vec![]; for media in medias.iter().map(|t| t.as_ref()) { @@ -608,103 +654,120 @@ impl NeoMediaSender { } } } - if !vid_buffers.is_empty() { - if let Some(appsrc) = self.vid.clone() { - let buffers = { - let mut buffers = gstreamer::BufferList::new_sized(vid_buffers.len()); - { - let buffers_ref = buffers.get_mut().unwrap(); - for (time, buf) in vid_buffers.drain(..) { - let runtime = self.buftime_to_runtime(time); + tokio::task::yield_now().await; + tokio::try_join!( + async { + if !vid_buffers.is_empty() { + if let Some(appsrc) = self.vid.clone() { + let buffers = { + let mut buffers = + gstreamer::BufferList::new_sized(vid_buffers.len()); + { + let buffers_ref = buffers.get_mut().unwrap(); + for (time, buf) in vid_buffers.drain(..) { + debug!("Sending vid frame at time {}", time); + tokio::task::yield_now().await; + let runtime = self.buftime_to_runtime(time); - let gst_buf = { - let mut gst_buf = - gstreamer::Buffer::with_size(buf.len()).unwrap(); - { - let gst_buf_mut = gst_buf.get_mut().unwrap(); + let gst_buf = { + let mut gst_buf = + gstreamer::Buffer::with_size(buf.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); - let time = - ClockTime::from_useconds(runtime.try_into().unwrap()); - gst_buf_mut.set_dts(time); - let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); - gst_buf_data.copy_from_slice(buf.as_slice()); + let time = ClockTime::from_useconds( + runtime.try_into().unwrap(), + ); + gst_buf_mut.set_dts(time); + let mut gst_buf_data = + gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(buf.as_slice()); + } + gst_buf + }; + buffers_ref.add(gst_buf); } - gst_buf - }; - buffers_ref.add(gst_buf); - } + } + buffers + }; + + let res = tokio::task::spawn_blocking(move || { + appsrc + .push_buffer_list(buffers.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await; + match &res { + Err(e) => { + debug!("Paniced on send buffer list: {:?}", e); + } + Ok(Err(e)) => { + debug!("Failed to send buffer list: {:?}", e); + } + Ok(Ok(_)) => {} + }; + res??; } - buffers - }; + } + AnyResult::Ok(()) + }, + async { + if !aud_buffers.is_empty() { + if let Some(appsrc) = self.aud.clone() { + let buffers = { + let mut buffers = + gstreamer::BufferList::new_sized(aud_buffers.len()); + { + let buffers_ref = buffers.get_mut().unwrap(); + for (time, buf) in aud_buffers.drain(..) { + tokio::task::yield_now().await; + let runtime = self.buftime_to_runtime(time); - let res = tokio::task::spawn_blocking(move || { - appsrc - .push_buffer_list(buffers.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) - }) - .await; - match &res { - Err(e) => { - debug!("Paniced on send buffer list: {:?}", e); - } - Ok(Err(e)) => { - debug!("Failed to send buffer list: {:?}", e); - } - Ok(Ok(_)) => {} - }; - res??; - } - } + let gst_buf = { + let mut gst_buf = + gstreamer::Buffer::with_size(buf.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); - if !aud_buffers.is_empty() { - if let Some(appsrc) = self.aud.clone() { - let buffers = { - let mut buffers = gstreamer::BufferList::new_sized(aud_buffers.len()); - { - let buffers_ref = buffers.get_mut().unwrap(); - for (time, buf) in aud_buffers.drain(..) { - let runtime = self.buftime_to_runtime(time); - - let gst_buf = { - let mut gst_buf = - gstreamer::Buffer::with_size(buf.len()).unwrap(); - { - let gst_buf_mut = gst_buf.get_mut().unwrap(); - - let time = - ClockTime::from_useconds(runtime.try_into().unwrap()); - gst_buf_mut.set_dts(time); - let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); - gst_buf_data.copy_from_slice(buf.as_slice()); + let time = ClockTime::from_useconds( + runtime.try_into().unwrap(), + ); + gst_buf_mut.set_dts(time); + let mut gst_buf_data = + gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(buf.as_slice()); + } + gst_buf + }; + buffers_ref.add(gst_buf); } - gst_buf - }; - buffers_ref.add(gst_buf); - } - } - buffers - }; + } + buffers + }; - let res = tokio::task::spawn_blocking(move || { - appsrc - .push_buffer_list(buffers.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) - }) - .await; - match &res { - Err(e) => { - debug!("Paniced on send buffer list: {:?}", e); + let res = tokio::task::spawn_blocking(move || { + appsrc + .push_buffer_list(buffers.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await; + match &res { + Err(e) => { + debug!("Paniced on send buffer list: {:?}", e); + } + Ok(Err(e)) => { + debug!("Failed to send buffer list: {:?}", e); + } + Ok(Ok(_)) => {} + }; + res??; } - Ok(Err(e)) => { - debug!("Failed to send buffer list: {:?}", e); - } - Ok(Ok(_)) => {} - }; - res??; + } + AnyResult::Ok(()) } - } + )?; } Ok(()) } diff --git a/src/rtsp/spring.rs b/src/rtsp/spring.rs index a610c48..c295a2f 100644 --- a/src/rtsp/spring.rs +++ b/src/rtsp/spring.rs @@ -34,9 +34,9 @@ impl Spring { ); } - pub(crate) fn value(&self) -> f64 { - self.value - } + // pub(crate) fn value(&self) -> f64 { + // self.value + // } // pub(crate) fn value_u32(&self) -> u32 { // self.value.max(std::u32::MIN as f64).floor().min(std::u32::MAX as f64) as u32 diff --git a/src/rtsp/states/loggedin.rs b/src/rtsp/states/loggedin.rs index 87ae651..5b194b2 100644 --- a/src/rtsp/states/loggedin.rs +++ b/src/rtsp/states/loggedin.rs @@ -32,22 +32,28 @@ impl Camera { pub(crate) async fn manage(&self) -> Result<()> { let cam_time = self.state.camera.get_time().await?; + let mut update = false; if let Some(time) = cam_time { info!( "{}: Camera time is already set: {}", self.shared.config.name, time ); + if self.shared.config.update_time { + update = true; + } } else { + update = true; + warn!( + "{}: Camera has no time set, Updating", + self.shared.config.name + ); + } + if update { use time::OffsetDateTime; - // We'd like now_local() but it's deprecated - try to get the local time, but if no - // time zone, fall back to UTC. let new_time = OffsetDateTime::now_local().unwrap_or_else(|_| OffsetDateTime::now_utc()); - warn!( - "{}: Camera has no time set, setting to {}", - self.shared.config.name, new_time - ); + info!("{}: Setting time to {}", self.shared.config.name, new_time); match self.state.camera.set_time(new_time).await { Ok(_) => { let cam_time = self.state.camera.get_time().await?; From 15f9993c1205645f952efd40c6fa9febde815512 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 9 May 2023 14:35:01 +0700 Subject: [PATCH 49/64] Manual seeking --- src/rtsp/gst/factory.rs | 32 +++++++++---- src/rtsp/gst/sender.rs | 103 ++++++++++++++++++++++++++-------------- src/rtsp/mod.rs | 33 ++++++++----- 3 files changed, 112 insertions(+), 56 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 75901a3..3272db0 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -323,11 +323,20 @@ impl NeoMediaFactoryImpl { .enough_data(move |_appsrc| { let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); }) - .seek_data(move |_appsrc, seek_pos| { + .seek_data(move |appsrc, seek_pos| { debug!("Send seek H265"); - let _ = - seek_command.blocking_send(NeoMediaSenderCommand::Seek(seek_pos)); - true + let runtime = appsrc.clock().and_then(|clock| { + clock.time().and_then(|time| { + appsrc.base_time().map(|base_time| { + time.saturating_sub(base_time).useconds() as i64 + }) + }) + }); + let res = seek_command + .blocking_send(NeoMediaSenderCommand::Seek(runtime, seek_pos)) + .is_ok(); + debug!("Sent seek H265: {}", res); + res }) .build(), ); @@ -371,15 +380,22 @@ impl NeoMediaFactoryImpl { source.set_callbacks( AppSrcCallbacks::builder() .need_data(move |_appsrc, _amt| { - let _ = need_command.try_send(NeoMediaSenderCommand::Resume); + let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume); }) .enough_data(move |_appsrc| { - let _ = enough_command.try_send(NeoMediaSenderCommand::Pause); + let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause); }) - .seek_data(move |_appsrc, seek_pos| { + .seek_data(move |appsrc, seek_pos| { debug!("Send seek H264: {}", seek_pos); + let runtime = appsrc.clock().and_then(|clock| { + clock.time().and_then(|time| { + appsrc.base_time().map(|base_time| { + time.saturating_sub(base_time).useconds() as i64 + }) + }) + }); let result = seek_command - .try_send(NeoMediaSenderCommand::Seek(seek_pos)) + .blocking_send(NeoMediaSenderCommand::Seek(runtime, seek_pos)) .is_ok(); debug!(" - Sent seek H264: {}", result); result diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index e6c9ba0..641f035 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -100,14 +100,14 @@ impl NeoBuffer { // .map(|i| i as FrameTime) // } - // fn start_time(&self) -> Option { - // let (fronts, backs) = self.buf.as_slices(); - // fronts - // .iter() - // .chain(&mut backs.iter()) - // .map(|frame| frame.time) - // .next() - // } + fn start_time(&self) -> Option { + let (fronts, backs) = self.buf.as_slices(); + fronts + .iter() + .chain(&mut backs.iter()) + .map(|frame| frame.time) + .next() + } fn end_time(&self) -> Option { let (fronts, backs) = self.buf.as_slices(); @@ -385,7 +385,7 @@ impl NeoMediaSenders { } pub(super) enum NeoMediaSenderCommand { - Seek(u64), + Seek(Option, u64), Pause, Resume, } @@ -401,19 +401,12 @@ pub(super) struct NeoMediaSender { playing: bool, } -#[allow(dead_code)] -enum JumpMethod { - MiddleIFrame, - BufferPerunit(f32), -} - impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(30); Self { - start_time: Spring::new(0.0, 0.0, 2.5), + start_time: Spring::new(0.0, 0.0, 5.5), buffer: NeoBuffer::default(), - vid: None, aud: None, command_reciever: rx, @@ -461,9 +454,28 @@ impl NeoMediaSender { async fn update_starttime(&mut self) -> AnyResult<()> { self.start_time.update().await; - let target_frame = self.buffer.buf.len().saturating_sub(BUFFER_SIZE / 3); - let target_time = self.buffer.buf.get(target_frame).map(|frame| frame.time); - if let (Some(runtime), Some(target_time)) = (self.get_runtime(), target_time) { + let target_idx = BUFFER_SIZE / 3; + let target_time_target_frame = if self.buffer.buf.len() >= target_idx { + let target_frame = self.buffer.buf.len().saturating_sub(target_idx); + self.buffer + .buf + .get(target_frame) + .map(|frame| (frame.time, target_frame as i64)) + } else { + // Approximate it's location + let fraction = target_idx as f64 / self.buffer.buf.len() as f64; + if let (Some(st), Some(et)) = (self.buffer.start_time(), self.buffer.end_time()) { + Some(( + et - ((et - st) as f64 * fraction) as FrameTime, + -((target_idx - self.buffer.buf.len()) as i64), + )) + } else { + None + } + }; + if let (Some(runtime), Some((target_time, target_frame))) = + (self.get_runtime(), target_time_target_frame) + { debug!( "Target frame: {}, Target time: {}, Target start time: {}, Current start time: {}", target_frame, @@ -478,22 +490,31 @@ impl NeoMediaSender { async fn seek( &mut self, + _original_runtime: Option, target_runtime: FrameTime, - master_buffer: &NeoBuffer, + _master_buffer: &NeoBuffer, ) -> AnyResult<()> { - let target_buftime = self.runtime_to_buftime(target_runtime); - self.buffer.buf.clear(); - - // Master buffer has historical data - // Refill our buffer with the frames it wants - for master_frame in master_buffer - .buf - .iter() - .filter(|frame| frame.time >= target_buftime) - { - self.buffer.push(master_frame.clone()); + if let Some(appsrc) = self.vid.as_ref() { + if let Some(clock) = appsrc.clock() { + if let Some(time) = clock.time() { + if appsrc.base_time().is_some() { + // let current_runtime = time.saturating_sub(base_time); + let target_runtime = ClockTime::from_useconds(target_runtime as u64); + appsrc.set_base_time(time.saturating_sub(target_runtime)); + } + } + } + } + if let Some(appsrc) = self.aud.as_ref() { + if let Some(clock) = appsrc.clock() { + if let Some(time) = clock.time() { + if appsrc.base_time().is_some() { + let target_runtime = ClockTime::from_useconds(target_runtime as u64); + appsrc.set_base_time(time.saturating_sub(target_runtime)); + } + } + } } - // Then accelerate us to to live self.jump_to_live().await?; Ok(()) } @@ -502,9 +523,9 @@ impl NeoMediaSender { if self.inited { if let Ok(command) = self.command_reciever.try_recv() { match command { - NeoMediaSenderCommand::Seek(dest) => { + NeoMediaSenderCommand::Seek(runtime, dest) => { debug!("Got Seek Request: {:?}", Duration::from_micros(dest)); - self.seek(dest as FrameTime, master_buffer).await?; + self.seek(runtime, dest as FrameTime, master_buffer).await?; } NeoMediaSenderCommand::Pause => { if self.playing { @@ -658,6 +679,7 @@ impl NeoMediaSender { tokio::try_join!( async { if !vid_buffers.is_empty() { + debug!("Sending video buffers: {}", vid_buffers.len()); if let Some(appsrc) = self.vid.clone() { let buffers = { let mut buffers = @@ -665,9 +687,17 @@ impl NeoMediaSender { { let buffers_ref = buffers.get_mut().unwrap(); for (time, buf) in vid_buffers.drain(..) { - debug!("Sending vid frame at time {}", time); tokio::task::yield_now().await; let runtime = self.buftime_to_runtime(time); + let actual_runtime = self + .get_runtime() + .map(|i| Duration::from_micros(i as u64)); + debug!( + " - Sending vid frame at time {} ({:?} Expect: {:?})", + time, + Duration::from_micros(runtime as u64), + actual_runtime + ); let gst_buf = { let mut gst_buf = @@ -692,6 +722,7 @@ impl NeoMediaSender { }; let res = tokio::task::spawn_blocking(move || { + debug!(" - Pushing buffer: {}", buffers.len()); appsrc .push_buffer_list(buffers.copy()) .map(|_| ()) diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index b5da075..4d9295e 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -199,19 +199,28 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi .map_err(CameraFailureKind::Retry)?; // Wait for buffers to be prepared - let mut waiter = tokio::time::interval(Duration::from_micros(500)); - loop { - waiter.tick().await; - if tags - .iter() - .map(|tag| rtsp_thread.buffer_ready(tag)) - .collect::>() - .all(|f| f.unwrap_or(false)) - .await - { - break; - } + tokio::select! { + v = async { + let mut waiter = tokio::time::interval(Duration::from_micros(500)); + loop { + waiter.tick().await; + if tags + .iter() + .map(|tag| rtsp_thread.buffer_ready(tag)) + .collect::>() + .all(|f| f.unwrap_or(false)) + .await + { + break; + } + } + Ok(()) + } => v, + // Or for stream to error + v = streaming.join() => {v}, } + .with_context(|| format!("{}: Error while waiting for buffers", name)) + .map_err(CameraFailureKind::Retry)?; tags.iter() .map(|tag| rtsp_thread.jump_to_live(tag)) From 55d357ecf8f088dedece4b2e5c3b80fb04619a46 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 9 May 2023 15:22:51 +0700 Subject: [PATCH 50/64] Async sender code --- src/rtsp/gst/sender.rs | 96 +++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 641f035..1601ac1 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -1,17 +1,17 @@ //! The component that handles passing BcMedia into //! gstreamer media stream use anyhow::anyhow; -use futures::stream::StreamExt; +use futures::{ + future::TryFutureExt, + stream::{FuturesUnordered, StreamExt}, +}; use gstreamer::{prelude::*, ClockTime}; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use log::*; use neolink_core::bcmedia::model::*; use std::{ - collections::{ - VecDeque, - {hash_map::Entry, HashMap}, - }, + collections::{HashMap, VecDeque}, convert::TryInto, sync::{ atomic::{AtomicU64, Ordering}, @@ -271,52 +271,70 @@ impl NeoMediaSenders { } async fn init_clients(&mut self) -> AnyResult<()> { - for key in self.client_data.keys().copied().collect::>() { - tokio::task::yield_now().await; - match self.client_data.entry(key) { - Entry::Occupied(mut occ) => { - if let Err(e) = occ.get_mut().initialise(&self.buffer).await { + let (client_data, buffer) = (&mut self.client_data, &self.buffer); + let keys_to_remove = client_data + .iter_mut() + .map(|(&key, client)| { + client + .initialise(buffer) + .map_ok(|e| { debug!("Could not init client: {:?}", e); - occ.remove(); - continue; - } - } - Entry::Vacant(_) => {} - } + None + }) + .unwrap_or_else(move |_| Some(key)) + }) + .collect::>() + .filter_map(|a| async move { a }) + .collect::>() + .await; + for key in keys_to_remove.iter() { + client_data.remove(key); } Ok(()) } async fn process_client_commands(&mut self) -> AnyResult<()> { - for key in self.client_data.keys().copied().collect::>() { - tokio::task::yield_now().await; - match self.client_data.entry(key) { - Entry::Occupied(mut occ) => { - if let Err(e) = occ.get_mut().process_commands(&self.buffer).await { + let (client_data, buffer) = (&mut self.client_data, &self.buffer); + let keys_to_remove = client_data + .iter_mut() + .map(|(&key, client)| { + client + .process_commands(buffer) + .map_ok(|e| { debug!("Could not process client command: {:?}", e); - occ.remove(); - continue; - } - } - Entry::Vacant(_) => {} - } + None + }) + .unwrap_or_else(move |_| Some(key)) + }) + .collect::>() + .filter_map(|a| async move { a }) + .collect::>() + .await; + for key in keys_to_remove.iter() { + client_data.remove(key); } Ok(()) } async fn process_client_update(&mut self) -> AnyResult<()> { - for key in self.client_data.keys().copied().collect::>() { - tokio::task::yield_now().await; - match self.client_data.entry(key) { - Entry::Occupied(mut occ) => { - if let Err(e) = occ.get_mut().update().await { + let client_data = &mut self.client_data; + let keys_to_remove = client_data + .iter_mut() + .map(|(&key, client)| { + client + .update() + .map_ok(|e| { debug!("Could not update client: {:?}", e); - occ.remove(); - continue; - } - } - Entry::Vacant(_) => {} - } + None + }) + .unwrap_or_else(move |_| Some(key)) + }) + .collect::>() + .filter_map(|a| async move { a }) + .collect::>() + .await; + for key in keys_to_remove.iter() { + client_data.remove(key); } Ok(()) } @@ -624,6 +642,7 @@ impl NeoMediaSender { .map(|data| data.time <= buftime) .unwrap_or(false) { + tokio::task::yield_now().await; match self.buffer.buf.pop_front() { Some(frame) => { buffers.push(frame); @@ -649,6 +668,7 @@ impl NeoMediaSender { let mut vid_buffers: Vec<(FrameTime, Vec)> = vec![]; let mut aud_buffers: Vec<(FrameTime, Vec)> = vec![]; for media in medias.iter().map(|t| t.as_ref()) { + tokio::task::yield_now().await; let buffer = match &media.data { BcMedia::Iframe(_) | BcMedia::Pframe(_) => Some(&mut vid_buffers), BcMedia::Aac(_) | BcMedia::Adpcm(_) => Some(&mut aud_buffers), From d2daad0bf21a58e03f3adf3b7efb3241939729ca Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 9 May 2023 15:47:06 +0700 Subject: [PATCH 51/64] Just seek to live --- src/rtsp/gst/sender.rs | 82 +++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 53 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 1601ac1..a35d580 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -277,11 +277,11 @@ impl NeoMediaSenders { .map(|(&key, client)| { client .initialise(buffer) - .map_ok(|e| { + .map_ok(|_| None) + .unwrap_or_else(move |e| { debug!("Could not init client: {:?}", e); - None + Some(key) }) - .unwrap_or_else(move |_| Some(key)) }) .collect::>() .filter_map(|a| async move { a }) @@ -300,11 +300,11 @@ impl NeoMediaSenders { .map(|(&key, client)| { client .process_commands(buffer) - .map_ok(|e| { + .map_ok(|_| None) + .unwrap_or_else(move |e| { debug!("Could not process client command: {:?}", e); - None + Some(key) }) - .unwrap_or_else(move |_| Some(key)) }) .collect::>() .filter_map(|a| async move { a }) @@ -321,13 +321,10 @@ impl NeoMediaSenders { let keys_to_remove = client_data .iter_mut() .map(|(&key, client)| { - client - .update() - .map_ok(|e| { - debug!("Could not update client: {:?}", e); - None - }) - .unwrap_or_else(move |_| Some(key)) + client.update().map_ok(|_| None).unwrap_or_else(move |e| { + debug!("Could not update client: {:?}", e); + Some(key) + }) }) .collect::>() .filter_map(|a| async move { a }) @@ -491,16 +488,16 @@ impl NeoMediaSender { None } }; - if let (Some(runtime), Some((target_time, target_frame))) = + if let (Some(runtime), Some((target_time, _target_frame))) = (self.get_runtime(), target_time_target_frame) { - debug!( - "Target frame: {}, Target time: {}, Target start time: {}, Current start time: {}", - target_frame, - target_time, - target_time - runtime, - self.start_time.value_i64(), - ); + // debug!( + // "Target frame: {}, Target time: {}, Target start time: {}, Current start time: {}", + // target_frame, + // target_time, + // target_time - runtime, + // self.start_time.value_i64(), + // ); self.start_time.set_target((target_time - runtime) as f64); } Ok(()) @@ -509,31 +506,10 @@ impl NeoMediaSender { async fn seek( &mut self, _original_runtime: Option, - target_runtime: FrameTime, + _target_runtime: FrameTime, _master_buffer: &NeoBuffer, ) -> AnyResult<()> { - if let Some(appsrc) = self.vid.as_ref() { - if let Some(clock) = appsrc.clock() { - if let Some(time) = clock.time() { - if appsrc.base_time().is_some() { - // let current_runtime = time.saturating_sub(base_time); - let target_runtime = ClockTime::from_useconds(target_runtime as u64); - appsrc.set_base_time(time.saturating_sub(target_runtime)); - } - } - } - } - if let Some(appsrc) = self.aud.as_ref() { - if let Some(clock) = appsrc.clock() { - if let Some(time) = clock.time() { - if appsrc.base_time().is_some() { - let target_runtime = ClockTime::from_useconds(target_runtime as u64); - appsrc.set_base_time(time.saturating_sub(target_runtime)); - } - } - } - } - self.jump_to_live().await?; + self.jump_to_live().await; Ok(()) } @@ -699,7 +675,7 @@ impl NeoMediaSender { tokio::try_join!( async { if !vid_buffers.is_empty() { - debug!("Sending video buffers: {}", vid_buffers.len()); + // debug!("Sending video buffers: {}", vid_buffers.len()); if let Some(appsrc) = self.vid.clone() { let buffers = { let mut buffers = @@ -709,15 +685,15 @@ impl NeoMediaSender { for (time, buf) in vid_buffers.drain(..) { tokio::task::yield_now().await; let runtime = self.buftime_to_runtime(time); - let actual_runtime = self + let _actual_runtime = self .get_runtime() .map(|i| Duration::from_micros(i as u64)); - debug!( - " - Sending vid frame at time {} ({:?} Expect: {:?})", - time, - Duration::from_micros(runtime as u64), - actual_runtime - ); + // debug!( + // " - Sending vid frame at time {} ({:?} Expect: {:?})", + // time, + // Duration::from_micros(runtime as u64), + // actual_runtime + // ); let gst_buf = { let mut gst_buf = @@ -742,7 +718,7 @@ impl NeoMediaSender { }; let res = tokio::task::spawn_blocking(move || { - debug!(" - Pushing buffer: {}", buffers.len()); + // debug!(" - Pushing buffer: {}", buffers.len()); appsrc .push_buffer_list(buffers.copy()) .map(|_| ()) From eec01faa0ad1a5c988dca533cba9ef6754ee5364 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 9 May 2023 18:25:07 +0700 Subject: [PATCH 52/64] 250ms Latency --- src/rtsp/gst/sender.rs | 57 +++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index a35d580..d3fabe6 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -420,7 +420,7 @@ impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(30); Self { - start_time: Spring::new(0.0, 0.0, 5.5), + start_time: Spring::new(0.0, 0.0, 1.5), buffer: NeoBuffer::default(), vid: None, aud: None, @@ -448,10 +448,25 @@ impl NeoMediaSender { self.command_sender.clone() } + fn target_live(&self) -> Option { + let target_idx = BUFFER_SIZE / 3; + if self.buffer.buf.len() >= target_idx { + let target_frame = self.buffer.buf.len().saturating_sub(target_idx); + self.buffer.buf.get(target_frame).map(|frame| frame.time) + } else { + // Approximate it's location + let fraction = target_idx as f64 / self.buffer.buf.len() as f64; + if let (Some(st), Some(et)) = (self.buffer.start_time(), self.buffer.end_time()) { + Some(et - ((et - st) as f64 * fraction) as FrameTime) + } else { + None + } + } + } + async fn jump_to_live(&mut self) -> AnyResult<()> { if self.inited { - let target_frame = self.buffer.buf.len().saturating_sub(BUFFER_SIZE / 3); - let target_time = self.buffer.buf.get(target_frame).map(|frame| frame.time); + let target_time = self.target_live(); if let Some(target_time) = target_time { let runtime = self.get_runtime().unwrap_or(0); @@ -469,35 +484,8 @@ impl NeoMediaSender { async fn update_starttime(&mut self) -> AnyResult<()> { self.start_time.update().await; - let target_idx = BUFFER_SIZE / 3; - let target_time_target_frame = if self.buffer.buf.len() >= target_idx { - let target_frame = self.buffer.buf.len().saturating_sub(target_idx); - self.buffer - .buf - .get(target_frame) - .map(|frame| (frame.time, target_frame as i64)) - } else { - // Approximate it's location - let fraction = target_idx as f64 / self.buffer.buf.len() as f64; - if let (Some(st), Some(et)) = (self.buffer.start_time(), self.buffer.end_time()) { - Some(( - et - ((et - st) as f64 * fraction) as FrameTime, - -((target_idx - self.buffer.buf.len()) as i64), - )) - } else { - None - } - }; - if let (Some(runtime), Some((target_time, _target_frame))) = - (self.get_runtime(), target_time_target_frame) - { - // debug!( - // "Target frame: {}, Target time: {}, Target start time: {}, Current start time: {}", - // target_frame, - // target_time, - // target_time - runtime, - // self.start_time.value_i64(), - // ); + + if let (Some(runtime), Some(target_time)) = (self.get_runtime(), self.target_live()) { self.start_time.set_target((target_time - runtime) as f64); } Ok(()) @@ -509,7 +497,7 @@ impl NeoMediaSender { _target_runtime: FrameTime, _master_buffer: &NeoBuffer, ) -> AnyResult<()> { - self.jump_to_live().await; + self.jump_to_live().await?; Ok(()) } @@ -609,7 +597,8 @@ impl NeoMediaSender { { return Err(anyhow!("Vid src is closed")); } - if let Some(buftime) = self.get_buftime() { + const LATENCY: FrameTime = Duration::from_millis(250).as_micros() as FrameTime; + if let Some(buftime) = self.get_buftime().map(|i| i.saturating_add(LATENCY)) { // debug!("Update: buftime: {}", buftime); while self .buffer From f9407d8d8d3ae81a15e13affd84ee67e2932d3ef Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 09:31:13 +0700 Subject: [PATCH 53/64] Seek within buffer, not RANDOM_ACCESS --- src/rtsp/gst/factory.rs | 6 +-- src/rtsp/gst/sender.rs | 87 ++++++++++++++++++++++++++++++++--------- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 3272db0..298048a 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -310,7 +310,7 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::RandomAccess); + source.set_stream_type(AppStreamType::Seekable); let need_command = client_data.get_commader(); let enough_command = client_data.get_commader(); @@ -372,7 +372,7 @@ impl NeoMediaFactoryImpl { // source.set_property("max-buffers", 0u32); // v1.20 // source.set_property("max-time", Duration::from_secs_f32(2.25).as_nanos() as u64); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::RandomAccess); + source.set_stream_type(AppStreamType::Seekable); let need_command = client_data.get_commader(); let enough_command = client_data.get_commader(); @@ -522,7 +522,7 @@ impl NeoMediaFactoryImpl { source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); - source.set_stream_type(AppStreamType::RandomAccess); + source.set_stream_type(AppStreamType::Seekable); source.set_caps(Some( &Caps::builder("audio/x-adpcm") .field("layout", "div") diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index d3fabe6..bcb110e 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -31,19 +31,19 @@ type FrameTime = i64; const BUFFER_SIZE: usize = 100; -#[derive(Debug)] +#[derive(Debug, Clone)] struct Stamped { time: FrameTime, - data: BcMedia, + data: Arc, } #[derive(Default, Debug)] struct NeoBuffer { - buf: VecDeque>, + buf: VecDeque, } impl NeoBuffer { - fn push(&mut self, item: Arc) { + fn push(&mut self, item: Stamped) { // Sort time // debug!("sorting"); let mut sorting_vec = vec![]; @@ -245,19 +245,36 @@ impl NeoMediaSenders { _ => self.buffer.end_time().unwrap_or(0), }; - let data = Arc::new(Stamped { time, data }); + let data = Stamped { + time, + data: Arc::new(data), + }; for client_data in self.client_data.values_mut() { client_data.add_data(data.clone()).await?; } let end_time = self.buffer.end_time(); let frame_time = data.time; + // Ocassionally the camera will make a jump in timestamps of about 15s + // This could mean that it runs on some 15s buffer if let Some(end_time) = end_time { - let delta_time = end_time - frame_time; + let delta_time = end_time - frame_time - 1; let delta_duration = Duration::from_micros(delta_time.unsigned_abs()); if delta_duration > Duration::from_secs(1) { - debug!("Clearing buffer due to jump"); - self.clear_buffer().await?; + debug!( + "Reschedule buffer due to jump: {:?}, Prev: {}, New: {}", + delta_duration, end_time, frame_time + ); + for frame in self.buffer.buf.iter_mut() { + frame.time = frame.time.saturating_add(delta_time); + } + + for (_, client) in self.client_data.iter_mut() { + for frame in client.buffer.buf.iter_mut() { + frame.time = frame.time.saturating_add(delta_time); + } + client.jump_to_live().await?; + } } } @@ -407,6 +424,7 @@ pub(super) enum NeoMediaSenderCommand { #[derive(Debug)] pub(super) struct NeoMediaSender { start_time: Spring, + live_offset: FrameTime, buffer: NeoBuffer, vid: Option, aud: Option, @@ -420,7 +438,8 @@ impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(30); Self { - start_time: Spring::new(0.0, 0.0, 1.5), + start_time: Spring::new(0.0, 0.0, 10.0), + live_offset: 0, buffer: NeoBuffer::default(), vid: None, aud: None, @@ -431,7 +450,7 @@ impl NeoMediaSender { } } - async fn add_data(&mut self, data: Arc) -> AnyResult<()> { + async fn add_data(&mut self, data: Stamped) -> AnyResult<()> { self.buffer.push(data); Ok(()) } @@ -469,6 +488,12 @@ impl NeoMediaSender { let target_time = self.target_live(); if let Some(target_time) = target_time { + if let Some(et) = self.buffer.end_time() { + debug!( + "Expected Latency: {:?}", + Duration::from_micros(et.saturating_sub(target_time).max(0) as u64) + ); + } let runtime = self.get_runtime().unwrap_or(0); self.start_time.reset_to((target_time - runtime) as f64); @@ -494,9 +519,24 @@ impl NeoMediaSender { async fn seek( &mut self, _original_runtime: Option, - _target_runtime: FrameTime, - _master_buffer: &NeoBuffer, + target_runtime: FrameTime, + master_buffer: &NeoBuffer, ) -> AnyResult<()> { + if let Some(current_runtime) = self.get_runtime() { + self.live_offset = self + .live_offset + .saturating_add(target_runtime.saturating_sub(current_runtime)); + debug!("Old runtime: {}", current_runtime); + debug!("Target runtime: {}", target_runtime); + debug!("Offset: {}", self.live_offset); + debug!("New runtime: {:?}", self.get_runtime()); + if let Some(new_buftime) = self.get_buftime() { + self.buffer.buf.clear(); + for frame in master_buffer.buf.iter().filter(|f| f.time >= new_buftime) { + self.buffer.buf.push_back(frame.clone()); + } + } + } self.jump_to_live().await?; Ok(()) } @@ -536,8 +576,8 @@ impl NeoMediaSender { // Split buffer into 2/3 of preprocess // and one third for the future buffer let split_idx = buffer.buf.len() * 2 / 3; - let cloned = buffer.buf.iter().cloned().collect::>(); - let (preprocess, buffer) = cloned.split_at(split_idx); + let rebuf = buffer.buf.iter().cloned().collect::>(); + let (preprocess, buffer) = rebuf.split_at(split_idx); let start_ms = buffer .first() .map(|data| data.time as f64) @@ -566,7 +606,16 @@ impl NeoMediaSender { if let Some(time) = clock.time() { if let Some(base_time) = appsrc.base_time() { let runtime = time.saturating_sub(base_time); - return Some(runtime.useconds() as FrameTime); + let res = Some( + (runtime.useconds() as FrameTime) + .saturating_add(self.live_offset) + .max(0), + ); + debug!( + "Runtime: {:?}, Offset: {:?}, Offseted Runtime: {:?}", + runtime, self.live_offset, res + ); + return res; } } } @@ -624,7 +673,7 @@ impl NeoMediaSender { Ok(()) } - async fn send_buffers>(&mut self, medias: &[T]) -> AnyResult<()> { + async fn send_buffers(&mut self, medias: &[Stamped]) -> AnyResult<()> { if medias.is_empty() { return Ok(()); } @@ -632,14 +681,14 @@ impl NeoMediaSender { tokio::task::yield_now().await; let mut vid_buffers: Vec<(FrameTime, Vec)> = vec![]; let mut aud_buffers: Vec<(FrameTime, Vec)> = vec![]; - for media in medias.iter().map(|t| t.as_ref()) { + for media in medias.iter() { tokio::task::yield_now().await; - let buffer = match &media.data { + let buffer = match media.data.as_ref() { BcMedia::Iframe(_) | BcMedia::Pframe(_) => Some(&mut vid_buffers), BcMedia::Aac(_) | BcMedia::Adpcm(_) => Some(&mut aud_buffers), _ => None, }; - let data = match &media.data { + let data = match media.data.as_ref() { BcMedia::Iframe(data) => Some(&data.data), BcMedia::Pframe(data) => Some(&data.data), BcMedia::Aac(data) => Some(&data.data), From b09a590fb6c2de067996d913b09f83dfc3f5ef7f Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 09:39:59 +0700 Subject: [PATCH 54/64] Cleanup logs --- src/rtsp/gst/sender.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index bcb110e..4eee4de 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -490,7 +490,7 @@ impl NeoMediaSender { if let Some(target_time) = target_time { if let Some(et) = self.buffer.end_time() { debug!( - "Expected Latency: {:?}", + "Minimum Latency: {:?}", Duration::from_micros(et.saturating_sub(target_time).max(0) as u64) ); } @@ -526,10 +526,10 @@ impl NeoMediaSender { self.live_offset = self .live_offset .saturating_add(target_runtime.saturating_sub(current_runtime)); - debug!("Old runtime: {}", current_runtime); - debug!("Target runtime: {}", target_runtime); - debug!("Offset: {}", self.live_offset); - debug!("New runtime: {:?}", self.get_runtime()); + trace!("Old runtime: {}", current_runtime); + trace!("Target runtime: {}", target_runtime); + trace!("Offset: {}", self.live_offset); + trace!("New runtime: {:?}", self.get_runtime()); if let Some(new_buftime) = self.get_buftime() { self.buffer.buf.clear(); for frame in master_buffer.buf.iter().filter(|f| f.time >= new_buftime) { @@ -611,9 +611,11 @@ impl NeoMediaSender { .saturating_add(self.live_offset) .max(0), ); - debug!( + trace!( "Runtime: {:?}, Offset: {:?}, Offseted Runtime: {:?}", - runtime, self.live_offset, res + runtime, + self.live_offset, + res ); return res; } From e712e3b47c4c019463841a70c9eabba3bee42d3e Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 10:26:53 +0700 Subject: [PATCH 55/64] Dedicated thread for the BC connection layer --- crates/core/src/bc_protocol/connection/bcconn.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/core/src/bc_protocol/connection/bcconn.rs b/crates/core/src/bc_protocol/connection/bcconn.rs index 2512770..b2038d7 100644 --- a/crates/core/src/bc_protocol/connection/bcconn.rs +++ b/crates/core/src/bc_protocol/connection/bcconn.rs @@ -48,10 +48,17 @@ impl BcConnection { let mut rx_thread = JoinSet::>::new(); let thread_poll_commander = poll_commander.clone(); + let handle = tokio::task::spawn_blocking(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + runtime.block_on( + PollSender::new(thread_poll_commander) + .send_all(&mut source.map(|bc| Ok(PollCommand::Bc(Box::new(bc))))), + ) + }); rx_thread.spawn(async move { - PollSender::new(thread_poll_commander) - .send_all(&mut source.map(|bc| Ok(PollCommand::Bc(Box::new(bc))))) - .await?; + handle.await??; Ok(()) }); From d411e9c25b350b24a1279ea301356056e3293bf3 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 11:38:13 +0700 Subject: [PATCH 56/64] Restamp over clear buffer during reconnect --- src/rtsp/gst/server.rs | 1 + src/rtsp/mod.rs | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 2a24280..6109145 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -283,6 +283,7 @@ impl NeoRtspServer { } // Clear buffers on all senders of a tag + #[allow(dead_code)] // Old function, was used to clear buffer on reconnect pub(crate) async fn clear_buffer>(&self, tag: T) -> AnyResult<()> { if let Some(sender) = self.imp().get_sender(tag).await { sender.send(FactoryCommand::ClearBuffer).await?; diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 4d9295e..9733008 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -185,11 +185,15 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi let rtsp_thread = loggedin.get_rtsp(); // Clear all buffers present - tags.iter() - .map(|tag| rtsp_thread.clear_buffer(tag)) - .collect::>() - .collect::>() - .await; + // Uncomment to clear buffers. This is now handlled in the buffer itself, + // instead of clearing it restamps it whenever there is a jump in the + // timestamps of >1s + // + // tags.iter() + // .map(|tag| rtsp_thread.clear_buffer(tag)) + // .collect::>() + // .collect::>() + // .await; // Start pulling data from the camera let mut streaming = loggedin From 004405e868d5ae62b0b98c76a73858e3a0d25ba6 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 11:47:18 +0700 Subject: [PATCH 57/64] Correct delta on buffer schedule --- src/rtsp/gst/sender.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 4eee4de..900937a 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -255,10 +255,10 @@ impl NeoMediaSenders { let end_time = self.buffer.end_time(); let frame_time = data.time; - // Ocassionally the camera will make a jump in timestamps of about 15s - // This could mean that it runs on some 15s buffer + // Ocassionally the camera will make a jump in timestamps of about 15s (on sub 9s on main) + // This could mean that it runs on some fixed sized buffer if let Some(end_time) = end_time { - let delta_time = end_time - frame_time - 1; + let delta_time = frame_time - end_time - -1; let delta_duration = Duration::from_micros(delta_time.unsigned_abs()); if delta_duration > Duration::from_secs(1) { debug!( From 2921ede080cc657065ed6cab364a7f9edcadfd70 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 13:11:56 +0700 Subject: [PATCH 58/64] Reinit client buffers when master is restamped --- src/rtsp/gst/sender.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 900937a..3f4e132 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -265,15 +265,15 @@ impl NeoMediaSenders { "Reschedule buffer due to jump: {:?}, Prev: {}, New: {}", delta_duration, end_time, frame_time ); + debug!("Adjusting master: {}", self.buffer.buf.len()); for frame in self.buffer.buf.iter_mut() { frame.time = frame.time.saturating_add(delta_time); + debug!(" - New frame time: {} -> {}", frame.time, frame_time); } for (_, client) in self.client_data.iter_mut() { - for frame in client.buffer.buf.iter_mut() { - frame.time = frame.time.saturating_add(delta_time); - } - client.jump_to_live().await?; + client.buffer.buf.clear(); + client.inited = false; } } } From e2d5eb061f3ed8764ae6798d65116ef9f70077a0 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 13:25:51 +0700 Subject: [PATCH 59/64] Add time out to map discovery --- .../core/src/bc_protocol/connection/discovery.rs | 15 ++++++++++----- .../core/src/bc_protocol/connection/udpsource.rs | 7 ++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/core/src/bc_protocol/connection/discovery.rs b/crates/core/src/bc_protocol/connection/discovery.rs index e44e022..9daabe4 100644 --- a/crates/core/src/bc_protocol/connection/discovery.rs +++ b/crates/core/src/bc_protocol/connection/discovery.rs @@ -189,11 +189,16 @@ impl Discoverer { F: Fn(UdpDiscovery, SocketAddr) -> Option, { let mut reply = ReceiverStream::new(self.subscribe(0).await?); - loop { - let (reply, addr) = reply.next().await.ok_or(Error::ConnectionUnavaliable)??; - if let Some(result) = map(reply, addr) { - return Ok(result); - } + tokio::select! { + v = async { + loop { + let (reply, addr) = reply.next().await.ok_or(Error::ConnectionUnavaliable)??; + if let Some(result) = map(reply, addr) { + return Ok(result); + } + } + } => v, + _ = tokio::time::sleep(*MAXIMUM_WAIT) => Err::(Error::DiscoveryTimeout), } } diff --git a/crates/core/src/bc_protocol/connection/udpsource.rs b/crates/core/src/bc_protocol/connection/udpsource.rs index b57ae0f..cb35b4b 100644 --- a/crates/core/src/bc_protocol/connection/udpsource.rs +++ b/crates/core/src/bc_protocol/connection/udpsource.rs @@ -198,6 +198,7 @@ impl Sink<(BcUdp, SocketAddr)> for BcUdpSource { } } +#[allow(dead_code)] enum State { Normal, // Normal recieve Flushing, // Used to send ack packets and things in the buffer @@ -298,15 +299,15 @@ impl Stream for UdpPayloadSource { let mut this = self.get_mut(); match this.state { State::Normal => { - this.state = State::YieldNow; + // this.state = State::YieldNow; + // Handle resend events + this.maintanence(cx); // Data ready to go if let Some(payload) = this.recieved.remove(&this.packets_want) { this.packets_want += 1; // error!("packets_want: {}", this.packets_want); return Poll::Ready(Some(Ok(payload))); } - // Handle resend events - this.maintanence(cx); // Normal behaviors match this.inner.poll_next_unpin(cx) { Poll::Ready(Some(Ok(( From df3462875bcc4b2d2c59f6b3727738f8e0332878 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 13:54:39 +0700 Subject: [PATCH 60/64] Add timeout to p2p address resolution --- .../src/bc_protocol/connection/discovery.rs | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/crates/core/src/bc_protocol/connection/discovery.rs b/crates/core/src/bc_protocol/connection/discovery.rs index 9daabe4..9354956 100644 --- a/crates/core/src/bc_protocol/connection/discovery.rs +++ b/crates/core/src/bc_protocol/connection/discovery.rs @@ -222,6 +222,7 @@ impl Discoverer { where F: Fn(UdpDiscovery, SocketAddr) -> Option, { + trace!("Retrying send"); disc.tid = 0; // Must be random to avoid simulatenous subscription errors let mut set = FuturesUnordered::new(); for dest in dests.iter() { @@ -230,6 +231,7 @@ impl Discoverer { // Get what ever completes first while let Some(result) = set.next().await { + trace!("Result: {}", result.is_ok()); if let Ok(ret) = result { return Ok(ret); } @@ -361,15 +363,20 @@ impl Discoverer { /// On success it returns the M2cQr that the p2p relay /// server has about the UID async fn uid_loopup(&self, uid: &str) -> Result { - let mut addrs = vec![]; - for p2p_relay in P2P_RELAY_HOSTNAMES.iter() { - addrs.append( - &mut format!("{}:9999", p2p_relay) - .to_socket_addrs() - .map(|i| i.collect::>()) - .unwrap_or_else(|_| vec![]), - ); - } + let task = tokio::task::spawn_blocking(move || { + let mut addrs = vec![]; + for p2p_relay in P2P_RELAY_HOSTNAMES.iter() { + addrs.append( + &mut format!("{}:9999", p2p_relay) + .to_socket_addrs() + .map(|i| i.collect::>()) + .unwrap_or_else(|_| vec![]), + ); + } + addrs + }); + let addrs = timeout(*MAXIMUM_WAIT, task).await??; + trace!("Uid lookup to: {:?}", addrs); let msg = UdpDiscovery { tid: 0, payload: UdpXml { @@ -380,6 +387,7 @@ impl Discoverer { ..Default::default() }, }; + trace!("Sending look up {:?}", msg); let (packet, _) = self .retry_send_multi(msg, addrs.as_slice(), |bc, addr| match bc { UdpDiscovery { @@ -393,7 +401,7 @@ impl Discoverer { _ => None, }) .await?; - + trace!("Look up complete"); Ok(UidLookupResults { reg: SocketAddr::new(packet.reg.ip.parse()?, packet.reg.port), relay: SocketAddr::new(packet.relay.ip.parse()?, packet.relay.port), @@ -1009,20 +1017,25 @@ impl Discovery { // directly #[allow(unused)] pub(crate) async fn remote(uid: &str) -> Result { + trace!("Start remote"); let mut discoverer = Discoverer::new().await?; let client_id = generate_cid(); + trace!("client_id: {}", client_id); let lookup = discoverer.uid_loopup(uid).await?; + trace!("lookup: {:?}", lookup); let reg_addr = lookup.reg; let relay_addr = lookup.relay; let reg_result = discoverer.register_address(uid, client_id, &lookup).await?; + trace!("reg_result: {:?}", reg_result); let connect_result = tokio::select! { v = discoverer.client_initiated_dev(®_result) => {v}, v = discoverer.device_initiated_dev(®_result) => {v}, }?; + trace!("connect_result: {:?}", connect_result); let (socket, keep_alive_tasks) = discoverer.into_socket().await; Ok(DiscoveryResult { From 5513fce18f6023e43fd00714875d3f48aef33b88 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 20:37:06 +0700 Subject: [PATCH 61/64] Disable time smoothing --- src/rtsp/gst/sender.rs | 8 ++++---- src/rtsp/spring.rs | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 3f4e132..e1bc87e 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -438,7 +438,7 @@ impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(30); Self { - start_time: Spring::new(0.0, 0.0, 10.0), + start_time: Spring::new(0.0, 0.0, 0.0), live_offset: 0, buffer: NeoBuffer::default(), vid: None, @@ -510,9 +510,9 @@ impl NeoMediaSender { async fn update_starttime(&mut self) -> AnyResult<()> { self.start_time.update().await; - if let (Some(runtime), Some(target_time)) = (self.get_runtime(), self.target_live()) { - self.start_time.set_target((target_time - runtime) as f64); - } + // if let (Some(runtime), Some(target_time)) = (self.get_runtime(), self.target_live()) { + // self.start_time.set_target((target_time - runtime) as f64); + // } Ok(()) } diff --git a/src/rtsp/spring.rs b/src/rtsp/spring.rs index c295a2f..4eaf09e 100644 --- a/src/rtsp/spring.rs +++ b/src/rtsp/spring.rs @@ -54,6 +54,7 @@ impl Spring { .min(std::u64::MAX as f64) as u64 } + #[allow(dead_code)] pub(crate) fn set_target(&mut self, target: f64) { self.target = target; } @@ -83,11 +84,16 @@ fn fast_negexp(x: f64) -> f64 { } fn spring_update(value: &mut f64, target: f64, velocity: &mut f64, halflife: f64, dt: f64) { - let y = halflife_to_damping(halflife) / 2.0; - let j0 = *value - target; - let j1 = *velocity + j0 * y; - let eydt = fast_negexp(y * dt); + if halflife.abs() < 1e-3 { + *value = target; + *velocity = 0.0; + } else { + let y = halflife_to_damping(halflife) / 2.0; + let j0 = *value - target; + let j1 = *velocity + j0 * y; + let eydt = fast_negexp(y * dt); - *value = eydt * (j0 + j1 * dt) + target; - *velocity = eydt * (*velocity - j1 * y * dt); + *value = eydt * (j0 + j1 * dt) + target; + *velocity = eydt * (*velocity - j1 * y * dt); + } } From bc3f9b65a0de9240c5fae53e6312a3271bfc4152 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 10 May 2023 20:43:53 +0700 Subject: [PATCH 62/64] Add messages about buffer size --- src/rtsp/gst/sender.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index e1bc87e..9ec1f9b 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -648,6 +648,11 @@ impl NeoMediaSender { { return Err(anyhow!("Vid src is closed")); } + if self.buffer.buf.len() < 4 { + error!("Buffer exhausted. Not enough data from Camera."); + } else { + debug!("Buffer size: {}", self.buffer.buf.len()); + } const LATENCY: FrameTime = Duration::from_millis(250).as_micros() as FrameTime; if let Some(buftime) = self.get_buftime().map(|i| i.saturating_add(LATENCY)) { // debug!("Update: buftime: {}", buftime); From c3c58b3f67ada437aa963d1d3defab4e329550dc Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Thu, 11 May 2023 13:28:21 +0700 Subject: [PATCH 63/64] Put back in smoothing and bump up the buffer size --- src/rtsp/gst/sender.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 9ec1f9b..f038130 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -29,7 +29,7 @@ use crate::rtsp::Spring; type FrameTime = i64; -const BUFFER_SIZE: usize = 100; +const BUFFER_SIZE: usize = 300; #[derive(Debug, Clone)] struct Stamped { @@ -438,7 +438,7 @@ impl NeoMediaSender { pub(super) fn new() -> Self { let (tx, rx) = channel(30); Self { - start_time: Spring::new(0.0, 0.0, 0.0), + start_time: Spring::new(0.0, 0.0, 10.0), live_offset: 0, buffer: NeoBuffer::default(), vid: None, @@ -510,9 +510,9 @@ impl NeoMediaSender { async fn update_starttime(&mut self) -> AnyResult<()> { self.start_time.update().await; - // if let (Some(runtime), Some(target_time)) = (self.get_runtime(), self.target_live()) { - // self.start_time.set_target((target_time - runtime) as f64); - // } + if let (Some(runtime), Some(target_time)) = (self.get_runtime(), self.target_live()) { + self.start_time.set_target((target_time - runtime) as f64); + } Ok(()) } @@ -651,7 +651,7 @@ impl NeoMediaSender { if self.buffer.buf.len() < 4 { error!("Buffer exhausted. Not enough data from Camera."); } else { - debug!("Buffer size: {}", self.buffer.buf.len()); + trace!("Buffer size: {}", self.buffer.buf.len()); } const LATENCY: FrameTime = Duration::from_millis(250).as_micros() as FrameTime; if let Some(buftime) = self.get_buftime().map(|i| i.saturating_add(LATENCY)) { From 84163dcd2aa16dcaa8e507c4aed5ebaeee9357d1 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Fri, 12 May 2023 09:26:31 +0700 Subject: [PATCH 64/64] v0.5.9 --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- crates/core/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef39f9d..e086e3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1402,7 +1402,7 @@ dependencies = [ [[package]] name = "neolink" -version = "0.5.8" +version = "0.5.9" dependencies = [ "anyhow", "byte-slice-cast", @@ -1432,7 +1432,7 @@ dependencies = [ [[package]] name = "neolink_core" -version = "0.5.8" +version = "0.5.9" dependencies = [ "aes", "assert_matches", diff --git a/Cargo.toml b/Cargo.toml index c069078..2abf21e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "neolink" description = "A standards-compliant bridge to Reolink IP cameras" -version = "0.5.8" +version = "0.5.9" authors = ["George Hilliard ", "Andrew King "] edition = "2018" license = "AGPL-3.0-or-later" @@ -25,7 +25,7 @@ gstreamer-rtsp = "0.20.0" gstreamer-rtsp-server = { version = "0.20.3", features = ["v1_16"] } lazy_static = "1.4.0" log = { version = "0.4.17", features = [ "release_max_level_debug" ] } -neolink_core = { path = "crates/core", version = "0.5.8" } +neolink_core = { path = "crates/core", version = "0.5.9" } regex = "1.7.3" rumqttc = "0.20.0" serde = { version = "1.0.160", features = ["derive"] } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 079047c..f56dc1f 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "neolink_core" description = "Core services and structure for Reolink IP cameras" -version = "0.5.8" +version = "0.5.9" authors = ["George Hilliard ", "Andrew King "] edition = "2018" license = "AGPL-3.0-or-later"