From 37049f7ca4a6967d33ceaf67f95f39a0f2c70fac Mon Sep 17 00:00:00 2001 From: Andrew Date: Sat, 15 Apr 2023 09:43:08 +0700 Subject: [PATCH 1/9] Fix bytes remaining on stream --- crates/core/src/bcudp/codex.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/core/src/bcudp/codex.rs b/crates/core/src/bcudp/codex.rs index 74ad60c..747a3ff 100644 --- a/crates/core/src/bcudp/codex.rs +++ b/crates/core/src/bcudp/codex.rs @@ -33,6 +33,15 @@ impl Decoder for BcUdpCodex { type Item = BcUdp; type Error = Error; + /// Since frames can cross EOF boundaries we overload this so it dosen't error if + /// there are bytes left on the stream + fn decode_eof(&mut self, buf: &mut BytesMut) -> Result> { + match self.decode(buf)? { + Some(frame) => Ok(Some(frame)), + None => Ok(None), + } + } + fn decode(&mut self, src: &mut BytesMut) -> Result> { match { BcUdp::deserialize(src) } { Ok(BcUdp::Discovery(UdpDiscovery { From f61fef9b32ec9429b7c5a79d27634f9fb0cd5d09 Mon Sep 17 00:00:00 2001 From: Andrew Date: Sat, 15 Apr 2023 14:30:25 +0700 Subject: [PATCH 2/9] Allow decode across EOF --- crates/core/src/bc/codex.rs | 19 +++++++++ .../src/bc_protocol/connection/udpsource.rs | 40 +++++-------------- crates/core/src/bc_protocol/stream.rs | 16 +++++++- crates/core/src/bcmedia/codex.rs | 9 +++++ src/rtsp/states/streaming.rs | 23 ++++++----- 5 files changed, 64 insertions(+), 43 deletions(-) diff --git a/crates/core/src/bc/codex.rs b/crates/core/src/bc/codex.rs index b19decb..203a143 100644 --- a/crates/core/src/bc/codex.rs +++ b/crates/core/src/bc/codex.rs @@ -7,6 +7,7 @@ use crate::bc::model::*; use crate::bc::xml::*; use crate::{Credentials, Error, Result}; use bytes::BytesMut; +use std::io; use tokio_util::codec::{Decoder, Encoder}; pub(crate) struct BcCodex { @@ -45,6 +46,24 @@ impl Encoder for BcCodex { impl Decoder for BcCodex { type Item = Bc; type Error = Error; + + fn decode_eof(&mut self, buf: &mut BytesMut) -> Result> { + match self.decode(buf)? { + Some(frame) => Ok(Some(frame)), + None => { + if buf.is_empty() { + Ok(None) + } else { + Err(io::Error::new( + io::ErrorKind::Other, + format!("bytes remaining on BC stream: {:X?}", buf), + ) + .into()) + } + } + } + } + fn decode(&mut self, src: &mut BytesMut) -> Result> { // trace!("Decoding: {:X?}", src); let bc = Bc::deserialize(&self.context, src); diff --git a/crates/core/src/bc_protocol/connection/udpsource.rs b/crates/core/src/bc_protocol/connection/udpsource.rs index f236078..a05553f 100644 --- a/crates/core/src/bc_protocol/connection/udpsource.rs +++ b/crates/core/src/bc_protocol/connection/udpsource.rs @@ -277,14 +277,12 @@ impl UdpPayloadSource { fn maintanence(&mut self, cx: &mut Context<'_>) { // Check for periodic resends if self.resend_interval.poll_tick(cx).is_ready() { - trace!("UDPSource.RecievedPacket: Resend"); for (_, resend) in self.sent.iter() { self.send_buffer.push_back(BcUdp::Data(resend.clone())); } } if self.ack_interval.poll_tick(cx).is_ready() { let ack = BcUdp::Ack(self.build_send_ack()); - trace!("UDPSource.RecievedPacket: QueueingAck"); self.send_buffer.push_back(ack); self.state = State::Flushing; } @@ -295,15 +293,12 @@ impl Stream for UdpPayloadSource { type Item = IoResult>; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - trace!("UdpPayloadSource.poll_next"); let camera_addr = self.inner.addr; let mut this = self.get_mut(); match this.state { State::Normal => { - trace!("UdpPayloadSource.State: Normal"); // Data ready to go if let Some(payload) = this.recieved.remove(&this.packets_want) { - trace!("UdpPayloadSource.Data: PreReady: {}", this.packets_want); this.packets_want += 1; // error!("packets_want: {}", this.packets_want); return Poll::Ready(Some(Ok(payload))); @@ -324,20 +319,12 @@ impl Stream for UdpPayloadSource { && packet_id >= this.packets_want => { if packet_id == this.packets_want { - trace!( - "UdpPayloadSource.RecievedPacker: NewData: {}", - this.packets_want - ); this.packets_want += 1; // error!("packets_want: {}", this.packets_want); let ack = BcUdp::Ack(this.build_send_ack()); this.send_buffer.push_back(ack); return Poll::Ready(Some(Ok(payload))); } else { - trace!( - "UdpPayloadSource.RecievedPacker: Future Data: {}", - packet_id - ); this.recieved.insert(packet_id, payload); } } @@ -358,17 +345,14 @@ impl Stream for UdpPayloadSource { BcUdp::Ack(ack @ UdpAck { connection_id, .. }), addr, )))) if connection_id == this.client_id && addr == camera_addr => { - trace!("UdpPayloadSource.RecievedPacket: Ack: {:?}", ack); this.handle_ack(ack); // Rather then immediatly flush wait for the next call // this.state = State::Flushing; } Poll::Ready(Some(Err(e))) => { - trace!("UdpPayloadSource.RecievedPacker: Error"); return Poll::Ready(Some(Err(IoError::new(ErrorKind::Other, e)))); } Poll::Ready(None) => { - trace!("UdpPayloadSource.RecievedPacker: Empty"); return Poll::Ready(None); } Poll::Pending => { @@ -386,22 +370,18 @@ impl Stream for UdpPayloadSource { // } } } - State::Flushing => { - trace!("UdpPayloadSource.State: Flushing"); - match this.poll_flush_unpin(cx) { - Poll::Ready(Ok(())) => { - this.state = State::Normal; - } - Poll::Ready(Err(e)) => { - return Poll::Ready(Some(Err(e))); - } - Poll::Pending => { - return Poll::Pending; - } + State::Flushing => match this.poll_flush_unpin(cx) { + Poll::Ready(Ok(())) => { + this.state = State::Normal; } - } + Poll::Ready(Err(e)) => { + return Poll::Ready(Some(Err(e))); + } + Poll::Pending => { + return Poll::Pending; + } + }, State::Closed => { - trace!("UdpPayloadSource.State: Closed"); return Poll::Ready(Some(Err(IoError::from(ErrorKind::ConnectionAborted)))); } } diff --git a/crates/core/src/bc_protocol/stream.rs b/crates/core/src/bc_protocol/stream.rs index ef3f6df..6457d36 100644 --- a/crates/core/src/bc_protocol/stream.rs +++ b/crates/core/src/bc_protocol/stream.rs @@ -31,7 +31,6 @@ pub enum StreamKind { /// /// When this object is dropped the streaming is stopped pub struct StreamData { - #[allow(dead_code)] handle: Option>>, rx: Receiver>, abort_handle: Arc, @@ -41,9 +40,22 @@ impl StreamData { /// Pull data from the camera's buffer /// This returns raw BcMedia packets pub async fn get_data(&mut self) -> Result> { + if let Some(handle) = self.handle.as_mut() { + if handle.is_finished() { + self.abort_handle.store(true, Ordering::Relaxed); + handle.await??; + return Err(Error::DroppedConnection); + } + } else { + self.abort_handle.store(true, Ordering::Relaxed); + return Err(Error::DroppedConnection); + } match self.rx.recv().await { Some(data) => Ok(data), - None => Err(Error::DroppedConnection), + None => { + self.abort_handle.store(true, Ordering::Relaxed); + Err(Error::DroppedConnection) + } } } } diff --git a/crates/core/src/bcmedia/codex.rs b/crates/core/src/bcmedia/codex.rs index 46dadb7..b006532 100644 --- a/crates/core/src/bcmedia/codex.rs +++ b/crates/core/src/bcmedia/codex.rs @@ -36,6 +36,15 @@ impl Decoder for BcMediaCodex { type Item = BcMedia; type Error = Error; + /// Since frames can cross EOF boundaries we overload this so it dosen't error if + /// there are bytes left on the stream + fn decode_eof(&mut self, buf: &mut BytesMut) -> Result> { + match self.decode(buf)? { + Some(frame) => Ok(Some(frame)), + None => Ok(None), + } + } + fn decode(&mut self, src: &mut BytesMut) -> Result> { let mut amount_skipped: usize = 0; loop { diff --git a/src/rtsp/states/streaming.rs b/src/rtsp/states/streaming.rs index b98e12d..f6b5394 100644 --- a/src/rtsp/states/streaming.rs +++ b/src/rtsp/states/streaming.rs @@ -4,6 +4,7 @@ use anyhow::{anyhow, Error, Result}; use log::*; +use neolink_core::bcmedia::model::BcMedia; use tokio::time::{timeout, Duration}; use tokio::{sync::RwLock, task::JoinSet}; @@ -43,7 +44,7 @@ impl Camera { let stream_thead = *stream; let strict_thread = loggedin.shared.config.strict; - // let tag_thread = tag.clone(); + let tag_thread = tag.clone(); let mut stream_data = loggedin .state .camera @@ -51,17 +52,17 @@ impl Camera { .await?; set.spawn(async move { loop { - // debug!("{}: BcMediaStreamRecv", &tag_thread); + debug!("{}: BcMediaStreamRecv", &tag_thread); let data = timeout(Duration::from_secs(15), stream_data.get_data()).await??; - // match &data { - // Ok(BcMedia::InfoV1(_)) => debug!("{}: - InfoV1", &tag_thread), - // Ok(BcMedia::InfoV2(_)) => debug!("{}: - InfoV2", &tag_thread), - // Ok(BcMedia::Iframe(_)) => debug!("{}: - Iframe", &tag_thread), - // Ok(BcMedia::Pframe(_)) => debug!("{}: - Pframe", &tag_thread), - // Ok(BcMedia::Aac(_)) => debug!("{}: - Aac", &tag_thread), - // Ok(BcMedia::Adpcm(_)) => debug!("{}: - Adpcm", &tag_thread), - // Err(_) => debug!(" - Error"), - // } + match &data { + Ok(BcMedia::InfoV1(_)) => debug!("{}: - InfoV1", &tag_thread), + Ok(BcMedia::InfoV2(_)) => debug!("{}: - InfoV2", &tag_thread), + Ok(BcMedia::Iframe(_)) => debug!("{}: - Iframe", &tag_thread), + Ok(BcMedia::Pframe(_)) => debug!("{}: - Pframe", &tag_thread), + Ok(BcMedia::Aac(_)) => debug!("{}: - Aac", &tag_thread), + Ok(BcMedia::Adpcm(_)) => debug!("{}: - Adpcm", &tag_thread), + Err(_) => debug!(" - Error"), + } sender.send(data?).await?; } }); From 824206faaac196fc0253454e8e76747620be20b3 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 16 Apr 2023 15:54:47 +0700 Subject: [PATCH 3/9] Reducing blocking on gstraemer pushes --- .../core/src/bc_protocol/connection/bcconn.rs | 3 +- .../src/bc_protocol/connection/udpsource.rs | 14 +- crates/core/src/bc_protocol/stream.rs | 13 +- src/rtsp/gst/factory.rs | 14 +- src/rtsp/gst/sender.rs | 141 ++++++++++++------ src/rtsp/states/streaming.rs | 5 +- 6 files changed, 133 insertions(+), 57 deletions(-) diff --git a/crates/core/src/bc_protocol/connection/bcconn.rs b/crates/core/src/bc_protocol/connection/bcconn.rs index 790473b..d4b2160 100644 --- a/crates/core/src/bc_protocol/connection/bcconn.rs +++ b/crates/core/src/bc_protocol/connection/bcconn.rs @@ -8,6 +8,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use tokio::sync::mpsc::{channel, Sender}; use tokio::sync::Mutex; +use tokio::task::yield_now; use tokio::{sync::RwLock, task::JoinSet}; @@ -46,12 +47,12 @@ impl BcConnection { let sink_thread = sink.clone(); rx_thread.spawn(async move { loop { + yield_now().await; // Give other tasks a chance to run before pulling next packet let packet = source.next().await; let bc = match packet { Some(res) => res, None => continue, }; - Self::poll(bc, &subs, &sink_thread).await? } }); diff --git a/crates/core/src/bc_protocol/connection/udpsource.rs b/crates/core/src/bc_protocol/connection/udpsource.rs index a05553f..b57ae0f 100644 --- a/crates/core/src/bc_protocol/connection/udpsource.rs +++ b/crates/core/src/bc_protocol/connection/udpsource.rs @@ -199,9 +199,10 @@ impl Sink<(BcUdp, SocketAddr)> for BcUdpSource { } enum State { - Normal, - Flushing, - Closed, + Normal, // Normal recieve + Flushing, // Used to send ack packets and things in the buffer + Closed, // Used to shutdown + YieldNow, // Used to ensure we rest between polling packets so as to not starve the runtime } pub(crate) struct UdpPayloadSource { @@ -297,6 +298,7 @@ impl Stream for UdpPayloadSource { let mut this = self.get_mut(); match this.state { State::Normal => { + this.state = State::YieldNow; // Data ready to go if let Some(payload) = this.recieved.remove(&this.packets_want) { this.packets_want += 1; @@ -384,6 +386,9 @@ impl Stream for UdpPayloadSource { State::Closed => { return Poll::Ready(Some(Err(IoError::from(ErrorKind::ConnectionAborted)))); } + State::YieldNow => { + this.state = State::Normal; + } } cx.waker().wake_by_ref(); Poll::Pending @@ -399,7 +404,7 @@ impl Sink> for UdpPayloadSource { ) -> Poll> { let this = self.get_mut(); match this.state { - State::Normal => this + State::Normal | State::YieldNow => this .inner .poll_ready_unpin(cx) .map_err(|e| IoError::new(ErrorKind::Other, e)), @@ -476,6 +481,7 @@ impl Sink> for UdpPayloadSource { cx.waker().wake_by_ref(); Poll::Pending } + fn poll_close( self: Pin<&mut Self>, cx: &mut Context<'_>, diff --git a/crates/core/src/bc_protocol/stream.rs b/crates/core/src/bc_protocol/stream.rs index 6457d36..01424dd 100644 --- a/crates/core/src/bc_protocol/stream.rs +++ b/crates/core/src/bc_protocol/stream.rs @@ -4,6 +4,7 @@ use crate::{ bcmedia::model::*, }; use futures::stream::StreamExt; +use log::*; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -50,9 +51,14 @@ impl StreamData { self.abort_handle.store(true, Ordering::Relaxed); return Err(Error::DroppedConnection); } + debug!("StreamData: Get"); match self.rx.recv().await { - Some(data) => Ok(data), + Some(data) => { + debug!("StreamData: Got"); + Ok(data) + } None => { + debug!("StreamData: Drop"); self.abort_handle.store(true, Ordering::Relaxed); Err(Error::DroppedConnection) } @@ -176,11 +182,16 @@ impl BcCamera { let mut media_sub = sub_video.bcmedia_stream(strict); while !abort_handle_thread.load(Ordering::Relaxed) { + debug!("Stream: Get"); if let Some(bc_media) = media_sub.next().await { + debug!("Stream: Got"); // We now have a complete interesting packet. Send it to on the callback + debug!("Stream: Send"); if tx.send(bc_media).await.is_err() { + debug!("Stream: Dropped"); break; // Connection dropped } + debug!("Stream: Sent"); } else { break; } diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index af318e6..956b25b 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -230,7 +230,7 @@ impl NeoMediaFactoryImpl { self.shared.microseconds.load(Ordering::Relaxed), )); source.set_is_live(true); - source.set_block(true); + source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); @@ -242,6 +242,7 @@ impl NeoMediaFactoryImpl { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; + queue.set_property_from_str("leaky", "downstream"); let parser = make_element("h265parse", "parser")?; let payload = make_element("rtph265pay", "pay0")?; bin.add_many(&[&source, &queue, &parser, &payload])?; @@ -261,7 +262,7 @@ impl NeoMediaFactoryImpl { self.shared.microseconds.load(Ordering::Relaxed), )); source.set_is_live(true); - source.set_block(true); + source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); @@ -273,6 +274,8 @@ impl NeoMediaFactoryImpl { .dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; + queue.set_property_from_str("leaky", "downstream"); + let parser = make_element("h264parse", "parser")?; let payload = make_element("rtph264pay", "pay0")?; bin.add_many(&[&source, &queue, &parser, &payload])?; @@ -288,6 +291,7 @@ impl NeoMediaFactoryImpl { let source = make_element("videotestsrc", "vidsrc")?; source.set_property_from_str("pattern", "snow"); let queue = make_element("queue", "queue0")?; + queue.set_property_from_str("leaky", "downstream"); let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -324,7 +328,7 @@ impl NeoMediaFactoryImpl { self.shared.microseconds.load(Ordering::Relaxed), )); source.set_is_live(true); - source.set_block(true); + source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); @@ -334,6 +338,7 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); let parser = make_element("aacparse", "audparser")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; @@ -372,7 +377,7 @@ impl NeoMediaFactoryImpl { self.shared.microseconds.load(Ordering::Relaxed), )); source.set_is_live(true); - source.set_block(true); + source.set_block(false); source.set_property("emit-signals", false); source.set_max_bytes(52428800); source.set_do_timestamp(false); @@ -390,6 +395,7 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "audqueue")?; + queue.set_property_from_str("leaky", "downstream"); let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 9cab641..8afc8dc 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -5,8 +5,9 @@ use futures::stream::StreamExt; use gstreamer::ClockTime; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; +use log::*; use neolink_core::bcmedia::model::*; -use std::collections::HashMap; +use std::collections::{hash_map::Entry, HashMap}; use std::sync::{ atomic::{AtomicU64, Ordering}, Arc, @@ -54,8 +55,10 @@ impl NeoMediaSender { self.shared .buffer_ready .store(!buffer_read.is_empty(), Ordering::Relaxed); + debug!("Sender: Get"); tokio::select! { v = self.data_source.next() => { + debug!("Sender: Got Data"); // debug!("data_source recieved"); if let Some(bc_media) = v { if ! self.skip_bcmedia(&bc_media)? { @@ -74,7 +77,7 @@ impl NeoMediaSender { ms: frame.microseconds as u64, data: frame.data, }; - self.process_vidbuffer(&new_data)?; + self.process_vidbuffer(&new_data).await?; buffer_write.push(new_data); } BcMedia::Pframe(frame) => { @@ -82,7 +85,7 @@ impl NeoMediaSender { ms: frame.microseconds as u64, data: frame.data, }; - self.process_vidbuffer(&new_data)?; + self.process_vidbuffer(&new_data).await?; buffer_write.push(new_data); } BcMedia::Aac(aac) => { @@ -91,7 +94,18 @@ impl NeoMediaSender { ms: last.ms, data: aac.data, }; - self.process_audbuffer(&new_data)?; + self.process_audbuffer(&new_data).await?; + audbuffer_write.push(new_data); + } + + } + BcMedia::Adpcm(adpcm) => { + if let Some(last) = buffer_write.last().as_ref() { + let new_data = StampedData { + ms: last.ms, + data: adpcm.data, + }; + self.process_audbuffer(&new_data).await?; audbuffer_write.push(new_data); } @@ -104,6 +118,7 @@ impl NeoMediaSender { } } v = self.clientsource.next() => { + debug!("Sender: Got Client"); if let Some(mut clientdata) = v { // Resend the video and audio buffers if !clientdata.inited { @@ -119,7 +134,7 @@ impl NeoMediaSender { next_vid.data.as_slice(), *idx as u64, 0, - ).is_err() { + ).await.is_err() { break; }; } @@ -132,7 +147,7 @@ impl NeoMediaSender { next_aud.data.as_slice(), *idx as u64, 0, - ).is_err() { + ).await.is_err() { break; }; } @@ -161,47 +176,77 @@ impl NeoMediaSender { Ok(()) } - fn process_vidbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { - self.clientdata.retain(|_, data| { - if let Some(vidsrc) = data.vidsrc.as_ref() { - Self::send_buffer( - vidsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.start_time, - ) - .is_ok() - // If ok retain is true - } else { - // Audio only - true + async fn process_vidbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { + for key in self + .clientdata + .keys() + .cloned() + .collect::>() + .drain(..) + { + match self.clientdata.entry(key) { + Entry::Occupied(data) => { + if let Some(vidsrc) = data.get().vidsrc.as_ref() { + if Self::send_buffer( + vidsrc, + stamped_data.data.as_slice(), + stamped_data.ms, + data.get().start_time, + ) + .await + .is_err() + { + data.remove(); + } + } else { + data.remove(); + } + } + Entry::Vacant(_) => {} } - }); + } + Ok(()) + } + + async fn process_audbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { + for key in self + .clientdata + .keys() + .cloned() + .collect::>() + .drain(..) + { + match self.clientdata.entry(key) { + Entry::Occupied(data) => { + if let Some(audsrc) = data.get().audsrc.as_ref() { + if Self::send_buffer( + audsrc, + stamped_data.data.as_slice(), + stamped_data.ms, + data.get().start_time, + ) + .await + .is_err() + { + data.remove(); + } + } else { + data.remove(); + } + } + Entry::Vacant(_) => {} + } + } Ok(()) } - fn process_audbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { - self.clientdata.retain(|_, data| { - if let Some(audsrc) = data.audsrc.as_ref() { - Self::send_buffer( - audsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.start_time, - ) - .is_ok() - // If ok retain is true - } else { - // video only - true - } - }); - - Ok(()) - } - - fn send_buffer(appsrc: &AppSrc, buf: &[u8], frame_ms: u64, start_time: u64) -> AnyResult<()> { + async fn send_buffer( + appsrc: &AppSrc, + buf: &[u8], + frame_ms: u64, + start_time: u64, + ) -> AnyResult<()> { let micros = if frame_ms > start_time { frame_ms - start_time } else { @@ -222,10 +267,14 @@ impl NeoMediaSender { gst_buf_data.copy_from_slice(buf); } // debug!("Buffer pushed"); - appsrc - .push_buffer(gst_buf.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) + let thread_appsrc = appsrc.clone(); // GObjects are refcounted + tokio::task::spawn_blocking(move || { + thread_appsrc + .push_buffer(gst_buf.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await? } fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult { diff --git a/src/rtsp/states/streaming.rs b/src/rtsp/states/streaming.rs index f6b5394..36ce2df 100644 --- a/src/rtsp/states/streaming.rs +++ b/src/rtsp/states/streaming.rs @@ -52,8 +52,9 @@ impl Camera { .await?; set.spawn(async move { loop { - debug!("{}: BcMediaStreamRecv", &tag_thread); + debug!("Straming: Get"); let data = timeout(Duration::from_secs(15), stream_data.get_data()).await??; + debug!("Straming: Got"); match &data { Ok(BcMedia::InfoV1(_)) => debug!("{}: - InfoV1", &tag_thread), Ok(BcMedia::InfoV2(_)) => debug!("{}: - InfoV2", &tag_thread), @@ -63,7 +64,9 @@ impl Camera { Ok(BcMedia::Adpcm(_)) => debug!("{}: - Adpcm", &tag_thread), Err(_) => debug!(" - Error"), } + debug!("Straming: Send"); sender.send(data?).await?; + debug!("Straming: Sent"); } }); } From 78186a2725c03ff63ae742f5e914e6e47989da61 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 16 Apr 2023 16:13:00 +0700 Subject: [PATCH 4/9] Debug dockerprune --- .github/workflows/dockerprune.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerprune.yml b/.github/workflows/dockerprune.yml index ba8c0ad..f3a3272 100644 --- a/.github/workflows/dockerprune.yml +++ b/.github/workflows/dockerprune.yml @@ -36,7 +36,7 @@ jobs: #Script will delete all images in all repositories of your docker hub account which are older than x days - set -e + set -e -x # set username and password From 5e9783fed333510c7fd51ff23b3c63fd53a7d419 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 16 Apr 2023 16:23:42 +0700 Subject: [PATCH 5/9] Dockerprune script token --- .github/workflows/dockerprune.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerprune.yml b/.github/workflows/dockerprune.yml index f3a3272..3a1226d 100644 --- a/.github/workflows/dockerprune.yml +++ b/.github/workflows/dockerprune.yml @@ -15,7 +15,7 @@ jobs: if [ -n $SECRET ]; then HAS_SECRET='true' ; fi echo "HAS_SECRET_TOKEN=${HAS_SECRET}" >> $GITHUB_OUTPUT env: - SECRET: "${{ secrets.DOCKER_TOKEN }}" + SECRET: "${{ secrets.DOCKER_PRUNE_TOKEN }}" - name: Convert username to lower case for docker id: string_user if: ${{ steps.vars.outputs.HAS_SECRET_TOKEN }} @@ -124,4 +124,4 @@ jobs: DAYS: "100" REPO_NAME: ${{ steps.string_repo.outputs.lowercase }} OWNER: ${{ steps.string_user.outputs.lowercase }} - TOKEN: ${{ secrets.DOCKER_TOKEN }} + TOKEN: ${{ secrets.DOCKER_PRUNE_TOKEN }} From f8b48544f8191808ac1dcf282aff02d9dcc5b0c7 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 17 Apr 2023 19:45:54 +0700 Subject: [PATCH 6/9] Switch too non-locking, fail skipping, signal reading --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/core/src/bc_protocol.rs | 5 + .../core/src/bc_protocol/connection/bcconn.rs | 22 ++-- .../core/src/bc_protocol/connection/bcsub.rs | 17 ++- .../src/bc_protocol/connection/discovery.rs | 20 +++- crates/core/src/bc_protocol/errors.rs | 14 +++ crates/core/src/bc_protocol/login.rs | 2 +- crates/core/src/bc_protocol/stream.rs | 21 ++-- crates/core/src/bcmedia/codex.rs | 7 +- src/cmdline.rs | 4 +- src/rtsp/gst/factory.rs | 52 +++++++- src/rtsp/gst/sender.rs | 112 +++++++++++------- src/rtsp/gst/shared.rs | 4 + src/rtsp/mod.rs | 8 +- src/rtsp/states/streaming.rs | 22 ++-- 16 files changed, 220 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e81c085..552a254 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -209,6 +209,7 @@ dependencies = [ "anstyle", "bitflags", "clap_lex", + "once_cell", "strsim", ] diff --git a/Cargo.toml b/Cargo.toml index a7e7e6a..c98fcb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ [dependencies] anyhow = "1.0.70" byte-slice-cast = "1.2.2" -clap = { version = "4.2.2", features = ["derive"] } +clap = { version = "4.2.2", features = ["derive", "cargo"] } crossbeam-channel = "0.5.8" env_logger = "*" futures = "0.3.28" diff --git a/crates/core/src/bc_protocol.rs b/crates/core/src/bc_protocol.rs index b6c5af0..11760ac 100644 --- a/crates/core/src/bc_protocol.rs +++ b/crates/core/src/bc_protocol.rs @@ -256,6 +256,7 @@ impl BcCamera { return Ok(CameraLocation::Tcp(addr)); } } + debug!("TCP fail"); // Tcp failed see if Local faired any better // Wait for all local to finish while let Some(result) = local_set.join_next().await { @@ -263,6 +264,7 @@ impl BcCamera { return Ok(CameraLocation::Udp(discovery)); } } + debug!("Local fail"); // Local failed see if Remote faired any better // Wait for all remote to finish while let Some(result) = remote_set.join_next().await { @@ -270,6 +272,7 @@ impl BcCamera { return Ok(CameraLocation::Udp(discovery)); } } + debug!("Remote fail"); // Remote failed see if Map faired any better // Wait for all Map to finish while let Some(result) = map_set.join_next().await { @@ -277,6 +280,7 @@ impl BcCamera { return Ok(CameraLocation::Udp(discovery)); } } + debug!("Map fail"); // Map failed see if Relay faired any better // Wait for all Relay to finish while let Some(result) = relay_set.join_next().await { @@ -284,6 +288,7 @@ impl BcCamera { return Ok(CameraLocation::Udp(discovery)); } } + debug!("Relay (All) fail"); // Nothing works Err(Error::CannotInitCamera) } diff --git a/crates/core/src/bc_protocol/connection/bcconn.rs b/crates/core/src/bc_protocol/connection/bcconn.rs index d4b2160..770a093 100644 --- a/crates/core/src/bc_protocol/connection/bcconn.rs +++ b/crates/core/src/bc_protocol/connection/bcconn.rs @@ -7,8 +7,8 @@ use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::sync::Arc; use tokio::sync::mpsc::{channel, Sender}; -use tokio::sync::Mutex; use tokio::task::yield_now; +use tokio_stream::wrappers::ReceiverStream; use tokio::{sync::RwLock, task::JoinSet}; @@ -30,21 +30,19 @@ pub(crate) type BcConnSource = Box> + Send + Sync + /// /// There can be only one subscriber per kind of message at a time. pub struct BcConnection { - sink: Arc>, + sink: Sender>, subscribers: Arc, #[allow(dead_code)] // Not dead we just need to hold a reference to keep it alive rx_thread: RwLock>>, } impl BcConnection { - pub async fn new(sink: BcConnSink, mut source: BcConnSource) -> Result { + pub async fn new(mut sink: BcConnSink, mut source: BcConnSource) -> Result { let subscribers: Arc = Default::default(); - let sink = Arc::new(Mutex::new(sink)); - let subs = subscribers.clone(); - + let (sinker, sinker_rx) = channel::>(100); let mut rx_thread = JoinSet::new(); - let sink_thread = sink.clone(); + let sink_thread = sinker.clone(); rx_thread.spawn(async move { loop { yield_now().await; // Give other tasks a chance to run before pulling next packet @@ -57,15 +55,17 @@ impl BcConnection { } }); + rx_thread.spawn(async move { sink.send_all(&mut ReceiverStream::new(sinker_rx)).await }); + Ok(BcConnection { - sink, + sink: sinker, subscribers, rx_thread: RwLock::new(rx_thread), }) } pub(super) async fn send(&self, bc: Bc) -> crate::Result<()> { - self.sink.lock().await.send(bc).await?; + self.sink.send(Ok(bc)).await?; Ok(()) } @@ -114,7 +114,7 @@ impl BcConnection { async fn poll( response: Result, subscribers: &Subscriber, - sink: &Mutex, + sink: &Sender>, ) -> Result<()> { match response { Ok(response) => { @@ -129,7 +129,7 @@ impl BcConnection { (Some(occ), _) => { if let Some(reply) = occ(&response) { assert!(reply.meta.msg_num == response.meta.msg_num); - sink.lock().await.send(reply).await?; + sink.send(Ok(reply)).await?; } } (None, Some(occ)) => { diff --git a/crates/core/src/bc_protocol/connection/bcsub.rs b/crates/core/src/bc_protocol/connection/bcsub.rs index a65fb3e..02c89ac 100644 --- a/crates/core/src/bc_protocol/connection/bcsub.rs +++ b/crates/core/src/bc_protocol/connection/bcsub.rs @@ -48,6 +48,7 @@ impl<'a> Stream for BcPayloadStream<'a> { mut self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll>>> { + // log::debug!("PayloadStream: Poll"); match Pin::new(&mut self.rx).poll_next(cx) { Poll::Ready(Some(Ok(Bc { body: @@ -57,17 +58,27 @@ impl<'a> Stream for BcPayloadStream<'a> { }), .. }))) => { + // log::debug!("PayloadStream: Data"); return Poll::Ready(Some(Ok(data))); } Poll::Ready(Some(Ok(_bc))) => { // trace!("Got other BC in payload stream"); + // log::debug!("PayloadStream: Other Data!"); } Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(Err(IoError::new(ErrorKind::Other, e)))) + // log::debug!("PayloadStream: Err"); + return Poll::Ready(Some(Err(IoError::new(ErrorKind::Other, e)))); + } + Poll::Ready(None) => { + // log::debug!("PayloadStream: None"); + return Poll::Ready(None); + } + Poll::Pending => { + // log::debug!("PayloadStream: Pend"); + return Poll::Pending; } - Poll::Ready(None) => return Poll::Ready(None), - Poll::Pending => return Poll::Pending, } + // log::debug!("PayloadStream: Default Pend and Wake"); cx.waker().wake_by_ref(); Poll::Pending } diff --git a/crates/core/src/bc_protocol/connection/discovery.rs b/crates/core/src/bc_protocol/connection/discovery.rs index ef7e696..efebdf8 100644 --- a/crates/core/src/bc_protocol/connection/discovery.rs +++ b/crates/core/src/bc_protocol/connection/discovery.rs @@ -11,7 +11,7 @@ use crate::bcudp::xml::*; use crate::{Error, Result}; use futures::{ sink::SinkExt, - stream::{FuturesUnordered, SplitSink, StreamExt}, + stream::{FuturesUnordered, StreamExt}, }; use lazy_static::lazy_static; use local_ip_address::local_ip; @@ -24,7 +24,7 @@ use tokio::{ net::UdpSocket, sync::{ mpsc::{channel, Receiver, Sender}, - Mutex, RwLock, + RwLock, }, task::JoinSet, time::{interval, timeout, Duration}, @@ -83,7 +83,7 @@ type ArcFramedSocket = UdpFramed>; pub(crate) struct Discoverer { socket: Arc, handle: RwLock>, - writer: Mutex>, + writer: Sender>, subsribers: Subscriber, local_addr: SocketAddr, } @@ -94,7 +94,7 @@ impl Discoverer { let local_addr = socket.local_addr()?; let inner: ArcFramedSocket = UdpFramed::new(socket.clone(), BcUdpCodex::new()); - let (writer, mut reader) = inner.split(); + let (mut writer, mut reader) = inner.split(); let mut set = JoinSet::new(); let subsribers: Subscriber = Default::default(); @@ -138,10 +138,15 @@ impl Discoverer { } }); + let (sinker, sinker_rx) = channel(100); + set.spawn(async move { + let _ = writer.send_all(&mut ReceiverStream::new(sinker_rx)).await; + }); + Ok(Discoverer { socket, handle: RwLock::new(set), - writer: Mutex::new(writer), + writer: sinker, subsribers, local_addr, }) @@ -192,7 +197,10 @@ impl Discoverer { } async fn send(&self, disc: BcUdp, addr: SocketAddr) -> Result<()> { - self.writer.lock().await.send((disc, addr)).await + self.writer + .send(Ok((disc, addr))) + .await + .map_err(Error::from) } fn local_addr(&self) -> &SocketAddr { diff --git a/crates/core/src/bc_protocol/errors.rs b/crates/core/src/bc_protocol/errors.rs index 81a1b09..d7a5687 100644 --- a/crates/core/src/bc_protocol/errors.rs +++ b/crates/core/src/bc_protocol/errors.rs @@ -39,6 +39,10 @@ pub enum Error { #[error(display = "Camera responded with Service Unavaliable")] CameraServiceUnavaliable, + /// Raised when the camera responds with a status code over than OK during login + #[error(display = "Camera responded with Err during login")] + CameraLoginFail, + /// Raised when a connection is dropped. #[error(display = "Dropped connection")] DroppedConnection, @@ -51,6 +55,10 @@ pub enum Error { #[error(display = "Dropped connection")] BroadcastDroppedConnectionTry(#[error(source)] tokio::sync::broadcast::error::TryRecvError), + /// Raised when a connection is dropped during a TryRecv event + #[error(display = "Send Error")] + TokioBcSendError, + /// Raised when the TIMEOUT is reach #[error(display = "Timeout")] Timeout(#[error(source)] std::sync::Arc), @@ -194,6 +202,12 @@ impl From for Error { } } +impl From> for Error { + fn from(_: tokio::sync::mpsc::error::SendError) -> Self { + Error::TokioBcSendError + } +} + impl From for Error { fn from(k: cookie_factory::GenError) -> Self { Error::GenError(std::sync::Arc::new(k)) diff --git a/crates/core/src/bc_protocol/login.rs b/crates/core/src/bc_protocol/login.rs index 4291727..ea9a10a 100644 --- a/crates/core/src/bc_protocol/login.rs +++ b/crates/core/src/bc_protocol/login.rs @@ -136,7 +136,7 @@ impl BcCamera { sub_login.send(modern_login).await?; let modern_reply = sub_login.recv().await?; if modern_reply.meta.response_code != 200 { - return Err(Error::CameraServiceUnavaliable); + return Err(Error::CameraLoginFail); } match modern_reply.body { diff --git a/crates/core/src/bc_protocol/stream.rs b/crates/core/src/bc_protocol/stream.rs index 01424dd..4528c2c 100644 --- a/crates/core/src/bc_protocol/stream.rs +++ b/crates/core/src/bc_protocol/stream.rs @@ -4,7 +4,6 @@ use crate::{ bcmedia::model::*, }; use futures::stream::StreamExt; -use log::*; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -51,14 +50,14 @@ impl StreamData { self.abort_handle.store(true, Ordering::Relaxed); return Err(Error::DroppedConnection); } - debug!("StreamData: Get"); + // debug!("StreamData: Get"); match self.rx.recv().await { Some(data) => { - debug!("StreamData: Got"); + // debug!("StreamData: Got"); Ok(data) } None => { - debug!("StreamData: Drop"); + // debug!("StreamData: Drop"); self.abort_handle.store(true, Ordering::Relaxed); Err(Error::DroppedConnection) } @@ -182,22 +181,23 @@ impl BcCamera { let mut media_sub = sub_video.bcmedia_stream(strict); while !abort_handle_thread.load(Ordering::Relaxed) { - debug!("Stream: Get"); + // debug!("Stream: Get"); if let Some(bc_media) = media_sub.next().await { - debug!("Stream: Got"); + // debug!("Stream: Got"); // We now have a complete interesting packet. Send it to on the callback - debug!("Stream: Send"); + // debug!("Stream: Send"); if tx.send(bc_media).await.is_err() { - debug!("Stream: Dropped"); + // debug!("Stream: Dropped"); break; // Connection dropped } - debug!("Stream: Sent"); + // debug!("Stream: Sent"); } else { break; } } } + // debug!("Stream: Stopping"); let stop_video = Bc::new_from_xml( BcMeta { msg_id: MSG_ID_VIDEO_STOP, @@ -217,8 +217,9 @@ impl BcCamera { ..Default::default() }, ); - + // debug!("Stream: Send Stop"); sub_video.send(stop_video).await?; + // debug!("Stream: Sent Stop"); Ok(()) }); diff --git a/crates/core/src/bcmedia/codex.rs b/crates/core/src/bcmedia/codex.rs index b006532..4fc40bc 100644 --- a/crates/core/src/bcmedia/codex.rs +++ b/crates/core/src/bcmedia/codex.rs @@ -4,7 +4,7 @@ //! use crate::bcmedia::model::*; use crate::{Error, Result}; -use bytes::{Buf, BytesMut}; +use bytes::BytesMut; use log::*; use tokio_util::codec::{Decoder, Encoder}; @@ -71,8 +71,9 @@ impl Decoder for BcMediaCodex { debug!("Error in stream attempting to restore"); trace!(" Stream Error: {:?}", e); } - src.advance(1); - amount_skipped += 1; + // Drop the whole packet and wait for a packet that starts with magic + amount_skipped += src.len(); + src.clear(); continue; } } diff --git a/src/cmdline.rs b/src/cmdline.rs index 7552ccc..ea2f319 100644 --- a/src/cmdline.rs +++ b/src/cmdline.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{crate_authors, crate_version, Parser}; use std::path::PathBuf; use std::str::FromStr; @@ -7,7 +7,7 @@ use std::str::FromStr; /// Neolink is free software released under the GNU AGPL v3. /// You can find its source code at https://github.com/thirtythreeforty/neolink #[derive(Parser, Debug)] -#[command(name = "neolink", arg_required_else_help = true)] +#[command(name = "neolink", arg_required_else_help = true, version = crate_version!(), author = crate_authors!("\n"))] pub struct Opt { #[arg(short, long, global = true, value_parser = PathBuf::from_str)] pub config: Option, diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 956b25b..7655a60 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -12,7 +12,7 @@ use gstreamer::{ Structure, }; use gstreamer::{Bin, Caps, ClockTime, Element, ElementFactory}; -use gstreamer_app::{AppSrc, AppStreamType}; +use gstreamer_app::{AppSrc, AppSrcCallbacks, AppStreamType}; use gstreamer_rtsp::RTSPUrl; use gstreamer_rtsp_server::prelude::*; use gstreamer_rtsp_server::subclass::prelude::*; @@ -139,7 +139,11 @@ impl Default for NeoMediaFactoryImpl { clientdata: Default::default(), waiting_for_iframe: true, }; - threads.spawn(async move { sender.run().await }); + threads.spawn(async move { + let res = sender.run().await; + error!("Factory Run thread stopped: {:?}", res); + res + }); Self { sender: datasender, @@ -209,6 +213,7 @@ impl NeoMediaFactoryImpl { let bin = media .dynamic_cast::() .map_err(|_| anyhow!("Media source's element should be a bin"))?; + let clock = bin.clock(); // Clear the autogenerated ones for element in bin.iterate_elements().into_iter().flatten() { bin.remove(&element)?; @@ -216,6 +221,7 @@ impl NeoMediaFactoryImpl { let mut client_data = ClientPipelineData { start_time: self.shared.microseconds.load(Ordering::Relaxed), + clock, ..Default::default() }; @@ -235,6 +241,25 @@ impl NeoMediaFactoryImpl { source.set_max_bytes(52428800); source.set_do_timestamp(false); source.set_stream_type(AppStreamType::Stream); + + let enough_data_start = client_data.enough_data.clone(); + let enough_data_stop = client_data.enough_data.clone(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + if enough_data_stop.load(Ordering::Relaxed) { + log::debug!("Client is in need of data"); + } + enough_data_stop.store(false, Ordering::Relaxed); + }) + .enough_data(move |_appsrc| { + if !enough_data_start.load(Ordering::Relaxed) { + log::debug!("Client is full of of data"); + } + enough_data_start.store(true, Ordering::Relaxed); + }) + .build(), + ); // source.set_caps(Some( // &Caps::builder("video/x-h265").field("parsed", false).build(), // )); @@ -243,6 +268,7 @@ impl NeoMediaFactoryImpl { .map_err(|_| anyhow!("Cannot cast back"))?; let queue = make_element("queue", "source_queue")?; queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); let parser = make_element("h265parse", "parser")?; let payload = make_element("rtph265pay", "pay0")?; bin.add_many(&[&source, &queue, &parser, &payload])?; @@ -267,6 +293,25 @@ impl NeoMediaFactoryImpl { source.set_max_bytes(52428800); source.set_do_timestamp(false); source.set_stream_type(AppStreamType::Stream); + + let enough_data_start = client_data.enough_data.clone(); + let enough_data_stop = client_data.enough_data.clone(); + source.set_callbacks( + AppSrcCallbacks::builder() + .need_data(move |_appsrc, _amt| { + if enough_data_stop.load(Ordering::Relaxed) { + log::debug!("Client is in need of data"); + } + enough_data_stop.store(false, Ordering::Relaxed); + }) + .enough_data(move |_appsrc| { + if !enough_data_start.load(Ordering::Relaxed) { + log::debug!("Client is full of of data"); + } + enough_data_start.store(true, Ordering::Relaxed); + }) + .build(), + ); // source.set_caps(Some( // &Caps::builder("video/x-h264").field("parsed", false).build(), // )); @@ -292,6 +337,7 @@ impl NeoMediaFactoryImpl { source.set_property_from_str("pattern", "snow"); let queue = make_element("queue", "queue0")?; queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); let overlay = make_element("textoverlay", "overlay")?; overlay.set_property("text", "Stream not Ready"); overlay.set_property_from_str("valignment", "top"); @@ -339,6 +385,7 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "audqueue")?; queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); let parser = make_element("aacparse", "audparser")?; let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; @@ -396,6 +443,7 @@ impl NeoMediaFactoryImpl { let queue = make_element("queue", "audqueue")?; queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 104857600u32); let decoder = make_element("decodebin", "auddecoder")?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 8afc8dc..3c62485 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -2,7 +2,7 @@ //! gstreamer media stream use anyhow::anyhow; use futures::stream::StreamExt; -use gstreamer::ClockTime; +use gstreamer::{prelude::*, ClockTime}; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use log::*; @@ -55,16 +55,16 @@ impl NeoMediaSender { self.shared .buffer_ready .store(!buffer_read.is_empty(), Ordering::Relaxed); - debug!("Sender: Get"); + // debug!("Sender: Get"); tokio::select! { v = self.data_source.next() => { - debug!("Sender: Got Data"); + // debug!("Sender: Got Data"); // debug!("data_source recieved"); if let Some(bc_media) = v { - if ! self.skip_bcmedia(&bc_media)? { + if ! self.skip_bcmedia(&bc_media) { // debug!("Not skipped"); // resend_pause.reset(); - self.inspect_bcmedia(&bc_media).await?; + self.inspect_bcmedia(&bc_media).await; match bc_media { BcMedia::Iframe(frame) => { if buffer_write.len() > 100 { @@ -77,7 +77,7 @@ impl NeoMediaSender { ms: frame.microseconds as u64, data: frame.data, }; - self.process_vidbuffer(&new_data).await?; + self.process_vidbuffer(&new_data).await; buffer_write.push(new_data); } BcMedia::Pframe(frame) => { @@ -85,7 +85,7 @@ impl NeoMediaSender { ms: frame.microseconds as u64, data: frame.data, }; - self.process_vidbuffer(&new_data).await?; + self.process_vidbuffer(&new_data).await; buffer_write.push(new_data); } BcMedia::Aac(aac) => { @@ -94,7 +94,7 @@ impl NeoMediaSender { ms: last.ms, data: aac.data, }; - self.process_audbuffer(&new_data).await?; + self.process_audbuffer(&new_data).await; audbuffer_write.push(new_data); } @@ -105,7 +105,7 @@ impl NeoMediaSender { ms: last.ms, data: adpcm.data, }; - self.process_audbuffer(&new_data).await?; + self.process_audbuffer(&new_data).await; audbuffer_write.push(new_data); } @@ -118,7 +118,7 @@ impl NeoMediaSender { } } v = self.clientsource.next() => { - debug!("Sender: Got Client"); + // debug!("Sender: Got Client"); if let Some(mut clientdata) = v { // Resend the video and audio buffers if !clientdata.inited { @@ -176,7 +176,7 @@ impl NeoMediaSender { Ok(()) } - async fn process_vidbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { + async fn process_vidbuffer(&mut self, stamped_data: &StampedData) { for key in self .clientdata .keys() @@ -187,16 +187,35 @@ impl NeoMediaSender { match self.clientdata.entry(key) { Entry::Occupied(data) => { if let Some(vidsrc) = data.get().vidsrc.as_ref() { - if Self::send_buffer( - vidsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.get().start_time, - ) - .await - .is_err() - { - data.remove(); + if !data.get().enough_data.load(Ordering::Relaxed) { + let ms = stamped_data.ms; + let st = data.get().start_time; + let micros = ms - st; + if let Some(clock) = data.get().clock.as_ref() { + if let Some(time) = clock.time() { + let buffer_time = ClockTime::from_useconds(micros); + log::debug!( + "Buffer time: {:?}. Clock time: {:?}", + buffer_time, + time + ); + } + } + debug!( + "Pushed Video Buffer: ms: {}, st: {}, delta: {}", + ms, st, micros + ); + if Self::send_buffer( + vidsrc, + stamped_data.data.as_slice(), + stamped_data.ms, + data.get().start_time, + ) + .await + .is_err() + { + data.remove(); + } } } else { data.remove(); @@ -205,10 +224,9 @@ impl NeoMediaSender { Entry::Vacant(_) => {} } } - Ok(()) } - async fn process_audbuffer(&mut self, stamped_data: &StampedData) -> AnyResult<()> { + async fn process_audbuffer(&mut self, stamped_data: &StampedData) { for key in self .clientdata .keys() @@ -219,16 +237,19 @@ impl NeoMediaSender { match self.clientdata.entry(key) { Entry::Occupied(data) => { if let Some(audsrc) = data.get().audsrc.as_ref() { - if Self::send_buffer( - audsrc, - stamped_data.data.as_slice(), - stamped_data.ms, - data.get().start_time, - ) - .await - .is_err() - { - data.remove(); + if !data.get().enough_data.load(Ordering::Relaxed) { + debug!("Pushed Audio Buffer"); + if Self::send_buffer( + audsrc, + stamped_data.data.as_slice(), + stamped_data.ms, + data.get().start_time, + ) + .await + .is_err() + { + data.remove(); + } } } else { data.remove(); @@ -237,8 +258,6 @@ impl NeoMediaSender { Entry::Vacant(_) => {} } } - - Ok(()) } async fn send_buffer( @@ -268,28 +287,38 @@ impl NeoMediaSender { } // debug!("Buffer pushed"); let thread_appsrc = appsrc.clone(); // GObjects are refcounted - tokio::task::spawn_blocking(move || { + let res = tokio::task::spawn_blocking(move || { thread_appsrc .push_buffer(gst_buf.copy()) .map(|_| ()) .map_err(|_| anyhow!("Could not push buffer to appsrc")) }) - .await? + .await; + match &res { + Err(e) => { + error!("Failed to send buffer: {:?}", e); + } + Ok(Err(e)) => { + error!("Failed to send buffer: {:?}", e); + } + Ok(Ok(_)) => {} + }; + res? } - fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult { + fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> bool { if self.waiting_for_iframe { if let BcMedia::Iframe(_) = bc_media { self.waiting_for_iframe = false; } else { log::debug!("Skipping bcmedia"); - return Ok(true); + return true; } } - Ok(false) + false } - async fn inspect_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult<()> { + async fn inspect_bcmedia(&mut self, bc_media: &BcMedia) { let old_vid = *self.shared.vid_format.read().await; let old_aud = *self.shared.aud_format.read().await; match bc_media { @@ -343,6 +372,5 @@ impl NeoMediaSender { if old_aud != new_aud { log::debug!("Audio format set to: {:?} from {:?}", new_aud, old_aud); } - Ok(()) } } diff --git a/src/rtsp/gst/shared.rs b/src/rtsp/gst/shared.rs index f626d98..847119d 100644 --- a/src/rtsp/gst/shared.rs +++ b/src/rtsp/gst/shared.rs @@ -1,8 +1,10 @@ //! Data shared between the various //! components that manage a media stream +use gstreamer::Clock; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize}; +use std::sync::Arc; use tokio::sync::RwLock; #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -45,4 +47,6 @@ pub(super) struct ClientPipelineData { pub(super) audsrc: Option, pub(super) start_time: u64, pub(super) inited: bool, + pub(super) clock: Option, + pub(super) enough_data: Arc, } diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 07f8e9d..5957635 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -162,7 +162,13 @@ async fn camera_main(camera: Camera) -> Result<(), CameraFailureKi .login() .await .with_context(|| format!("{}: Could not login to camera", name)) - .map_err(CameraFailureKind::Fatal)?; + .map_err(|e| { + let e_inner = e.downcast_ref::().unwrap(); + match e_inner { + neolink_core::Error::CameraLoginFail => CameraFailureKind::Fatal(e), + _ => CameraFailureKind::Retry(e), + } + })?; let _ = loggedin.manage().await; diff --git a/src/rtsp/states/streaming.rs b/src/rtsp/states/streaming.rs index 36ce2df..2cfaa97 100644 --- a/src/rtsp/states/streaming.rs +++ b/src/rtsp/states/streaming.rs @@ -52,21 +52,21 @@ impl Camera { .await?; set.spawn(async move { loop { - debug!("Straming: Get"); + // debug!("Straming: Get"); let data = timeout(Duration::from_secs(15), stream_data.get_data()).await??; - debug!("Straming: Got"); + // debug!("Straming: Got"); match &data { - Ok(BcMedia::InfoV1(_)) => debug!("{}: - InfoV1", &tag_thread), - Ok(BcMedia::InfoV2(_)) => debug!("{}: - InfoV2", &tag_thread), - Ok(BcMedia::Iframe(_)) => debug!("{}: - Iframe", &tag_thread), - Ok(BcMedia::Pframe(_)) => debug!("{}: - Pframe", &tag_thread), - Ok(BcMedia::Aac(_)) => debug!("{}: - Aac", &tag_thread), - Ok(BcMedia::Adpcm(_)) => debug!("{}: - Adpcm", &tag_thread), - Err(_) => debug!(" - Error"), + Ok(BcMedia::InfoV1(_)) => trace!("{}: - InfoV1", &tag_thread), + Ok(BcMedia::InfoV2(_)) => trace!("{}: - InfoV2", &tag_thread), + Ok(BcMedia::Iframe(_)) => trace!("{}: - Iframe", &tag_thread), + Ok(BcMedia::Pframe(_)) => trace!("{}: - Pframe", &tag_thread), + Ok(BcMedia::Aac(_)) => trace!("{}: - Aac", &tag_thread), + Ok(BcMedia::Adpcm(_)) => trace!("{}: - Adpcm", &tag_thread), + Err(_) => trace!(" - Error"), } - debug!("Straming: Send"); + // debug!("Straming: Send"); sender.send(data?).await?; - debug!("Straming: Sent"); + // debug!("Straming: Sent"); } }); } From 233a73e1dc3a8b837593ccb506e05797f9471ac9 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 17 Apr 2023 19:49:50 +0700 Subject: [PATCH 7/9] Backoff on retries --- src/rtsp/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 5957635..723376c 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -95,6 +95,7 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { set.spawn(async move { let shared = camera.shared.clone(); let name = camera.get_name(); + let mut backoff = Duration::from_micros(125); loop { let failure = camera_main(camera).await; match failure { @@ -104,6 +105,10 @@ pub(crate) async fn main(_opt: Opt, mut config: Config) -> Result<()> { } Err(CameraFailureKind::Retry(e)) => { warn!("{}: Retryable error: {:X?}", name, e); + tokio::time::sleep(backoff).await; + if backoff < Duration::from_secs(5) { + backoff *= 2; + } camera = Camera { shared: shared.clone(), state: Disconnected {}, From 74274ae30e3e15b0acb58f61915dff1d1484b023 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 18 Apr 2023 08:15:31 +0700 Subject: [PATCH 8/9] Add restarts to the send thread --- src/rtsp/gst/factory.rs | 17 +++++++++++++---- src/rtsp/gst/sender.rs | 4 ++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 7655a60..81bd512 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -124,7 +124,7 @@ impl Drop for NeoMediaFactoryImpl { impl Default for NeoMediaFactoryImpl { fn default() -> Self { - warn!("Constructing Factor Impl"); + debug!("Constructing Factor Impl"); let (datasender, datarx) = channel(3); let (clientsender, rx_clientsender) = channel(3); let shared: Arc = Default::default(); @@ -140,9 +140,18 @@ impl Default for NeoMediaFactoryImpl { waiting_for_iframe: true, }; threads.spawn(async move { - let res = sender.run().await; - error!("Factory Run thread stopped: {:?}", res); - res + loop { + match sender.run().await { + Err(e) => { + warn!("Media send thead failed... restarting. Cause: {:?}", e); + } + Ok(()) => { + break; + } + } + sender.waiting_for_iframe = true; + } + Ok(()) }); Self { diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 3c62485..a5a2973 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -296,10 +296,10 @@ impl NeoMediaSender { .await; match &res { Err(e) => { - error!("Failed to send buffer: {:?}", e); + debug!("Failed to send buffer: {:?}", e); } Ok(Err(e)) => { - error!("Failed to send buffer: {:?}", e); + debug!("Failed to send buffer: {:?}", e); } Ok(Ok(_)) => {} }; From c1445ea1e15a1b0b57000980419b6753765a1f32 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Tue, 18 Apr 2023 08:17:52 +0700 Subject: [PATCH 9/9] Remove debug print --- .github/workflows/dockerprune.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerprune.yml b/.github/workflows/dockerprune.yml index 3a1226d..33251c6 100644 --- a/.github/workflows/dockerprune.yml +++ b/.github/workflows/dockerprune.yml @@ -36,7 +36,7 @@ jobs: #Script will delete all images in all repositories of your docker hub account which are older than x days - set -e -x + set -e # set username and password