From b728ba14c78bd23957199ade41090d394ca2278e Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sat, 11 Mar 2023 13:28:12 +0700 Subject: [PATCH] Added AAC --- .../src/bc_protocol/connection/udpsource.rs | 12 +- crates/core/src/bc_protocol/stream.rs | 1 + crates/core/src/bcmedia/codex.rs | 6 +- crates/core/src/bcudp/codex.rs | 10 +- src/rtsp/gst/factory.rs | 76 +++++++++++-- src/rtsp/gst/sender.rs | 105 ++++++++++++++---- src/rtsp/gst/shared.rs | 4 +- src/rtsp/states/streaming.rs | 17 +-- 8 files changed, 177 insertions(+), 54 deletions(-) diff --git a/crates/core/src/bc_protocol/connection/udpsource.rs b/crates/core/src/bc_protocol/connection/udpsource.rs index 62f1125..b1e2ceb 100644 --- a/crates/core/src/bc_protocol/connection/udpsource.rs +++ b/crates/core/src/bc_protocol/connection/udpsource.rs @@ -226,17 +226,17 @@ pub(crate) struct UdpPayloadSource { impl UdpPayloadSource { fn build_send_ack(&self) -> UdpAck { if self.packets_want > 0 { - let mut start: u32 = self.packets_want - 1; - while self.recieved.contains_key(&start) { + let mut first_missing: u32 = self.packets_want; + while self.recieved.contains_key(&first_missing) { // Happens if we have recieved but not consumed yet - start += 1; + first_missing += 1; } let missing_ids = if let Some(end) = self.recieved.keys().max() { let mut vec = vec![]; // From last contiguous packet to last recieved packet // create a payload of `00` (unreceived) and `01` (received) // that can be used to form the `UdpAck` packet - for i in (start + 1)..(end + 1) { + for i in (first_missing)..(end + 1) { if self.recieved.contains_key(&i) { vec.push(1) } else { @@ -250,7 +250,7 @@ impl UdpPayloadSource { UdpAck { connection_id: self.camera_id, - packet_id: start, + packet_id: first_missing - 1, // Last we actually have is first_missing - 1 group_id: 0, payload: missing_ids, } @@ -306,8 +306,6 @@ impl Stream for UdpPayloadSource { trace!("UdpPayloadSource.Data: PreReady: {}", this.packets_want); this.packets_want += 1; // error!("packets_want: {}", this.packets_want); - let ack = BcUdp::Ack(this.build_send_ack()); - this.send_buffer.push_back(ack); return Poll::Ready(Some(Ok(payload))); } // Handle resend events diff --git a/crates/core/src/bc_protocol/stream.rs b/crates/core/src/bc_protocol/stream.rs index ef3f6df..068ae2a 100644 --- a/crates/core/src/bc_protocol/stream.rs +++ b/crates/core/src/bc_protocol/stream.rs @@ -173,6 +173,7 @@ impl BcCamera { break; } } + log::error!("Stopping media_sub"); } let stop_video = Bc::new_from_xml( diff --git a/crates/core/src/bcmedia/codex.rs b/crates/core/src/bcmedia/codex.rs index 7a2e75a..d9d2ef1 100644 --- a/crates/core/src/bcmedia/codex.rs +++ b/crates/core/src/bcmedia/codex.rs @@ -42,13 +42,13 @@ impl Decoder for BcMediaCodex { match { BcMedia::deserialize(src) } { Ok(bc) => { if amount_skipped > 0 { - trace!("Amount skipped to restore stream: {}", amount_skipped); + debug!("Amount skipped to restore stream: {}", amount_skipped); } return Ok(Some(bc)); } Err(Error::NomIncomplete(_)) => { if amount_skipped > 0 { - trace!("Amount skipped to restore stream: {}", amount_skipped); + debug!("Amount skipped to restore stream: {}", amount_skipped); } return Ok(None); } @@ -59,7 +59,7 @@ impl Decoder for BcMediaCodex { return Ok(None); } else { if amount_skipped == 0 { - trace!("Error in stream attempting to restore: {:?}", e); + debug!("Error in stream attempting to restore: {:?}", e); } src.advance(1); amount_skipped += 1; diff --git a/crates/core/src/bcudp/codex.rs b/crates/core/src/bcudp/codex.rs index 1ef79e5..5ba9d06 100644 --- a/crates/core/src/bcudp/codex.rs +++ b/crates/core/src/bcudp/codex.rs @@ -40,13 +40,19 @@ impl Decoder for BcUdpCodex { r2c_disc: Some(_), .. }, .. - })) => Err(Error::RelayTerminate), + })) => { + log::error!("Relay terminate"); + Err(Error::RelayTerminate) + } Ok(BcUdp::Discovery(UdpDiscovery { payload: UdpXml { d2c_disc: Some(_), .. }, .. - })) => Err(Error::CameraTerminate), + })) => { + log::error!("Camera terminate"); + Err(Error::CameraTerminate) + } Ok(bc) => Ok(Some(bc)), Err(Error::NomIncomplete(_)) => Ok(None), Err(e) => Err(e), diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 548d303..2e2e6b2 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -100,7 +100,8 @@ unsafe impl Sync for NeoMediaFactory {} pub(crate) struct NeoMediaFactoryImpl { sender: Sender, - appsender: Sender, + vidsender: Sender, + audsender: Sender, shared: Arc, #[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj threads: JoinSet>, @@ -116,23 +117,28 @@ impl Default for NeoMediaFactoryImpl { fn default() -> Self { warn!("Constructing Factor Impl"); let (datasender, datarx) = channel(3); - let (appsender, rx_appsender) = channel(3); + let (vidsender, rx_vidsender) = channel(3); + let (audsender, rx_audsender) = channel(3); let shared: Arc = Default::default(); // Prepare thread that sends data into the appsrcs let mut threads: JoinSet> = Default::default(); let mut sender = NeoMediaSender { data_source: ReceiverStream::new(datarx), - app_source: ReceiverStream::new(rx_appsender), + vidsource: ReceiverStream::new(rx_vidsender), + audsource: ReceiverStream::new(rx_audsender), shared: shared.clone(), - appsrcs: Default::default(), + uid: Default::default(), + vidsrcs: Default::default(), + audsrcs: Default::default(), waiting_for_iframe: true, }; threads.spawn(async move { sender.run().await }); Self { sender: datasender, - appsender, + vidsender, + audsender, shared, threads, } @@ -195,7 +201,6 @@ impl NeoMediaFactoryImpl { bin.remove(&element)?; } - debug!("self.shared owners: {}", Arc::strong_count(&self.shared)); // Now contruct the actual ones match VidFormats::from(self.shared.vid_format.load(Ordering::Relaxed)) { VidFormats::H265 => { @@ -227,7 +232,7 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - self.appsender.blocking_send(source)?; + self.vidsender.blocking_send(source)?; } VidFormats::H264 => { debug!("Building H264 Pipeline"); @@ -258,7 +263,7 @@ impl NeoMediaFactoryImpl { let source = source .dynamic_cast::() .map_err(|_| anyhow!("Cannot convert appsrc"))?; - self.appsender.blocking_send(source)?; + self.vidsender.blocking_send(source)?; } VidFormats::Unknown => { debug!("Building Unknown Pipeline"); @@ -290,6 +295,61 @@ impl NeoMediaFactoryImpl { } } + match AudFormats::from(self.shared.aud_format.load(Ordering::Relaxed)) { + AudFormats::Unknown => {} + AudFormats::Aac => { + debug!("Building Aac pipeline"); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + source.set_base_time(ClockTime::from_mseconds( + self.shared.microseconds.load(Ordering::Relaxed), + )); + source.set_is_live(true); + source.set_block(true); + source.set_property("emit-signals", &false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Stream); + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; + + let queue = make_element("queue", "audqueue")?; + let parser = make_element("aacparse", "audparser")?; + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; + + bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &parser, &decoder])?; + Element::link_many(&[&encoder, &payload])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link AAC decoder to encoder"); + }); + + // decoder.link_filtered( + // &encoder, + // & Caps::builder("audio/x-raw") + // .field("format", "S16BE") + // .field("layout", "interleaved") + // // .field("channels", 1) + // .build(), + // )?; + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + self.audsender.blocking_send(source)?; + } + AudFormats::Adpcm => {} + } + bin.dynamic_cast::() .map_err(|_| anyhow!("Cannot cast back")) } diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 5611ca3..0ff3ba7 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -6,8 +6,10 @@ use gstreamer::ClockTime; use gstreamer_app::AppSrc; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use neolink_core::bcmedia::model::*; +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; use std::sync::{atomic::Ordering, Arc}; -use tokio::time::{interval, Duration, MissedTickBehavior}; +// use tokio::time::{interval, Duration, MissedTickBehavior}; use tokio_stream::wrappers::ReceiverStream; use super::{shared::*, AnyResult}; @@ -16,13 +18,20 @@ use super::{shared::*, AnyResult}; pub(super) struct ClientData { pub(super) appsrc: AppSrc, pub(super) start_time: u64, + pub(super) inited: bool, } pub(super) struct NeoMediaSender { pub(super) data_source: ReceiverStream, - pub(super) app_source: ReceiverStream, + pub(super) vidsource: ReceiverStream, + pub(super) audsource: ReceiverStream, pub(super) shared: Arc, - pub(super) appsrcs: std::collections::HashSet, + // Used to generate the key for the hashmaps + pub(super) uid: AtomicU64, + // Hashmap so we can alter the data in ClientData while also allowing for remove at any point + // If you know a better collections pm me + pub(super) vidsrcs: HashMap, + pub(super) audsrcs: HashMap, pub(super) waiting_for_iframe: bool, } @@ -38,15 +47,15 @@ struct ProcessData<'a> { impl NeoMediaSender { pub(super) async fn run(&mut self) -> AnyResult<()> { - let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32)); - resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip); + // let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32)); + // resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip); let mut buffer: Vec = Default::default(); loop { tokio::select! { v = self.data_source.next() => { if let Some(bc_media) = v { if ! self.skip_bcmedia(&bc_media)? { - resend_pause.reset(); + // resend_pause.reset(); self.inspect_bcmedia(&bc_media)?; match bc_media { BcMedia::Iframe(frame) => { @@ -57,7 +66,7 @@ impl NeoMediaSender { data: frame.data.clone(), } ); - self.process_buffer(&ProcessData { + self.process_vidbuffer(&ProcessData { live: StampedData{ ms: frame.microseconds as u64, data: frame.data, @@ -72,7 +81,7 @@ impl NeoMediaSender { data: frame.data.clone(), } ); - self.process_buffer(&ProcessData { + self.process_vidbuffer(&ProcessData { live: StampedData{ ms: frame.microseconds as u64, data: frame.data, @@ -80,6 +89,12 @@ impl NeoMediaSender { resend: buffer.as_slice() })?; } + BcMedia::Aac(aac) => { + if let Some(last) = buffer.last().as_ref() { + self.process_audbuffer(aac.data.as_slice(), last.ms)?; + } + + } _ => {} } } @@ -87,36 +102,49 @@ impl NeoMediaSender { break; } } - v = self.app_source.next() => { + v = self.vidsource.next() => { if let Some(appsrc) = v { - self.appsrcs.insert(ClientData { + self.vidsrcs.insert(self.uid.fetch_add(1, Ordering::Relaxed) , ClientData { appsrc, - start_time: self.shared.microseconds.load(Ordering::Relaxed) + start_time: self.shared.microseconds.load(Ordering::Relaxed), + inited: false, }); } else { break; } }, - _ = resend_pause.tick() => { - if let Some(live) = buffer.last().cloned() { - self.process_buffer(&ProcessData { - live, - resend: buffer.as_slice() - })?; + v = self.audsource.next() => { + if let Some(appsrc) = v { + self.audsrcs.insert(self.uid.fetch_add(1, Ordering::Relaxed), ClientData { + appsrc, + start_time: self.shared.microseconds.load(Ordering::Relaxed), + inited: false, + }); + } else { + break; } - } + }, + // _ = resend_pause.tick() => { + // if let Some(live) = buffer.last().cloned() { + // self.process_vidbuffer(&ProcessData { + // live, + // resend: buffer.as_slice() + // })?; + // } + // } } } Ok(()) } - fn process_buffer(&mut self, process_data: &ProcessData) -> AnyResult<()> { - let frame_ms = process_data.live.ms; - self.appsrcs.retain(|data| { - let start_time = data.start_time; - // If frame_ms < our start time then we just joined the stream + fn process_vidbuffer(&mut self, process_data: &ProcessData) -> AnyResult<()> { + self.vidsrcs.retain(|_, data| { + // If ! inited then we just joined the stream // Send all data from the last iframe to catch up - if frame_ms < start_time { + // This should eliminate visual artifact that arise from recieving a pframe + // before an iframe + if !data.inited { + data.inited = true; for (idx, resend_buffer) in process_data.resend.iter().enumerate() { if Self::send_buffer(&data.appsrc, resend_buffer.data.as_slice(), idx as u64, 0) .is_err() @@ -137,6 +165,15 @@ impl NeoMediaSender { Ok(()) } + fn process_audbuffer(&mut self, buf: &[u8], frame_ms: u64) -> AnyResult<()> { + self.audsrcs.retain(|_, data| { + Self::send_buffer(&data.appsrc, buf, frame_ms, data.start_time).is_ok() + // If ok retain is true + }); + + Ok(()) + } + fn send_buffer(appsrc: &AppSrc, buf: &[u8], frame_ms: u64, start_time: u64) -> AnyResult<()> { let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { @@ -158,11 +195,13 @@ impl NeoMediaSender { .map(|_| ()) .map_err(|_| anyhow!("Could not push buffer to appsrc")) } + fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult { if self.waiting_for_iframe { if let BcMedia::Iframe(_) = bc_media { self.waiting_for_iframe = false; } else { + log::debug!("Skipping bcmedia"); return Ok(true); } } @@ -170,6 +209,8 @@ impl NeoMediaSender { } fn inspect_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult<()> { + let old_vid = self.shared.vid_format.load(Ordering::Relaxed); + let old_aud = self.shared.aud_format.load(Ordering::Relaxed); match bc_media { BcMedia::Iframe(frame) => { match frame.video_type { @@ -217,6 +258,22 @@ impl NeoMediaSender { } _ => {} } + let new_vid = self.shared.vid_format.load(Ordering::Relaxed); + if old_vid != new_vid { + log::debug!( + "Video format set to: {:?} from {:?}", + VidFormats::from(new_vid), + VidFormats::from(old_vid) + ); + } + let new_aud = self.shared.aud_format.load(Ordering::Relaxed); + if old_aud != new_aud { + log::debug!( + "Audio format set to: {:?} from {:?}", + AudFormats::from(new_aud), + AudFormats::from(old_aud) + ); + } Ok(()) } } diff --git a/src/rtsp/gst/shared.rs b/src/rtsp/gst/shared.rs index 8fdabf5..9470a1e 100644 --- a/src/rtsp/gst/shared.rs +++ b/src/rtsp/gst/shared.rs @@ -4,7 +4,7 @@ pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use std::convert::{From, Into}; use std::sync::atomic::AtomicU64; -#[derive(PartialEq)] +#[derive(PartialEq, Eq, Debug)] pub(super) enum VidFormats { Unknown = 0, H264, @@ -32,7 +32,7 @@ impl From for VidFormats { } } -#[derive(PartialEq)] +#[derive(PartialEq, Eq, Debug)] pub(super) enum AudFormats { Unknown = 0, Aac, diff --git a/src/rtsp/states/streaming.rs b/src/rtsp/states/streaming.rs index e925f9b..54b861e 100644 --- a/src/rtsp/states/streaming.rs +++ b/src/rtsp/states/streaming.rs @@ -54,19 +54,20 @@ impl CameraState for Streaming { .start_video(stream_thead, 0, strict_thread) .await?; while thread_abort_handle.is_live() { - trace!("BcMediaStreamRecv"); + debug!("BcMediaStreamRecv"); let data = stream_data.get_data().await?; match &data { - Ok(BcMedia::InfoV1(_)) => trace!(" - InfoV1"), - Ok(BcMedia::InfoV2(_)) => trace!(" - InfoV2"), - Ok(BcMedia::Iframe(_)) => trace!(" - Iframe"), - Ok(BcMedia::Pframe(_)) => trace!(" - Pframe"), - Ok(BcMedia::Aac(_)) => trace!(" - Aac"), - Ok(BcMedia::Adpcm(_)) => trace!(" - Adpcm"), - Err(_) => trace!(" - Error"), + Ok(BcMedia::InfoV1(_)) => debug!(" - InfoV1"), + Ok(BcMedia::InfoV2(_)) => debug!(" - InfoV2"), + Ok(BcMedia::Iframe(_)) => debug!(" - Iframe"), + Ok(BcMedia::Pframe(_)) => debug!(" - Pframe"), + Ok(BcMedia::Aac(_)) => debug!(" - Aac"), + Ok(BcMedia::Adpcm(_)) => debug!(" - Adpcm"), + Err(_) => debug!(" - Error"), } sender.send(data?).await?; } + debug!("Shutting down streaming thread"); Ok(()) });