Added AAC

This commit is contained in:
QuantumEntangledAndy authored and Andrew committed 2023-04-05 18:19:29 +07:00
1 parent ceff4c2495
commit b728ba14c7
8 files changed
+177 -54

No files matched your search

@@ -226,17 +226,17 @@ pub(crate) struct UdpPayloadSource {
impl UdpPayloadSource {
fn build_send_ack(&self) -> UdpAck {
if self.packets_want > 0 {
let mut start: u32 = self.packets_want - 1;
while self.recieved.contains_key(&start) {
let mut first_missing: u32 = self.packets_want;
while self.recieved.contains_key(&first_missing) {
// Happens if we have recieved but not consumed yet
start += 1;
first_missing += 1;
}
let missing_ids = if let Some(end) = self.recieved.keys().max() {
let mut vec = vec![];
// From last contiguous packet to last recieved packet
// create a payload of `00` (unreceived) and `01` (received)
// that can be used to form the `UdpAck` packet
for i in (start + 1)..(end + 1) {
for i in (first_missing)..(end + 1) {
if self.recieved.contains_key(&i) {
vec.push(1)
} else {
@@ -250,7 +250,7 @@ impl UdpPayloadSource {
UdpAck {
connection_id: self.camera_id,
packet_id: start,
packet_id: first_missing - 1, // Last we actually have is first_missing - 1
group_id: 0,
payload: missing_ids,
}
@@ -306,8 +306,6 @@ impl Stream for UdpPayloadSource {
trace!("UdpPayloadSource.Data: PreReady: {}", this.packets_want);
this.packets_want += 1;
// error!("packets_want: {}", this.packets_want);
let ack = BcUdp::Ack(this.build_send_ack());
this.send_buffer.push_back(ack);
return Poll::Ready(Some(Ok(payload)));
}
// Handle resend events
+1
View File
@@ -173,6 +173,7 @@ impl BcCamera {
break;
}
}
log::error!("Stopping media_sub");
}
let stop_video = Bc::new_from_xml(
+3 -3
View File
@@ -42,13 +42,13 @@ impl Decoder for BcMediaCodex {
match { BcMedia::deserialize(src) } {
Ok(bc) => {
if amount_skipped > 0 {
trace!("Amount skipped to restore stream: {}", amount_skipped);
debug!("Amount skipped to restore stream: {}", amount_skipped);
}
return Ok(Some(bc));
}
Err(Error::NomIncomplete(_)) => {
if amount_skipped > 0 {
trace!("Amount skipped to restore stream: {}", amount_skipped);
debug!("Amount skipped to restore stream: {}", amount_skipped);
}
return Ok(None);
}
@@ -59,7 +59,7 @@ impl Decoder for BcMediaCodex {
return Ok(None);
} else {
if amount_skipped == 0 {
trace!("Error in stream attempting to restore: {:?}", e);
debug!("Error in stream attempting to restore: {:?}", e);
}
src.advance(1);
amount_skipped += 1;
+8 -2
View File
@@ -40,13 +40,19 @@ impl Decoder for BcUdpCodex {
r2c_disc: Some(_), ..
},
..
})) => Err(Error::RelayTerminate),
})) => {
log::error!("Relay terminate");
Err(Error::RelayTerminate)
}
Ok(BcUdp::Discovery(UdpDiscovery {
payload: UdpXml {
d2c_disc: Some(_), ..
},
..
})) => Err(Error::CameraTerminate),
})) => {
log::error!("Camera terminate");
Err(Error::CameraTerminate)
}
Ok(bc) => Ok(Some(bc)),
Err(Error::NomIncomplete(_)) => Ok(None),
Err(e) => Err(e),
+68 -8
View File
@@ -100,7 +100,8 @@ unsafe impl Sync for NeoMediaFactory {}
pub(crate) struct NeoMediaFactoryImpl {
sender: Sender<BcMedia>,
appsender: Sender<AppSrc>,
vidsender: Sender<AppSrc>,
audsender: Sender<AppSrc>,
shared: Arc<NeoMediaShared>,
#[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj
threads: JoinSet<AnyResult<()>>,
@@ -116,23 +117,28 @@ impl Default for NeoMediaFactoryImpl {
fn default() -> Self {
warn!("Constructing Factor Impl");
let (datasender, datarx) = channel(3);
let (appsender, rx_appsender) = channel(3);
let (vidsender, rx_vidsender) = channel(3);
let (audsender, rx_audsender) = channel(3);
let shared: Arc<NeoMediaShared> = Default::default();
// Prepare thread that sends data into the appsrcs
let mut threads: JoinSet<AnyResult<()>> = Default::default();
let mut sender = NeoMediaSender {
data_source: ReceiverStream::new(datarx),
app_source: ReceiverStream::new(rx_appsender),
vidsource: ReceiverStream::new(rx_vidsender),
audsource: ReceiverStream::new(rx_audsender),
shared: shared.clone(),
appsrcs: Default::default(),
uid: Default::default(),
vidsrcs: Default::default(),
audsrcs: Default::default(),
waiting_for_iframe: true,
};
threads.spawn(async move { sender.run().await });
Self {
sender: datasender,
appsender,
vidsender,
audsender,
shared,
threads,
}
@@ -195,7 +201,6 @@ impl NeoMediaFactoryImpl {
bin.remove(&element)?;
}
debug!("self.shared owners: {}", Arc::strong_count(&self.shared));
// Now contruct the actual ones
match VidFormats::from(self.shared.vid_format.load(Ordering::Relaxed)) {
VidFormats::H265 => {
@@ -227,7 +232,7 @@ impl NeoMediaFactoryImpl {
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
self.appsender.blocking_send(source)?;
self.vidsender.blocking_send(source)?;
}
VidFormats::H264 => {
debug!("Building H264 Pipeline");
@@ -258,7 +263,7 @@ impl NeoMediaFactoryImpl {
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
self.appsender.blocking_send(source)?;
self.vidsender.blocking_send(source)?;
}
VidFormats::Unknown => {
debug!("Building Unknown Pipeline");
@@ -290,6 +295,61 @@ impl NeoMediaFactoryImpl {
}
}
match AudFormats::from(self.shared.aud_format.load(Ordering::Relaxed)) {
AudFormats::Unknown => {}
AudFormats::Aac => {
debug!("Building Aac pipeline");
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_base_time(ClockTime::from_mseconds(
self.shared.microseconds.load(Ordering::Relaxed),
));
source.set_is_live(true);
source.set_block(true);
source.set_property("emit-signals", &false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Stream);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_element("queue", "audqueue")?;
let parser = make_element("aacparse", "audparser")?;
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
Element::link_many(&[&source, &queue, &parser, &decoder])?;
Element::link_many(&[&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link AAC decoder to encoder");
});
// decoder.link_filtered(
// &encoder,
// & Caps::builder("audio/x-raw")
// .field("format", "S16BE")
// .field("layout", "interleaved")
// // .field("channels", 1)
// .build(),
// )?;
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
self.audsender.blocking_send(source)?;
}
AudFormats::Adpcm => {}
}
bin.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))
}
+81 -24
View File
@@ -6,8 +6,10 @@ use gstreamer::ClockTime;
use gstreamer_app::AppSrc;
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
use neolink_core::bcmedia::model::*;
use std::collections::HashMap;
use std::sync::atomic::AtomicU64;
use std::sync::{atomic::Ordering, Arc};
use tokio::time::{interval, Duration, MissedTickBehavior};
// use tokio::time::{interval, Duration, MissedTickBehavior};
use tokio_stream::wrappers::ReceiverStream;
use super::{shared::*, AnyResult};
@@ -16,13 +18,20 @@ use super::{shared::*, AnyResult};
pub(super) struct ClientData {
pub(super) appsrc: AppSrc,
pub(super) start_time: u64,
pub(super) inited: bool,
}
pub(super) struct NeoMediaSender {
pub(super) data_source: ReceiverStream<BcMedia>,
pub(super) app_source: ReceiverStream<AppSrc>,
pub(super) vidsource: ReceiverStream<AppSrc>,
pub(super) audsource: ReceiverStream<AppSrc>,
pub(super) shared: Arc<NeoMediaShared>,
pub(super) appsrcs: std::collections::HashSet<ClientData>,
// Used to generate the key for the hashmaps
pub(super) uid: AtomicU64,
// Hashmap so we can alter the data in ClientData while also allowing for remove at any point
// If you know a better collections pm me
pub(super) vidsrcs: HashMap<u64, ClientData>,
pub(super) audsrcs: HashMap<u64, ClientData>,
pub(super) waiting_for_iframe: bool,
}
@@ -38,15 +47,15 @@ struct ProcessData<'a> {
impl NeoMediaSender {
pub(super) async fn run(&mut self) -> AnyResult<()> {
let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32));
resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip);
// let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32));
// resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip);
let mut buffer: Vec<StampedData> = Default::default();
loop {
tokio::select! {
v = self.data_source.next() => {
if let Some(bc_media) = v {
if ! self.skip_bcmedia(&bc_media)? {
resend_pause.reset();
// resend_pause.reset();
self.inspect_bcmedia(&bc_media)?;
match bc_media {
BcMedia::Iframe(frame) => {
@@ -57,7 +66,7 @@ impl NeoMediaSender {
data: frame.data.clone(),
}
);
self.process_buffer(&ProcessData {
self.process_vidbuffer(&ProcessData {
live: StampedData{
ms: frame.microseconds as u64,
data: frame.data,
@@ -72,7 +81,7 @@ impl NeoMediaSender {
data: frame.data.clone(),
}
);
self.process_buffer(&ProcessData {
self.process_vidbuffer(&ProcessData {
live: StampedData{
ms: frame.microseconds as u64,
data: frame.data,
@@ -80,6 +89,12 @@ impl NeoMediaSender {
resend: buffer.as_slice()
})?;
}
BcMedia::Aac(aac) => {
if let Some(last) = buffer.last().as_ref() {
self.process_audbuffer(aac.data.as_slice(), last.ms)?;
}
}
_ => {}
}
}
@@ -87,36 +102,49 @@ impl NeoMediaSender {
break;
}
}
v = self.app_source.next() => {
v = self.vidsource.next() => {
if let Some(appsrc) = v {
self.appsrcs.insert(ClientData {
self.vidsrcs.insert(self.uid.fetch_add(1, Ordering::Relaxed) , ClientData {
appsrc,
start_time: self.shared.microseconds.load(Ordering::Relaxed)
start_time: self.shared.microseconds.load(Ordering::Relaxed),
inited: false,
});
} else {
break;
}
},
_ = resend_pause.tick() => {
if let Some(live) = buffer.last().cloned() {
self.process_buffer(&ProcessData {
live,
resend: buffer.as_slice()
})?;
v = self.audsource.next() => {
if let Some(appsrc) = v {
self.audsrcs.insert(self.uid.fetch_add(1, Ordering::Relaxed), ClientData {
appsrc,
start_time: self.shared.microseconds.load(Ordering::Relaxed),
inited: false,
});
} else {
break;
}
}
},
// _ = resend_pause.tick() => {
// if let Some(live) = buffer.last().cloned() {
// self.process_vidbuffer(&ProcessData {
// live,
// resend: buffer.as_slice()
// })?;
// }
// }
}
}
Ok(())
}
fn process_buffer(&mut self, process_data: &ProcessData) -> AnyResult<()> {
let frame_ms = process_data.live.ms;
self.appsrcs.retain(|data| {
let start_time = data.start_time;
// If frame_ms < our start time then we just joined the stream
fn process_vidbuffer(&mut self, process_data: &ProcessData) -> AnyResult<()> {
self.vidsrcs.retain(|_, data| {
// If ! inited then we just joined the stream
// Send all data from the last iframe to catch up
if frame_ms < start_time {
// This should eliminate visual artifact that arise from recieving a pframe
// before an iframe
if !data.inited {
data.inited = true;
for (idx, resend_buffer) in process_data.resend.iter().enumerate() {
if Self::send_buffer(&data.appsrc, resend_buffer.data.as_slice(), idx as u64, 0)
.is_err()
@@ -137,6 +165,15 @@ impl NeoMediaSender {
Ok(())
}
fn process_audbuffer(&mut self, buf: &[u8], frame_ms: u64) -> AnyResult<()> {
self.audsrcs.retain(|_, data| {
Self::send_buffer(&data.appsrc, buf, frame_ms, data.start_time).is_ok()
// If ok retain is true
});
Ok(())
}
fn send_buffer(appsrc: &AppSrc, buf: &[u8], frame_ms: u64, start_time: u64) -> AnyResult<()> {
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
@@ -158,11 +195,13 @@ impl NeoMediaSender {
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
}
fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult<bool> {
if self.waiting_for_iframe {
if let BcMedia::Iframe(_) = bc_media {
self.waiting_for_iframe = false;
} else {
log::debug!("Skipping bcmedia");
return Ok(true);
}
}
@@ -170,6 +209,8 @@ impl NeoMediaSender {
}
fn inspect_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult<()> {
let old_vid = self.shared.vid_format.load(Ordering::Relaxed);
let old_aud = self.shared.aud_format.load(Ordering::Relaxed);
match bc_media {
BcMedia::Iframe(frame) => {
match frame.video_type {
@@ -217,6 +258,22 @@ impl NeoMediaSender {
}
_ => {}
}
let new_vid = self.shared.vid_format.load(Ordering::Relaxed);
if old_vid != new_vid {
log::debug!(
"Video format set to: {:?} from {:?}",
VidFormats::from(new_vid),
VidFormats::from(old_vid)
);
}
let new_aud = self.shared.aud_format.load(Ordering::Relaxed);
if old_aud != new_aud {
log::debug!(
"Audio format set to: {:?} from {:?}",
AudFormats::from(new_aud),
AudFormats::from(old_aud)
);
}
Ok(())
}
}
+2 -2
View File
@@ -4,7 +4,7 @@ pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
use std::convert::{From, Into};
use std::sync::atomic::AtomicU64;
#[derive(PartialEq)]
#[derive(PartialEq, Eq, Debug)]
pub(super) enum VidFormats {
Unknown = 0,
H264,
@@ -32,7 +32,7 @@ impl From<u64> for VidFormats {
}
}
#[derive(PartialEq)]
#[derive(PartialEq, Eq, Debug)]
pub(super) enum AudFormats {
Unknown = 0,
Aac,
+9 -8
View File
@@ -54,19 +54,20 @@ impl CameraState for Streaming {
.start_video(stream_thead, 0, strict_thread)
.await?;
while thread_abort_handle.is_live() {
trace!("BcMediaStreamRecv");
debug!("BcMediaStreamRecv");
let data = stream_data.get_data().await?;
match &data {
Ok(BcMedia::InfoV1(_)) => trace!(" - InfoV1"),
Ok(BcMedia::InfoV2(_)) => trace!(" - InfoV2"),
Ok(BcMedia::Iframe(_)) => trace!(" - Iframe"),
Ok(BcMedia::Pframe(_)) => trace!(" - Pframe"),
Ok(BcMedia::Aac(_)) => trace!(" - Aac"),
Ok(BcMedia::Adpcm(_)) => trace!(" - Adpcm"),
Err(_) => trace!(" - Error"),
Ok(BcMedia::InfoV1(_)) => debug!(" - InfoV1"),
Ok(BcMedia::InfoV2(_)) => debug!(" - InfoV2"),
Ok(BcMedia::Iframe(_)) => debug!(" - Iframe"),
Ok(BcMedia::Pframe(_)) => debug!(" - Pframe"),
Ok(BcMedia::Aac(_)) => debug!(" - Aac"),
Ok(BcMedia::Adpcm(_)) => debug!(" - Adpcm"),
Err(_) => debug!(" - Error"),
}
sender.send(data?).await?;
}
debug!("Shutting down streaming thread");
Ok(())
});