Added AAC
This commit is contained in:
1 parent
ceff4c2495
commit
b728ba14c7
8 files changed
+177
-54
No files matched your search
@@ -226,17 +226,17 @@ pub(crate) struct UdpPayloadSource {
|
||||
impl UdpPayloadSource {
|
||||
fn build_send_ack(&self) -> UdpAck {
|
||||
if self.packets_want > 0 {
|
||||
let mut start: u32 = self.packets_want - 1;
|
||||
while self.recieved.contains_key(&start) {
|
||||
let mut first_missing: u32 = self.packets_want;
|
||||
while self.recieved.contains_key(&first_missing) {
|
||||
// Happens if we have recieved but not consumed yet
|
||||
start += 1;
|
||||
first_missing += 1;
|
||||
}
|
||||
let missing_ids = if let Some(end) = self.recieved.keys().max() {
|
||||
let mut vec = vec![];
|
||||
// From last contiguous packet to last recieved packet
|
||||
// create a payload of `00` (unreceived) and `01` (received)
|
||||
// that can be used to form the `UdpAck` packet
|
||||
for i in (start + 1)..(end + 1) {
|
||||
for i in (first_missing)..(end + 1) {
|
||||
if self.recieved.contains_key(&i) {
|
||||
vec.push(1)
|
||||
} else {
|
||||
@@ -250,7 +250,7 @@ impl UdpPayloadSource {
|
||||
|
||||
UdpAck {
|
||||
connection_id: self.camera_id,
|
||||
packet_id: start,
|
||||
packet_id: first_missing - 1, // Last we actually have is first_missing - 1
|
||||
group_id: 0,
|
||||
payload: missing_ids,
|
||||
}
|
||||
@@ -306,8 +306,6 @@ impl Stream for UdpPayloadSource {
|
||||
trace!("UdpPayloadSource.Data: PreReady: {}", this.packets_want);
|
||||
this.packets_want += 1;
|
||||
// error!("packets_want: {}", this.packets_want);
|
||||
let ack = BcUdp::Ack(this.build_send_ack());
|
||||
this.send_buffer.push_back(ack);
|
||||
return Poll::Ready(Some(Ok(payload)));
|
||||
}
|
||||
// Handle resend events
|
||||
|
||||
@@ -173,6 +173,7 @@ impl BcCamera {
|
||||
break;
|
||||
}
|
||||
}
|
||||
log::error!("Stopping media_sub");
|
||||
}
|
||||
|
||||
let stop_video = Bc::new_from_xml(
|
||||
|
||||
@@ -42,13 +42,13 @@ impl Decoder for BcMediaCodex {
|
||||
match { BcMedia::deserialize(src) } {
|
||||
Ok(bc) => {
|
||||
if amount_skipped > 0 {
|
||||
trace!("Amount skipped to restore stream: {}", amount_skipped);
|
||||
debug!("Amount skipped to restore stream: {}", amount_skipped);
|
||||
}
|
||||
return Ok(Some(bc));
|
||||
}
|
||||
Err(Error::NomIncomplete(_)) => {
|
||||
if amount_skipped > 0 {
|
||||
trace!("Amount skipped to restore stream: {}", amount_skipped);
|
||||
debug!("Amount skipped to restore stream: {}", amount_skipped);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -59,7 +59,7 @@ impl Decoder for BcMediaCodex {
|
||||
return Ok(None);
|
||||
} else {
|
||||
if amount_skipped == 0 {
|
||||
trace!("Error in stream attempting to restore: {:?}", e);
|
||||
debug!("Error in stream attempting to restore: {:?}", e);
|
||||
}
|
||||
src.advance(1);
|
||||
amount_skipped += 1;
|
||||
|
||||
@@ -40,13 +40,19 @@ impl Decoder for BcUdpCodex {
|
||||
r2c_disc: Some(_), ..
|
||||
},
|
||||
..
|
||||
})) => Err(Error::RelayTerminate),
|
||||
})) => {
|
||||
log::error!("Relay terminate");
|
||||
Err(Error::RelayTerminate)
|
||||
}
|
||||
Ok(BcUdp::Discovery(UdpDiscovery {
|
||||
payload: UdpXml {
|
||||
d2c_disc: Some(_), ..
|
||||
},
|
||||
..
|
||||
})) => Err(Error::CameraTerminate),
|
||||
})) => {
|
||||
log::error!("Camera terminate");
|
||||
Err(Error::CameraTerminate)
|
||||
}
|
||||
Ok(bc) => Ok(Some(bc)),
|
||||
Err(Error::NomIncomplete(_)) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
|
||||
+68
-8
@@ -100,7 +100,8 @@ unsafe impl Sync for NeoMediaFactory {}
|
||||
|
||||
pub(crate) struct NeoMediaFactoryImpl {
|
||||
sender: Sender<BcMedia>,
|
||||
appsender: Sender<AppSrc>,
|
||||
vidsender: Sender<AppSrc>,
|
||||
audsender: Sender<AppSrc>,
|
||||
shared: Arc<NeoMediaShared>,
|
||||
#[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj
|
||||
threads: JoinSet<AnyResult<()>>,
|
||||
@@ -116,23 +117,28 @@ impl Default for NeoMediaFactoryImpl {
|
||||
fn default() -> Self {
|
||||
warn!("Constructing Factor Impl");
|
||||
let (datasender, datarx) = channel(3);
|
||||
let (appsender, rx_appsender) = channel(3);
|
||||
let (vidsender, rx_vidsender) = channel(3);
|
||||
let (audsender, rx_audsender) = channel(3);
|
||||
let shared: Arc<NeoMediaShared> = Default::default();
|
||||
|
||||
// Prepare thread that sends data into the appsrcs
|
||||
let mut threads: JoinSet<AnyResult<()>> = Default::default();
|
||||
let mut sender = NeoMediaSender {
|
||||
data_source: ReceiverStream::new(datarx),
|
||||
app_source: ReceiverStream::new(rx_appsender),
|
||||
vidsource: ReceiverStream::new(rx_vidsender),
|
||||
audsource: ReceiverStream::new(rx_audsender),
|
||||
shared: shared.clone(),
|
||||
appsrcs: Default::default(),
|
||||
uid: Default::default(),
|
||||
vidsrcs: Default::default(),
|
||||
audsrcs: Default::default(),
|
||||
waiting_for_iframe: true,
|
||||
};
|
||||
threads.spawn(async move { sender.run().await });
|
||||
|
||||
Self {
|
||||
sender: datasender,
|
||||
appsender,
|
||||
vidsender,
|
||||
audsender,
|
||||
shared,
|
||||
threads,
|
||||
}
|
||||
@@ -195,7 +201,6 @@ impl NeoMediaFactoryImpl {
|
||||
bin.remove(&element)?;
|
||||
}
|
||||
|
||||
debug!("self.shared owners: {}", Arc::strong_count(&self.shared));
|
||||
// Now contruct the actual ones
|
||||
match VidFormats::from(self.shared.vid_format.load(Ordering::Relaxed)) {
|
||||
VidFormats::H265 => {
|
||||
@@ -227,7 +232,7 @@ impl NeoMediaFactoryImpl {
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
self.appsender.blocking_send(source)?;
|
||||
self.vidsender.blocking_send(source)?;
|
||||
}
|
||||
VidFormats::H264 => {
|
||||
debug!("Building H264 Pipeline");
|
||||
@@ -258,7 +263,7 @@ impl NeoMediaFactoryImpl {
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
self.appsender.blocking_send(source)?;
|
||||
self.vidsender.blocking_send(source)?;
|
||||
}
|
||||
VidFormats::Unknown => {
|
||||
debug!("Building Unknown Pipeline");
|
||||
@@ -290,6 +295,61 @@ impl NeoMediaFactoryImpl {
|
||||
}
|
||||
}
|
||||
|
||||
match AudFormats::from(self.shared.aud_format.load(Ordering::Relaxed)) {
|
||||
AudFormats::Unknown => {}
|
||||
AudFormats::Aac => {
|
||||
debug!("Building Aac pipeline");
|
||||
let source = make_element("appsrc", "audsrc")?
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
|
||||
source.set_base_time(ClockTime::from_mseconds(
|
||||
self.shared.microseconds.load(Ordering::Relaxed),
|
||||
));
|
||||
source.set_is_live(true);
|
||||
source.set_block(true);
|
||||
source.set_property("emit-signals", &false);
|
||||
source.set_max_bytes(52428800);
|
||||
source.set_do_timestamp(false);
|
||||
source.set_stream_type(AppStreamType::Stream);
|
||||
let source = source
|
||||
.dynamic_cast::<Element>()
|
||||
.map_err(|_| anyhow!("Cannot cast back"))?;
|
||||
|
||||
let queue = make_element("queue", "audqueue")?;
|
||||
let parser = make_element("aacparse", "audparser")?;
|
||||
let decoder = make_element("decodebin", "auddecoder")?;
|
||||
let encoder = make_element("audioconvert", "audencoder")?;
|
||||
let payload = make_element("rtpL16pay", "pay1")?;
|
||||
|
||||
bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
|
||||
Element::link_many(&[&source, &queue, &parser, &decoder])?;
|
||||
Element::link_many(&[&encoder, &payload])?;
|
||||
decoder.connect_pad_added(move |_element, pad| {
|
||||
debug!("Linking encoder to decoder: {:?}", pad.caps());
|
||||
let sink_pad = encoder
|
||||
.static_pad("sink")
|
||||
.expect("Encoder is missing its pad");
|
||||
pad.link(&sink_pad)
|
||||
.expect("Failed to link AAC decoder to encoder");
|
||||
});
|
||||
|
||||
// decoder.link_filtered(
|
||||
// &encoder,
|
||||
// & Caps::builder("audio/x-raw")
|
||||
// .field("format", "S16BE")
|
||||
// .field("layout", "interleaved")
|
||||
// // .field("channels", 1)
|
||||
// .build(),
|
||||
// )?;
|
||||
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
self.audsender.blocking_send(source)?;
|
||||
}
|
||||
AudFormats::Adpcm => {}
|
||||
}
|
||||
|
||||
bin.dynamic_cast::<Element>()
|
||||
.map_err(|_| anyhow!("Cannot cast back"))
|
||||
}
|
||||
|
||||
+81
-24
@@ -6,8 +6,10 @@ use gstreamer::ClockTime;
|
||||
use gstreamer_app::AppSrc;
|
||||
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
|
||||
use neolink_core::bcmedia::model::*;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::{atomic::Ordering, Arc};
|
||||
use tokio::time::{interval, Duration, MissedTickBehavior};
|
||||
// use tokio::time::{interval, Duration, MissedTickBehavior};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
use super::{shared::*, AnyResult};
|
||||
@@ -16,13 +18,20 @@ use super::{shared::*, AnyResult};
|
||||
pub(super) struct ClientData {
|
||||
pub(super) appsrc: AppSrc,
|
||||
pub(super) start_time: u64,
|
||||
pub(super) inited: bool,
|
||||
}
|
||||
|
||||
pub(super) struct NeoMediaSender {
|
||||
pub(super) data_source: ReceiverStream<BcMedia>,
|
||||
pub(super) app_source: ReceiverStream<AppSrc>,
|
||||
pub(super) vidsource: ReceiverStream<AppSrc>,
|
||||
pub(super) audsource: ReceiverStream<AppSrc>,
|
||||
pub(super) shared: Arc<NeoMediaShared>,
|
||||
pub(super) appsrcs: std::collections::HashSet<ClientData>,
|
||||
// Used to generate the key for the hashmaps
|
||||
pub(super) uid: AtomicU64,
|
||||
// Hashmap so we can alter the data in ClientData while also allowing for remove at any point
|
||||
// If you know a better collections pm me
|
||||
pub(super) vidsrcs: HashMap<u64, ClientData>,
|
||||
pub(super) audsrcs: HashMap<u64, ClientData>,
|
||||
pub(super) waiting_for_iframe: bool,
|
||||
}
|
||||
|
||||
@@ -38,15 +47,15 @@ struct ProcessData<'a> {
|
||||
|
||||
impl NeoMediaSender {
|
||||
pub(super) async fn run(&mut self) -> AnyResult<()> {
|
||||
let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32));
|
||||
resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip);
|
||||
// let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32));
|
||||
// resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip);
|
||||
let mut buffer: Vec<StampedData> = Default::default();
|
||||
loop {
|
||||
tokio::select! {
|
||||
v = self.data_source.next() => {
|
||||
if let Some(bc_media) = v {
|
||||
if ! self.skip_bcmedia(&bc_media)? {
|
||||
resend_pause.reset();
|
||||
// resend_pause.reset();
|
||||
self.inspect_bcmedia(&bc_media)?;
|
||||
match bc_media {
|
||||
BcMedia::Iframe(frame) => {
|
||||
@@ -57,7 +66,7 @@ impl NeoMediaSender {
|
||||
data: frame.data.clone(),
|
||||
}
|
||||
);
|
||||
self.process_buffer(&ProcessData {
|
||||
self.process_vidbuffer(&ProcessData {
|
||||
live: StampedData{
|
||||
ms: frame.microseconds as u64,
|
||||
data: frame.data,
|
||||
@@ -72,7 +81,7 @@ impl NeoMediaSender {
|
||||
data: frame.data.clone(),
|
||||
}
|
||||
);
|
||||
self.process_buffer(&ProcessData {
|
||||
self.process_vidbuffer(&ProcessData {
|
||||
live: StampedData{
|
||||
ms: frame.microseconds as u64,
|
||||
data: frame.data,
|
||||
@@ -80,6 +89,12 @@ impl NeoMediaSender {
|
||||
resend: buffer.as_slice()
|
||||
})?;
|
||||
}
|
||||
BcMedia::Aac(aac) => {
|
||||
if let Some(last) = buffer.last().as_ref() {
|
||||
self.process_audbuffer(aac.data.as_slice(), last.ms)?;
|
||||
}
|
||||
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
@@ -87,36 +102,49 @@ impl NeoMediaSender {
|
||||
break;
|
||||
}
|
||||
}
|
||||
v = self.app_source.next() => {
|
||||
v = self.vidsource.next() => {
|
||||
if let Some(appsrc) = v {
|
||||
self.appsrcs.insert(ClientData {
|
||||
self.vidsrcs.insert(self.uid.fetch_add(1, Ordering::Relaxed) , ClientData {
|
||||
appsrc,
|
||||
start_time: self.shared.microseconds.load(Ordering::Relaxed)
|
||||
start_time: self.shared.microseconds.load(Ordering::Relaxed),
|
||||
inited: false,
|
||||
});
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
},
|
||||
_ = resend_pause.tick() => {
|
||||
if let Some(live) = buffer.last().cloned() {
|
||||
self.process_buffer(&ProcessData {
|
||||
live,
|
||||
resend: buffer.as_slice()
|
||||
})?;
|
||||
v = self.audsource.next() => {
|
||||
if let Some(appsrc) = v {
|
||||
self.audsrcs.insert(self.uid.fetch_add(1, Ordering::Relaxed), ClientData {
|
||||
appsrc,
|
||||
start_time: self.shared.microseconds.load(Ordering::Relaxed),
|
||||
inited: false,
|
||||
});
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
// _ = resend_pause.tick() => {
|
||||
// if let Some(live) = buffer.last().cloned() {
|
||||
// self.process_vidbuffer(&ProcessData {
|
||||
// live,
|
||||
// resend: buffer.as_slice()
|
||||
// })?;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn process_buffer(&mut self, process_data: &ProcessData) -> AnyResult<()> {
|
||||
let frame_ms = process_data.live.ms;
|
||||
self.appsrcs.retain(|data| {
|
||||
let start_time = data.start_time;
|
||||
// If frame_ms < our start time then we just joined the stream
|
||||
fn process_vidbuffer(&mut self, process_data: &ProcessData) -> AnyResult<()> {
|
||||
self.vidsrcs.retain(|_, data| {
|
||||
// If ! inited then we just joined the stream
|
||||
// Send all data from the last iframe to catch up
|
||||
if frame_ms < start_time {
|
||||
// This should eliminate visual artifact that arise from recieving a pframe
|
||||
// before an iframe
|
||||
if !data.inited {
|
||||
data.inited = true;
|
||||
for (idx, resend_buffer) in process_data.resend.iter().enumerate() {
|
||||
if Self::send_buffer(&data.appsrc, resend_buffer.data.as_slice(), idx as u64, 0)
|
||||
.is_err()
|
||||
@@ -137,6 +165,15 @@ impl NeoMediaSender {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn process_audbuffer(&mut self, buf: &[u8], frame_ms: u64) -> AnyResult<()> {
|
||||
self.audsrcs.retain(|_, data| {
|
||||
Self::send_buffer(&data.appsrc, buf, frame_ms, data.start_time).is_ok()
|
||||
// If ok retain is true
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn send_buffer(appsrc: &AppSrc, buf: &[u8], frame_ms: u64, start_time: u64) -> AnyResult<()> {
|
||||
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
|
||||
{
|
||||
@@ -158,11 +195,13 @@ impl NeoMediaSender {
|
||||
.map(|_| ())
|
||||
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
|
||||
}
|
||||
|
||||
fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult<bool> {
|
||||
if self.waiting_for_iframe {
|
||||
if let BcMedia::Iframe(_) = bc_media {
|
||||
self.waiting_for_iframe = false;
|
||||
} else {
|
||||
log::debug!("Skipping bcmedia");
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
@@ -170,6 +209,8 @@ impl NeoMediaSender {
|
||||
}
|
||||
|
||||
fn inspect_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult<()> {
|
||||
let old_vid = self.shared.vid_format.load(Ordering::Relaxed);
|
||||
let old_aud = self.shared.aud_format.load(Ordering::Relaxed);
|
||||
match bc_media {
|
||||
BcMedia::Iframe(frame) => {
|
||||
match frame.video_type {
|
||||
@@ -217,6 +258,22 @@ impl NeoMediaSender {
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let new_vid = self.shared.vid_format.load(Ordering::Relaxed);
|
||||
if old_vid != new_vid {
|
||||
log::debug!(
|
||||
"Video format set to: {:?} from {:?}",
|
||||
VidFormats::from(new_vid),
|
||||
VidFormats::from(old_vid)
|
||||
);
|
||||
}
|
||||
let new_aud = self.shared.aud_format.load(Ordering::Relaxed);
|
||||
if old_aud != new_aud {
|
||||
log::debug!(
|
||||
"Audio format set to: {:?} from {:?}",
|
||||
AudFormats::from(new_aud),
|
||||
AudFormats::from(old_aud)
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
|
||||
use std::convert::{From, Into};
|
||||
use std::sync::atomic::AtomicU64;
|
||||
|
||||
#[derive(PartialEq)]
|
||||
#[derive(PartialEq, Eq, Debug)]
|
||||
pub(super) enum VidFormats {
|
||||
Unknown = 0,
|
||||
H264,
|
||||
@@ -32,7 +32,7 @@ impl From<u64> for VidFormats {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq)]
|
||||
#[derive(PartialEq, Eq, Debug)]
|
||||
pub(super) enum AudFormats {
|
||||
Unknown = 0,
|
||||
Aac,
|
||||
|
||||
@@ -54,19 +54,20 @@ impl CameraState for Streaming {
|
||||
.start_video(stream_thead, 0, strict_thread)
|
||||
.await?;
|
||||
while thread_abort_handle.is_live() {
|
||||
trace!("BcMediaStreamRecv");
|
||||
debug!("BcMediaStreamRecv");
|
||||
let data = stream_data.get_data().await?;
|
||||
match &data {
|
||||
Ok(BcMedia::InfoV1(_)) => trace!(" - InfoV1"),
|
||||
Ok(BcMedia::InfoV2(_)) => trace!(" - InfoV2"),
|
||||
Ok(BcMedia::Iframe(_)) => trace!(" - Iframe"),
|
||||
Ok(BcMedia::Pframe(_)) => trace!(" - Pframe"),
|
||||
Ok(BcMedia::Aac(_)) => trace!(" - Aac"),
|
||||
Ok(BcMedia::Adpcm(_)) => trace!(" - Adpcm"),
|
||||
Err(_) => trace!(" - Error"),
|
||||
Ok(BcMedia::InfoV1(_)) => debug!(" - InfoV1"),
|
||||
Ok(BcMedia::InfoV2(_)) => debug!(" - InfoV2"),
|
||||
Ok(BcMedia::Iframe(_)) => debug!(" - Iframe"),
|
||||
Ok(BcMedia::Pframe(_)) => debug!(" - Pframe"),
|
||||
Ok(BcMedia::Aac(_)) => debug!(" - Aac"),
|
||||
Ok(BcMedia::Adpcm(_)) => debug!(" - Adpcm"),
|
||||
Err(_) => debug!(" - Error"),
|
||||
}
|
||||
sender.send(data?).await?;
|
||||
}
|
||||
debug!("Shutting down streaming thread");
|
||||
Ok(())
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user