Complete rework of buffer time

This commit is contained in:
QuantumEntangledAndy committed 2023-04-25 14:28:35 +07:00
1 parent c4c35d1bde
commit 107902ef3a
3 files changed
+706 -562

No files matched your search

+202 -219
View File
@@ -11,7 +11,7 @@ use gstreamer::{
glib::{self, Object},
Structure,
};
use gstreamer::{Bin, Caps, ClockTime, Element, ElementFactory};
use gstreamer::{Bin, Caps, Element, ElementFactory};
use gstreamer_app::{AppSrc, AppSrcCallbacks, AppStreamType};
use gstreamer_rtsp::RTSPUrl;
use gstreamer_rtsp_server::prelude::*;
@@ -23,14 +23,12 @@ use log::*;
use neolink_core::bcmedia::model::*;
use std::{
collections::HashSet,
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
sync::{atomic::Ordering, Arc},
};
use tokio::{
sync::mpsc::{channel, Sender},
task::JoinSet,
time::Duration,
};
use tokio_stream::wrappers::ReceiverStream;
@@ -113,7 +111,7 @@ unsafe impl Sync for NeoMediaFactory {}
pub(crate) struct NeoMediaFactoryImpl {
sender: Sender<BcMedia>,
clientsender: Sender<ClientPipelineData>,
clientsender: Sender<NeoMediaSender>,
shared: Arc<NeoMediaShared>,
#[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj
threads: JoinSet<AnyResult<()>>,
@@ -134,14 +132,11 @@ impl Default for NeoMediaFactoryImpl {
// Prepare thread that sends data into the appsrcs
let mut threads: JoinSet<AnyResult<()>> = Default::default();
let mut sender = NeoMediaSender {
data_source: ReceiverStream::new(datarx),
clientsource: ReceiverStream::new(rx_clientsender),
shared: shared.clone(),
uid: Default::default(),
clientdata: Default::default(),
waiting_for_iframe: true,
};
let mut sender = NeoMediaSenders::new(
shared.clone(),
ReceiverStream::new(datarx),
ReceiverStream::new(rx_clientsender),
);
threads.spawn(async move {
loop {
tokio::task::yield_now().await;
@@ -153,7 +148,6 @@ impl Default for NeoMediaFactoryImpl {
break;
}
}
sender.waiting_for_iframe = true;
}
Ok(())
});
@@ -231,12 +225,7 @@ impl NeoMediaFactoryImpl {
bin.remove(&element)?;
}
let mut client_data = ClientPipelineData {
start_time: Arc::new(AtomicU64::new(
self.shared.microseconds.load(Ordering::Relaxed),
)),
..Default::default()
};
let mut client_data = NeoMediaSender::new();
// Now contruct the actual ones
match *self.shared.vid_format.blocking_read() {
@@ -245,38 +234,28 @@ impl NeoMediaFactoryImpl {
let source = make_element("appsrc", "vidsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_base_time(ClockTime::from_mseconds(
self.shared.microseconds.load(Ordering::Relaxed),
));
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
source.set_stream_type(AppStreamType::RandomAccess);
let enough_data_start = client_data.enough_data.clone();
let enough_data_stop = client_data.enough_data.clone();
let seek_shared = self.shared.clone();
let seek_start_time = client_data.start_time.clone();
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
let seek_command = client_data.get_commader();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
if enough_data_stop.load(Ordering::Relaxed) {
log::debug!("Client is in need of data");
}
enough_data_stop.store(false, Ordering::Relaxed);
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
})
.enough_data(move |_appsrc| {
if !enough_data_start.load(Ordering::Relaxed) {
log::debug!("Client is full of of data");
}
enough_data_start.store(true, Ordering::Relaxed);
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
})
.seek_data(move |_appsrc, seek_pos| {
log::debug!("Seeking to {}", &seek_pos);
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
debug!("Send seek H265");
let _ =
seek_command.blocking_send(NeoMediaSenderCommand::Seek(seek_pos));
true
})
.build(),
@@ -290,7 +269,6 @@ impl NeoMediaFactoryImpl {
let queue = make_element("queue", "source_queue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
queue.set_property("min-threshold-time", 500u64 * 1000u64);
let parser = make_element("h265parse", "parser")?;
// parser.set_property("config-interval", 5i32);
let payload = make_element("rtph265pay", "pay0")?;
@@ -300,46 +278,47 @@ impl NeoMediaFactoryImpl {
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.vidsrc.replace(source);
client_data.update_vid(source);
}
VidFormats::H264 => {
debug!("Building H264 Pipeline");
let source = make_element("appsrc", "vidsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_base_time(ClockTime::from_mseconds(
self.shared.microseconds.load(Ordering::Relaxed),
));
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_max_bytes(50000000u64); // 50MB
// source.set_property("max-buffers", 0u32); // v1.20
// source.set_property("max-time", Duration::from_secs_f32(2.25).as_nanos() as u64);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
source.set_stream_type(AppStreamType::RandomAccess);
let clock = source.clock();
if let Some(clock) = clock {
debug!("Clock: {:?}", clock.time());
} else {
debug!("No clock");
}
let enough_data_start = client_data.enough_data.clone();
let enough_data_stop = client_data.enough_data.clone();
let seek_shared = self.shared.clone();
let seek_start_time = client_data.start_time.clone();
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
let seek_command = client_data.get_commader();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
if enough_data_stop.load(Ordering::Relaxed) {
log::debug!("Client is in need of data");
}
enough_data_stop.store(false, Ordering::Relaxed);
let _ = need_command.try_send(NeoMediaSenderCommand::Resume);
})
.enough_data(move |_appsrc| {
if !enough_data_start.load(Ordering::Relaxed) {
log::debug!("Client is full of of data");
}
enough_data_start.store(true, Ordering::Relaxed);
let _ = enough_command.try_send(NeoMediaSenderCommand::Pause);
})
.seek_data(move |_appsrc, seek_pos| {
log::debug!("Seeking to {}", &seek_pos);
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
true
debug!("Send seek H264: {}", seek_pos);
let result = seek_command
.try_send(NeoMediaSenderCommand::Seek(seek_pos))
.is_ok();
debug!(" - Sent seek H264: {}", result);
result
})
.build(),
);
@@ -351,18 +330,36 @@ impl NeoMediaFactoryImpl {
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_element("queue", "source_queue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("min-threshold-time", 500u64 * 1000u64);
// queue.set_property("max-size-bytes", 0u32);
// queue.set_property("max-size-buffers", 0u32);
queue.set_property("max-size-time", Duration::from_secs(2).as_nanos() as u64);
let parser = make_element("h264parse", "parser")?;
// parser.set_property("update-timecode", true);
let payload = make_element("rtph264pay", "pay0")?;
// payload.set_property("config-interval", 1i32);
// let storage = make_element("rtpstorage", "vidstorage")?;
// storage.set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64);
// let jitter = make_element("rtpjitterbuffer", "pay0")?;
// jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32);
// payload.set_property("config-interval", 5i32);
bin.add_many(&[&source, &queue, &parser, &payload])?;
Element::link_many(&[&source, &queue, &parser, &payload])?;
bin.add_many(&[
&source, &queue, &parser,
&payload,
// &storage,
// &jitter,
])?;
Element::link_many(&[
&source, &queue, &parser,
&payload,
// &storage,
// &jitter,
])?;
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.vidsrc.replace(source);
client_data.update_vid(source);
}
VidFormats::Unknown => {
debug!("Building Unknown Pipeline");
@@ -371,7 +368,6 @@ impl NeoMediaFactoryImpl {
let queue = make_element("queue", "queue0")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
queue.set_property("min-threshold-time", 500u64 * 1000u64 * 1000u64);
let overlay = make_element("textoverlay", "overlay")?;
overlay.set_property("text", "Stream not Ready");
overlay.set_property_from_str("valignment", "top");
@@ -391,171 +387,158 @@ impl NeoMediaFactoryImpl {
.build(),
)?;
// source.link(&queue)?;
queue.link(&overlay)?;
overlay.link(&encoder)?;
encoder.link(&payload)?;
Element::link_many(&[&queue, &overlay, &encoder, &payload])?;
}
}
match *self.shared.aud_format.blocking_read() {
AudFormats::Unknown => {}
AudFormats::Aac => {
debug!("Building Aac pipeline");
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_base_time(ClockTime::from_mseconds(
self.shared.microseconds.load(Ordering::Relaxed),
));
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
let do_aud = false;
if do_aud {
match *self.shared.aud_format.blocking_read() {
AudFormats::Unknown => {}
AudFormats::Aac => {
debug!("Building Aac pipeline");
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
let enough_data_start = client_data.enough_data.clone();
let enough_data_stop = client_data.enough_data.clone();
let seek_shared = self.shared.clone();
let seek_start_time = client_data.start_time.clone();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
if enough_data_stop.load(Ordering::Relaxed) {
log::debug!("Client is in need of data");
}
enough_data_stop.store(false, Ordering::Relaxed);
})
.enough_data(move |_appsrc| {
if !enough_data_start.load(Ordering::Relaxed) {
log::debug!("Client is full of of data");
}
enough_data_start.store(true, Ordering::Relaxed);
})
.seek_data(move |_appsrc, seek_pos| {
log::debug!("Seeking to {}", &seek_pos);
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
true
})
.build(),
);
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
let seek_command = client_data.get_commader();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
})
.enough_data(move |_appsrc| {
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
})
.seek_data(move |_appsrc, seek_pos| {
debug!("Send seek AAC");
let _ = seek_command
.blocking_send(NeoMediaSenderCommand::Seek(seek_pos));
true
})
.build(),
);
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
queue.set_property("min-threshold-time", 500u64 * 1000u64);
let parser = make_element("aacparse", "audparser")?;
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
Element::link_many(&[&source, &queue, &parser, &decoder])?;
Element::link_many(&[&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link AAC decoder to encoder");
});
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
let parser = make_element("aacparse", "audparser")?;
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "audpayload")?;
let storage = make_element("rtpstorage", "audstorage")?;
storage
.set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64);
let jitter = make_element("rtpjitterbuffer", "pay1")?;
jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32);
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.audsrc.replace(source);
}
AudFormats::Adpcm(block_size) => {
debug!("Building Adpcm pipeline");
// Original command line
// caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000
// ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024
// ! adpcmdec
// ! audioconvert
// ! rtpL16pay name=pay1
bin.add_many(&[
&source, &queue, &parser, &decoder, &encoder, &payload, &storage, &jitter,
])?;
Element::link_many(&[&source, &queue, &parser, &decoder])?;
Element::link_many(&[&encoder, &payload, &storage, &jitter])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link AAC decoder to encoder");
});
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_base_time(ClockTime::from_mseconds(
self.shared.microseconds.load(Ordering::Relaxed),
));
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
source.set_caps(Some(
&Caps::builder("audio/x-adpcm")
.field("layout", "div")
.field("block_align", block_size as i32)
.field("channels", 1i32)
.field("rate", 8000i32)
.build(),
));
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.update_aud(source);
}
AudFormats::Adpcm(block_size) => {
debug!("Building Adpcm pipeline");
// Original command line
// caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000
// ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024
// ! adpcmdec
// ! audioconvert
// ! rtpL16pay name=pay1
let enough_data_start = client_data.enough_data.clone();
let enough_data_stop = client_data.enough_data.clone();
let seek_shared = self.shared.clone();
let seek_start_time = client_data.start_time.clone();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
if enough_data_stop.load(Ordering::Relaxed) {
log::debug!("Client is in need of data");
}
enough_data_stop.store(false, Ordering::Relaxed);
})
.enough_data(move |_appsrc| {
if !enough_data_start.load(Ordering::Relaxed) {
log::debug!("Client is full of of data");
}
enough_data_start.store(true, Ordering::Relaxed);
})
.seek_data(move |_appsrc, seek_pos| {
log::debug!("Seeking to {}", &seek_pos);
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
true
})
.build(),
);
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_is_live(true);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::RandomAccess);
source.set_caps(Some(
&Caps::builder("audio/x-adpcm")
.field("layout", "div")
.field("block_align", block_size as i32)
.field("channels", 1i32)
.field("rate", 8000i32)
.build(),
));
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let need_command = client_data.get_commader();
let enough_command = client_data.get_commader();
let seek_command = client_data.get_commader();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
})
.enough_data(move |_appsrc| {
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
})
.seek_data(move |_appsrc, seek_pos| {
debug!("Send seek Adpcm");
let _ = seek_command
.blocking_send(NeoMediaSenderCommand::Seek(seek_pos));
true
})
.build(),
);
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
queue.set_property("min-threshold-time", 500u64 * 1000u64);
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?;
Element::link_many(&[&source, &queue, &decoder])?;
Element::link_many(&[&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link ADPCM decoder to encoder");
});
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.audsrc.replace(source);
bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?;
Element::link_many(&[&source, &queue, &decoder])?;
Element::link_many(&[&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link ADPCM decoder to encoder");
});
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
client_data.update_aud(source);
}
}
}
+503 -320
View File
@@ -2,363 +2,546 @@
//! gstreamer media stream
use anyhow::anyhow;
use futures::stream::StreamExt;
use gstreamer::ClockTime;
use gstreamer::{prelude::*, ClockTime};
use gstreamer_app::AppSrc;
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
use log::*;
use neolink_core::bcmedia::model::*;
use std::collections::{hash_map::Entry, HashMap};
use std::sync::{
atomic::{AtomicU64, Ordering},
Arc,
use std::collections::{
VecDeque,
{hash_map::Entry, HashMap},
};
use std::{
iter::Iterator,
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
};
// use tokio::time::{interval, Duration, MissedTickBehavior};
use tokio::{
sync::mpsc::{channel, Receiver, Sender},
time::Duration,
};
use tokio_stream::wrappers::ReceiverStream;
use super::{shared::*, AnyResult};
pub(super) struct NeoMediaSender {
pub(super) data_source: ReceiverStream<BcMedia>,
pub(super) clientsource: ReceiverStream<ClientPipelineData>,
pub(super) shared: Arc<NeoMediaShared>,
// Used to generate the key for the hashmaps
pub(super) uid: AtomicU64,
// Hashmap so we can alter the data in ClientData while also allowing for remove at any point
// If you know a better collections pm me
pub(super) clientdata: HashMap<u64, ClientPipelineData>,
pub(super) waiting_for_iframe: bool,
#[derive(Default)]
struct NeoBuffer {
buf: VecDeque<Vec<BcMedia>>,
}
#[derive(Clone)]
struct StampedData {
ms: u64,
data: Vec<u8>,
impl NeoBuffer {
fn push(&mut self, item: BcMedia) {
match item {
iframe @ BcMedia::Iframe(_) => self.buf.push_back(vec![iframe]),
pframe @ BcMedia::Pframe(_) => {
if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() {
last.push(pframe);
}
}
aac @ BcMedia::Aac(_) => {
if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() {
last.push(aac);
}
}
adpcm @ BcMedia::Adpcm(_) => {
if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() {
last.push(adpcm);
}
}
BcMedia::InfoV1(_) | BcMedia::InfoV2(_) => {}
}
while self.buf.len() > 25 {
// 25 iframes
let _ = self.buf.pop_front();
}
}
fn start_time(&self) -> Option<u64> {
if let Some(BcMedia::Iframe(data)) =
self.buf.front().and_then(|inner_buf| inner_buf.first())
{
Some(data.microseconds as u64)
} else {
None
}
}
fn end_time(&self) -> Option<u64> {
if let Some(innerbuffer) = self.buf.back() {
let mut last_ms = None;
for frame in innerbuffer.iter() {
match frame {
BcMedia::Iframe(frame) => {
let v = last_ms.get_or_insert(frame.microseconds as u64);
if *v < frame.microseconds as u64 {
*v = frame.microseconds as u64;
}
}
BcMedia::Pframe(frame) => {
let v = last_ms.get_or_insert(frame.microseconds as u64);
if *v < frame.microseconds as u64 {
*v = frame.microseconds as u64;
}
}
_ => {}
}
}
last_ms
} else {
None
}
}
}
pub(super) struct NeoMediaSenders {
data_source: ReceiverStream<BcMedia>,
client_source: ReceiverStream<NeoMediaSender>,
shared: Arc<NeoMediaShared>,
uid: AtomicU64,
client_data: HashMap<u64, NeoMediaSender>,
buffer: NeoBuffer,
}
impl NeoMediaSenders {
pub(super) fn new(
shared: Arc<NeoMediaShared>,
data_source: ReceiverStream<BcMedia>,
client_source: ReceiverStream<NeoMediaSender>,
) -> Self {
Self {
data_source,
client_source,
shared,
uid: Default::default(),
client_data: Default::default(),
buffer: Default::default(),
}
}
async fn handle_new_client(&mut self, mut client: NeoMediaSender) -> AnyResult<()> {
client.initialise(&self.buffer).await?;
self.client_data
.insert(self.uid.fetch_add(1, Ordering::Relaxed), client);
Ok(())
}
async fn update_mediatypes(&self, data: &BcMedia) {
match data {
BcMedia::Iframe(BcMediaIframe {
video_type: VideoType::H264,
..
}) => {
if !matches!(*self.shared.vid_format.read().await, VidFormats::H264) {
*self.shared.vid_format.write().await = VidFormats::H264;
}
}
BcMedia::Pframe(BcMediaPframe {
video_type: VideoType::H264,
..
}) => {
if !matches!(*self.shared.vid_format.read().await, VidFormats::H264) {
*self.shared.vid_format.write().await = VidFormats::H264;
}
}
BcMedia::Iframe(BcMediaIframe {
video_type: VideoType::H265,
..
}) => {
if !matches!(*self.shared.vid_format.read().await, VidFormats::H265) {
*self.shared.vid_format.write().await = VidFormats::H265;
}
}
BcMedia::Pframe(BcMediaPframe {
video_type: VideoType::H265,
..
}) => {
if !matches!(*self.shared.vid_format.read().await, VidFormats::H265) {
*self.shared.vid_format.write().await = VidFormats::H265;
}
}
BcMedia::Aac(_) => {
if !matches!(*self.shared.aud_format.read().await, AudFormats::Aac) {
*self.shared.aud_format.write().await = AudFormats::Aac;
}
}
BcMedia::Adpcm(data) => {
if !matches!(*self.shared.aud_format.read().await, AudFormats::Adpcm(_)) {
*self.shared.aud_format.write().await =
AudFormats::Adpcm(data.data.len() as u16);
}
}
_ => {}
}
}
async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> {
// debug!("Handle new data");
self.update_mediatypes(&data).await;
for key in self.client_data.keys().copied().collect::<Vec<_>>() {
// debug!(" - Client: {}", key);
match self.client_data.entry(key) {
Entry::Occupied(mut occ) => {
if let Err(e) = occ.get_mut().initialise(&self.buffer).await {
debug!("Could not init client: {:?}", e);
occ.remove();
continue;
}
if let Err(e) = occ.get_mut().process_commands(&self.buffer).await {
debug!("Could not process client command: {:?}", e);
occ.remove();
continue;
}
if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await {
debug!("Could not handle jump to live: {:?}", e);
occ.remove();
continue;
}
if let Err(e) = occ.get_mut().process_buffer(&self.buffer).await {
debug!("Could not send client data: {:?}", e);
occ.remove();
continue;
}
}
Entry::Vacant(_) => {}
}
}
self.buffer.push(data);
Ok(())
}
pub(super) async fn run(&mut self) -> AnyResult<()> {
loop {
tokio::select! {
Some(v) = self.data_source.next() => {
self.handle_new_data(v).await
},
Some(v) = self.client_source.next() => {
self.handle_new_client(v).await
},
else => {
Err(anyhow!("Sender data source closed"))
}
}?;
}
}
}
pub(super) enum NeoMediaSenderCommand {
Seek(u64),
Pause,
Resume,
}
#[derive(Debug)]
pub(super) struct NeoMediaSender {
needle: u64,
start_time: u64,
vid: Option<AppSrc>,
aud: Option<AppSrc>,
command_reciever: Receiver<NeoMediaSenderCommand>,
command_sender: Sender<NeoMediaSenderCommand>,
inited: bool,
playing: bool,
}
impl NeoMediaSender {
pub(super) async fn run(&mut self) -> AnyResult<()> {
// let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32));
// resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip);
let mut buffer_a: Vec<StampedData> = Default::default();
let mut buffer_b: Vec<StampedData> = Default::default();
let buffer_read = &mut buffer_a;
let buffer_write = &mut buffer_b;
pub(super) fn new() -> Self {
let (tx, rx) = channel(3);
Self {
needle: 0,
start_time: 0,
vid: None,
aud: None,
command_reciever: rx,
command_sender: tx,
inited: false,
playing: true,
}
}
let mut audbuffer_a: Vec<StampedData> = Default::default();
let mut audbuffer_b: Vec<StampedData> = Default::default();
let audbuffer_read = &mut audbuffer_a;
let audbuffer_write = &mut audbuffer_b;
loop {
self.shared
.number_of_clients
.store(self.clientdata.len(), Ordering::Relaxed);
self.shared
.buffer_ready
.store(!buffer_read.is_empty(), Ordering::Relaxed);
// debug!("Sender: Get");
tokio::select! {
v = self.data_source.next() => {
// debug!("Sender: Got Data");
// debug!("data_source recieved");
if let Some(bc_media) = v {
if ! self.skip_bcmedia(&bc_media) {
// debug!("Not skipped");
// resend_pause.reset();
self.inspect_bcmedia(&bc_media).await;
match bc_media {
BcMedia::Iframe(frame) => {
if buffer_write.len() > 25 {
std::mem::swap(buffer_read, buffer_write);
buffer_write.clear();
std::mem::swap(audbuffer_read, audbuffer_write);
audbuffer_write.clear();
}
let new_data = StampedData {
ms: frame.microseconds as u64,
data: frame.data,
};
self.process_vidbuffer(&new_data).await;
buffer_write.push(new_data);
}
BcMedia::Pframe(frame) => {
let new_data = StampedData {
ms: frame.microseconds as u64,
data: frame.data,
};
self.process_vidbuffer(&new_data).await;
buffer_write.push(new_data);
}
BcMedia::Aac(aac) => {
if let Some(last) = buffer_write.last().as_ref() {
let new_data = StampedData {
ms: last.ms,
data: aac.data,
};
self.process_audbuffer(&new_data).await;
audbuffer_write.push(new_data);
}
pub(super) fn update_vid(&mut self, source: AppSrc) {
self.vid.replace(source);
}
}
BcMedia::Adpcm(adpcm) => {
if let Some(last) = buffer_write.last().as_ref() {
let new_data = StampedData {
ms: last.ms,
data: adpcm.data,
};
self.process_audbuffer(&new_data).await;
audbuffer_write.push(new_data);
}
pub(super) fn update_aud(&mut self, source: AppSrc) {
self.aud.replace(source);
}
}
_ => {}
}
pub(super) fn get_commader(&self) -> Sender<NeoMediaSenderCommand> {
self.command_sender.clone()
}
async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
if self.needle > self.start_time {
self.seek(self.needle - self.start_time, buffer).await
} else {
self.seek(0, buffer).await
}
}
async fn seek(&mut self, dest: u64, buffer: &NeoBuffer) -> AnyResult<()> {
if let (Some(first_ms), Some(last_ms)) = (buffer.start_time(), buffer.end_time()) {
let target_seek = self.start_time + dest;
if target_seek >= first_ms && target_seek <= last_ms {
debug!("Seeking inside buffer: {}", target_seek);
self.needle = target_seek;
self.process_buffer(buffer).await?;
debug!("Seeked inside buffer");
} else {
// Fake seek
debug!(
"Seeking outside buffer: {} - {} = {}",
self.needle,
target_seek,
self.needle - dest
);
self.start_time = self.needle - dest;
}
} else {
// Buffers arent valid yet
debug!("Seek buffers are not ready: {}", dest);
debug!(" - {:?}", (buffer.start_time(), buffer.end_time()));
}
Ok(())
}
async fn process_commands(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
if self.inited {
if let Ok(command) = self.command_reciever.try_recv() {
match command {
NeoMediaSenderCommand::Seek(dest) => {
debug!("Got Seek Request: {}", dest);
self.seek(dest, buffer).await?;
}
NeoMediaSenderCommand::Pause => {
if self.playing {
debug!("Pausing");
self.playing = false;
}
}
NeoMediaSenderCommand::Resume => {
if !self.playing {
debug!("Resuming");
self.playing = true;
self.jump_to_live(buffer).await?;
}
} else {
break;
}
}
v = self.clientsource.next() => {
// debug!("Sender: Got Client");
if let Some(mut clientdata) = v {
// Resend the video and audio buffers
if !clientdata.inited {
clientdata.inited = true;
if let Some(last) = buffer_read.first() {
debug!("Start PTS SET: {:?}", tokio::time::Duration::from_micros(last.ms));
clientdata.set_start_time(last.ms);
}
let mut vid_resend = buffer_read.iter().chain(buffer_write.iter());
let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter());
loop {
let next_vid = vid_resend.next();
if let Some(next_vid) = next_vid.as_ref() {
if let Some(src) = clientdata.vidsrc.as_ref() {
if Self::send_buffer(
src,
next_vid.data.as_slice(),
next_vid.ms,
clientdata.get_start_time(),
).await.is_err() {
break;
};
}
}
let next_aud = aud_resend.next();
if let Some(next_aud) = next_aud.as_ref() {
if next_aud.ms >= clientdata.get_start_time() {
if let Some(src) = clientdata.audsrc.as_ref() {
if Self::send_buffer(
src,
next_aud.data.as_slice(),
next_aud.ms,
clientdata.get_start_time(),
).await.is_err() {
break;
};
}
}
}
if next_vid.is_none() && next_aud.is_none() {
break;
}
}
debug!("Advance PTS Pushed");
}
// Save the client for future sends
self.clientdata.insert(self.uid.fetch_add(1, Ordering::Relaxed) , clientdata);
} else {
break;
}
},
// _ = resend_pause.tick() => {
// if let Some(live) = buffer.last().cloned() {
// self.process_vidbuffer(&ProcessData {
// live,
// resend: buffer.as_slice()
// })?;
// }
// }
}
}
Ok(())
}
async fn process_vidbuffer(&mut self, stamped_data: &StampedData) {
for key in self
.clientdata
.keys()
.cloned()
.collect::<Vec<_>>()
.drain(..)
{
match self.clientdata.entry(key) {
Entry::Occupied(data) => {
if let Some(vidsrc) = data.get().vidsrc.as_ref() {
if !data.get().enough_data.load(Ordering::Relaxed)
&& Self::send_buffer(
vidsrc,
stamped_data.data.as_slice(),
stamped_data.ms,
data.get().get_start_time(),
)
.await
.is_err()
{
data.remove();
async fn initialise(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
if !self.inited {
if let (Some(first_ms), Some(end_ms)) = (buffer.start_time(), buffer.end_time()) {
// if let Some(source) = self.vid.as_ref() {
// source.set_base_time(ClockTime::from_mseconds(0));
// }
// if let Some(source) = self.aud.as_ref() {
// source.set_base_time(ClockTime::from_mseconds(0));
// }
self.start_time = (first_ms + end_ms) / 2;
self.needle = self.start_time;
self.inited = true;
self.process_buffer(buffer).await?;
}
}
Ok(())
}
async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> {
if self.inited && self.playing {
let mut buf_it = buf.buf.iter().peekable();
while let Some(buffer) = buf_it.next() {
tokio::task::yield_now().await;
let next = buf_it.peek();
// Look at the next frame set and see if it is after the needle (or if empty)
let deep = next
.map(|buffer| {
buffer
.first()
.map(|frame| {
if let BcMedia::Iframe(data) = frame {
data.microseconds as u64 >= self.needle
} else {
unreachable!()
}
})
.unwrap_or(true) // No first data
})
.unwrap_or(true); // Next buffer is empty
// This frame set contains our data process it
if deep {
let mut send_it = false;
for frame in buffer.iter() {
if !send_it {
let microseconds = match frame {
BcMedia::Iframe(data) => Some(data.microseconds),
BcMedia::Pframe(data) => Some(data.microseconds),
_ => None,
};
if let Some(microseconds) = microseconds {
if microseconds as u64 > self.needle {
send_it = true;
}
}
}
} else {
data.remove();
}
}
Entry::Vacant(_) => {}
}
}
}
async fn process_audbuffer(&mut self, stamped_data: &StampedData) {
for key in self
.clientdata
.keys()
.cloned()
.collect::<Vec<_>>()
.drain(..)
{
match self.clientdata.entry(key) {
Entry::Occupied(data) => {
if let Some(audsrc) = data.get().audsrc.as_ref() {
if !data.get().enough_data.load(Ordering::Relaxed)
&& Self::send_buffer(
audsrc,
stamped_data.data.as_slice(),
stamped_data.ms,
data.get().get_start_time(),
)
.await
.is_err()
{
data.remove();
if send_it && !self.send_buffer(frame).await? {
// Buffer too far is ahead of needle early exit
break;
}
}
}
}
}
Ok(())
}
async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
if let Some(first_buftime) = buffer.start_time() {
if first_buftime > self.needle {
debug!("Jump to live first_buftime > self.needle");
self.jump_to_live(buffer).await?;
} else if let Some(runtime) = self.get_runtime() {
if first_buftime > self.start_time {
let buftime = Duration::from_micros(first_buftime - self.start_time);
if buftime > runtime {
debug!("Jump to live buftime > runtime");
self.jump_to_live(buffer).await?;
}
} else {
debug!("Jump to live first_buftime > self.start_time ");
self.jump_to_live(buffer).await?;
}
}
}
Ok(())
}
fn get_runtime(&self) -> Option<Duration> {
if let Some(appsrc) = self.vid.as_ref() {
if let Some(clock) = appsrc.clock() {
if let Some(time) = clock.time() {
if let Some(base_time) = appsrc.base_time() {
let runtime = time - base_time;
return Some(Duration::from_nanos(runtime.nseconds()));
}
}
}
}
None
}
async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult<bool> {
if self.inited && self.playing {
let microseconds = match media {
BcMedia::Iframe(data) => {
if data.microseconds as u64 > self.start_time {
data.microseconds as u64 - self.start_time
} else {
data.remove();
0
}
}
Entry::Vacant(_) => {}
}
}
}
async fn send_buffer(
appsrc: &AppSrc,
buf: &[u8],
frame_ms: u64,
start_time: u64,
) -> AnyResult<()> {
let micros = if frame_ms >= start_time {
frame_ms - start_time
} else {
0
};
debug!("PTS: {:?}", tokio::time::Duration::from_micros(micros));
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let time = ClockTime::from_useconds(micros);
gst_buf_mut.set_pts(time);
// debug!(
// "PTS set to: {:?} ({}-{}={:?})",
// time, frame_ms, start_time, micros
// );
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(buf);
}
// debug!("Buffer pushed");
let thread_appsrc = appsrc.clone(); // GObjects are refcounted
let res = tokio::task::spawn_blocking(move || {
thread_appsrc
.push_buffer(gst_buf.copy())
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
})
.await;
match &res {
Err(e) => {
debug!("Failed to send buffer: {:?}", e);
}
Ok(Err(e)) => {
debug!("Failed to send buffer: {:?}", e);
}
Ok(Ok(_)) => {}
};
res?
}
fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> bool {
if self.waiting_for_iframe {
if let BcMedia::Iframe(_) = bc_media {
self.waiting_for_iframe = false;
} else {
log::debug!("Skipping bcmedia");
return true;
}
}
false
}
async fn inspect_bcmedia(&mut self, bc_media: &BcMedia) {
let old_vid = *self.shared.vid_format.read().await;
let old_aud = *self.shared.aud_format.read().await;
match bc_media {
BcMedia::Iframe(frame) => {
match frame.video_type {
VideoType::H264 => {
(*self.shared.vid_format.write().await) = VidFormats::H264;
}
VideoType::H265 => {
(*self.shared.vid_format.write().await) = VidFormats::H265;
BcMedia::Pframe(data) => {
if data.microseconds as u64 > self.start_time {
data.microseconds as u64 - self.start_time
} else {
0
}
}
self.shared
.microseconds
.store(frame.microseconds as u64, Ordering::Relaxed);
// log::debug!(
// "Time set to {}",
// self.shared.microseconds.load(Ordering::Relaxed)
// );
}
BcMedia::Pframe(frame) => {
match frame.video_type {
VideoType::H264 => {
(*self.shared.vid_format.write().await) = VidFormats::H264;
}
VideoType::H265 => {
(*self.shared.vid_format.write().await) = VidFormats::H265;
_ => {
if self.needle > self.start_time {
self.needle - self.start_time
} else {
0
}
}
self.shared
.microseconds
.store(frame.microseconds as u64, Ordering::Relaxed);
// log::debug!(
// "Time set to {}",
// self.shared.microseconds.load(Ordering::Relaxed)
// );
};
let buf = match media {
BcMedia::Iframe(data) => Some(&data.data),
BcMedia::Pframe(data) => Some(&data.data),
BcMedia::Aac(data) => Some(&data.data),
BcMedia::Adpcm(data) => Some(&data.data),
_ => None,
};
let appsrc = match media {
BcMedia::Iframe(_) | BcMedia::Pframe(_) => self.vid.as_ref(),
BcMedia::Aac(_) | BcMedia::Adpcm(_) => self.aud.as_ref(),
_ => None,
};
if let (Some(buf), Some(appsrc)) = (buf, appsrc) {
if let Some(runtime) = self.get_runtime() {
let buftimr = Duration::from_micros(microseconds);
if buftimr > runtime + Duration::from_micros(2000) {
// Stop advancing the buffer
return Ok(false);
} else {
debug!(" - runtime: {:?}", runtime);
debug!(" - buftime: {:?}", buftimr);
}
}
debug!("PTS: {:?}", Duration::from_micros(microseconds));
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let time = ClockTime::from_useconds(microseconds);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(buf);
}
// debug!("Buffer pushed");
let thread_appsrc = appsrc.clone(); // GObjects are refcounted
let res = tokio::task::spawn_blocking(move || {
thread_appsrc
.push_buffer(gst_buf.copy())
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
})
.await;
match &res {
Err(e) => {
debug!("Failed to send buffer: {:?}", e);
}
Ok(Err(e)) => {
debug!("Failed to send buffer: {:?}", e);
}
Ok(Ok(_)) => {}
};
res??;
// Buffer sent sucessfully
//
// Update the needle position
match media {
BcMedia::Iframe(data) => {
if data.microseconds as u64 > self.needle {
// debug!("Updating needle to: {}", data.microseconds);
self.needle = data.microseconds as u64;
}
}
BcMedia::Pframe(data) => {
if data.microseconds as u64 > self.needle {
// debug!("Updating needle to: {}", data.microseconds);
self.needle = data.microseconds as u64;
}
}
_ => {}
}
}
BcMedia::Aac(_aac) => {
(*self.shared.aud_format.write().await) = AudFormats::Aac;
}
BcMedia::Adpcm(adpcm) => {
(*self.shared.aud_format.write().await) = AudFormats::Adpcm(adpcm.data.len() as u16)
}
_ => {}
}
let new_vid = *self.shared.vid_format.read().await;
if new_vid != old_vid {
log::debug!("Video format set to: {:?} from {:?}", new_vid, old_vid);
}
let new_aud = *self.shared.aud_format.read().await;
if old_aud != new_aud {
log::debug!("Audio format set to: {:?} from {:?}", new_aud, old_aud);
}
Ok(true)
}
}
+1 -23
View File
@@ -1,9 +1,7 @@
//! Data shared between the various
//! components that manage a media stream
use gstreamer_app::AppSrc;
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize};
use tokio::sync::RwLock;
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -23,7 +21,6 @@ pub(super) enum AudFormats {
pub(super) struct NeoMediaShared {
pub(super) vid_format: RwLock<VidFormats>,
pub(super) aud_format: RwLock<AudFormats>,
pub(super) microseconds: AtomicU64,
pub(super) number_of_clients: AtomicUsize,
pub(super) buffer_ready: AtomicBool,
}
@@ -33,27 +30,8 @@ impl Default for NeoMediaShared {
Self {
vid_format: RwLock::new(VidFormats::Unknown),
aud_format: RwLock::new(AudFormats::Unknown),
microseconds: AtomicU64::new(0),
number_of_clients: AtomicUsize::new(0),
buffer_ready: AtomicBool::new(false),
}
}
}
#[derive(Default, Debug)]
pub(super) struct ClientPipelineData {
pub(super) vidsrc: Option<AppSrc>,
pub(super) audsrc: Option<AppSrc>,
pub(super) start_time: Arc<AtomicU64>,
pub(super) inited: bool,
pub(super) enough_data: Arc<AtomicBool>,
}
impl ClientPipelineData {
pub(super) fn get_start_time(&self) -> u64 {
self.start_time.load(Ordering::Relaxed)
}
pub(super) fn set_start_time(&self, time: u64) {
self.start_time.store(time, Ordering::Relaxed)
}
}