Complete rework of buffer time
This commit is contained in:
3 files changed
+706
-562
No files matched your search
+202
-219
@@ -11,7 +11,7 @@ use gstreamer::{
|
||||
glib::{self, Object},
|
||||
Structure,
|
||||
};
|
||||
use gstreamer::{Bin, Caps, ClockTime, Element, ElementFactory};
|
||||
use gstreamer::{Bin, Caps, Element, ElementFactory};
|
||||
use gstreamer_app::{AppSrc, AppSrcCallbacks, AppStreamType};
|
||||
use gstreamer_rtsp::RTSPUrl;
|
||||
use gstreamer_rtsp_server::prelude::*;
|
||||
@@ -23,14 +23,12 @@ use log::*;
|
||||
use neolink_core::bcmedia::model::*;
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
sync::{atomic::Ordering, Arc},
|
||||
};
|
||||
use tokio::{
|
||||
sync::mpsc::{channel, Sender},
|
||||
task::JoinSet,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
@@ -113,7 +111,7 @@ unsafe impl Sync for NeoMediaFactory {}
|
||||
|
||||
pub(crate) struct NeoMediaFactoryImpl {
|
||||
sender: Sender<BcMedia>,
|
||||
clientsender: Sender<ClientPipelineData>,
|
||||
clientsender: Sender<NeoMediaSender>,
|
||||
shared: Arc<NeoMediaShared>,
|
||||
#[allow(dead_code)] // Not dead just need a handle to keep it alive and drop with this obj
|
||||
threads: JoinSet<AnyResult<()>>,
|
||||
@@ -134,14 +132,11 @@ impl Default for NeoMediaFactoryImpl {
|
||||
|
||||
// Prepare thread that sends data into the appsrcs
|
||||
let mut threads: JoinSet<AnyResult<()>> = Default::default();
|
||||
let mut sender = NeoMediaSender {
|
||||
data_source: ReceiverStream::new(datarx),
|
||||
clientsource: ReceiverStream::new(rx_clientsender),
|
||||
shared: shared.clone(),
|
||||
uid: Default::default(),
|
||||
clientdata: Default::default(),
|
||||
waiting_for_iframe: true,
|
||||
};
|
||||
let mut sender = NeoMediaSenders::new(
|
||||
shared.clone(),
|
||||
ReceiverStream::new(datarx),
|
||||
ReceiverStream::new(rx_clientsender),
|
||||
);
|
||||
threads.spawn(async move {
|
||||
loop {
|
||||
tokio::task::yield_now().await;
|
||||
@@ -153,7 +148,6 @@ impl Default for NeoMediaFactoryImpl {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sender.waiting_for_iframe = true;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
@@ -231,12 +225,7 @@ impl NeoMediaFactoryImpl {
|
||||
bin.remove(&element)?;
|
||||
}
|
||||
|
||||
let mut client_data = ClientPipelineData {
|
||||
start_time: Arc::new(AtomicU64::new(
|
||||
self.shared.microseconds.load(Ordering::Relaxed),
|
||||
)),
|
||||
..Default::default()
|
||||
};
|
||||
let mut client_data = NeoMediaSender::new();
|
||||
|
||||
// Now contruct the actual ones
|
||||
match *self.shared.vid_format.blocking_read() {
|
||||
@@ -245,38 +234,28 @@ impl NeoMediaFactoryImpl {
|
||||
let source = make_element("appsrc", "vidsrc")?
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
|
||||
source.set_base_time(ClockTime::from_mseconds(
|
||||
self.shared.microseconds.load(Ordering::Relaxed),
|
||||
));
|
||||
source.set_is_live(true);
|
||||
source.set_block(false);
|
||||
source.set_property("emit-signals", false);
|
||||
source.set_max_bytes(52428800);
|
||||
source.set_do_timestamp(false);
|
||||
source.set_stream_type(AppStreamType::Seekable);
|
||||
source.set_stream_type(AppStreamType::RandomAccess);
|
||||
|
||||
let enough_data_start = client_data.enough_data.clone();
|
||||
let enough_data_stop = client_data.enough_data.clone();
|
||||
let seek_shared = self.shared.clone();
|
||||
let seek_start_time = client_data.start_time.clone();
|
||||
let need_command = client_data.get_commader();
|
||||
let enough_command = client_data.get_commader();
|
||||
let seek_command = client_data.get_commader();
|
||||
source.set_callbacks(
|
||||
AppSrcCallbacks::builder()
|
||||
.need_data(move |_appsrc, _amt| {
|
||||
if enough_data_stop.load(Ordering::Relaxed) {
|
||||
log::debug!("Client is in need of data");
|
||||
}
|
||||
enough_data_stop.store(false, Ordering::Relaxed);
|
||||
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
|
||||
})
|
||||
.enough_data(move |_appsrc| {
|
||||
if !enough_data_start.load(Ordering::Relaxed) {
|
||||
log::debug!("Client is full of of data");
|
||||
}
|
||||
enough_data_start.store(true, Ordering::Relaxed);
|
||||
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
|
||||
})
|
||||
.seek_data(move |_appsrc, seek_pos| {
|
||||
log::debug!("Seeking to {}", &seek_pos);
|
||||
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
|
||||
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
|
||||
debug!("Send seek H265");
|
||||
let _ =
|
||||
seek_command.blocking_send(NeoMediaSenderCommand::Seek(seek_pos));
|
||||
true
|
||||
})
|
||||
.build(),
|
||||
@@ -290,7 +269,6 @@ impl NeoMediaFactoryImpl {
|
||||
let queue = make_element("queue", "source_queue")?;
|
||||
queue.set_property_from_str("leaky", "downstream");
|
||||
queue.set_property("max-size-bytes", 104857600u32);
|
||||
queue.set_property("min-threshold-time", 500u64 * 1000u64);
|
||||
let parser = make_element("h265parse", "parser")?;
|
||||
// parser.set_property("config-interval", 5i32);
|
||||
let payload = make_element("rtph265pay", "pay0")?;
|
||||
@@ -300,46 +278,47 @@ impl NeoMediaFactoryImpl {
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
client_data.vidsrc.replace(source);
|
||||
client_data.update_vid(source);
|
||||
}
|
||||
VidFormats::H264 => {
|
||||
debug!("Building H264 Pipeline");
|
||||
let source = make_element("appsrc", "vidsrc")?
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
|
||||
source.set_base_time(ClockTime::from_mseconds(
|
||||
self.shared.microseconds.load(Ordering::Relaxed),
|
||||
));
|
||||
|
||||
source.set_is_live(true);
|
||||
source.set_block(false);
|
||||
source.set_property("emit-signals", false);
|
||||
source.set_max_bytes(52428800);
|
||||
source.set_max_bytes(50000000u64); // 50MB
|
||||
// source.set_property("max-buffers", 0u32); // v1.20
|
||||
// source.set_property("max-time", Duration::from_secs_f32(2.25).as_nanos() as u64);
|
||||
source.set_do_timestamp(false);
|
||||
source.set_stream_type(AppStreamType::Seekable);
|
||||
source.set_stream_type(AppStreamType::RandomAccess);
|
||||
let clock = source.clock();
|
||||
if let Some(clock) = clock {
|
||||
debug!("Clock: {:?}", clock.time());
|
||||
} else {
|
||||
debug!("No clock");
|
||||
}
|
||||
|
||||
let enough_data_start = client_data.enough_data.clone();
|
||||
let enough_data_stop = client_data.enough_data.clone();
|
||||
let seek_shared = self.shared.clone();
|
||||
let seek_start_time = client_data.start_time.clone();
|
||||
let need_command = client_data.get_commader();
|
||||
let enough_command = client_data.get_commader();
|
||||
let seek_command = client_data.get_commader();
|
||||
source.set_callbacks(
|
||||
AppSrcCallbacks::builder()
|
||||
.need_data(move |_appsrc, _amt| {
|
||||
if enough_data_stop.load(Ordering::Relaxed) {
|
||||
log::debug!("Client is in need of data");
|
||||
}
|
||||
enough_data_stop.store(false, Ordering::Relaxed);
|
||||
let _ = need_command.try_send(NeoMediaSenderCommand::Resume);
|
||||
})
|
||||
.enough_data(move |_appsrc| {
|
||||
if !enough_data_start.load(Ordering::Relaxed) {
|
||||
log::debug!("Client is full of of data");
|
||||
}
|
||||
enough_data_start.store(true, Ordering::Relaxed);
|
||||
let _ = enough_command.try_send(NeoMediaSenderCommand::Pause);
|
||||
})
|
||||
.seek_data(move |_appsrc, seek_pos| {
|
||||
log::debug!("Seeking to {}", &seek_pos);
|
||||
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
|
||||
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
|
||||
true
|
||||
debug!("Send seek H264: {}", seek_pos);
|
||||
let result = seek_command
|
||||
.try_send(NeoMediaSenderCommand::Seek(seek_pos))
|
||||
.is_ok();
|
||||
debug!(" - Sent seek H264: {}", result);
|
||||
result
|
||||
})
|
||||
.build(),
|
||||
);
|
||||
@@ -351,18 +330,36 @@ impl NeoMediaFactoryImpl {
|
||||
.map_err(|_| anyhow!("Cannot cast back"))?;
|
||||
let queue = make_element("queue", "source_queue")?;
|
||||
queue.set_property_from_str("leaky", "downstream");
|
||||
queue.set_property("min-threshold-time", 500u64 * 1000u64);
|
||||
// queue.set_property("max-size-bytes", 0u32);
|
||||
// queue.set_property("max-size-buffers", 0u32);
|
||||
queue.set_property("max-size-time", Duration::from_secs(2).as_nanos() as u64);
|
||||
let parser = make_element("h264parse", "parser")?;
|
||||
// parser.set_property("update-timecode", true);
|
||||
let payload = make_element("rtph264pay", "pay0")?;
|
||||
// payload.set_property("config-interval", 1i32);
|
||||
// let storage = make_element("rtpstorage", "vidstorage")?;
|
||||
// storage.set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64);
|
||||
// let jitter = make_element("rtpjitterbuffer", "pay0")?;
|
||||
// jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32);
|
||||
|
||||
// payload.set_property("config-interval", 5i32);
|
||||
bin.add_many(&[&source, &queue, &parser, &payload])?;
|
||||
Element::link_many(&[&source, &queue, &parser, &payload])?;
|
||||
bin.add_many(&[
|
||||
&source, &queue, &parser,
|
||||
&payload,
|
||||
// &storage,
|
||||
// &jitter,
|
||||
])?;
|
||||
Element::link_many(&[
|
||||
&source, &queue, &parser,
|
||||
&payload,
|
||||
// &storage,
|
||||
// &jitter,
|
||||
])?;
|
||||
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
client_data.vidsrc.replace(source);
|
||||
client_data.update_vid(source);
|
||||
}
|
||||
VidFormats::Unknown => {
|
||||
debug!("Building Unknown Pipeline");
|
||||
@@ -371,7 +368,6 @@ impl NeoMediaFactoryImpl {
|
||||
let queue = make_element("queue", "queue0")?;
|
||||
queue.set_property_from_str("leaky", "downstream");
|
||||
queue.set_property("max-size-bytes", 104857600u32);
|
||||
queue.set_property("min-threshold-time", 500u64 * 1000u64 * 1000u64);
|
||||
let overlay = make_element("textoverlay", "overlay")?;
|
||||
overlay.set_property("text", "Stream not Ready");
|
||||
overlay.set_property_from_str("valignment", "top");
|
||||
@@ -391,171 +387,158 @@ impl NeoMediaFactoryImpl {
|
||||
.build(),
|
||||
)?;
|
||||
// source.link(&queue)?;
|
||||
queue.link(&overlay)?;
|
||||
overlay.link(&encoder)?;
|
||||
encoder.link(&payload)?;
|
||||
Element::link_many(&[&queue, &overlay, &encoder, &payload])?;
|
||||
}
|
||||
}
|
||||
|
||||
match *self.shared.aud_format.blocking_read() {
|
||||
AudFormats::Unknown => {}
|
||||
AudFormats::Aac => {
|
||||
debug!("Building Aac pipeline");
|
||||
let source = make_element("appsrc", "audsrc")?
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
|
||||
source.set_base_time(ClockTime::from_mseconds(
|
||||
self.shared.microseconds.load(Ordering::Relaxed),
|
||||
));
|
||||
source.set_is_live(true);
|
||||
source.set_block(false);
|
||||
source.set_property("emit-signals", false);
|
||||
source.set_max_bytes(52428800);
|
||||
source.set_do_timestamp(false);
|
||||
source.set_stream_type(AppStreamType::Seekable);
|
||||
let do_aud = false;
|
||||
if do_aud {
|
||||
match *self.shared.aud_format.blocking_read() {
|
||||
AudFormats::Unknown => {}
|
||||
AudFormats::Aac => {
|
||||
debug!("Building Aac pipeline");
|
||||
let source = make_element("appsrc", "audsrc")?
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
|
||||
|
||||
let enough_data_start = client_data.enough_data.clone();
|
||||
let enough_data_stop = client_data.enough_data.clone();
|
||||
let seek_shared = self.shared.clone();
|
||||
let seek_start_time = client_data.start_time.clone();
|
||||
source.set_callbacks(
|
||||
AppSrcCallbacks::builder()
|
||||
.need_data(move |_appsrc, _amt| {
|
||||
if enough_data_stop.load(Ordering::Relaxed) {
|
||||
log::debug!("Client is in need of data");
|
||||
}
|
||||
enough_data_stop.store(false, Ordering::Relaxed);
|
||||
})
|
||||
.enough_data(move |_appsrc| {
|
||||
if !enough_data_start.load(Ordering::Relaxed) {
|
||||
log::debug!("Client is full of of data");
|
||||
}
|
||||
enough_data_start.store(true, Ordering::Relaxed);
|
||||
})
|
||||
.seek_data(move |_appsrc, seek_pos| {
|
||||
log::debug!("Seeking to {}", &seek_pos);
|
||||
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
|
||||
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
|
||||
true
|
||||
})
|
||||
.build(),
|
||||
);
|
||||
source.set_is_live(true);
|
||||
source.set_block(false);
|
||||
source.set_property("emit-signals", false);
|
||||
source.set_max_bytes(52428800);
|
||||
source.set_do_timestamp(false);
|
||||
source.set_stream_type(AppStreamType::Seekable);
|
||||
|
||||
let source = source
|
||||
.dynamic_cast::<Element>()
|
||||
.map_err(|_| anyhow!("Cannot cast back"))?;
|
||||
let need_command = client_data.get_commader();
|
||||
let enough_command = client_data.get_commader();
|
||||
let seek_command = client_data.get_commader();
|
||||
source.set_callbacks(
|
||||
AppSrcCallbacks::builder()
|
||||
.need_data(move |_appsrc, _amt| {
|
||||
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
|
||||
})
|
||||
.enough_data(move |_appsrc| {
|
||||
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
|
||||
})
|
||||
.seek_data(move |_appsrc, seek_pos| {
|
||||
debug!("Send seek AAC");
|
||||
let _ = seek_command
|
||||
.blocking_send(NeoMediaSenderCommand::Seek(seek_pos));
|
||||
true
|
||||
})
|
||||
.build(),
|
||||
);
|
||||
|
||||
let queue = make_element("queue", "audqueue")?;
|
||||
queue.set_property_from_str("leaky", "downstream");
|
||||
queue.set_property("max-size-bytes", 104857600u32);
|
||||
queue.set_property("min-threshold-time", 500u64 * 1000u64);
|
||||
let parser = make_element("aacparse", "audparser")?;
|
||||
let decoder = make_element("decodebin", "auddecoder")?;
|
||||
let encoder = make_element("audioconvert", "audencoder")?;
|
||||
let payload = make_element("rtpL16pay", "pay1")?;
|
||||
let source = source
|
||||
.dynamic_cast::<Element>()
|
||||
.map_err(|_| anyhow!("Cannot cast back"))?;
|
||||
|
||||
bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
|
||||
Element::link_many(&[&source, &queue, &parser, &decoder])?;
|
||||
Element::link_many(&[&encoder, &payload])?;
|
||||
decoder.connect_pad_added(move |_element, pad| {
|
||||
debug!("Linking encoder to decoder: {:?}", pad.caps());
|
||||
let sink_pad = encoder
|
||||
.static_pad("sink")
|
||||
.expect("Encoder is missing its pad");
|
||||
pad.link(&sink_pad)
|
||||
.expect("Failed to link AAC decoder to encoder");
|
||||
});
|
||||
let queue = make_element("queue", "audqueue")?;
|
||||
queue.set_property_from_str("leaky", "downstream");
|
||||
queue.set_property("max-size-bytes", 104857600u32);
|
||||
let parser = make_element("aacparse", "audparser")?;
|
||||
let decoder = make_element("decodebin", "auddecoder")?;
|
||||
let encoder = make_element("audioconvert", "audencoder")?;
|
||||
let payload = make_element("rtpL16pay", "audpayload")?;
|
||||
let storage = make_element("rtpstorage", "audstorage")?;
|
||||
storage
|
||||
.set_property("size-time", Duration::from_secs_f32(2.25).as_nanos() as u64);
|
||||
let jitter = make_element("rtpjitterbuffer", "pay1")?;
|
||||
jitter.set_property("latency", Duration::from_secs_f32(2.0).as_nanos() as u32);
|
||||
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
client_data.audsrc.replace(source);
|
||||
}
|
||||
AudFormats::Adpcm(block_size) => {
|
||||
debug!("Building Adpcm pipeline");
|
||||
// Original command line
|
||||
// caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000
|
||||
// ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024
|
||||
// ! adpcmdec
|
||||
// ! audioconvert
|
||||
// ! rtpL16pay name=pay1
|
||||
bin.add_many(&[
|
||||
&source, &queue, &parser, &decoder, &encoder, &payload, &storage, &jitter,
|
||||
])?;
|
||||
Element::link_many(&[&source, &queue, &parser, &decoder])?;
|
||||
Element::link_many(&[&encoder, &payload, &storage, &jitter])?;
|
||||
decoder.connect_pad_added(move |_element, pad| {
|
||||
debug!("Linking encoder to decoder: {:?}", pad.caps());
|
||||
let sink_pad = encoder
|
||||
.static_pad("sink")
|
||||
.expect("Encoder is missing its pad");
|
||||
pad.link(&sink_pad)
|
||||
.expect("Failed to link AAC decoder to encoder");
|
||||
});
|
||||
|
||||
let source = make_element("appsrc", "audsrc")?
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
|
||||
source.set_base_time(ClockTime::from_mseconds(
|
||||
self.shared.microseconds.load(Ordering::Relaxed),
|
||||
));
|
||||
source.set_is_live(true);
|
||||
source.set_block(false);
|
||||
source.set_property("emit-signals", false);
|
||||
source.set_max_bytes(52428800);
|
||||
source.set_do_timestamp(false);
|
||||
source.set_stream_type(AppStreamType::Seekable);
|
||||
source.set_caps(Some(
|
||||
&Caps::builder("audio/x-adpcm")
|
||||
.field("layout", "div")
|
||||
.field("block_align", block_size as i32)
|
||||
.field("channels", 1i32)
|
||||
.field("rate", 8000i32)
|
||||
.build(),
|
||||
));
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
client_data.update_aud(source);
|
||||
}
|
||||
AudFormats::Adpcm(block_size) => {
|
||||
debug!("Building Adpcm pipeline");
|
||||
// Original command line
|
||||
// caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000
|
||||
// ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024
|
||||
// ! adpcmdec
|
||||
// ! audioconvert
|
||||
// ! rtpL16pay name=pay1
|
||||
|
||||
let enough_data_start = client_data.enough_data.clone();
|
||||
let enough_data_stop = client_data.enough_data.clone();
|
||||
let seek_shared = self.shared.clone();
|
||||
let seek_start_time = client_data.start_time.clone();
|
||||
source.set_callbacks(
|
||||
AppSrcCallbacks::builder()
|
||||
.need_data(move |_appsrc, _amt| {
|
||||
if enough_data_stop.load(Ordering::Relaxed) {
|
||||
log::debug!("Client is in need of data");
|
||||
}
|
||||
enough_data_stop.store(false, Ordering::Relaxed);
|
||||
})
|
||||
.enough_data(move |_appsrc| {
|
||||
if !enough_data_start.load(Ordering::Relaxed) {
|
||||
log::debug!("Client is full of of data");
|
||||
}
|
||||
enough_data_start.store(true, Ordering::Relaxed);
|
||||
})
|
||||
.seek_data(move |_appsrc, seek_pos| {
|
||||
log::debug!("Seeking to {}", &seek_pos);
|
||||
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
|
||||
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
|
||||
true
|
||||
})
|
||||
.build(),
|
||||
);
|
||||
let source = make_element("appsrc", "audsrc")?
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
|
||||
source.set_is_live(true);
|
||||
source.set_block(false);
|
||||
source.set_property("emit-signals", false);
|
||||
source.set_max_bytes(52428800);
|
||||
source.set_do_timestamp(false);
|
||||
source.set_stream_type(AppStreamType::RandomAccess);
|
||||
source.set_caps(Some(
|
||||
&Caps::builder("audio/x-adpcm")
|
||||
.field("layout", "div")
|
||||
.field("block_align", block_size as i32)
|
||||
.field("channels", 1i32)
|
||||
.field("rate", 8000i32)
|
||||
.build(),
|
||||
));
|
||||
|
||||
let source = source
|
||||
.dynamic_cast::<Element>()
|
||||
.map_err(|_| anyhow!("Cannot cast back"))?;
|
||||
let need_command = client_data.get_commader();
|
||||
let enough_command = client_data.get_commader();
|
||||
let seek_command = client_data.get_commader();
|
||||
source.set_callbacks(
|
||||
AppSrcCallbacks::builder()
|
||||
.need_data(move |_appsrc, _amt| {
|
||||
let _ = need_command.blocking_send(NeoMediaSenderCommand::Resume);
|
||||
})
|
||||
.enough_data(move |_appsrc| {
|
||||
let _ = enough_command.blocking_send(NeoMediaSenderCommand::Pause);
|
||||
})
|
||||
.seek_data(move |_appsrc, seek_pos| {
|
||||
debug!("Send seek Adpcm");
|
||||
let _ = seek_command
|
||||
.blocking_send(NeoMediaSenderCommand::Seek(seek_pos));
|
||||
true
|
||||
})
|
||||
.build(),
|
||||
);
|
||||
|
||||
let queue = make_element("queue", "audqueue")?;
|
||||
queue.set_property_from_str("leaky", "downstream");
|
||||
queue.set_property("max-size-bytes", 104857600u32);
|
||||
queue.set_property("min-threshold-time", 500u64 * 1000u64);
|
||||
let decoder = make_element("decodebin", "auddecoder")?;
|
||||
let encoder = make_element("audioconvert", "audencoder")?;
|
||||
let payload = make_element("rtpL16pay", "pay1")?;
|
||||
let source = source
|
||||
.dynamic_cast::<Element>()
|
||||
.map_err(|_| anyhow!("Cannot cast back"))?;
|
||||
|
||||
bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?;
|
||||
Element::link_many(&[&source, &queue, &decoder])?;
|
||||
Element::link_many(&[&encoder, &payload])?;
|
||||
decoder.connect_pad_added(move |_element, pad| {
|
||||
debug!("Linking encoder to decoder: {:?}", pad.caps());
|
||||
let sink_pad = encoder
|
||||
.static_pad("sink")
|
||||
.expect("Encoder is missing its pad");
|
||||
pad.link(&sink_pad)
|
||||
.expect("Failed to link ADPCM decoder to encoder");
|
||||
});
|
||||
let queue = make_element("queue", "audqueue")?;
|
||||
queue.set_property_from_str("leaky", "downstream");
|
||||
queue.set_property("max-size-bytes", 104857600u32);
|
||||
let decoder = make_element("decodebin", "auddecoder")?;
|
||||
let encoder = make_element("audioconvert", "audencoder")?;
|
||||
let payload = make_element("rtpL16pay", "pay1")?;
|
||||
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
client_data.audsrc.replace(source);
|
||||
bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?;
|
||||
Element::link_many(&[&source, &queue, &decoder])?;
|
||||
Element::link_many(&[&encoder, &payload])?;
|
||||
decoder.connect_pad_added(move |_element, pad| {
|
||||
debug!("Linking encoder to decoder: {:?}", pad.caps());
|
||||
let sink_pad = encoder
|
||||
.static_pad("sink")
|
||||
.expect("Encoder is missing its pad");
|
||||
pad.link(&sink_pad)
|
||||
.expect("Failed to link ADPCM decoder to encoder");
|
||||
});
|
||||
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
|
||||
client_data.update_aud(source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+503
-320
@@ -2,363 +2,546 @@
|
||||
//! gstreamer media stream
|
||||
use anyhow::anyhow;
|
||||
use futures::stream::StreamExt;
|
||||
use gstreamer::ClockTime;
|
||||
use gstreamer::{prelude::*, ClockTime};
|
||||
use gstreamer_app::AppSrc;
|
||||
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
|
||||
use log::*;
|
||||
use neolink_core::bcmedia::model::*;
|
||||
use std::collections::{hash_map::Entry, HashMap};
|
||||
use std::sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
use std::collections::{
|
||||
VecDeque,
|
||||
{hash_map::Entry, HashMap},
|
||||
};
|
||||
use std::{
|
||||
iter::Iterator,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
// use tokio::time::{interval, Duration, MissedTickBehavior};
|
||||
use tokio::{
|
||||
sync::mpsc::{channel, Receiver, Sender},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
use super::{shared::*, AnyResult};
|
||||
|
||||
pub(super) struct NeoMediaSender {
|
||||
pub(super) data_source: ReceiverStream<BcMedia>,
|
||||
pub(super) clientsource: ReceiverStream<ClientPipelineData>,
|
||||
pub(super) shared: Arc<NeoMediaShared>,
|
||||
// Used to generate the key for the hashmaps
|
||||
pub(super) uid: AtomicU64,
|
||||
// Hashmap so we can alter the data in ClientData while also allowing for remove at any point
|
||||
// If you know a better collections pm me
|
||||
pub(super) clientdata: HashMap<u64, ClientPipelineData>,
|
||||
pub(super) waiting_for_iframe: bool,
|
||||
#[derive(Default)]
|
||||
struct NeoBuffer {
|
||||
buf: VecDeque<Vec<BcMedia>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct StampedData {
|
||||
ms: u64,
|
||||
data: Vec<u8>,
|
||||
impl NeoBuffer {
|
||||
fn push(&mut self, item: BcMedia) {
|
||||
match item {
|
||||
iframe @ BcMedia::Iframe(_) => self.buf.push_back(vec![iframe]),
|
||||
pframe @ BcMedia::Pframe(_) => {
|
||||
if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() {
|
||||
last.push(pframe);
|
||||
}
|
||||
}
|
||||
aac @ BcMedia::Aac(_) => {
|
||||
if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() {
|
||||
last.push(aac);
|
||||
}
|
||||
}
|
||||
adpcm @ BcMedia::Adpcm(_) => {
|
||||
if let Some(last) = self.buf.make_contiguous().last_mut().as_mut() {
|
||||
last.push(adpcm);
|
||||
}
|
||||
}
|
||||
BcMedia::InfoV1(_) | BcMedia::InfoV2(_) => {}
|
||||
}
|
||||
while self.buf.len() > 25 {
|
||||
// 25 iframes
|
||||
let _ = self.buf.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
fn start_time(&self) -> Option<u64> {
|
||||
if let Some(BcMedia::Iframe(data)) =
|
||||
self.buf.front().and_then(|inner_buf| inner_buf.first())
|
||||
{
|
||||
Some(data.microseconds as u64)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn end_time(&self) -> Option<u64> {
|
||||
if let Some(innerbuffer) = self.buf.back() {
|
||||
let mut last_ms = None;
|
||||
for frame in innerbuffer.iter() {
|
||||
match frame {
|
||||
BcMedia::Iframe(frame) => {
|
||||
let v = last_ms.get_or_insert(frame.microseconds as u64);
|
||||
if *v < frame.microseconds as u64 {
|
||||
*v = frame.microseconds as u64;
|
||||
}
|
||||
}
|
||||
BcMedia::Pframe(frame) => {
|
||||
let v = last_ms.get_or_insert(frame.microseconds as u64);
|
||||
if *v < frame.microseconds as u64 {
|
||||
*v = frame.microseconds as u64;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
last_ms
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct NeoMediaSenders {
|
||||
data_source: ReceiverStream<BcMedia>,
|
||||
client_source: ReceiverStream<NeoMediaSender>,
|
||||
shared: Arc<NeoMediaShared>,
|
||||
uid: AtomicU64,
|
||||
client_data: HashMap<u64, NeoMediaSender>,
|
||||
buffer: NeoBuffer,
|
||||
}
|
||||
|
||||
impl NeoMediaSenders {
|
||||
pub(super) fn new(
|
||||
shared: Arc<NeoMediaShared>,
|
||||
data_source: ReceiverStream<BcMedia>,
|
||||
client_source: ReceiverStream<NeoMediaSender>,
|
||||
) -> Self {
|
||||
Self {
|
||||
data_source,
|
||||
client_source,
|
||||
shared,
|
||||
uid: Default::default(),
|
||||
client_data: Default::default(),
|
||||
buffer: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_new_client(&mut self, mut client: NeoMediaSender) -> AnyResult<()> {
|
||||
client.initialise(&self.buffer).await?;
|
||||
self.client_data
|
||||
.insert(self.uid.fetch_add(1, Ordering::Relaxed), client);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_mediatypes(&self, data: &BcMedia) {
|
||||
match data {
|
||||
BcMedia::Iframe(BcMediaIframe {
|
||||
video_type: VideoType::H264,
|
||||
..
|
||||
}) => {
|
||||
if !matches!(*self.shared.vid_format.read().await, VidFormats::H264) {
|
||||
*self.shared.vid_format.write().await = VidFormats::H264;
|
||||
}
|
||||
}
|
||||
BcMedia::Pframe(BcMediaPframe {
|
||||
video_type: VideoType::H264,
|
||||
..
|
||||
}) => {
|
||||
if !matches!(*self.shared.vid_format.read().await, VidFormats::H264) {
|
||||
*self.shared.vid_format.write().await = VidFormats::H264;
|
||||
}
|
||||
}
|
||||
BcMedia::Iframe(BcMediaIframe {
|
||||
video_type: VideoType::H265,
|
||||
..
|
||||
}) => {
|
||||
if !matches!(*self.shared.vid_format.read().await, VidFormats::H265) {
|
||||
*self.shared.vid_format.write().await = VidFormats::H265;
|
||||
}
|
||||
}
|
||||
BcMedia::Pframe(BcMediaPframe {
|
||||
video_type: VideoType::H265,
|
||||
..
|
||||
}) => {
|
||||
if !matches!(*self.shared.vid_format.read().await, VidFormats::H265) {
|
||||
*self.shared.vid_format.write().await = VidFormats::H265;
|
||||
}
|
||||
}
|
||||
BcMedia::Aac(_) => {
|
||||
if !matches!(*self.shared.aud_format.read().await, AudFormats::Aac) {
|
||||
*self.shared.aud_format.write().await = AudFormats::Aac;
|
||||
}
|
||||
}
|
||||
BcMedia::Adpcm(data) => {
|
||||
if !matches!(*self.shared.aud_format.read().await, AudFormats::Adpcm(_)) {
|
||||
*self.shared.aud_format.write().await =
|
||||
AudFormats::Adpcm(data.data.len() as u16);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_new_data(&mut self, data: BcMedia) -> AnyResult<()> {
|
||||
// debug!("Handle new data");
|
||||
self.update_mediatypes(&data).await;
|
||||
for key in self.client_data.keys().copied().collect::<Vec<_>>() {
|
||||
// debug!(" - Client: {}", key);
|
||||
match self.client_data.entry(key) {
|
||||
Entry::Occupied(mut occ) => {
|
||||
if let Err(e) = occ.get_mut().initialise(&self.buffer).await {
|
||||
debug!("Could not init client: {:?}", e);
|
||||
occ.remove();
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = occ.get_mut().process_commands(&self.buffer).await {
|
||||
debug!("Could not process client command: {:?}", e);
|
||||
occ.remove();
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = occ.get_mut().process_jump_to_live(&self.buffer).await {
|
||||
debug!("Could not handle jump to live: {:?}", e);
|
||||
occ.remove();
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = occ.get_mut().process_buffer(&self.buffer).await {
|
||||
debug!("Could not send client data: {:?}", e);
|
||||
occ.remove();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Entry::Vacant(_) => {}
|
||||
}
|
||||
}
|
||||
self.buffer.push(data);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn run(&mut self) -> AnyResult<()> {
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(v) = self.data_source.next() => {
|
||||
self.handle_new_data(v).await
|
||||
},
|
||||
Some(v) = self.client_source.next() => {
|
||||
self.handle_new_client(v).await
|
||||
},
|
||||
else => {
|
||||
Err(anyhow!("Sender data source closed"))
|
||||
}
|
||||
}?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) enum NeoMediaSenderCommand {
|
||||
Seek(u64),
|
||||
Pause,
|
||||
Resume,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub(super) struct NeoMediaSender {
|
||||
needle: u64,
|
||||
start_time: u64,
|
||||
vid: Option<AppSrc>,
|
||||
aud: Option<AppSrc>,
|
||||
command_reciever: Receiver<NeoMediaSenderCommand>,
|
||||
command_sender: Sender<NeoMediaSenderCommand>,
|
||||
inited: bool,
|
||||
playing: bool,
|
||||
}
|
||||
|
||||
impl NeoMediaSender {
|
||||
pub(super) async fn run(&mut self) -> AnyResult<()> {
|
||||
// let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32));
|
||||
// resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip);
|
||||
let mut buffer_a: Vec<StampedData> = Default::default();
|
||||
let mut buffer_b: Vec<StampedData> = Default::default();
|
||||
let buffer_read = &mut buffer_a;
|
||||
let buffer_write = &mut buffer_b;
|
||||
pub(super) fn new() -> Self {
|
||||
let (tx, rx) = channel(3);
|
||||
Self {
|
||||
needle: 0,
|
||||
start_time: 0,
|
||||
vid: None,
|
||||
aud: None,
|
||||
command_reciever: rx,
|
||||
command_sender: tx,
|
||||
inited: false,
|
||||
playing: true,
|
||||
}
|
||||
}
|
||||
|
||||
let mut audbuffer_a: Vec<StampedData> = Default::default();
|
||||
let mut audbuffer_b: Vec<StampedData> = Default::default();
|
||||
let audbuffer_read = &mut audbuffer_a;
|
||||
let audbuffer_write = &mut audbuffer_b;
|
||||
loop {
|
||||
self.shared
|
||||
.number_of_clients
|
||||
.store(self.clientdata.len(), Ordering::Relaxed);
|
||||
self.shared
|
||||
.buffer_ready
|
||||
.store(!buffer_read.is_empty(), Ordering::Relaxed);
|
||||
// debug!("Sender: Get");
|
||||
tokio::select! {
|
||||
v = self.data_source.next() => {
|
||||
// debug!("Sender: Got Data");
|
||||
// debug!("data_source recieved");
|
||||
if let Some(bc_media) = v {
|
||||
if ! self.skip_bcmedia(&bc_media) {
|
||||
// debug!("Not skipped");
|
||||
// resend_pause.reset();
|
||||
self.inspect_bcmedia(&bc_media).await;
|
||||
match bc_media {
|
||||
BcMedia::Iframe(frame) => {
|
||||
if buffer_write.len() > 25 {
|
||||
std::mem::swap(buffer_read, buffer_write);
|
||||
buffer_write.clear();
|
||||
std::mem::swap(audbuffer_read, audbuffer_write);
|
||||
audbuffer_write.clear();
|
||||
}
|
||||
let new_data = StampedData {
|
||||
ms: frame.microseconds as u64,
|
||||
data: frame.data,
|
||||
};
|
||||
self.process_vidbuffer(&new_data).await;
|
||||
buffer_write.push(new_data);
|
||||
}
|
||||
BcMedia::Pframe(frame) => {
|
||||
let new_data = StampedData {
|
||||
ms: frame.microseconds as u64,
|
||||
data: frame.data,
|
||||
};
|
||||
self.process_vidbuffer(&new_data).await;
|
||||
buffer_write.push(new_data);
|
||||
}
|
||||
BcMedia::Aac(aac) => {
|
||||
if let Some(last) = buffer_write.last().as_ref() {
|
||||
let new_data = StampedData {
|
||||
ms: last.ms,
|
||||
data: aac.data,
|
||||
};
|
||||
self.process_audbuffer(&new_data).await;
|
||||
audbuffer_write.push(new_data);
|
||||
}
|
||||
pub(super) fn update_vid(&mut self, source: AppSrc) {
|
||||
self.vid.replace(source);
|
||||
}
|
||||
|
||||
}
|
||||
BcMedia::Adpcm(adpcm) => {
|
||||
if let Some(last) = buffer_write.last().as_ref() {
|
||||
let new_data = StampedData {
|
||||
ms: last.ms,
|
||||
data: adpcm.data,
|
||||
};
|
||||
self.process_audbuffer(&new_data).await;
|
||||
audbuffer_write.push(new_data);
|
||||
}
|
||||
pub(super) fn update_aud(&mut self, source: AppSrc) {
|
||||
self.aud.replace(source);
|
||||
}
|
||||
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
pub(super) fn get_commader(&self) -> Sender<NeoMediaSenderCommand> {
|
||||
self.command_sender.clone()
|
||||
}
|
||||
|
||||
async fn jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
|
||||
if self.needle > self.start_time {
|
||||
self.seek(self.needle - self.start_time, buffer).await
|
||||
} else {
|
||||
self.seek(0, buffer).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn seek(&mut self, dest: u64, buffer: &NeoBuffer) -> AnyResult<()> {
|
||||
if let (Some(first_ms), Some(last_ms)) = (buffer.start_time(), buffer.end_time()) {
|
||||
let target_seek = self.start_time + dest;
|
||||
if target_seek >= first_ms && target_seek <= last_ms {
|
||||
debug!("Seeking inside buffer: {}", target_seek);
|
||||
self.needle = target_seek;
|
||||
self.process_buffer(buffer).await?;
|
||||
debug!("Seeked inside buffer");
|
||||
} else {
|
||||
// Fake seek
|
||||
debug!(
|
||||
"Seeking outside buffer: {} - {} = {}",
|
||||
self.needle,
|
||||
target_seek,
|
||||
self.needle - dest
|
||||
);
|
||||
self.start_time = self.needle - dest;
|
||||
}
|
||||
} else {
|
||||
// Buffers arent valid yet
|
||||
debug!("Seek buffers are not ready: {}", dest);
|
||||
debug!(" - {:?}", (buffer.start_time(), buffer.end_time()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn process_commands(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
|
||||
if self.inited {
|
||||
if let Ok(command) = self.command_reciever.try_recv() {
|
||||
match command {
|
||||
NeoMediaSenderCommand::Seek(dest) => {
|
||||
debug!("Got Seek Request: {}", dest);
|
||||
self.seek(dest, buffer).await?;
|
||||
}
|
||||
NeoMediaSenderCommand::Pause => {
|
||||
if self.playing {
|
||||
debug!("Pausing");
|
||||
self.playing = false;
|
||||
}
|
||||
}
|
||||
NeoMediaSenderCommand::Resume => {
|
||||
if !self.playing {
|
||||
debug!("Resuming");
|
||||
self.playing = true;
|
||||
self.jump_to_live(buffer).await?;
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
v = self.clientsource.next() => {
|
||||
// debug!("Sender: Got Client");
|
||||
if let Some(mut clientdata) = v {
|
||||
// Resend the video and audio buffers
|
||||
if !clientdata.inited {
|
||||
clientdata.inited = true;
|
||||
if let Some(last) = buffer_read.first() {
|
||||
debug!("Start PTS SET: {:?}", tokio::time::Duration::from_micros(last.ms));
|
||||
clientdata.set_start_time(last.ms);
|
||||
}
|
||||
let mut vid_resend = buffer_read.iter().chain(buffer_write.iter());
|
||||
let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter());
|
||||
loop {
|
||||
let next_vid = vid_resend.next();
|
||||
if let Some(next_vid) = next_vid.as_ref() {
|
||||
if let Some(src) = clientdata.vidsrc.as_ref() {
|
||||
if Self::send_buffer(
|
||||
src,
|
||||
next_vid.data.as_slice(),
|
||||
next_vid.ms,
|
||||
clientdata.get_start_time(),
|
||||
).await.is_err() {
|
||||
break;
|
||||
};
|
||||
}
|
||||
}
|
||||
let next_aud = aud_resend.next();
|
||||
if let Some(next_aud) = next_aud.as_ref() {
|
||||
if next_aud.ms >= clientdata.get_start_time() {
|
||||
if let Some(src) = clientdata.audsrc.as_ref() {
|
||||
if Self::send_buffer(
|
||||
src,
|
||||
next_aud.data.as_slice(),
|
||||
next_aud.ms,
|
||||
clientdata.get_start_time(),
|
||||
).await.is_err() {
|
||||
break;
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
if next_vid.is_none() && next_aud.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
debug!("Advance PTS Pushed");
|
||||
}
|
||||
// Save the client for future sends
|
||||
self.clientdata.insert(self.uid.fetch_add(1, Ordering::Relaxed) , clientdata);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
},
|
||||
// _ = resend_pause.tick() => {
|
||||
// if let Some(live) = buffer.last().cloned() {
|
||||
// self.process_vidbuffer(&ProcessData {
|
||||
// live,
|
||||
// resend: buffer.as_slice()
|
||||
// })?;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn process_vidbuffer(&mut self, stamped_data: &StampedData) {
|
||||
for key in self
|
||||
.clientdata
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
.drain(..)
|
||||
{
|
||||
match self.clientdata.entry(key) {
|
||||
Entry::Occupied(data) => {
|
||||
if let Some(vidsrc) = data.get().vidsrc.as_ref() {
|
||||
if !data.get().enough_data.load(Ordering::Relaxed)
|
||||
&& Self::send_buffer(
|
||||
vidsrc,
|
||||
stamped_data.data.as_slice(),
|
||||
stamped_data.ms,
|
||||
data.get().get_start_time(),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
data.remove();
|
||||
async fn initialise(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
|
||||
if !self.inited {
|
||||
if let (Some(first_ms), Some(end_ms)) = (buffer.start_time(), buffer.end_time()) {
|
||||
// if let Some(source) = self.vid.as_ref() {
|
||||
// source.set_base_time(ClockTime::from_mseconds(0));
|
||||
// }
|
||||
// if let Some(source) = self.aud.as_ref() {
|
||||
// source.set_base_time(ClockTime::from_mseconds(0));
|
||||
// }
|
||||
|
||||
self.start_time = (first_ms + end_ms) / 2;
|
||||
self.needle = self.start_time;
|
||||
|
||||
self.inited = true;
|
||||
|
||||
self.process_buffer(buffer).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn process_buffer(&mut self, buf: &NeoBuffer) -> AnyResult<()> {
|
||||
if self.inited && self.playing {
|
||||
let mut buf_it = buf.buf.iter().peekable();
|
||||
while let Some(buffer) = buf_it.next() {
|
||||
tokio::task::yield_now().await;
|
||||
let next = buf_it.peek();
|
||||
// Look at the next frame set and see if it is after the needle (or if empty)
|
||||
let deep = next
|
||||
.map(|buffer| {
|
||||
buffer
|
||||
.first()
|
||||
.map(|frame| {
|
||||
if let BcMedia::Iframe(data) = frame {
|
||||
data.microseconds as u64 >= self.needle
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
})
|
||||
.unwrap_or(true) // No first data
|
||||
})
|
||||
.unwrap_or(true); // Next buffer is empty
|
||||
// This frame set contains our data process it
|
||||
if deep {
|
||||
let mut send_it = false;
|
||||
for frame in buffer.iter() {
|
||||
if !send_it {
|
||||
let microseconds = match frame {
|
||||
BcMedia::Iframe(data) => Some(data.microseconds),
|
||||
BcMedia::Pframe(data) => Some(data.microseconds),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(microseconds) = microseconds {
|
||||
if microseconds as u64 > self.needle {
|
||||
send_it = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
data.remove();
|
||||
}
|
||||
}
|
||||
Entry::Vacant(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_audbuffer(&mut self, stamped_data: &StampedData) {
|
||||
for key in self
|
||||
.clientdata
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
.drain(..)
|
||||
{
|
||||
match self.clientdata.entry(key) {
|
||||
Entry::Occupied(data) => {
|
||||
if let Some(audsrc) = data.get().audsrc.as_ref() {
|
||||
if !data.get().enough_data.load(Ordering::Relaxed)
|
||||
&& Self::send_buffer(
|
||||
audsrc,
|
||||
stamped_data.data.as_slice(),
|
||||
stamped_data.ms,
|
||||
data.get().get_start_time(),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
data.remove();
|
||||
if send_it && !self.send_buffer(frame).await? {
|
||||
// Buffer too far is ahead of needle early exit
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn process_jump_to_live(&mut self, buffer: &NeoBuffer) -> AnyResult<()> {
|
||||
if let Some(first_buftime) = buffer.start_time() {
|
||||
if first_buftime > self.needle {
|
||||
debug!("Jump to live first_buftime > self.needle");
|
||||
self.jump_to_live(buffer).await?;
|
||||
} else if let Some(runtime) = self.get_runtime() {
|
||||
if first_buftime > self.start_time {
|
||||
let buftime = Duration::from_micros(first_buftime - self.start_time);
|
||||
if buftime > runtime {
|
||||
debug!("Jump to live buftime > runtime");
|
||||
self.jump_to_live(buffer).await?;
|
||||
}
|
||||
} else {
|
||||
debug!("Jump to live first_buftime > self.start_time ");
|
||||
self.jump_to_live(buffer).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_runtime(&self) -> Option<Duration> {
|
||||
if let Some(appsrc) = self.vid.as_ref() {
|
||||
if let Some(clock) = appsrc.clock() {
|
||||
if let Some(time) = clock.time() {
|
||||
if let Some(base_time) = appsrc.base_time() {
|
||||
let runtime = time - base_time;
|
||||
return Some(Duration::from_nanos(runtime.nseconds()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn send_buffer(&mut self, media: &BcMedia) -> AnyResult<bool> {
|
||||
if self.inited && self.playing {
|
||||
let microseconds = match media {
|
||||
BcMedia::Iframe(data) => {
|
||||
if data.microseconds as u64 > self.start_time {
|
||||
data.microseconds as u64 - self.start_time
|
||||
} else {
|
||||
data.remove();
|
||||
0
|
||||
}
|
||||
}
|
||||
Entry::Vacant(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_buffer(
|
||||
appsrc: &AppSrc,
|
||||
buf: &[u8],
|
||||
frame_ms: u64,
|
||||
start_time: u64,
|
||||
) -> AnyResult<()> {
|
||||
let micros = if frame_ms >= start_time {
|
||||
frame_ms - start_time
|
||||
} else {
|
||||
0
|
||||
};
|
||||
debug!("PTS: {:?}", tokio::time::Duration::from_micros(micros));
|
||||
|
||||
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
|
||||
{
|
||||
let gst_buf_mut = gst_buf.get_mut().unwrap();
|
||||
|
||||
let time = ClockTime::from_useconds(micros);
|
||||
gst_buf_mut.set_pts(time);
|
||||
// debug!(
|
||||
// "PTS set to: {:?} ({}-{}={:?})",
|
||||
// time, frame_ms, start_time, micros
|
||||
// );
|
||||
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
|
||||
gst_buf_data.copy_from_slice(buf);
|
||||
}
|
||||
// debug!("Buffer pushed");
|
||||
let thread_appsrc = appsrc.clone(); // GObjects are refcounted
|
||||
let res = tokio::task::spawn_blocking(move || {
|
||||
thread_appsrc
|
||||
.push_buffer(gst_buf.copy())
|
||||
.map(|_| ())
|
||||
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
|
||||
})
|
||||
.await;
|
||||
match &res {
|
||||
Err(e) => {
|
||||
debug!("Failed to send buffer: {:?}", e);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
debug!("Failed to send buffer: {:?}", e);
|
||||
}
|
||||
Ok(Ok(_)) => {}
|
||||
};
|
||||
res?
|
||||
}
|
||||
|
||||
fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> bool {
|
||||
if self.waiting_for_iframe {
|
||||
if let BcMedia::Iframe(_) = bc_media {
|
||||
self.waiting_for_iframe = false;
|
||||
} else {
|
||||
log::debug!("Skipping bcmedia");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn inspect_bcmedia(&mut self, bc_media: &BcMedia) {
|
||||
let old_vid = *self.shared.vid_format.read().await;
|
||||
let old_aud = *self.shared.aud_format.read().await;
|
||||
match bc_media {
|
||||
BcMedia::Iframe(frame) => {
|
||||
match frame.video_type {
|
||||
VideoType::H264 => {
|
||||
(*self.shared.vid_format.write().await) = VidFormats::H264;
|
||||
}
|
||||
VideoType::H265 => {
|
||||
(*self.shared.vid_format.write().await) = VidFormats::H265;
|
||||
BcMedia::Pframe(data) => {
|
||||
if data.microseconds as u64 > self.start_time {
|
||||
data.microseconds as u64 - self.start_time
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
self.shared
|
||||
.microseconds
|
||||
.store(frame.microseconds as u64, Ordering::Relaxed);
|
||||
// log::debug!(
|
||||
// "Time set to {}",
|
||||
// self.shared.microseconds.load(Ordering::Relaxed)
|
||||
// );
|
||||
}
|
||||
BcMedia::Pframe(frame) => {
|
||||
match frame.video_type {
|
||||
VideoType::H264 => {
|
||||
(*self.shared.vid_format.write().await) = VidFormats::H264;
|
||||
}
|
||||
VideoType::H265 => {
|
||||
(*self.shared.vid_format.write().await) = VidFormats::H265;
|
||||
_ => {
|
||||
if self.needle > self.start_time {
|
||||
self.needle - self.start_time
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
self.shared
|
||||
.microseconds
|
||||
.store(frame.microseconds as u64, Ordering::Relaxed);
|
||||
// log::debug!(
|
||||
// "Time set to {}",
|
||||
// self.shared.microseconds.load(Ordering::Relaxed)
|
||||
// );
|
||||
};
|
||||
let buf = match media {
|
||||
BcMedia::Iframe(data) => Some(&data.data),
|
||||
BcMedia::Pframe(data) => Some(&data.data),
|
||||
BcMedia::Aac(data) => Some(&data.data),
|
||||
BcMedia::Adpcm(data) => Some(&data.data),
|
||||
_ => None,
|
||||
};
|
||||
let appsrc = match media {
|
||||
BcMedia::Iframe(_) | BcMedia::Pframe(_) => self.vid.as_ref(),
|
||||
BcMedia::Aac(_) | BcMedia::Adpcm(_) => self.aud.as_ref(),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
if let (Some(buf), Some(appsrc)) = (buf, appsrc) {
|
||||
if let Some(runtime) = self.get_runtime() {
|
||||
let buftimr = Duration::from_micros(microseconds);
|
||||
if buftimr > runtime + Duration::from_micros(2000) {
|
||||
// Stop advancing the buffer
|
||||
return Ok(false);
|
||||
} else {
|
||||
debug!(" - runtime: {:?}", runtime);
|
||||
debug!(" - buftime: {:?}", buftimr);
|
||||
}
|
||||
}
|
||||
|
||||
debug!("PTS: {:?}", Duration::from_micros(microseconds));
|
||||
|
||||
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
|
||||
{
|
||||
let gst_buf_mut = gst_buf.get_mut().unwrap();
|
||||
|
||||
let time = ClockTime::from_useconds(microseconds);
|
||||
gst_buf_mut.set_pts(time);
|
||||
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
|
||||
gst_buf_data.copy_from_slice(buf);
|
||||
}
|
||||
// debug!("Buffer pushed");
|
||||
let thread_appsrc = appsrc.clone(); // GObjects are refcounted
|
||||
let res = tokio::task::spawn_blocking(move || {
|
||||
thread_appsrc
|
||||
.push_buffer(gst_buf.copy())
|
||||
.map(|_| ())
|
||||
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
|
||||
})
|
||||
.await;
|
||||
match &res {
|
||||
Err(e) => {
|
||||
debug!("Failed to send buffer: {:?}", e);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
debug!("Failed to send buffer: {:?}", e);
|
||||
}
|
||||
Ok(Ok(_)) => {}
|
||||
};
|
||||
res??;
|
||||
// Buffer sent sucessfully
|
||||
//
|
||||
// Update the needle position
|
||||
match media {
|
||||
BcMedia::Iframe(data) => {
|
||||
if data.microseconds as u64 > self.needle {
|
||||
// debug!("Updating needle to: {}", data.microseconds);
|
||||
self.needle = data.microseconds as u64;
|
||||
}
|
||||
}
|
||||
BcMedia::Pframe(data) => {
|
||||
if data.microseconds as u64 > self.needle {
|
||||
// debug!("Updating needle to: {}", data.microseconds);
|
||||
self.needle = data.microseconds as u64;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
BcMedia::Aac(_aac) => {
|
||||
(*self.shared.aud_format.write().await) = AudFormats::Aac;
|
||||
}
|
||||
BcMedia::Adpcm(adpcm) => {
|
||||
(*self.shared.aud_format.write().await) = AudFormats::Adpcm(adpcm.data.len() as u16)
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let new_vid = *self.shared.vid_format.read().await;
|
||||
if new_vid != old_vid {
|
||||
log::debug!("Video format set to: {:?} from {:?}", new_vid, old_vid);
|
||||
}
|
||||
let new_aud = *self.shared.aud_format.read().await;
|
||||
if old_aud != new_aud {
|
||||
log::debug!("Audio format set to: {:?} from {:?}", new_aud, old_aud);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
+1
-23
@@ -1,9 +1,7 @@
|
||||
//! Data shared between the various
|
||||
//! components that manage a media stream
|
||||
use gstreamer_app::AppSrc;
|
||||
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
@@ -23,7 +21,6 @@ pub(super) enum AudFormats {
|
||||
pub(super) struct NeoMediaShared {
|
||||
pub(super) vid_format: RwLock<VidFormats>,
|
||||
pub(super) aud_format: RwLock<AudFormats>,
|
||||
pub(super) microseconds: AtomicU64,
|
||||
pub(super) number_of_clients: AtomicUsize,
|
||||
pub(super) buffer_ready: AtomicBool,
|
||||
}
|
||||
@@ -33,27 +30,8 @@ impl Default for NeoMediaShared {
|
||||
Self {
|
||||
vid_format: RwLock::new(VidFormats::Unknown),
|
||||
aud_format: RwLock::new(AudFormats::Unknown),
|
||||
microseconds: AtomicU64::new(0),
|
||||
number_of_clients: AtomicUsize::new(0),
|
||||
buffer_ready: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub(super) struct ClientPipelineData {
|
||||
pub(super) vidsrc: Option<AppSrc>,
|
||||
pub(super) audsrc: Option<AppSrc>,
|
||||
pub(super) start_time: Arc<AtomicU64>,
|
||||
pub(super) inited: bool,
|
||||
pub(super) enough_data: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl ClientPipelineData {
|
||||
pub(super) fn get_start_time(&self) -> u64 {
|
||||
self.start_time.load(Ordering::Relaxed)
|
||||
}
|
||||
pub(super) fn set_start_time(&self, time: u64) {
|
||||
self.start_time.store(time, Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user