Implement fake seek for VLC

This commit is contained in:
Andrew committed 2023-04-20 15:40:58 +07:00
1 parent 81a573ba6d
commit c4c35d1bde
3 files changed
+140 -63

No files matched your search

+92 -10
View File
@@ -23,7 +23,10 @@ use log::*;
use neolink_core::bcmedia::model::*;
use std::{
collections::HashSet,
sync::{atomic::Ordering, Arc},
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
};
use tokio::{
sync::mpsc::{channel, Sender},
@@ -118,7 +121,7 @@ pub(crate) struct NeoMediaFactoryImpl {
impl Drop for NeoMediaFactoryImpl {
fn drop(&mut self) {
log::info!("Dopping NeoMediaFactoryImpl");
log::debug!("Dopping NeoMediaFactoryImpl");
}
}
@@ -223,15 +226,15 @@ impl NeoMediaFactoryImpl {
let bin = media
.dynamic_cast::<Bin>()
.map_err(|_| anyhow!("Media source's element should be a bin"))?;
let clock = bin.clock();
// Clear the autogenerated ones
for element in bin.iterate_elements().into_iter().flatten() {
bin.remove(&element)?;
}
let mut client_data = ClientPipelineData {
start_time: self.shared.microseconds.load(Ordering::Relaxed),
clock,
start_time: Arc::new(AtomicU64::new(
self.shared.microseconds.load(Ordering::Relaxed),
)),
..Default::default()
};
@@ -250,10 +253,12 @@ impl NeoMediaFactoryImpl {
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Stream);
source.set_stream_type(AppStreamType::Seekable);
let enough_data_start = client_data.enough_data.clone();
let enough_data_stop = client_data.enough_data.clone();
let seek_shared = self.shared.clone();
let seek_start_time = client_data.start_time.clone();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
@@ -268,6 +273,12 @@ impl NeoMediaFactoryImpl {
}
enough_data_start.store(true, Ordering::Relaxed);
})
.seek_data(move |_appsrc, seek_pos| {
log::debug!("Seeking to {}", &seek_pos);
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
true
})
.build(),
);
// source.set_caps(Some(
@@ -279,7 +290,9 @@ impl NeoMediaFactoryImpl {
let queue = make_element("queue", "source_queue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
queue.set_property("min-threshold-time", 500u64 * 1000u64);
let parser = make_element("h265parse", "parser")?;
// parser.set_property("config-interval", 5i32);
let payload = make_element("rtph265pay", "pay0")?;
bin.add_many(&[&source, &queue, &parser, &payload])?;
Element::link_many(&[&source, &queue, &parser, &payload])?;
@@ -302,10 +315,12 @@ impl NeoMediaFactoryImpl {
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Stream);
source.set_stream_type(AppStreamType::Seekable);
let enough_data_start = client_data.enough_data.clone();
let enough_data_stop = client_data.enough_data.clone();
let seek_shared = self.shared.clone();
let seek_start_time = client_data.start_time.clone();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
@@ -320,6 +335,12 @@ impl NeoMediaFactoryImpl {
}
enough_data_start.store(true, Ordering::Relaxed);
})
.seek_data(move |_appsrc, seek_pos| {
log::debug!("Seeking to {}", &seek_pos);
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
true
})
.build(),
);
// source.set_caps(Some(
@@ -330,9 +351,11 @@ impl NeoMediaFactoryImpl {
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_element("queue", "source_queue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("min-threshold-time", 500u64 * 1000u64);
let parser = make_element("h264parse", "parser")?;
// parser.set_property("update-timecode", true);
let payload = make_element("rtph264pay", "pay0")?;
// payload.set_property("config-interval", 5i32);
bin.add_many(&[&source, &queue, &parser, &payload])?;
Element::link_many(&[&source, &queue, &parser, &payload])?;
@@ -348,6 +371,7 @@ impl NeoMediaFactoryImpl {
let queue = make_element("queue", "queue0")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
queue.set_property("min-threshold-time", 500u64 * 1000u64 * 1000u64);
let overlay = make_element("textoverlay", "overlay")?;
overlay.set_property("text", "Stream not Ready");
overlay.set_property_from_str("valignment", "top");
@@ -388,7 +412,35 @@ impl NeoMediaFactoryImpl {
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Stream);
source.set_stream_type(AppStreamType::Seekable);
let enough_data_start = client_data.enough_data.clone();
let enough_data_stop = client_data.enough_data.clone();
let seek_shared = self.shared.clone();
let seek_start_time = client_data.start_time.clone();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
if enough_data_stop.load(Ordering::Relaxed) {
log::debug!("Client is in need of data");
}
enough_data_stop.store(false, Ordering::Relaxed);
})
.enough_data(move |_appsrc| {
if !enough_data_start.load(Ordering::Relaxed) {
log::debug!("Client is full of of data");
}
enough_data_start.store(true, Ordering::Relaxed);
})
.seek_data(move |_appsrc, seek_pos| {
log::debug!("Seeking to {}", &seek_pos);
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
true
})
.build(),
);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
@@ -396,6 +448,7 @@ impl NeoMediaFactoryImpl {
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
queue.set_property("min-threshold-time", 500u64 * 1000u64);
let parser = make_element("aacparse", "audparser")?;
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
@@ -438,7 +491,7 @@ impl NeoMediaFactoryImpl {
source.set_property("emit-signals", false);
source.set_max_bytes(52428800);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Stream);
source.set_stream_type(AppStreamType::Seekable);
source.set_caps(Some(
&Caps::builder("audio/x-adpcm")
.field("layout", "div")
@@ -447,6 +500,34 @@ impl NeoMediaFactoryImpl {
.field("rate", 8000i32)
.build(),
));
let enough_data_start = client_data.enough_data.clone();
let enough_data_stop = client_data.enough_data.clone();
let seek_shared = self.shared.clone();
let seek_start_time = client_data.start_time.clone();
source.set_callbacks(
AppSrcCallbacks::builder()
.need_data(move |_appsrc, _amt| {
if enough_data_stop.load(Ordering::Relaxed) {
log::debug!("Client is in need of data");
}
enough_data_stop.store(false, Ordering::Relaxed);
})
.enough_data(move |_appsrc| {
if !enough_data_start.load(Ordering::Relaxed) {
log::debug!("Client is full of of data");
}
enough_data_start.store(true, Ordering::Relaxed);
})
.seek_data(move |_appsrc, seek_pos| {
log::debug!("Seeking to {}", &seek_pos);
let current_time = seek_shared.microseconds.load(Ordering::Relaxed);
seek_start_time.store(current_time - seek_pos, Ordering::Relaxed);
true
})
.build(),
);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
@@ -454,6 +535,7 @@ impl NeoMediaFactoryImpl {
let queue = make_element("queue", "audqueue")?;
queue.set_property_from_str("leaky", "downstream");
queue.set_property("max-size-bytes", 104857600u32);
queue.set_property("min-threshold-time", 500u64 * 1000u64);
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
+37 -49
View File
@@ -2,7 +2,7 @@
//! gstreamer media stream
use anyhow::anyhow;
use futures::stream::StreamExt;
use gstreamer::{prelude::*, ClockTime};
use gstreamer::ClockTime;
use gstreamer_app::AppSrc;
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
use log::*;
@@ -67,7 +67,7 @@ impl NeoMediaSender {
self.inspect_bcmedia(&bc_media).await;
match bc_media {
BcMedia::Iframe(frame) => {
if buffer_write.len() > 100 {
if buffer_write.len() > 25 {
std::mem::swap(buffer_read, buffer_write);
buffer_write.clear();
std::mem::swap(audbuffer_read, audbuffer_write);
@@ -123,39 +123,46 @@ impl NeoMediaSender {
// Resend the video and audio buffers
if !clientdata.inited {
clientdata.inited = true;
let mut vid_resend = buffer_read.iter().chain(buffer_write.iter()).enumerate();
let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter()).enumerate();
if let Some(last) = buffer_read.first() {
debug!("Start PTS SET: {:?}", tokio::time::Duration::from_micros(last.ms));
clientdata.set_start_time(last.ms);
}
let mut vid_resend = buffer_read.iter().chain(buffer_write.iter());
let mut aud_resend = audbuffer_read.iter().chain(audbuffer_write.iter());
loop {
let next_vid = vid_resend.next();
if let Some((idx, next_vid)) = next_vid.as_ref() {
if let Some(next_vid) = next_vid.as_ref() {
if let Some(src) = clientdata.vidsrc.as_ref() {
if Self::send_buffer(
src,
next_vid.data.as_slice(),
*idx as u64,
0,
next_vid.ms,
clientdata.get_start_time(),
).await.is_err() {
break;
};
}
}
let next_aud = aud_resend.next();
if let Some((idx, next_aud)) = next_aud.as_ref() {
if let Some(src) = clientdata.audsrc.as_ref() {
if Self::send_buffer(
src,
next_aud.data.as_slice(),
*idx as u64,
0,
).await.is_err() {
break;
};
if let Some(next_aud) = next_aud.as_ref() {
if next_aud.ms >= clientdata.get_start_time() {
if let Some(src) = clientdata.audsrc.as_ref() {
if Self::send_buffer(
src,
next_aud.data.as_slice(),
next_aud.ms,
clientdata.get_start_time(),
).await.is_err() {
break;
};
}
}
}
if next_vid.is_none() && next_aud.is_none() {
break;
}
}
debug!("Advance PTS Pushed");
}
// Save the client for future sends
self.clientdata.insert(self.uid.fetch_add(1, Ordering::Relaxed) , clientdata);
@@ -187,35 +194,17 @@ impl NeoMediaSender {
match self.clientdata.entry(key) {
Entry::Occupied(data) => {
if let Some(vidsrc) = data.get().vidsrc.as_ref() {
if !data.get().enough_data.load(Ordering::Relaxed) {
let ms = stamped_data.ms;
let st = data.get().start_time;
let micros = ms - st;
if let Some(clock) = data.get().clock.as_ref() {
if let Some(time) = clock.time() {
let buffer_time = ClockTime::from_useconds(micros);
log::debug!(
"Buffer time: {:?}. Clock time: {:?}",
buffer_time,
time
);
}
}
debug!(
"Pushed Video Buffer: ms: {}, st: {}, delta: {}",
ms, st, micros
);
if Self::send_buffer(
if !data.get().enough_data.load(Ordering::Relaxed)
&& Self::send_buffer(
vidsrc,
stamped_data.data.as_slice(),
stamped_data.ms,
data.get().start_time,
data.get().get_start_time(),
)
.await
.is_err()
{
data.remove();
}
{
data.remove();
}
} else {
data.remove();
@@ -237,19 +226,17 @@ impl NeoMediaSender {
match self.clientdata.entry(key) {
Entry::Occupied(data) => {
if let Some(audsrc) = data.get().audsrc.as_ref() {
if !data.get().enough_data.load(Ordering::Relaxed) {
debug!("Pushed Audio Buffer");
if Self::send_buffer(
if !data.get().enough_data.load(Ordering::Relaxed)
&& Self::send_buffer(
audsrc,
stamped_data.data.as_slice(),
stamped_data.ms,
data.get().start_time,
data.get().get_start_time(),
)
.await
.is_err()
{
data.remove();
}
{
data.remove();
}
} else {
data.remove();
@@ -266,11 +253,12 @@ impl NeoMediaSender {
frame_ms: u64,
start_time: u64,
) -> AnyResult<()> {
let micros = if frame_ms > start_time {
let micros = if frame_ms >= start_time {
frame_ms - start_time
} else {
start_time
0
};
debug!("PTS: {:?}", tokio::time::Duration::from_micros(micros));
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
+11 -4
View File
@@ -1,9 +1,8 @@
//! Data shared between the various
//! components that manage a media stream
use gstreamer::Clock;
use gstreamer_app::AppSrc;
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::Arc;
use tokio::sync::RwLock;
@@ -45,8 +44,16 @@ impl Default for NeoMediaShared {
pub(super) struct ClientPipelineData {
pub(super) vidsrc: Option<AppSrc>,
pub(super) audsrc: Option<AppSrc>,
pub(super) start_time: u64,
pub(super) start_time: Arc<AtomicU64>,
pub(super) inited: bool,
pub(super) clock: Option<Clock>,
pub(super) enough_data: Arc<AtomicBool>,
}
impl ClientPipelineData {
pub(super) fn get_start_time(&self) -> u64 {
self.start_time.load(Ordering::Relaxed)
}
pub(super) fn set_start_time(&self, time: u64) {
self.start_time.store(time, Ordering::Relaxed)
}
}