Insert fillers when no stream

This commit is contained in:
QuantumEntangledAndy committed 2024-04-16 10:11:06 +07:00
1 parent e74e12268a
commit c85a7641cb
1 file changed
+155 -87
+155 -87
View File
@@ -413,8 +413,8 @@ async fn stream_run(
let mut thread_client_count = client_count.subscribe();
let thread_format = stream_config.vid_format;
// let fallback_time = Duration::from_secs(3);
// let framerate =
// Duration::from_millis(1000u64 / std::cmp::max(stream_config.fps as u64, 5u64));
let framerate =
Duration::from_millis(1000u64 / std::cmp::max(stream_config.fps as u64, 5u64));
if let Some(thread_vid) = thread_vid {
set.spawn(async move {
thread_client_count.activate().await?;
@@ -424,14 +424,18 @@ async fn stream_run(
},
v = send_to_appsrc(
pad_vid(
// frametime_stream(
// ensure_order(
wait_for_keyframe(
vid_data_rx,
),
// ),
// framerate
// ),
insert_filler(
frametime_stream(
// ensure_order(
wait_for_keyframe(
vid_data_rx,
),
// ),
framerate
),
thread_format,
framerate,
),
thread_format,
),
&thread_vid
@@ -449,8 +453,8 @@ async fn stream_run(
let thread_stream_cancel = stream_cancel.clone();
let aud_data_rx = BroadcastStream::new(aud_data_rx).filter(|f| f.is_ok()); // Filter to ignore lagged
let thread_aud = aud.clone();
// let aud_framerate =
// Duration::from_millis(1000u64 / std::cmp::max(stream_config.fps as u64, 5u64));
let aud_framerate =
Duration::from_millis(1000u64 / std::cmp::max(stream_config.fps as u64, 5u64));
if let Some(thread_aud) = thread_aud {
set.spawn(async move {
let r = tokio::select! {
@@ -458,13 +462,13 @@ async fn stream_run(
AnyResult::Ok(())
},
v = send_to_appsrc(
// frametime_stream(
frametime_stream(
// ensure_order(
wait_for_keyframe(
aud_data_rx
),
// ),
// aud_framerate),
aud_framerate),
&thread_aud) => {
v
},
@@ -490,9 +494,10 @@ fn check_live(app: &AppSrc) -> Result<()> {
.iter()
.all(|pad| pad.is_linked())
.then_some(())
.ok_or(anyhow!("App source is closed"))
.ok_or(anyhow!("App source is not linked"))
}
#[allow(dead_code)]
fn get_runtime(app: &AppSrc) -> Option<Duration> {
if let Some(clock) = app.clock() {
if let Some(time) = clock.time() {
@@ -601,6 +606,48 @@ fn frametime_stream<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
})
}
/// Insert filler into the video stream when no frames are comming
/// this should help the stream not be considered dead while
/// we wait for the reconnect
fn insert_filler<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
mut stream: T,
format: VidFormat,
delay: Duration,
) -> impl Stream<Item = AnyResult<StampedData>> + Unpin {
let mut last_ts = Duration::ZERO;
Box::pin(async_stream::stream! {
while let Some(frame) = tokio::select!{
v = stream.next() => {
v
},
_ = tokio::time::sleep(delay) => {
match format {
VidFormat::H264 => {
Some(Ok(StampedData {
data: Arc::new(h264_filler(4096)),
keyframe: false,
ts: last_ts,
}))
}
VidFormat::H265 => {
Some(Ok(StampedData {
data: Arc::new(h265_filler(4096)),
keyframe: false,
ts: last_ts,
}))
}
VidFormat::None => unreachable!(),
}
}
} {
if let Ok(frame) = frame {
last_ts = frame.ts;
yield Ok(frame);
}
}
})
}
fn h265_filler(size: usize) -> Vec<u8> {
assert!(size >= 6);
let mut buf = vec![0x0, 0x0, 0x1, 0b01001100, 0x0];
@@ -719,88 +766,108 @@ async fn send_to_appsrc<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
mut stream: T,
appsrc: &AppSrc,
) -> AnyResult<()> {
let mut rt = Duration::ZERO;
let mut ts_0 = Duration::MAX;
let mut wait_for_iframe = true;
let mut pools: HashMap<usize, gstreamer::BufferPool> = Default::default();
let mut buffer_inited = false;
let mut dts = 1;
appsrc.set_state(gstreamer::State::Paused).unwrap();
let mut keyframes_pushed = 0;
let mut inited = false;
// appsrc.set_state(gstreamer::State::Paused).unwrap();
let (tx, mut rx) = tokio::sync::mpsc::channel::<StampedData>(2000);
// Run blocking code on a seperate thread
let appsrc = appsrc.clone();
std::thread::spawn(move || {
while let Some(data) = rx.blocking_recv() {
check_live(&appsrc)?; // Stop if appsrc is dropped
if wait_for_iframe && !data.keyframe {
continue;
} else if wait_for_iframe {
wait_for_iframe = false;
}
let r = (move || {
while let Some(data) = rx.blocking_recv() {
check_live(&appsrc)?; // Stop if appsrc is dropped
if wait_for_iframe && !data.keyframe {
continue;
} else if wait_for_iframe {
wait_for_iframe = false;
}
if ts_0 > data.ts {
ts_0 = data.ts;
}
let rt = data.ts - ts_0;
// log::info!("rt: {:?}", rt);
let buf = {
// let mut gst_buf = pool.acquire_buffer(None).unwrap();
let msg_size = data.data.len();
let pool = pools.entry(msg_size).or_insert_with_key(|size| {
let pool = gstreamer::BufferPool::new();
let mut pool_config = pool.config();
pool_config.set_params(None, (*size) as u32, 8, 0);
pool.set_config(pool_config).unwrap();
// let (allocator, alloc_parms) = pool.allocator().unwrap();
pool.set_active(true).unwrap();
pool
});
let mut gst_buf = pool.acquire_buffer(None).unwrap();
// let mut gst_buf = gstreamer::Buffer::with_size(data.data.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let time = ClockTime::from_useconds(rt.as_micros() as u64);
// gst_buf_mut.set_dts(ClockTime::from_useconds(dts));
gst_buf_mut.set_dts(time);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(data.data.as_slice());
}
gst_buf
};
if let Some(rt_i) = get_runtime(&appsrc) {
rt = rt_i;
match appsrc.push_buffer(buf) {
Ok(_) => {
// log::info!(
// "Send {}{} on {}",
// data.data.len(),
// if data.keyframe { " (keyframe)" } else { "" },
// appsrc.name()
// );
Ok(())
}
Err(FlowError::Flushing) => {
// Buffer is full just skip
//
// But ensure we start with an iframe to reduce gray screens
wait_for_iframe = true;
log::info!("Buffer full on {}", appsrc.name());
Ok(())
}
Err(e) => Err(anyhow!("Error in streaming: {e:?}")),
}?;
if !inited {
if keyframes_pushed > 1
|| appsrc.current_level_bytes() >= appsrc.max_bytes() * 2 / 3
{
// log::info!("=== Playing state on {}", appsrc.name());
// log::info!(
// "Buffer filled to {} of {} ({} %) in {} keyframes",
// appsrc.current_level_bytes(),
// appsrc.max_bytes(),
// appsrc.current_level_bytes() * 100 / appsrc.max_bytes(),
// keyframes_pushed
// );
inited = true;
// appsrc.set_state(gstreamer::State::Playing).unwrap();
} else if data.keyframe {
// log::info!(
// "Buffer filling to {} of {} ({} %) in {} keyframes on {}",
// appsrc.current_level_bytes(),
// appsrc.max_bytes(),
// appsrc.current_level_bytes() * 100 / appsrc.max_bytes(),
// keyframes_pushed,
// appsrc.name()
// );
keyframes_pushed += 1;
}
}
}
let buf = {
// let mut gst_buf = pool.acquire_buffer(None).unwrap();
let msg_size = data.data.len();
let pool = pools.entry(msg_size).or_insert_with_key(|size| {
let pool = gstreamer::BufferPool::new();
let mut pool_config = pool.config();
pool_config.set_params(None, (*size) as u32, 8, 0);
pool.set_config(pool_config).unwrap();
// let (allocator, alloc_parms) = pool.allocator().unwrap();
pool.set_active(true).unwrap();
pool
});
let mut gst_buf = pool.acquire_buffer(None).unwrap();
// let mut gst_buf = gstreamer::Buffer::with_size(data.data.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let time = ClockTime::from_useconds(rt.as_micros() as u64);
gst_buf_mut.set_dts(ClockTime::from_useconds(dts));
dts += 1;
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(data.data.as_slice());
}
gst_buf
};
match appsrc.push_buffer(buf) {
Ok(_) => {
log::info!(
"Send {}{} on {}",
data.data.len(),
if data.keyframe { " (keyframe)" } else { "" },
appsrc.name()
);
Ok(())
}
Err(FlowError::Flushing) => {
// Buffer is full just skip
//
// But ensure we start with an iframe to reduce gray screens
wait_for_iframe = true;
log::info!("Buffer full on {}", appsrc.name());
Ok(())
}
Err(e) => Err(anyhow!("Error in streaming: {e:?}")),
}?;
if !buffer_inited && appsrc.current_level_bytes() >= (appsrc.max_bytes() * 3 / 2) {
appsrc.set_state(gstreamer::State::Playing).unwrap();
buffer_inited = true;
log::info!("Playing state on {}", appsrc.name());
} else if !buffer_inited {
log::info!(
"{} of {}",
appsrc.current_level_bytes(),
(appsrc.max_bytes() * 3 / 2)
);
}
}
AnyResult::Ok(())
AnyResult::Ok(())
})();
log::error!("r: {:?}", r);
r
});
// Send to the blocking thread
@@ -810,5 +877,6 @@ async fn send_to_appsrc<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
break;
}
}
log::error!("Exiting send_to_appsrc");
Ok(())
}