Refractor stream handlleing

This commit is contained in:
QuantumEntangledAndy committed 2023-10-08 17:13:04 +07:00
1 parent 0d9d6b97c9
commit 894f2e3644
6 files changed
+1351 -1470

No files matched your search

Generated
+313 -268
View File
File diff suppressed because it is too large. Load diff
+1
View File
@@ -13,6 +13,7 @@ members = [
[dependencies]
anyhow = "1.0.70"
async-stream = "0.3.5"
base64 = "0.21.2"
byte-slice-cast = "1.2.2"
clap = { version = "4.2.2", features = ["derive", "cargo"] }
+8 -3
View File
@@ -316,6 +316,7 @@ impl StreamInstance {
impl StreamData {
async fn new(name: StreamKind, instance: NeoInstance, strict: bool) -> Result<Self> {
const BUFFER_DURATION: Duration = Duration::from_secs(15);
let (vid, _) = broadcast::<StampedData>(100);
let (aud, _) = broadcast::<StampedData>(100);
let (vid_history, _) = watch::<VecDeque<StampedData>>(VecDeque::new());
@@ -420,6 +421,7 @@ impl StreamData {
let vid_history = vid_history.clone();
let aud_history = aud_history.clone();
let watchdog_tx = watchdog_tx.clone();
log::debug!("Running Stream Instance Task");
Box::pin(async move {
// use std::io::Write;
@@ -514,6 +516,7 @@ impl StreamData {
match data {
BcMedia::Iframe(BcMediaIframe{data, microseconds, ..}) => {
prev_ts = Duration::from_micros(microseconds as u64);
log::debug!("IFrame: {prev_ts:?}");
let d = StampedData{
keyframe: true,
data: Arc::new(data),
@@ -521,7 +524,7 @@ impl StreamData {
};
let _ = vid_tx.send(d.clone());
vid_history.send_modify(|history| {
let drop_time = d.ts.saturating_sub(Duration::from_secs(5));
let drop_time = d.ts.saturating_sub(BUFFER_DURATION);
history.push_back(d);
while history.front().is_some_and(|di| di.ts < drop_time) {
history.pop_front();
@@ -533,6 +536,7 @@ impl StreamData {
},
BcMedia::Pframe(BcMediaPframe{data, microseconds,..}) if recieved_iframe => {
prev_ts = Duration::from_micros(microseconds as u64);
log::debug!("PFrame: {prev_ts:?}");
// log::debug!("data: {data:02X?}");
let d = StampedData{
keyframe: false,
@@ -541,7 +545,7 @@ impl StreamData {
};
let _ = vid_tx.send(d.clone());
vid_history.send_modify(|history| {
let drop_time = d.ts.saturating_sub(Duration::from_secs(5));
let drop_time = d.ts.saturating_sub(BUFFER_DURATION);
history.push_back(d);
while history.front().is_some_and(|di| di.ts < drop_time) {
history.pop_front();
@@ -550,6 +554,7 @@ impl StreamData {
log::trace!("Sent Vid Frame");
}
BcMedia::Aac(BcMediaAac{data, ..}) | BcMedia::Adpcm(BcMediaAdpcm{data,..}) if recieved_iframe => {
log::debug!("Audio: {prev_ts:?}");
let d = StampedData{
keyframe: aud_keyframe,
data: Arc::new(data),
@@ -558,7 +563,7 @@ impl StreamData {
aud_keyframe = false;
let _ = aud_tx.send(d.clone())?;
aud_history.send_modify(|history| {
let drop_time = d.ts.saturating_sub(Duration::from_secs(5));
let drop_time = d.ts.saturating_sub(BUFFER_DURATION);
history.push_back(d);
while history.front().is_some_and(|di| di.ts < drop_time) {
history.pop_front();
+408
View File
@@ -0,0 +1,408 @@
use anyhow::{anyhow, Context, Result};
use gstreamer::{prelude::*, Bin, Caps, Element, ElementFactory, GhostPad};
use gstreamer_app::{AppSrc, AppSrcCallbacks, AppStreamType};
use log::*;
use tokio::sync::mpsc::{channel as mpsc, Receiver as MpscReceiver};
use crate::{
common::{AudFormat, StreamConfig, VidFormat},
rtsp::gst::NeoMediaFactory,
AnyResult,
};
pub(super) struct ClientSourceData {
pub(super) app: AppSrc,
}
pub(super) struct ClientData {
pub(super) vid: Option<ClientSourceData>,
pub(super) aud: Option<ClientSourceData>,
}
pub(super) async fn make_dummy_factory(use_splash: bool) -> AnyResult<NeoMediaFactory> {
NeoMediaFactory::new_with_callback(move |element| {
clear_bin(&element)?;
if !use_splash {
Ok(None)
} else {
build_unknown(&element)?;
Ok(Some(element))
}
})
.await
}
pub(super) async fn make_factory(
stream_config: &StreamConfig,
) -> AnyResult<(NeoMediaFactory, MpscReceiver<ClientData>)> {
let (client_tx, client_rx) = mpsc(100);
let factory = {
let stream_config = stream_config.clone();
NeoMediaFactory::new_with_callback(move |element| {
clear_bin(&element)?;
let vid = match stream_config.vid_format {
VidFormat::None => {
build_unknown(&element)?;
AnyResult::Ok(None)
}
VidFormat::H264 => {
let app = build_h264(&element, &stream_config)?;
app.set_callbacks(
AppSrcCallbacks::builder()
.seek_data(move |_, _seek_pos| true)
.build(),
);
AnyResult::Ok(Some(app))
}
VidFormat::H265 => {
let app = build_h265(&element, &stream_config)?;
app.set_callbacks(
AppSrcCallbacks::builder()
.seek_data(move |_, _seek_pos| true)
.build(),
);
AnyResult::Ok(Some(app))
}
}?;
let aud = if matches!(stream_config.vid_format, VidFormat::None) {
None
} else {
match stream_config.aud_format {
AudFormat::None => AnyResult::Ok(None),
AudFormat::Aac => {
let app = build_aac(&element, &stream_config)?;
app.set_callbacks(
AppSrcCallbacks::builder()
.seek_data(move |_, _seek_pos| true)
.build(),
);
AnyResult::Ok(Some(app))
}
AudFormat::Adpcm(block_size) => {
let app = build_adpcm(&element, block_size, &stream_config)?;
app.set_callbacks(
AppSrcCallbacks::builder()
.seek_data(move |_, _seek_pos| true)
.build(),
);
AnyResult::Ok(Some(app))
}
}?
};
client_tx.blocking_send(ClientData {
vid: vid.map(|app| ClientSourceData { app }),
aud: aud.map(|app| ClientSourceData { app }),
})?;
Ok(Some(element))
})
.await
}?;
Ok((factory, client_rx))
}
fn clear_bin(bin: &Element) -> Result<()> {
let bin = bin
.clone()
.dynamic_cast::<Bin>()
.map_err(|_| anyhow!("Media source's element should be a bin"))?;
// Clear the autogenerated ones
log::debug!("Clearing old elements");
for element in bin.iterate_elements().into_iter().flatten() {
bin.remove(&element)?;
}
Ok(())
}
fn build_unknown(bin: &Element) -> Result<()> {
let bin = bin
.clone()
.dynamic_cast::<Bin>()
.map_err(|_| anyhow!("Media source's element should be a bin"))?;
log::debug!("Building Unknown Pipeline");
let source = make_element("videotestsrc", "testvidsrc")?;
source.set_property_from_str("pattern", "snow");
source.set_property("num-buffers", 500i32); // Send buffers then EOS
let queue = make_queue("queue0", 1024 * 1024 * 4)?;
let overlay = make_element("textoverlay", "overlay")?;
overlay.set_property("text", "Stream not Ready");
overlay.set_property_from_str("valignment", "top");
overlay.set_property_from_str("halignment", "left");
overlay.set_property("font-desc", "Sans, 16");
let encoder = make_element("jpegenc", "encoder")?;
let payload = make_element("rtpjpegpay", "pay0")?;
bin.add_many([&source, &queue, &overlay, &encoder, &payload])?;
source.link_filtered(
&queue,
&Caps::builder("video/x-raw")
.field("format", "YUY2")
.field("width", 896i32)
.field("height", 512i32)
.field("framerate", gstreamer::Fraction::new(25, 1))
.build(),
)?;
Element::link_many([&queue, &overlay, &encoder, &payload])?;
Ok(())
}
fn build_h264(bin: &Element, stream_config: &StreamConfig) -> Result<AppSrc> {
let buffer_size = buffer_size(stream_config.bitrate);
log::debug!("buffer_size: {buffer_size}");
let bin = bin
.clone()
.dynamic_cast::<Bin>()
.map_err(|_| anyhow!("Media source's element should be a bin"))?;
log::debug!("Building H264 Pipeline");
let source = make_element("appsrc", "vidsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_is_live(true);
source.set_block(false);
source.set_min_latency(0);
source.set_property("emit-signals", false);
source.set_max_bytes(buffer_size as u64);
source.set_do_timestamp(true);
source.set_stream_type(AppStreamType::Seekable);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_queue("source_queue", buffer_size)?;
let parser = make_element("h264parse", "parser")?;
parser.set_property("config-interval", -1i32); // Send SPS and PPS every Iframe to imporove joining frame late
let payload = make_element("rtph264pay", "pay0")?;
bin.add_many([&source, &queue, &parser, &payload])?;
Element::link_many([&source, &queue, &parser, &payload])?;
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
Ok(source)
}
fn build_h265(bin: &Element, stream_config: &StreamConfig) -> Result<AppSrc> {
let buffer_size = buffer_size(stream_config.bitrate);
log::debug!("buffer_size: {buffer_size}");
let bin = bin
.clone()
.dynamic_cast::<Bin>()
.map_err(|_| anyhow!("Media source's element should be a bin"))?;
log::debug!("Building H265 Pipeline");
let source = make_element("appsrc", "vidsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_is_live(true);
source.set_block(false);
source.set_min_latency(0);
source.set_property("emit-signals", false);
source.set_max_bytes(buffer_size as u64);
source.set_do_timestamp(true);
source.set_stream_type(AppStreamType::Seekable);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_queue("source_queue", buffer_size)?;
let parser = make_element("h265parse", "parser")?;
let payload = make_element("rtph265pay", "pay0")?;
bin.add_many([&source, &queue, &parser, &payload])?;
Element::link_many([&source, &queue, &parser, &payload])?;
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
Ok(source)
}
fn build_aac(bin: &Element, stream_config: &StreamConfig) -> Result<AppSrc> {
let buffer_size = buffer_size(stream_config.bitrate);
log::debug!("buffer_size: {buffer_size}");
let bin = bin
.clone()
.dynamic_cast::<Bin>()
.map_err(|_| anyhow!("Media source's element should be a bin"))?;
log::debug!("Building Aac pipeline");
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_is_live(true);
source.set_block(false);
source.set_min_latency(0);
source.set_property("emit-signals", false);
source.set_max_bytes(buffer_size as u64);
source.set_do_timestamp(true);
source.set_stream_type(AppStreamType::Seekable);
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_queue("audqueue", buffer_size)?;
let parser = make_element("aacparse", "audparser")?;
let decoder = match make_element("faad", "auddecoder_faad") {
Ok(ele) => Ok(ele),
Err(_) => make_element("avdec_aac", "auddecoder_avdec_aac"),
}?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
bin.add_many([&source, &queue, &parser, &decoder, &encoder, &payload])?;
Element::link_many([&source, &queue, &parser, &decoder, &encoder, &payload])?;
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
Ok(source)
}
fn build_adpcm(bin: &Element, block_size: u32, stream_config: &StreamConfig) -> Result<AppSrc> {
let buffer_size = buffer_size(stream_config.bitrate);
log::debug!("buffer_size: {buffer_size}");
let bin = bin
.clone()
.dynamic_cast::<Bin>()
.map_err(|_| anyhow!("Media source's element should be a bin"))?;
log::debug!("Building Adpcm pipeline");
// Original command line
// caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000
// ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024
// ! adpcmdec
// ! audioconvert
// ! rtpL16pay name=pay1
let source = make_element("appsrc", "audsrc")?
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot cast to appsrc."))?;
source.set_is_live(true);
source.set_block(false);
source.set_min_latency(0);
source.set_property("emit-signals", false);
source.set_max_bytes(buffer_size as u64);
source.set_do_timestamp(true);
source.set_stream_type(AppStreamType::Seekable);
source.set_caps(Some(
&Caps::builder("audio/x-adpcm")
.field("layout", "div")
.field("block_align", block_size as i32)
.field("channels", 1i32)
.field("rate", 8000i32)
.build(),
));
let source = source
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_queue("audqueue", buffer_size)?;
let decoder = make_element("decodebin", "auddecoder")?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
bin.add_many([&source, &queue, &decoder, &encoder, &payload])?;
Element::link_many([&source, &queue, &decoder])?;
Element::link_many([&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link ADPCM decoder to encoder");
});
let source = source
.dynamic_cast::<AppSrc>()
.map_err(|_| anyhow!("Cannot convert appsrc"))?;
Ok(source)
}
// Convenice funcion to make an element or provide a message
// about what plugin is missing
fn make_element(kind: &str, name: &str) -> AnyResult<Element> {
ElementFactory::make_with_name(kind, Some(name)).with_context(|| {
let plugin = match kind {
"appsrc" => "app (gst-plugins-base)",
"audioconvert" => "audioconvert (gst-plugins-base)",
"adpcmdec" => "Required for audio",
"h264parse" => "videoparsersbad (gst-plugins-bad)",
"h265parse" => "videoparsersbad (gst-plugins-bad)",
"rtph264pay" => "rtp (gst-plugins-good)",
"rtph265pay" => "rtp (gst-plugins-good)",
"rtpjitterbuffer" => "rtp (gst-plugins-good)",
"aacparse" => "audioparsers (gst-plugins-good)",
"rtpL16pay" => "rtp (gst-plugins-good)",
"x264enc" => "x264 (gst-plugins-ugly)",
"x265enc" => "x265 (gst-plugins-bad)",
"avdec_h264" => "libav (gst-libav)",
"avdec_h265" => "libav (gst-libav)",
"videotestsrc" => "videotestsrc (gst-plugins-base)",
"imagefreeze" => "imagefreeze (gst-plugins-good)",
"audiotestsrc" => "audiotestsrc (gst-plugins-base)",
"decodebin" => "playback (gst-plugins-good)",
_ => "Unknown",
};
format!(
"Missing required gstreamer plugin `{}` for `{}` element",
plugin, kind
)
})
}
fn make_queue(name: &str, buffer_size: u32) -> AnyResult<Element> {
let queue = make_element("queue", &format!("queue1_{}", name))?;
queue.set_property("max-size-bytes", buffer_size);
queue.set_property("max-size-buffers", 0u32);
queue.set_property("max-size-time", 0u64);
// queue.set_property(
// "max-size-time",
// std::convert::TryInto::<u64>::try_into(tokio::time::Duration::from_secs(5).as_nanos())
// .unwrap_or(0),
// );
let queue2 = make_element("queue2", &format!("queue2_{}", name))?;
queue2.set_property("max-size-bytes", buffer_size * 2u32 / 3u32);
queue.set_property("max-size-buffers", 0u32);
queue.set_property("max-size-time", 0u64);
// queue2.set_property(
// "max-size-time",
// std::convert::TryInto::<u64>::try_into(tokio::time::Duration::from_secs(5).as_nanos())
// .unwrap_or(0),
// );
queue2.set_property("use-buffering", false);
let bin = gstreamer::Bin::builder().name(name).build();
bin.add_many([&queue, &queue2])?;
Element::link_many([&queue, &queue2])?;
let pad = queue
.static_pad("sink")
.expect("Failed to get a static pad from queue.");
let ghost_pad = GhostPad::builder_with_target(&pad).unwrap().build();
ghost_pad.set_active(true)?;
bin.add_pad(&ghost_pad)?;
let pad = queue2
.static_pad("src")
.expect("Failed to get a static pad from queue2.");
let ghost_pad = GhostPad::builder_with_target(&pad).unwrap().build();
ghost_pad.set_active(true)?;
bin.add_pad(&ghost_pad)?;
let bin = bin
.dynamic_cast::<Element>()
.map_err(|_| anyhow!("Cannot convert bin"))?;
Ok(bin)
}
fn buffer_size(bitrate: u32) -> u32 {
std::cmp::max(bitrate * 15u32 / 8u32, 4u32 * 1024u32 * 1024u32)
}
+9 -1199
View File
File diff suppressed because it is too large. Load diff
+612
View File
@@ -0,0 +1,612 @@
use anyhow::{anyhow, Result};
use gstreamer::{prelude::*, ClockTime};
use gstreamer_app::AppSrc;
use gstreamer_rtsp_server::prelude::*;
use std::collections::HashSet;
use tokio::{
sync::broadcast::channel as broadcast,
task::JoinSet,
time::{sleep, sleep_until, Duration, Instant},
};
use tokio_stream::{wrappers::BroadcastStream, Stream, StreamExt};
use tokio_util::sync::CancellationToken;
use crate::common::{Permit, StampedData, UseCounter};
use crate::{
common::{NeoInstance, StreamConfig, StreamInstance},
AnyResult,
};
use super::{factory::*, gst::NeoRtspServer};
/// This handles the stream by activating and deacivating it as required
pub(super) async fn stream_main(
mut stream_instance: StreamInstance,
camera: NeoInstance,
rtsp: &NeoRtspServer,
users: &HashSet<String>,
paths: &[String],
) -> Result<()> {
let mut camera_config = camera.config().await?.clone();
let name = camera_config.borrow().name.clone();
let mut curr_pause;
loop {
log::debug!("{}: Activating Stream", &name);
stream_instance.activate().await?;
// Wait for a valid stream format to be detected
log::debug!("{}: Waiting for Valid Stream", &name);
stream_instance
.config
.wait_for(|config| {
log::debug!("{:?}", config);
config.vid_ready()
})
.await?;
log::debug!("{}: Waiting for Valid Audio", &name);
// After vid give it 1s to look for audio
// Ignore timeout but check err
if let Ok(v) = tokio::time::timeout(
Duration::from_secs(1),
stream_instance.config.wait_for(|config| {
log::debug!("{:?}", config);
config.aud_ready()
}),
)
.await
{
v?;
}
curr_pause = camera_config.borrow().pause.clone();
let last_stream_config = stream_instance.config.borrow().clone();
let mut thread_stream_config = stream_instance.config.clone();
let mut set = JoinSet::<AnyResult<()>>::new();
log::debug!("{}: Creating Client Counters", &name);
// Handles the on off of the stream with the client pause
let client_counter = UseCounter::new().await;
let client_count = client_counter.create_deactivated().await?;
if curr_pause.on_disconnect && curr_pause.on_motion {
log::debug!("{}: Enabling Client&Motion Pause", &name);
// Take over activation
let mut client_activator = stream_instance.activator_handle().await;
client_activator.deactivate().await?; // Start deactivated
stream_instance.deactivate().await?;
// Get client counter
let client_count = client_counter.create_deactivated().await?;
// Get motion counter
let mut motion = camera.motion().await?;
let delta = Duration::from_secs_f64(curr_pause.motion_timeout);
let thread_name = name.clone();
set.spawn(async move {
loop {
// Wait for motion to start
motion.wait_for(|md| matches!(md, crate::common::MdState::Start(_))).await?;
tokio::select! {
v = client_count.aquired_users() => {
v?;
},
v = motion.wait_for(|md| matches!(md, crate::common::MdState::Stop(n) if n.elapsed()>delta)) => {
v?;
// Motion has stopped go back
continue;
}
}
log::debug!("{}: Activating Client&Motion", thread_name);
client_activator.activate().await?;
tokio::select! {
v = client_count.dropped_users() => {
v?;
},
v = motion.wait_for(|md| matches!(md, crate::common::MdState::Stop(n) if n.elapsed()>delta)) => {
v?;
}
}
// Deactivate time
log::debug!("{}: Pausing Client&Motion", thread_name);
client_activator.deactivate().await?;
}
});
// Push notfications
let mut pn = camera.push_notifications().await?;
let mut curr_pn = None;
let mut client_activator = stream_instance.activator_handle().await;
client_activator.deactivate().await?;
let thread_name = name.clone();
set.spawn(async move {
loop {
curr_pn = pn
.wait_for(|pn| pn != &curr_pn && pn.is_some())
.await?
.clone();
log::info!("{}: Enabling Push Notification", thread_name);
client_activator.activate().await?;
tokio::select! {
v = pn.wait_for(|pn| pn != &curr_pn && pn.is_some()) => {
v?;
// If another PN during wait then go back to wait more
continue;
}
_ = sleep(Duration::from_secs(30)) => {}
}
log::info!("{}: Pausing Push Notification", thread_name);
client_activator.deactivate().await?;
}
});
} else if curr_pause.on_disconnect {
log::debug!("{}: Enabling Client Pause", &name);
// Take over activation
let mut client_activator = stream_instance.activator_handle().await;
client_activator.deactivate().await?;
stream_instance.deactivate().await?;
let client_count = client_counter.create_deactivated().await?;
let thread_name = name.clone();
set.spawn(async move {
loop {
client_count.aquired_users().await?;
log::info!("{}: Activating Client", thread_name);
client_activator.activate().await?;
client_count.dropped_users().await?;
log::info!("{}: Pausing Client", thread_name);
client_activator.deactivate().await?;
}
});
} else if curr_pause.on_motion {
// Handles on motion pausing
log::debug!("{}: Activating Motion Pause", &name);
// Take over activation
let mut client_activator = stream_instance.activator_handle().await;
client_activator.deactivate().await?;
stream_instance.deactivate().await?;
let mut motion = camera.motion().await?;
let delta = Duration::from_secs_f64(curr_pause.motion_timeout);
let thread_name = name.clone();
set.spawn(async move {
loop {
motion
.wait_for(|md| matches!(md, crate::common::MdState::Start(_)))
.await?;
log::info!("{}: Enabling Motion", thread_name);
client_activator.activate().await?;
motion
.wait_for(
|md| matches!(md, crate::common::MdState::Stop(n) if n.elapsed()>delta),
)
.await?;
log::info!("{}: Pausing Motion", thread_name);
client_activator.deactivate().await?;
}
});
// Push notfications
let mut pn = camera.push_notifications().await?;
let mut curr_pn = None;
let mut client_activator = stream_instance.activator_handle().await;
client_activator.deactivate().await?;
let thread_name = name.clone();
set.spawn(async move {
loop {
curr_pn = pn
.wait_for(|pn| pn != &curr_pn && pn.is_some())
.await?
.clone();
log::info!("{}: Enabling Push Notification", thread_name);
client_activator.activate().await?;
tokio::select! {
v = pn.wait_for(|pn| pn != &curr_pn && pn.is_some()) => {
v?;
// If another PN during wait then go back to wait more
continue;
}
_ = sleep(Duration::from_secs(30)) => {}
}
log::info!("{}: Pausing Push Notification", thread_name);
client_activator.deactivate().await?;
}
});
}
// This thread jsut keeps it active for 5s after an initial start to build the buffer
let mut init_activator = stream_instance.activator_handle().await;
set.spawn(async move {
init_activator.activate().await?;
sleep(Duration::from_secs(5)).await;
init_activator.deactivate().await?;
AnyResult::Ok(())
});
// Task to just report the number of clients for debug purposes
let counter = client_counter.create_deactivated().await?;
let mut cur_count = 0;
let thread_name = name.clone();
set.spawn(async move {
loop {
cur_count = *counter.get_counter().wait_for(|v| v != &cur_count).await?;
log::debug!("{thread_name}: Number of rtsp clients: {cur_count}");
}
});
// This runs the actual stream.
// The select will restart if the stream's config updates
log::debug!("{}: Stream Activated", &name);
break tokio::select! {
v = thread_stream_config.wait_for(|new_conf| new_conf != &last_stream_config) => {
let v = v?;
// If stream config changes we reload the stream
log::info!("{}: Stream Configuration Changed. Reloading Streams", &name);
log::trace!(" From {:?} to {:?}", last_stream_config, v.clone());
continue;
},
v = camera_config.wait_for(|new_conf| new_conf.pause != curr_pause ) => {
v?;
// If pause config changes restart
log::info!("{}: Pause Configuration Changed. Reloading Streams", &name);
continue;
},
v = stream_run(&name, &stream_instance, rtsp, &last_stream_config, users, paths, client_count) => v,
};
}
}
/// This handles the stream itself by creating the factory and pushing messages into it
async fn stream_run(
name: &str,
stream_instance: &StreamInstance,
rtsp: &NeoRtspServer,
stream_config: &StreamConfig,
users: &HashSet<String>,
paths: &[String],
client_count: Permit,
) -> AnyResult<()> {
let vidstream = stream_instance.vid.resubscribe();
let audstream = stream_instance.aud.resubscribe();
let vid_history = stream_instance.vid_history.clone();
let aud_history = stream_instance.aud_history.clone();
// Finally ready to create the factory and connect the stream
let mounts = rtsp
.mount_points()
.ok_or(anyhow!("RTSP server lacks mount point"))?;
// Create the factory
let (factory, mut client_rx) = make_factory(stream_config).await?;
factory.add_permitted_roles(users);
for path in paths.iter() {
log::debug!("Path: {}", path);
mounts.add_factory(path, factory.clone());
}
log::info!("{}: Avaliable at {}", name, paths.join(", "));
let stream_cancel = CancellationToken::new();
let drop_guard = stream_cancel.clone().drop_guard();
let mut set = JoinSet::new();
// Wait for new media client data to come in from the factory
while let Some(mut client_data) = client_rx.recv().await {
log::debug!("New media");
// New media created
let vid = client_data.vid.take().map(|data| data.app);
let aud = client_data.aud.take().map(|data| data.app);
// This is the data that gets sent to gstreamer thread
// It represents the combination of the camera stream and the appsrc seek messages
let (aud_data_tx, aud_data_rx) = broadcast(100);
let (vid_data_tx, vid_data_rx) = broadcast(100);
// This thread takes the video data from the cam and passed it into the stream
let mut vidstream = BroadcastStream::new(vidstream.resubscribe());
let thread_vid_data_tx = vid_data_tx.clone();
let thread_stream_cancel = stream_cancel.clone();
let thread_vid_history = vid_history.clone();
set.spawn(async move {
let r = tokio::select! {
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
v = async {
// Send Initial
for data in thread_vid_history.borrow().iter() {
thread_vid_data_tx.send(
data.clone()
)?;
}
// Send new
while let Some(frame) = vidstream.next().await {
if let Ok(data) = frame {
thread_vid_data_tx.send(
data
)?;
}
};
AnyResult::Ok(())
} => v,
};
log::trace!("Stream Vid Media End {r:?}");
AnyResult::Ok(())
});
// This thread takes the audio data from the cam and passed it into the stream
let mut audstream = BroadcastStream::new(audstream.resubscribe());
let thread_stream_cancel = stream_cancel.clone();
let thread_aud_data_tx = aud_data_tx.clone();
let thread_aud_history = aud_history.clone();
set.spawn(async move {
let r = tokio::select! {
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
v = async {
// Send Initial
for data in thread_aud_history.borrow().iter() {
thread_aud_data_tx.send(
data.clone()
)?;
}
// Send new
while let Some(frame) = audstream.next().await {
if let Ok(data) = frame {
thread_aud_data_tx.send(
data
)?;
}
};
AnyResult::Ok(())
} => v,
};
log::trace!("Stream Aud Media End: {r:?}");
AnyResult::Ok(())
});
// Handles sending the video data into gstreamer
let thread_stream_cancel = stream_cancel.clone();
let vid_data_rx = BroadcastStream::new(vid_data_rx).filter(|f| f.is_ok()); // Filter to ignore lagged
let thread_vid = vid.clone();
let mut thread_client_count = client_count.subscribe();
if let Some(thread_vid) = thread_vid {
set.spawn(async move {
thread_client_count.activate().await?;
let r = tokio::select! {
_ = thread_stream_cancel.cancelled() => {
AnyResult::Ok(())
},
v = send_to_appsrc(
repeat_keyframe(
frametime_stream(
hold_stream(
wait_for_keyframe(
vid_data_rx,
)
)
),
Duration::from_secs(1)
), &thread_vid) => {
v
},
};
drop(thread_client_count);
let _ = thread_vid.end_of_stream();
log::debug!("Vid Thread End: {:?}", r);
r
});
}
// Handles the audio data into gstreamer
let thread_stream_cancel = stream_cancel.clone();
let aud_data_rx = BroadcastStream::new(aud_data_rx).filter(|f| f.is_ok()); // Filter to ignore lagged
let thread_aud = aud.clone();
if let Some(thread_aud) = thread_aud {
set.spawn(async move {
let r = tokio::select! {
_ = thread_stream_cancel.cancelled() => {
AnyResult::Ok(())
},
v = send_to_appsrc(
frametime_stream(
hold_stream(
wait_for_keyframe(
aud_data_rx
)
)
), &thread_aud) => {
v
},
};
let _ = thread_aud.end_of_stream();
log::debug!("Aud Thread End: {:?}", r);
r
});
}
}
log::debug!("Cleaning up streams");
// At this point the factory has been destroyed
// Cancel any remaining threads that are trying to send data
// Although it should be finished already when the appsrcs are dropped
stream_cancel.cancel();
drop(drop_guard);
while set.join_next().await.is_some() {}
log::trace!("Stream done");
AnyResult::Ok(())
}
fn check_live(app: &AppSrc) -> Result<()> {
// log::debug!("Checking Live: {:?}", app.bus());
app.bus().map(|_| ()).ok_or(anyhow!("App source is closed"))
}
fn get_runtime(app: &AppSrc) -> Option<Duration> {
if let Some(clock) = app.clock() {
if let Some(time) = clock.time() {
// log::debug!("time: {time:?}");
if let Some(base_time) = app.base_time() {
// log::debug!("base_time: {base_time:?}");
let runtime = time.saturating_sub(base_time);
// log::debug!("runtime: {runtime:?}");
return Some(Duration::from_micros(runtime.useconds()));
}
}
}
None
}
// This ensures we start at a keyframe
fn wait_for_keyframe<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
mut stream: T,
) -> impl Stream<Item = AnyResult<StampedData>> + Unpin {
Box::pin(async_stream::stream! {
let mut found_key = false;
while let Some(frame) = stream.next().await {
if let Ok(frame) = frame {
if frame.keyframe || found_key {
found_key = true;
yield Ok(frame);
}
}
}
})
}
// Take a stream of stamped data and release them
// in waves when a new key frame is found
// this ensure that the last frame sent is always an IFrame
fn hold_stream<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
mut stream: T,
) -> impl Stream<Item = AnyResult<StampedData>> + Unpin {
Box::pin(async_stream::stream! {
let mut held_frames = vec![];
while let Some(frame) = stream.next().await {
if let Ok(frame) = frame {
if frame.keyframe {
// Release
log::debug!("Yielding: {}", held_frames.len());
for held_frame in held_frames.drain(..) {
yield Ok(held_frame);
}
log::debug!("Yielded");
yield Ok(frame);
} else {
// Hold
held_frames.push(frame);
}
}
}
})
}
// Take a stream of stamped data pause until
// it is time to display it
fn frametime_stream<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
mut stream: T,
) -> impl Stream<Item = AnyResult<StampedData>> + Unpin {
Box::pin(async_stream::stream! {
const MIN_FPS_DELTA: Duration = Duration::from_millis(1000/5);
let mut last_release = Instant::now();
let mut cached_prev_ts = None;
while let Some(frame) = stream.next().await {
if let Ok(frame) = frame {
let curr_ts = frame.ts;
let mut prev_ts = cached_prev_ts.unwrap_or(curr_ts);
// Check if we have gone backwards
if curr_ts < prev_ts {
// If we have reset things
prev_ts = curr_ts;
}
let delta_ts = std::cmp::min(curr_ts - prev_ts, MIN_FPS_DELTA);
log::debug!("curr_ts: {curr_ts:?}, {prev_ts:?} delta_ts: {delta_ts:?}");
sleep_until(last_release + delta_ts).await;
last_release = Instant::now();
cached_prev_ts = Some(curr_ts);
yield Ok(frame);
}
}
})
}
// This will take a stream and if there is a notibable lack of data
// then it will repeat the last keyframe (if there have been no)
// pframes in between
fn repeat_keyframe<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
mut stream: T,
fallback_time: Duration,
) -> impl Stream<Item = Result<StampedData, E>> + Unpin {
Box::pin(async_stream::stream! {
while let Some(frame) = stream.next().await {
if let Ok(frame) = frame {
if frame.keyframe {
log::debug!("Key Frame");
let repeater = frame.clone();
yield Ok(frame);
// Wait for either timeout or a new frame
loop {
tokio::select!{
v = stream.next() => {
if let Some(frame) = v {
if let Ok(frame) = frame {
log::debug!("Key Frame: Resume");
yield Ok(frame);
break;
}
} else {
break;
}
},
_ = sleep(fallback_time) => {
log::debug!("Inserting Skip Frame");
yield Ok(repeater.clone());
}
}
}
} else {
// P frames go through as-is
yield Ok(frame);
}
}
}
})
}
/// Takes a stream and sends it to an appsrc
async fn send_to_appsrc<E, T: Stream<Item = Result<StampedData, E>> + Unpin>(
mut stream: T,
appsrc: &AppSrc,
) -> AnyResult<()> {
let mut rt = Duration::ZERO;
while let Some(Ok(data)) = stream.next().await {
check_live(appsrc)?; // Stop if appsrc is dropped
if let Some(rt_i) = get_runtime(appsrc) {
rt = rt_i;
}
let buf = {
let mut gst_buf = gstreamer::Buffer::with_size(data.data.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
// log::debug!("Setting PTS: {ts:?}, Runtime: {ts:?}");
let time = ClockTime::from_useconds(rt.as_micros() as u64);
gst_buf_mut.set_dts(time);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(data.data.as_slice());
}
gst_buf
};
appsrc
.push_buffer(buf)
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))?;
}
Ok(())
}