Add a watchdog to the stream

This commit is contained in:
QuantumEntangledAndy committed 2023-09-16 10:41:42 +07:00
1 parent e894b7cb4a
commit dc6ca3532b
4 files changed
+161 -42

No files matched your search

+5 -2
View File
@@ -88,8 +88,11 @@ impl StreamData {
impl Drop for StreamData {
fn drop(&mut self) {
log::debug!("SteamData::drop Cancel");
self.abort_handle.cancel();
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let _ = self.shutdown().await;
});
});
}
}
+63 -20
View File
@@ -5,7 +5,7 @@
use futures::stream::{FuturesUnordered, StreamExt};
use std::{
collections::{hash_map::Entry, HashMap},
collections::{hash_map::Entry, HashMap, VecDeque},
sync::Arc,
};
use tokio::{
@@ -18,7 +18,7 @@ use tokio::{
watch::{channel as watch, Receiver as WatchReceiver, Sender as WatchSender},
},
task::JoinHandle,
time::Duration,
time::{timeout, Duration},
};
use tokio_util::sync::CancellationToken;
@@ -144,13 +144,8 @@ impl NeoCamStreamThread {
// Grab it
if let Entry::Occupied(occ) = self.streams.entry(name) {
result = Some(
StreamInstance {
name,
vid: occ.get().vid.subscribe(),
aud: occ.get().aud.subscribe(),
config: occ.get().config.subscribe(),
in_use: occ.get().users.create_activated().await?,
});
StreamInstance::new(occ.get()).await?
);
break;
}
}
@@ -369,6 +364,8 @@ impl Drop for CountUses {
pub(crate) struct StreamData {
vid: BroadcastSender<StampedData>,
aud: BroadcastSender<StampedData>,
vid_history: Arc<WatchSender<VecDeque<StampedData>>>,
aud_history: Arc<WatchSender<VecDeque<StampedData>>>,
config: Arc<WatchSender<StreamConfig>>,
name: StreamKind,
instance: NeoInstance,
@@ -400,14 +397,16 @@ pub(crate) struct StreamConfig {
#[derive(Debug, Clone)]
pub(crate) struct StampedData {
pub(crate) data: Vec<u8>,
pub(crate) data: Arc<Vec<u8>>,
pub(crate) ts: Duration,
}
pub(crate) struct StreamInstance {
pub(crate) name: StreamKind,
pub(crate) vid: BroadcastReceiver<StampedData>,
pub(crate) vid_history: WatchReceiver<VecDeque<StampedData>>,
pub(crate) aud: BroadcastReceiver<StampedData>,
pub(crate) aud_history: WatchReceiver<VecDeque<StampedData>>,
pub(crate) config: WatchReceiver<StreamConfig>,
in_use: CountUses,
}
@@ -417,7 +416,9 @@ impl StreamInstance {
Ok(Self {
name: data.name,
vid: data.vid.subscribe(),
vid_history: data.vid_history.subscribe(),
aud: data.aud.subscribe(),
aud_history: data.aud_history.subscribe(),
config: data.config.subscribe(),
in_use: data.users.create_activated().await?,
})
@@ -448,6 +449,10 @@ impl StreamData {
async fn new(name: StreamKind, instance: NeoInstance, strict: bool) -> Result<Self> {
let (vid, _) = broadcast::<StampedData>(100);
let (aud, _) = broadcast::<StampedData>(100);
let (vid_history, _) = watch::<VecDeque<StampedData>>(VecDeque::new());
let vid_history = Arc::new(vid_history);
let (aud_history, _) = watch::<VecDeque<StampedData>>(VecDeque::new());
let aud_history = Arc::new(aud_history);
let resolution = instance
.run_task(|cam| {
Box::pin(async move {
@@ -478,7 +483,9 @@ impl StreamData {
cancel: CancellationToken::new(),
config: Arc::new(config_tx),
vid,
vid_history,
aud,
aud_history,
instance,
handle: None,
strict,
@@ -493,30 +500,52 @@ impl StreamData {
let strict = me.strict;
let config = me.config.clone();
let thread_inuse = me.users.create_deactivated().await?;
let vid_history = me.vid_history.clone();
let aud_history = me.aud_history.clone();
me.handle = Some(tokio::task::spawn(async move {
tokio::select! {
let r = tokio::select! {
_ = cancel.cancelled() => {
Result::<(), anyhow::Error>::Ok(())
},
v = async {
loop {
let (watchdog_tx, mut watchdog_rx) = mpsc(1);
tokio::select! {
v = thread_inuse.dropped_users() => {
// Handles the stop and restart when no active users
log::debug!("Streaming STOP");
v?;
thread_inuse.aquired_users().await?; // Wait for new users of the stream
log::debug!("Streaming START");
AnyResult::Ok(())
},
v = async {
loop {
let check_timeout = timeout(Duration::from_secs(5), watchdog_rx.recv()).await;
if let Err(_)| Ok(None) = check_timeout {
// Timeout
// Reply with Ok to trigger the restart
log::debug!("Watchdog kicking the stream");
break AnyResult::Ok(());
}
}
} => v,
result = instance.run_task(|camera| {
let vid_tx = vid.clone();
let aud_tx = aud.clone();
let stream_config = config.clone();
let vid_history = vid_history.clone();
let aud_history = aud_history.clone();
let watchdog_tx = watchdog_tx.clone();
log::debug!("Running Stream Instance Task");
Box::pin(async move {
let res = async {
watchdog_tx.send(()).await?; // Feed the watchdog
let mut prev_ts = Duration::ZERO;
let mut stream_data = camera.start_video(name, 0, strict).await?;
loop {
watchdog_tx.send(()).await?; // Feed the watchdog
let data = stream_data.get_data().await??;
// Update the stream config with any information
@@ -597,19 +626,31 @@ impl StreamData {
match data {
BcMedia::Iframe(BcMediaIframe{data, microseconds, ..}) | BcMedia::Pframe(BcMediaPframe{data, microseconds,..}) => {
prev_ts = Duration::from_micros(microseconds as u64);
let _ = vid_tx.send(
StampedData{
data,
let d = StampedData{
data: Arc::new(data),
ts: prev_ts
};
let _ = vid_tx.send(d.clone());
vid_history.send_modify(|history| {
history.push_back(d);
while history.len() > 100 {
history.pop_front();
}
});
log::trace!("Sent Vid Frame");
}
BcMedia::Aac(BcMediaAac{data, ..}) | BcMedia::Adpcm(BcMediaAdpcm{data,..}) => {
let _ = aud_tx.send(
StampedData{
data,
let d = StampedData{
data: Arc::new(data),
ts: prev_ts,
})?;
};
let _ = aud_tx.send(d.clone())?;
aud_history.send_modify(|history| {
history.push_back(d);
while history.len() > 100 {
history.pop_front();
}
});
log::trace!("Sent Aud Frame");
},
_ => {},
@@ -636,8 +677,10 @@ impl StreamData {
},
}?;
}
} => v,
}
} => v,
};
log::debug!("Stream Thead Stopped: {:?}", r);
r
}));
Ok(me)
+2 -2
View File
@@ -15,7 +15,7 @@ use crate::common::VidFormat;
#[derive(Debug)]
enum GstControl {
Data(Vec<u8>),
Data(std::sync::Arc<Vec<u8>>),
Eos,
}
@@ -26,7 +26,7 @@ pub(super) struct GstSender {
}
impl GstSender {
pub(super) async fn send(&self, buf: Vec<u8>) -> Result<()> {
pub(super) async fn send(&self, buf: std::sync::Arc<Vec<u8>>) -> Result<()> {
self.sender
.send(GstControl::Data(buf))
.await
+91 -18
View File
@@ -62,7 +62,7 @@ use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use tokio::{
sync::{
broadcast::{channel as broadcast, Receiver as BroadcastReceiver},
broadcast::channel as broadcast,
mpsc::{channel as mpsc, Receiver as MpscReceiver, Sender as MpscSender},
},
task::JoinSet,
@@ -74,7 +74,7 @@ use tokio_util::sync::CancellationToken;
mod cmdline;
mod gst;
use crate::common::{CountUses, StampedData, UseCounter};
use crate::common::{CountUses, UseCounter};
use crate::{
common::{AudFormat, NeoInstance, NeoReactor, StreamConfig, StreamInstance, VidFormat},
rtsp::gst::NeoMediaFactory,
@@ -410,25 +410,27 @@ async fn stream_main(
let name = camera_config.borrow().name.clone();
let mut curr_pause;
loop {
log::trace!("{}: Activating Stream", &name);
log::debug!("{}: Activating Stream", &name);
stream_instance.activate().await?;
// Wait for a valid stream format to be detected
log::trace!("{}: Waiting for Valid Stream", &name);
log::debug!("{}: Waiting for Valid Stream", &name);
stream_instance
.config
.wait_for(|config| {
log::debug!("{:?}", config);
!matches!(config.vid_format, VidFormat::None)
&& config.resolution[0] > 0
&& config.resolution[1] > 0
})
.await?;
log::debug!("{}: Waiting for Valid Audio", &name);
// After vid give it 1s to look for audio
// Ignore timeout but check err
if let Ok(v) = tokio::time::timeout(
Duration::from_secs(1),
stream_instance.config.wait_for(|config| {
log::debug!("{:?}", config);
!matches!(config.vid_format, VidFormat::None)
&& !matches!(config.aud_format, AudFormat::None)
&& config.resolution[0] > 0
@@ -442,19 +444,16 @@ async fn stream_main(
curr_pause = camera_config.borrow().pause.clone();
let vid = stream_instance.vid.resubscribe();
let aud = stream_instance.aud.resubscribe();
let last_stream_config = stream_instance.config.borrow().clone();
let mut thread_stream_config = stream_instance.config.clone();
let mut set = JoinSet::<AnyResult<()>>::new();
log::trace!("{}: Creating Client Counters", &name);
log::debug!("{}: Creating Client Counters", &name);
// Handles the on off of the stream with the client pause
let client_counter = UseCounter::new().await;
let client_count = client_counter.create_deactivated().await?;
if curr_pause.on_disconnect {
log::trace!("{}: Enabling Client Pause", &name);
log::debug!("{}: Enabling Client Pause", &name);
// Take over activation
let mut client_activator = stream_instance.activator_handle().await;
client_activator.activate().await?;
@@ -475,7 +474,7 @@ async fn stream_main(
// Handles on motion pausing
if curr_pause.on_motion {
log::trace!("{}: Activating Motion Pause", &name);
log::debug!("{}: Activating Motion Pause", &name);
// Take over activation
let mut client_activator = stream_instance.activator_handle().await;
stream_instance.deactivate().await?;
@@ -494,8 +493,18 @@ async fn stream_main(
});
}
// This select is for the streams config updates
log::trace!("{}: Stream Activated", &name);
// This thread jsut keeps it active for 2s after an initial start to build the buffer
let mut init_activator = stream_instance.activator_handle().await;
set.spawn(async move {
init_activator.activate().await?;
sleep(Duration::from_secs(2)).await;
init_activator.deactivate().await?;
AnyResult::Ok(())
});
// This runs the actual stream.
// The select will restart if the stream's config updates
log::debug!("{}: Stream Activated", &name);
break tokio::select! {
v = thread_stream_config.wait_for(|new_conf| new_conf != &last_stream_config) => {
let v = v?;
@@ -510,7 +519,7 @@ async fn stream_main(
log::info!("{}: Pause Configuration Changed. Reloading Streams", &name);
continue;
},
v = stream_run(&name, vid, aud, rtsp, &last_stream_config, users, paths, client_count) => v,
v = stream_run(&name, &stream_instance, rtsp, &last_stream_config, users, paths, client_count) => v,
};
}
}
@@ -518,14 +527,18 @@ async fn stream_main(
#[allow(clippy::too_many_arguments)] // Seriously clippy needs to be done
async fn stream_run(
name: &str,
vidstream: BroadcastReceiver<StampedData>,
audstream: BroadcastReceiver<StampedData>,
stream_instance: &StreamInstance,
rtsp: &NeoRtspServer,
stream_config: &StreamConfig,
users: &HashSet<String>,
paths: &[String],
client_count: CountUses,
) -> AnyResult<()> {
let vidstream = stream_instance.vid.resubscribe();
let audstream = stream_instance.aud.resubscribe();
let vid_history = stream_instance.vid_history.clone();
let aud_history = stream_instance.aud_history.clone();
// Finally ready to create the factory and connect the stream
let mounts = rtsp
.mount_points()
@@ -565,15 +578,27 @@ async fn stream_run(
let mut vidstream = BroadcastStream::new(vidstream.resubscribe());
let thread_vid_data_tx = vid_data_tx.clone();
let thread_stream_cancel = stream_cancel.clone();
let thread_vid_history = vid_history.clone();
set.spawn(async move {
let r = tokio::select! {
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
v = async {
// Send Initial
for data in thread_vid_history.borrow().iter() {
thread_vid_data_tx.send(
StreamData::Media {
data: data.data.clone(),
ts: Duration::ZERO,
}
)?;
}
// Send new
while let Some(data) = vidstream.next().await {
if let Ok(data) = data {
thread_vid_data_tx.send(
StreamData::Media {
data: Arc::new(data.data),
data: data.data,
ts: data.ts
}
)?;
@@ -591,15 +616,26 @@ async fn stream_run(
let mut audstream = BroadcastStream::new(audstream.resubscribe());
let thread_stream_cancel = stream_cancel.clone();
let thread_aud_data_tx = aud_data_tx.clone();
let thread_aud_history = aud_history.clone();
set.spawn(async move {
let r = tokio::select! {
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
v = async {
// Send Initial
for data in thread_aud_history.borrow().iter() {
thread_aud_data_tx.send(
StreamData::Media {
data: data.data.clone(),
ts: Duration::ZERO,
}
)?;
}
// Send new
while let Some(data) = audstream.next().await {
if let Ok(data) = data {
thread_aud_data_tx.send(
StreamData::Media {
data: Arc::new(data.data),
data: data.data,
ts: data.ts
}
)?;
@@ -615,13 +651,32 @@ async fn stream_run(
// This thread takes the seek data for the vid and passed it into the stream
let thread_stream_cancel = stream_cancel.clone();
let thread_vid_history = vid_history.clone();
set.spawn(async move {
let r = tokio::select! {
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
v = async {
if let Some(vid_seek) = vid_seek.as_mut() {
while let Some(data) = vid_seek.recv().await {
let seek_ts = if let StreamData::Seek{ts, ..} = &data {
Some(*ts)
} else {
None
};
// Send seek
vid_data_tx.send(data)?;
// Send initial buffer
if let Some(seek_ts) = seek_ts {
// Send Initial
for data in thread_vid_history.borrow().iter() {
vid_data_tx.send(
StreamData::Media {
data: data.data.clone(),
ts: seek_ts,
}
)?;
}
}
}
}
AnyResult::Ok(())
@@ -633,13 +688,31 @@ async fn stream_run(
// This thread takes the seek data for the aud and passed it into the stream
let thread_stream_cancel = stream_cancel.clone();
let thread_aud_history = aud_history.clone();
set.spawn(async move {
let r = tokio::select! {
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
v = async {
if let Some(aud_seek) = aud_seek.as_mut() {
while let Some(data) = aud_seek.recv().await {
let seek_ts = if let StreamData::Seek{ts, ..} = &data {
Some(*ts)
} else {
None
};
aud_data_tx.send(data)?;
// Send initial buffer
if let Some(seek_ts) = seek_ts {
// Send Initial
for data in thread_aud_history.borrow().iter() {
aud_data_tx.send(
StreamData::Media {
data: data.data.clone(),
ts: seek_ts,
}
)?;
}
}
}
}
AnyResult::Ok(())