Add a watchdog to the stream
This commit is contained in:
4 files changed
+161
-42
No files matched your search
@@ -88,8 +88,11 @@ impl StreamData {
|
||||
|
||||
impl Drop for StreamData {
|
||||
fn drop(&mut self) {
|
||||
log::debug!("SteamData::drop Cancel");
|
||||
self.abort_handle.cancel();
|
||||
tokio::task::block_in_place(|| {
|
||||
tokio::runtime::Handle::current().block_on(async {
|
||||
let _ = self.shutdown().await;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+63
-20
@@ -5,7 +5,7 @@
|
||||
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use std::{
|
||||
collections::{hash_map::Entry, HashMap},
|
||||
collections::{hash_map::Entry, HashMap, VecDeque},
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio::{
|
||||
@@ -18,7 +18,7 @@ use tokio::{
|
||||
watch::{channel as watch, Receiver as WatchReceiver, Sender as WatchSender},
|
||||
},
|
||||
task::JoinHandle,
|
||||
time::Duration,
|
||||
time::{timeout, Duration},
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -144,13 +144,8 @@ impl NeoCamStreamThread {
|
||||
// Grab it
|
||||
if let Entry::Occupied(occ) = self.streams.entry(name) {
|
||||
result = Some(
|
||||
StreamInstance {
|
||||
name,
|
||||
vid: occ.get().vid.subscribe(),
|
||||
aud: occ.get().aud.subscribe(),
|
||||
config: occ.get().config.subscribe(),
|
||||
in_use: occ.get().users.create_activated().await?,
|
||||
});
|
||||
StreamInstance::new(occ.get()).await?
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -369,6 +364,8 @@ impl Drop for CountUses {
|
||||
pub(crate) struct StreamData {
|
||||
vid: BroadcastSender<StampedData>,
|
||||
aud: BroadcastSender<StampedData>,
|
||||
vid_history: Arc<WatchSender<VecDeque<StampedData>>>,
|
||||
aud_history: Arc<WatchSender<VecDeque<StampedData>>>,
|
||||
config: Arc<WatchSender<StreamConfig>>,
|
||||
name: StreamKind,
|
||||
instance: NeoInstance,
|
||||
@@ -400,14 +397,16 @@ pub(crate) struct StreamConfig {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct StampedData {
|
||||
pub(crate) data: Vec<u8>,
|
||||
pub(crate) data: Arc<Vec<u8>>,
|
||||
pub(crate) ts: Duration,
|
||||
}
|
||||
|
||||
pub(crate) struct StreamInstance {
|
||||
pub(crate) name: StreamKind,
|
||||
pub(crate) vid: BroadcastReceiver<StampedData>,
|
||||
pub(crate) vid_history: WatchReceiver<VecDeque<StampedData>>,
|
||||
pub(crate) aud: BroadcastReceiver<StampedData>,
|
||||
pub(crate) aud_history: WatchReceiver<VecDeque<StampedData>>,
|
||||
pub(crate) config: WatchReceiver<StreamConfig>,
|
||||
in_use: CountUses,
|
||||
}
|
||||
@@ -417,7 +416,9 @@ impl StreamInstance {
|
||||
Ok(Self {
|
||||
name: data.name,
|
||||
vid: data.vid.subscribe(),
|
||||
vid_history: data.vid_history.subscribe(),
|
||||
aud: data.aud.subscribe(),
|
||||
aud_history: data.aud_history.subscribe(),
|
||||
config: data.config.subscribe(),
|
||||
in_use: data.users.create_activated().await?,
|
||||
})
|
||||
@@ -448,6 +449,10 @@ impl StreamData {
|
||||
async fn new(name: StreamKind, instance: NeoInstance, strict: bool) -> Result<Self> {
|
||||
let (vid, _) = broadcast::<StampedData>(100);
|
||||
let (aud, _) = broadcast::<StampedData>(100);
|
||||
let (vid_history, _) = watch::<VecDeque<StampedData>>(VecDeque::new());
|
||||
let vid_history = Arc::new(vid_history);
|
||||
let (aud_history, _) = watch::<VecDeque<StampedData>>(VecDeque::new());
|
||||
let aud_history = Arc::new(aud_history);
|
||||
let resolution = instance
|
||||
.run_task(|cam| {
|
||||
Box::pin(async move {
|
||||
@@ -478,7 +483,9 @@ impl StreamData {
|
||||
cancel: CancellationToken::new(),
|
||||
config: Arc::new(config_tx),
|
||||
vid,
|
||||
vid_history,
|
||||
aud,
|
||||
aud_history,
|
||||
instance,
|
||||
handle: None,
|
||||
strict,
|
||||
@@ -493,30 +500,52 @@ impl StreamData {
|
||||
let strict = me.strict;
|
||||
let config = me.config.clone();
|
||||
let thread_inuse = me.users.create_deactivated().await?;
|
||||
let vid_history = me.vid_history.clone();
|
||||
let aud_history = me.aud_history.clone();
|
||||
|
||||
me.handle = Some(tokio::task::spawn(async move {
|
||||
tokio::select! {
|
||||
let r = tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
Result::<(), anyhow::Error>::Ok(())
|
||||
},
|
||||
v = async {
|
||||
loop {
|
||||
let (watchdog_tx, mut watchdog_rx) = mpsc(1);
|
||||
tokio::select! {
|
||||
v = thread_inuse.dropped_users() => {
|
||||
// Handles the stop and restart when no active users
|
||||
log::debug!("Streaming STOP");
|
||||
v?;
|
||||
thread_inuse.aquired_users().await?; // Wait for new users of the stream
|
||||
log::debug!("Streaming START");
|
||||
AnyResult::Ok(())
|
||||
},
|
||||
v = async {
|
||||
loop {
|
||||
let check_timeout = timeout(Duration::from_secs(5), watchdog_rx.recv()).await;
|
||||
if let Err(_)| Ok(None) = check_timeout {
|
||||
// Timeout
|
||||
// Reply with Ok to trigger the restart
|
||||
log::debug!("Watchdog kicking the stream");
|
||||
break AnyResult::Ok(());
|
||||
}
|
||||
}
|
||||
} => v,
|
||||
result = instance.run_task(|camera| {
|
||||
let vid_tx = vid.clone();
|
||||
let aud_tx = aud.clone();
|
||||
let stream_config = config.clone();
|
||||
let vid_history = vid_history.clone();
|
||||
let aud_history = aud_history.clone();
|
||||
let watchdog_tx = watchdog_tx.clone();
|
||||
log::debug!("Running Stream Instance Task");
|
||||
Box::pin(async move {
|
||||
let res = async {
|
||||
watchdog_tx.send(()).await?; // Feed the watchdog
|
||||
let mut prev_ts = Duration::ZERO;
|
||||
let mut stream_data = camera.start_video(name, 0, strict).await?;
|
||||
loop {
|
||||
watchdog_tx.send(()).await?; // Feed the watchdog
|
||||
let data = stream_data.get_data().await??;
|
||||
|
||||
// Update the stream config with any information
|
||||
@@ -597,19 +626,31 @@ impl StreamData {
|
||||
match data {
|
||||
BcMedia::Iframe(BcMediaIframe{data, microseconds, ..}) | BcMedia::Pframe(BcMediaPframe{data, microseconds,..}) => {
|
||||
prev_ts = Duration::from_micros(microseconds as u64);
|
||||
let _ = vid_tx.send(
|
||||
StampedData{
|
||||
data,
|
||||
let d = StampedData{
|
||||
data: Arc::new(data),
|
||||
ts: prev_ts
|
||||
};
|
||||
let _ = vid_tx.send(d.clone());
|
||||
vid_history.send_modify(|history| {
|
||||
history.push_back(d);
|
||||
while history.len() > 100 {
|
||||
history.pop_front();
|
||||
}
|
||||
});
|
||||
log::trace!("Sent Vid Frame");
|
||||
}
|
||||
BcMedia::Aac(BcMediaAac{data, ..}) | BcMedia::Adpcm(BcMediaAdpcm{data,..}) => {
|
||||
let _ = aud_tx.send(
|
||||
StampedData{
|
||||
data,
|
||||
let d = StampedData{
|
||||
data: Arc::new(data),
|
||||
ts: prev_ts,
|
||||
})?;
|
||||
};
|
||||
let _ = aud_tx.send(d.clone())?;
|
||||
aud_history.send_modify(|history| {
|
||||
history.push_back(d);
|
||||
while history.len() > 100 {
|
||||
history.pop_front();
|
||||
}
|
||||
});
|
||||
log::trace!("Sent Aud Frame");
|
||||
},
|
||||
_ => {},
|
||||
@@ -636,8 +677,10 @@ impl StreamData {
|
||||
},
|
||||
}?;
|
||||
}
|
||||
} => v,
|
||||
}
|
||||
} => v,
|
||||
};
|
||||
log::debug!("Stream Thead Stopped: {:?}", r);
|
||||
r
|
||||
}));
|
||||
|
||||
Ok(me)
|
||||
|
||||
+2
-2
@@ -15,7 +15,7 @@ use crate::common::VidFormat;
|
||||
|
||||
#[derive(Debug)]
|
||||
enum GstControl {
|
||||
Data(Vec<u8>),
|
||||
Data(std::sync::Arc<Vec<u8>>),
|
||||
Eos,
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ pub(super) struct GstSender {
|
||||
}
|
||||
|
||||
impl GstSender {
|
||||
pub(super) async fn send(&self, buf: Vec<u8>) -> Result<()> {
|
||||
pub(super) async fn send(&self, buf: std::sync::Arc<Vec<u8>>) -> Result<()> {
|
||||
self.sender
|
||||
.send(GstControl::Data(buf))
|
||||
.await
|
||||
|
||||
+91
-18
@@ -62,7 +62,7 @@ use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use tokio::{
|
||||
sync::{
|
||||
broadcast::{channel as broadcast, Receiver as BroadcastReceiver},
|
||||
broadcast::channel as broadcast,
|
||||
mpsc::{channel as mpsc, Receiver as MpscReceiver, Sender as MpscSender},
|
||||
},
|
||||
task::JoinSet,
|
||||
@@ -74,7 +74,7 @@ use tokio_util::sync::CancellationToken;
|
||||
mod cmdline;
|
||||
mod gst;
|
||||
|
||||
use crate::common::{CountUses, StampedData, UseCounter};
|
||||
use crate::common::{CountUses, UseCounter};
|
||||
use crate::{
|
||||
common::{AudFormat, NeoInstance, NeoReactor, StreamConfig, StreamInstance, VidFormat},
|
||||
rtsp::gst::NeoMediaFactory,
|
||||
@@ -410,25 +410,27 @@ async fn stream_main(
|
||||
let name = camera_config.borrow().name.clone();
|
||||
let mut curr_pause;
|
||||
loop {
|
||||
log::trace!("{}: Activating Stream", &name);
|
||||
log::debug!("{}: Activating Stream", &name);
|
||||
stream_instance.activate().await?;
|
||||
|
||||
// Wait for a valid stream format to be detected
|
||||
log::trace!("{}: Waiting for Valid Stream", &name);
|
||||
log::debug!("{}: Waiting for Valid Stream", &name);
|
||||
stream_instance
|
||||
.config
|
||||
.wait_for(|config| {
|
||||
log::debug!("{:?}", config);
|
||||
!matches!(config.vid_format, VidFormat::None)
|
||||
&& config.resolution[0] > 0
|
||||
&& config.resolution[1] > 0
|
||||
})
|
||||
.await?;
|
||||
|
||||
log::debug!("{}: Waiting for Valid Audio", &name);
|
||||
// After vid give it 1s to look for audio
|
||||
// Ignore timeout but check err
|
||||
if let Ok(v) = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
stream_instance.config.wait_for(|config| {
|
||||
log::debug!("{:?}", config);
|
||||
!matches!(config.vid_format, VidFormat::None)
|
||||
&& !matches!(config.aud_format, AudFormat::None)
|
||||
&& config.resolution[0] > 0
|
||||
@@ -442,19 +444,16 @@ async fn stream_main(
|
||||
|
||||
curr_pause = camera_config.borrow().pause.clone();
|
||||
|
||||
let vid = stream_instance.vid.resubscribe();
|
||||
let aud = stream_instance.aud.resubscribe();
|
||||
|
||||
let last_stream_config = stream_instance.config.borrow().clone();
|
||||
let mut thread_stream_config = stream_instance.config.clone();
|
||||
|
||||
let mut set = JoinSet::<AnyResult<()>>::new();
|
||||
log::trace!("{}: Creating Client Counters", &name);
|
||||
log::debug!("{}: Creating Client Counters", &name);
|
||||
// Handles the on off of the stream with the client pause
|
||||
let client_counter = UseCounter::new().await;
|
||||
let client_count = client_counter.create_deactivated().await?;
|
||||
if curr_pause.on_disconnect {
|
||||
log::trace!("{}: Enabling Client Pause", &name);
|
||||
log::debug!("{}: Enabling Client Pause", &name);
|
||||
// Take over activation
|
||||
let mut client_activator = stream_instance.activator_handle().await;
|
||||
client_activator.activate().await?;
|
||||
@@ -475,7 +474,7 @@ async fn stream_main(
|
||||
|
||||
// Handles on motion pausing
|
||||
if curr_pause.on_motion {
|
||||
log::trace!("{}: Activating Motion Pause", &name);
|
||||
log::debug!("{}: Activating Motion Pause", &name);
|
||||
// Take over activation
|
||||
let mut client_activator = stream_instance.activator_handle().await;
|
||||
stream_instance.deactivate().await?;
|
||||
@@ -494,8 +493,18 @@ async fn stream_main(
|
||||
});
|
||||
}
|
||||
|
||||
// This select is for the streams config updates
|
||||
log::trace!("{}: Stream Activated", &name);
|
||||
// This thread jsut keeps it active for 2s after an initial start to build the buffer
|
||||
let mut init_activator = stream_instance.activator_handle().await;
|
||||
set.spawn(async move {
|
||||
init_activator.activate().await?;
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
init_activator.deactivate().await?;
|
||||
AnyResult::Ok(())
|
||||
});
|
||||
|
||||
// This runs the actual stream.
|
||||
// The select will restart if the stream's config updates
|
||||
log::debug!("{}: Stream Activated", &name);
|
||||
break tokio::select! {
|
||||
v = thread_stream_config.wait_for(|new_conf| new_conf != &last_stream_config) => {
|
||||
let v = v?;
|
||||
@@ -510,7 +519,7 @@ async fn stream_main(
|
||||
log::info!("{}: Pause Configuration Changed. Reloading Streams", &name);
|
||||
continue;
|
||||
},
|
||||
v = stream_run(&name, vid, aud, rtsp, &last_stream_config, users, paths, client_count) => v,
|
||||
v = stream_run(&name, &stream_instance, rtsp, &last_stream_config, users, paths, client_count) => v,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -518,14 +527,18 @@ async fn stream_main(
|
||||
#[allow(clippy::too_many_arguments)] // Seriously clippy needs to be done
|
||||
async fn stream_run(
|
||||
name: &str,
|
||||
vidstream: BroadcastReceiver<StampedData>,
|
||||
audstream: BroadcastReceiver<StampedData>,
|
||||
stream_instance: &StreamInstance,
|
||||
rtsp: &NeoRtspServer,
|
||||
stream_config: &StreamConfig,
|
||||
users: &HashSet<String>,
|
||||
paths: &[String],
|
||||
client_count: CountUses,
|
||||
) -> AnyResult<()> {
|
||||
let vidstream = stream_instance.vid.resubscribe();
|
||||
let audstream = stream_instance.aud.resubscribe();
|
||||
let vid_history = stream_instance.vid_history.clone();
|
||||
let aud_history = stream_instance.aud_history.clone();
|
||||
|
||||
// Finally ready to create the factory and connect the stream
|
||||
let mounts = rtsp
|
||||
.mount_points()
|
||||
@@ -565,15 +578,27 @@ async fn stream_run(
|
||||
let mut vidstream = BroadcastStream::new(vidstream.resubscribe());
|
||||
let thread_vid_data_tx = vid_data_tx.clone();
|
||||
let thread_stream_cancel = stream_cancel.clone();
|
||||
let thread_vid_history = vid_history.clone();
|
||||
set.spawn(async move {
|
||||
let r = tokio::select! {
|
||||
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
|
||||
v = async {
|
||||
// Send Initial
|
||||
for data in thread_vid_history.borrow().iter() {
|
||||
thread_vid_data_tx.send(
|
||||
StreamData::Media {
|
||||
data: data.data.clone(),
|
||||
ts: Duration::ZERO,
|
||||
}
|
||||
)?;
|
||||
}
|
||||
|
||||
// Send new
|
||||
while let Some(data) = vidstream.next().await {
|
||||
if let Ok(data) = data {
|
||||
thread_vid_data_tx.send(
|
||||
StreamData::Media {
|
||||
data: Arc::new(data.data),
|
||||
data: data.data,
|
||||
ts: data.ts
|
||||
}
|
||||
)?;
|
||||
@@ -591,15 +616,26 @@ async fn stream_run(
|
||||
let mut audstream = BroadcastStream::new(audstream.resubscribe());
|
||||
let thread_stream_cancel = stream_cancel.clone();
|
||||
let thread_aud_data_tx = aud_data_tx.clone();
|
||||
let thread_aud_history = aud_history.clone();
|
||||
set.spawn(async move {
|
||||
let r = tokio::select! {
|
||||
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
|
||||
v = async {
|
||||
// Send Initial
|
||||
for data in thread_aud_history.borrow().iter() {
|
||||
thread_aud_data_tx.send(
|
||||
StreamData::Media {
|
||||
data: data.data.clone(),
|
||||
ts: Duration::ZERO,
|
||||
}
|
||||
)?;
|
||||
}
|
||||
// Send new
|
||||
while let Some(data) = audstream.next().await {
|
||||
if let Ok(data) = data {
|
||||
thread_aud_data_tx.send(
|
||||
StreamData::Media {
|
||||
data: Arc::new(data.data),
|
||||
data: data.data,
|
||||
ts: data.ts
|
||||
}
|
||||
)?;
|
||||
@@ -615,13 +651,32 @@ async fn stream_run(
|
||||
|
||||
// This thread takes the seek data for the vid and passed it into the stream
|
||||
let thread_stream_cancel = stream_cancel.clone();
|
||||
let thread_vid_history = vid_history.clone();
|
||||
set.spawn(async move {
|
||||
let r = tokio::select! {
|
||||
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
|
||||
v = async {
|
||||
if let Some(vid_seek) = vid_seek.as_mut() {
|
||||
while let Some(data) = vid_seek.recv().await {
|
||||
let seek_ts = if let StreamData::Seek{ts, ..} = &data {
|
||||
Some(*ts)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Send seek
|
||||
vid_data_tx.send(data)?;
|
||||
// Send initial buffer
|
||||
if let Some(seek_ts) = seek_ts {
|
||||
// Send Initial
|
||||
for data in thread_vid_history.borrow().iter() {
|
||||
vid_data_tx.send(
|
||||
StreamData::Media {
|
||||
data: data.data.clone(),
|
||||
ts: seek_ts,
|
||||
}
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
AnyResult::Ok(())
|
||||
@@ -633,13 +688,31 @@ async fn stream_run(
|
||||
|
||||
// This thread takes the seek data for the aud and passed it into the stream
|
||||
let thread_stream_cancel = stream_cancel.clone();
|
||||
let thread_aud_history = aud_history.clone();
|
||||
set.spawn(async move {
|
||||
let r = tokio::select! {
|
||||
_ = thread_stream_cancel.cancelled() => AnyResult::Ok(()),
|
||||
v = async {
|
||||
if let Some(aud_seek) = aud_seek.as_mut() {
|
||||
while let Some(data) = aud_seek.recv().await {
|
||||
let seek_ts = if let StreamData::Seek{ts, ..} = &data {
|
||||
Some(*ts)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
aud_data_tx.send(data)?;
|
||||
// Send initial buffer
|
||||
if let Some(seek_ts) = seek_ts {
|
||||
// Send Initial
|
||||
for data in thread_aud_history.borrow().iter() {
|
||||
aud_data_tx.send(
|
||||
StreamData::Media {
|
||||
data: data.data.clone(),
|
||||
ts: seek_ts,
|
||||
}
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
AnyResult::Ok(())
|
||||
|
||||
Reference in new issue
Block a user