From 62a694fbe4c5e9c9363c1a309fc16c7f38dde4b7 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Thu, 16 Sep 2021 11:45:09 +0700 Subject: [PATCH 01/11] Added basic motion based streaming --- Cargo.lock | 1 + Cargo.toml | 1 + crates/core/src/bc/de.rs | 6 +- crates/core/src/bc/model.rs | 2 + crates/core/src/bc/xml.rs | 2 +- crates/core/src/bc_protocol/connection/mod.rs | 4 + crates/core/src/bc_protocol/stream.rs | 84 +++++++++++++- src/rtsp/abort.rs | 29 +++++ src/rtsp/gst.rs | 43 ++++++- src/rtsp/mod.rs | 105 +++++++++++++++++- src/rtsp/motion.rs | 32 ++++++ 11 files changed, 290 insertions(+), 19 deletions(-) create mode 100644 src/rtsp/abort.rs create mode 100644 src/rtsp/motion.rs diff --git a/Cargo.lock b/Cargo.lock index e828ff0..693548a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -958,6 +958,7 @@ dependencies = [ "cfb-mode", "cookie-factory", "crossbeam", + "crossbeam-channel 0.5.1", "env_logger", "err-derive", "gstreamer", diff --git a/Cargo.toml b/Cargo.toml index 3b2de33..1912e6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ aes = "0.6" cfb-mode = "0.6" cookie-factory = "0.3" crossbeam = "0.7" +crossbeam-channel = "0.5.1" err-derive = "0.2" env_logger = "*" gstreamer = "0.17" diff --git a/crates/core/src/bc/de.rs b/crates/core/src/bc/de.rs index dd07d29..f91a281 100644 --- a/crates/core/src/bc/de.rs +++ b/crates/core/src/bc/de.rs @@ -431,7 +431,7 @@ mod tests { .. })), }), - }) if version == "1.1" && stream_type == "externStream" + }) if version == "1.1" && stream_type == Some("externStream".to_string()) ); } @@ -475,7 +475,7 @@ mod tests { .. })), }), - }) if version == "1.1" && stream_type == "subStream" + }) if version == "1.1" && stream_type == Some("subStream".to_string()) ); } @@ -519,7 +519,7 @@ mod tests { .. })), }), - }) if version == "1.1" && stream_type == "mainStream" + }) if version == "1.1" && stream_type == Some("mainStream".to_string()) ); } } diff --git a/crates/core/src/bc/model.rs b/crates/core/src/bc/model.rs index 37b8f6b..e2992ca 100644 --- a/crates/core/src/bc/model.rs +++ b/crates/core/src/bc/model.rs @@ -10,6 +10,8 @@ pub const MSG_ID_LOGIN: u32 = 1; pub const MSG_ID_LOGOUT: u32 = 2; /// Video and Audio Streams messages have this ID pub const MSG_ID_VIDEO: u32 = 3; +/// ID used to stop the video stream +pub const MSG_ID_VIDEO_STOP: u32 = 4; /// TalkAbility messages have this ID pub const MSG_ID_TALKABILITY: u32 = 10; /// TalkReset messages have this ID diff --git a/crates/core/src/bc/xml.rs b/crates/core/src/bc/xml.rs index f4a216a..ef7bb7f 100644 --- a/crates/core/src/bc/xml.rs +++ b/crates/core/src/bc/xml.rs @@ -199,7 +199,7 @@ pub struct Preview { pub handle: u32, /// Either `"mainStream"` or `"subStream"` #[yaserde(rename = "streamType")] - pub stream_type: String, + pub stream_type: Option, } /// Extension xml diff --git a/crates/core/src/bc_protocol/connection/mod.rs b/crates/core/src/bc_protocol/connection/mod.rs index 4d6a235..504ae23 100644 --- a/crates/core/src/bc_protocol/connection/mod.rs +++ b/crates/core/src/bc_protocol/connection/mod.rs @@ -7,6 +7,7 @@ //! use crate::bc; use crate::bc::model::*; +use crate::bcmedia; use crate::bcudp; use err_derive::Error; use log::*; @@ -45,6 +46,9 @@ pub enum Error { #[error(display = "Deserialization error")] Deserialization(#[error(source)] bc::de::Error), + #[error(display = "BcMedia Deserialization error")] + BcMediaDeserialization(#[error(source)] bcmedia::de::Error), + #[error(display = "Serialization error")] Serialization(#[error(source)] bc::ser::Error), diff --git a/crates/core/src/bc_protocol/stream.rs b/crates/core/src/bc_protocol/stream.rs index 6c0b24e..316c9e9 100644 --- a/crates/core/src/bc_protocol/stream.rs +++ b/crates/core/src/bc_protocol/stream.rs @@ -1,4 +1,4 @@ -use super::{BcCamera, BinarySubscriber, Result}; +use super::{BcCamera, BinarySubscriber, Error, Result, RX_TIMEOUT}; use crate::{ bc::{model::*, xml::*}, bcmedia::model::*, @@ -112,7 +112,7 @@ impl BcCamera { version: xml_ver(), channel_id: self.channel_id, handle, - stream_type: stream_name, + stream_type: Some(stream_name), }), ..Default::default() }, @@ -122,14 +122,90 @@ impl BcCamera { let mut media_sub = BinarySubscriber::from_bc_sub(&sub_video); + let result; + loop { let bc_media = BcMedia::deserialize(&mut media_sub)?; // We now have a complete interesting packet. Send it to on the callback match data_outs.stream_recv(bc_media) { Ok(true) => {} - Ok(false) => return Ok(()), - Err(e) => return Err(e), + Ok(false) => { + result = Ok(()); + break; + } + Err(e) => { + result = Err(e); + break; + } }; } + + println!("Stopping video: {:?}", result); + self.stop_stream(stream)?; + println!("Video stopped"); + result + } + + /// + /// Sends the stop stream message to the camera + /// + pub fn stop_stream(&self, stream: Stream) -> Result<()> { + let connection = self + .connection + .as_ref() + .expect("Must be connected to stop video"); + let sub = connection.subscribe(MSG_ID_VIDEO_STOP)?; + + // Theses are the numbers used with the offical client + // On an E1 and swann cameras: + // - mainStream always has a value of 0 + // - subStream always has a value of 1 + // - There is no externStram + // On a B800: + // - mainStream is 0 + // - subStream is 256 + // - externStram is 1024 + let handle = match stream { + Stream::Main => 0, + Stream::Sub => 256, + Stream::Extern => 1024, + }; + + let msg = Bc::new_from_xml( + BcMeta { + msg_id: MSG_ID_VIDEO_STOP, + channel_id: self.channel_id, + msg_num: self.new_message_num(), + stream_type: 0, + response_code: 0, + class: 0x6414, // IDK why + }, + BcXml { + preview: Some(Preview { + version: xml_ver(), + channel_id: self.channel_id, + handle, + stream_type: None, + }), + ..Default::default() + }, + ); + + sub.send(msg)?; + + let msg = sub.rx.recv_timeout(RX_TIMEOUT)?; + + if let BcMeta { + response_code: 200, .. + } = msg.meta + { + } else { + return Err(Error::UnintelligibleReply { + reply: msg, + why: "The camera did not accept the stream stop command.", + }); + } + + Ok(()) } } diff --git a/src/rtsp/abort.rs b/src/rtsp/abort.rs new file mode 100644 index 0000000..4dab182 --- /dev/null +++ b/src/rtsp/abort.rs @@ -0,0 +1,29 @@ +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +#[derive(Clone)] +pub(crate) struct AbortHandle { + live: Arc, +} + +impl AbortHandle { + pub(crate) fn new() -> Self { + Self { + live: Arc::new(AtomicBool::new(true)), + } + } + + pub(crate) fn abort(&self) { + self.live.store(false, Ordering::Relaxed); + } + + pub(crate) fn reset(&self) { + self.live.store(true, Ordering::Relaxed); + } + + pub(crate) fn is_live(&self) -> bool { + self.live.load(Ordering::Relaxed) + } +} diff --git a/src/rtsp/gst.rs b/src/rtsp/gst.rs index 77a67a1..d9a4872 100644 --- a/src/rtsp/gst.rs +++ b/src/rtsp/gst.rs @@ -7,6 +7,8 @@ use gstreamer::prelude::Cast; use gstreamer::{Bin, Structure}; use gstreamer_app::AppSrc; //use gstreamer_rtsp::RTSPLowerTrans; +use super::abort::AbortHandle; +use anyhow::anyhow; use gstreamer_rtsp::RTSPAuthMethod; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; use gstreamer_rtsp_server::glib; @@ -27,6 +29,7 @@ use std::io; use std::io::Write; type Result = std::result::Result; +type AnyResult = std::result::Result; pub(crate) struct RtspServer { server: GstRTSPServer, @@ -38,6 +41,8 @@ pub(crate) struct GstOutputs { video_format: Option, audio_format: Option, factory: RTSPMediaFactory, + live: AbortHandle, + last_iframe: Option>, } // The stream from the camera will be using one of these formats @@ -58,6 +63,7 @@ enum StreamFormat { impl StreamOutput for GstOutputs { fn stream_recv(&mut self, media: BcMedia) -> StreamOutputError { + let mut should_continue = true; match media { BcMedia::Iframe(payload) => { let video_type = match payload.video_type { @@ -66,6 +72,11 @@ impl StreamOutput for GstOutputs { }; self.set_format(Some(video_type)); self.vidsrc.write_all(&payload.data)?; + self.last_iframe = Some(payload.data); + // Only stop on an iframe so we have the + // last frame to show + should_continue = self.live.is_live(); + debug!("Iframe continue: {}", should_continue); } BcMedia::Pframe(payload) => { let video_type = match payload.video_type { @@ -87,8 +98,8 @@ impl StreamOutput for GstOutputs { //Ignore other BcMedia like InfoV1 and InfoV2 } } - - Ok(true) + debug!("Output continue: {}", should_continue); + Ok(should_continue) } } @@ -100,11 +111,31 @@ impl GstOutputs { video_format: None, audio_format: None, factory: RTSPMediaFactory::new(), + live: AbortHandle::new(), + last_iframe: Default::default(), }; result.apply_format(); result } + pub(crate) fn get_abort_handle(&self) -> AbortHandle { + self.live.clone() + } + + pub(crate) fn has_last_iframe(&self) -> bool { + self.last_iframe.is_some() + } + + pub(crate) fn write_last_iframe(&mut self) -> AnyResult<()> { + self.vidsrc.write_all( + self.last_iframe + .as_ref() + .ok_or_else(|| anyhow!("No iframe data avaliable"))? + .as_slice(), + )?; + Ok(()) + } + fn set_format(&mut self, format: Option) { match format { Some(StreamFormat::H264) | Some(StreamFormat::H265) => { @@ -140,8 +171,7 @@ impl GstOutputs { _ => "! fakesink".to_string(), }; - self.factory.set_launch( - &vec![ + let launch_str = &vec![ "( ", "appsrc name=vidsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks launch_vid, @@ -149,8 +179,9 @@ impl GstOutputs { &launch_aud, ")" ] - .join(" "), - ); + .join(" "); + debug!("Gstreamer launch str: {:?}", launch_str); + self.factory.set_launch(launch_str); } } diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 06f0a60..e016248 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -30,24 +30,27 @@ /// use anyhow::{Context, Result}; use log::*; -use neolink_core::bc_protocol::{BcCamera, Stream}; +use neolink_core::bc_protocol::{BcCamera, MotionStatus, Stream}; use std::collections::HashSet; use std::fs; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Duration; use validator::Validate; // mod adpcm; -/// The command line parameters for this subcommand +// The command line parameters for this subcommand mod cmdline; mod config; -/// The errors this subcommand can raise +// The gstreamer implementation +mod abort; mod gst; +mod motion; use crate::utils::AddressOrUid; pub(crate) use cmdline::Opt; use config::{CameraConfig, Config, UserConfig}; use gst::{GstOutputs, RtspServer, TlsAuthenticationMode}; +use motion::MotionStream; /// Entry point for the rtsp subcommand /// @@ -278,7 +281,99 @@ fn camera_main( "{}: Starting video stream {}", camera_config.name, stream_display_name ); - camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", camera_config.name)) + + // Used to get the error from the thread + let thread_error: Arc>> = Default::default(); + + crossbeam::scope(|s| { + let (mut motion, rx) = MotionStream::new(); + + + let motionthread_motion_abort = motion.get_abort_handle(); + let motionthread_stream_abort = outputs.get_abort_handle(); + let rxthread_motion_abort = motion.get_abort_handle(); + let rxthread_stream_abort = outputs.get_abort_handle(); + let streamthread_stream_abort = outputs.get_abort_handle(); + let streamthread_motion_abort = motion.get_abort_handle(); + + // Start in the aborted state because motion defaults to still + streamthread_stream_abort.abort(); + + let arc_camera = Arc::new(camera); + + // Motion detection thread + let motion_camera = arc_camera.clone(); + let motion_camname = camera_config.name.to_string(); + let motion_error = thread_error.clone(); + s.spawn(move |_| { + if let Err(e) = motion_camera.listen_on_motion(&mut motion).with_context(|| format!("Error while reading motion {}", motion_camname)) { + // When motion errors set the thread error + *motion_error.lock().unwrap() = Some(e); + } + // Always abort the streams/motion when motion stops regardless of why + // so that this thread can reset the camera + motionthread_motion_abort.abort(); + motionthread_stream_abort.abort(); + debug!("Motion stopped"); + }); + + // Motion rx -> stream thread + s.spawn(move |_| { + // Stops when motion is stopped + // which happens when either motion or stream errors + while rxthread_motion_abort.is_live() { + let msg = rx.try_recv(); + if let Ok(msg) = msg { + match msg { + MotionStatus::Start => { + debug!("Motion start"); + rxthread_stream_abort.reset(); + } + MotionStatus::Stop => { + debug!("Motion stop"); + rxthread_stream_abort.abort(); + } + _ => {} + } + } + } + println!("Motion RX Stopped"); + }); + + // Stream thread + let stream_camera = arc_camera; + let stream_camname = camera_config.name.to_string(); + let stream_error = thread_error.clone(); + s.spawn(move |_| { + loop { + if streamthread_stream_abort.is_live() || ! outputs.has_last_iframe() { + debug!("Stream resumed: waiting for motion to stop"); + // Will block until the rx thread send the abort + if let Err(e) = stream_camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", stream_camname)) { + // When stream errors set the thread error and abort the motion + *stream_error.lock().unwrap() = Some(e); + break; + } + debug!("Stream paused: waiting for motion to start"); + } else { + // While stream is aborted we send the blank image + if let Err(e) = outputs.write_last_iframe().with_context(|| format!("Failed to write no motion image for {}", stream_camname)) { + *stream_error.lock().unwrap() = Some(e); + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + } + streamthread_stream_abort.abort(); + streamthread_motion_abort.abort(); + println!("Stream stopped"); + }); + }).unwrap(); + + // Should be the only one holding the thread_error so unwrap it + // all the way to an actual error + Err(Arc::try_unwrap(thread_error).unwrap().into_inner().unwrap().unwrap()) + })().map_err(|e| CameraErr{ connected, login_fail, diff --git a/src/rtsp/motion.rs b/src/rtsp/motion.rs new file mode 100644 index 0000000..920fd1e --- /dev/null +++ b/src/rtsp/motion.rs @@ -0,0 +1,32 @@ +use super::abort::AbortHandle; +use crossbeam_channel::{unbounded, Receiver, Sender}; +use neolink_core::bc_protocol::{MotionOutput, MotionOutputError, MotionStatus}; + +pub(crate) struct MotionStream { + tx: Sender, + live: AbortHandle, +} + +impl MotionStream { + pub(crate) fn new() -> (Self, Receiver) { + let (tx, rx) = unbounded(); + ( + Self { + tx, + live: AbortHandle::new(), + }, + rx, + ) + } + + pub(crate) fn get_abort_handle(&self) -> AbortHandle { + self.live.clone() + } +} + +impl MotionOutput for MotionStream { + fn motion_recv(&mut self, motion_status: MotionStatus) -> MotionOutputError { + let _ = self.tx.send(motion_status); + Ok(self.live.is_live()) + } +} From 13690d723f5adb448206db7305ccc9a9165347c3 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Thu, 16 Sep 2021 12:32:40 +0700 Subject: [PATCH 02/11] Added a timeout to the motion based stream --- src/rtsp/config.rs | 13 ++++ src/rtsp/gst.rs | 3 +- src/rtsp/mod.rs | 166 ++++++++++++++++++++++++--------------------- src/rtsp/motion.rs | 38 ++++++++++- 4 files changed, 138 insertions(+), 82 deletions(-) diff --git a/src/rtsp/config.rs b/src/rtsp/config.rs index 013c3de..ee07e63 100644 --- a/src/rtsp/config.rs +++ b/src/rtsp/config.rs @@ -73,6 +73,9 @@ pub(crate) struct CameraConfig { #[validate(range(min = 0, max = 31, message = "Invalid channel", code = "channel_id"))] #[serde(default = "default_channel_id")] pub(crate) channel_id: u8, + + #[validate] + pub(crate) motion: Option, } #[derive(Debug, Deserialize, Validate, Clone)] @@ -85,6 +88,12 @@ pub(crate) struct UserConfig { pub(crate) pass: String, } +#[derive(Debug, Deserialize, Validate, Clone)] +pub(crate) struct MotionConfig { + #[serde(default = "default_motion_timeout", alias = "timeout_seconds")] + pub(crate) timeout: f64, +} + fn default_bind_addr() -> String { "0.0.0.0".to_string() } @@ -109,6 +118,10 @@ fn default_channel_id() -> u8 { 0 } +fn default_motion_timeout() -> f64 { + 1. +} + pub(crate) static RESERVED_NAMES: &[&str] = &["anyone", "anonymous"]; fn validate_username(name: &str) -> Result<(), ValidationError> { if name.trim().is_empty() { diff --git a/src/rtsp/gst.rs b/src/rtsp/gst.rs index d9a4872..a5db706 100644 --- a/src/rtsp/gst.rs +++ b/src/rtsp/gst.rs @@ -76,7 +76,6 @@ impl StreamOutput for GstOutputs { // Only stop on an iframe so we have the // last frame to show should_continue = self.live.is_live(); - debug!("Iframe continue: {}", should_continue); } BcMedia::Pframe(payload) => { let video_type = match payload.video_type { @@ -98,7 +97,7 @@ impl StreamOutput for GstOutputs { //Ignore other BcMedia like InfoV1 and InfoV2 } } - debug!("Output continue: {}", should_continue); + Ok(should_continue) } } diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index e016248..b893ed7 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -285,94 +285,104 @@ fn camera_main( // Used to get the error from the thread let thread_error: Arc>> = Default::default(); - crossbeam::scope(|s| { - let (mut motion, rx) = MotionStream::new(); + match &camera_config.motion { + Some(motion_config) => { + crossbeam::scope(|s| { + let (mut motion, rx) = MotionStream::new(motion_config.timeout); - let motionthread_motion_abort = motion.get_abort_handle(); - let motionthread_stream_abort = outputs.get_abort_handle(); - let rxthread_motion_abort = motion.get_abort_handle(); - let rxthread_stream_abort = outputs.get_abort_handle(); - let streamthread_stream_abort = outputs.get_abort_handle(); - let streamthread_motion_abort = motion.get_abort_handle(); + let motionthread_motion_abort = motion.get_abort_handle(); + let motionthread_stream_abort = outputs.get_abort_handle(); + let rxthread_motion_abort = motion.get_abort_handle(); + let rxthread_stream_abort = outputs.get_abort_handle(); + let streamthread_stream_abort = outputs.get_abort_handle(); + let streamthread_motion_abort = motion.get_abort_handle(); - // Start in the aborted state because motion defaults to still - streamthread_stream_abort.abort(); + // Start in the aborted state because motion defaults to still + streamthread_stream_abort.abort(); - let arc_camera = Arc::new(camera); + let arc_camera = Arc::new(camera); - // Motion detection thread - let motion_camera = arc_camera.clone(); - let motion_camname = camera_config.name.to_string(); - let motion_error = thread_error.clone(); - s.spawn(move |_| { - if let Err(e) = motion_camera.listen_on_motion(&mut motion).with_context(|| format!("Error while reading motion {}", motion_camname)) { - // When motion errors set the thread error - *motion_error.lock().unwrap() = Some(e); - } - // Always abort the streams/motion when motion stops regardless of why - // so that this thread can reset the camera - motionthread_motion_abort.abort(); - motionthread_stream_abort.abort(); - debug!("Motion stopped"); - }); + // Motion detection thread + let motion_camera = arc_camera.clone(); + let motion_camname = camera_config.name.to_string(); + let motion_error = thread_error.clone(); + s.spawn(move |_| { + if let Err(e) = motion_camera.listen_on_motion(&mut motion).with_context(|| format!("Error while reading motion {}", motion_camname)) { + // When motion errors set the thread error + *motion_error.lock().unwrap() = Some(e); + } + // Always abort the streams/motion when motion stops regardless of why + // so that this thread can reset the camera + motionthread_motion_abort.abort(); + motionthread_stream_abort.abort(); + debug!("Motion stopped"); + }); - // Motion rx -> stream thread - s.spawn(move |_| { - // Stops when motion is stopped - // which happens when either motion or stream errors - while rxthread_motion_abort.is_live() { - let msg = rx.try_recv(); - if let Ok(msg) = msg { - match msg { - MotionStatus::Start => { - debug!("Motion start"); - rxthread_stream_abort.reset(); + // Motion rx -> stream thread + s.spawn(move |_| { + // Stops when motion is stopped + // which happens when either motion or stream errors + while rxthread_motion_abort.is_live() { + let msg = rx.try_recv(); + if let Ok(msg) = msg { + match msg { + MotionStatus::Start => { + debug!("Motion start"); + rxthread_stream_abort.reset(); + } + MotionStatus::Stop => { + debug!("Motion stop"); + rxthread_stream_abort.abort(); + } + _ => {} + } } - MotionStatus::Stop => { - debug!("Motion stop"); - rxthread_stream_abort.abort(); + } + println!("Motion RX Stopped"); + }); + + // Stream thread + let stream_camera = arc_camera; + let stream_camname = camera_config.name.to_string(); + let stream_error = thread_error.clone(); + s.spawn(move |_| { + loop { + if streamthread_stream_abort.is_live() || ! outputs.has_last_iframe() { + debug!("Stream resumed: waiting for motion to stop"); + // Will block until the rx thread send the abort + if let Err(e) = stream_camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", stream_camname)) { + // When stream errors set the thread error and abort the motion + *stream_error.lock().unwrap() = Some(e); + break; + } + debug!("Stream paused: waiting for motion to start"); + } else { + // While stream is aborted we send the blank image + if let Err(e) = outputs.write_last_iframe().with_context(|| format!("Failed to write no motion image for {}", stream_camname)) { + *stream_error.lock().unwrap() = Some(e); + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); } - _ => {} } - } - } - println!("Motion RX Stopped"); - }); + streamthread_stream_abort.abort(); + streamthread_motion_abort.abort(); + println!("Stream stopped"); + }); + }).unwrap(); - // Stream thread - let stream_camera = arc_camera; - let stream_camname = camera_config.name.to_string(); - let stream_error = thread_error.clone(); - s.spawn(move |_| { - loop { - if streamthread_stream_abort.is_live() || ! outputs.has_last_iframe() { - debug!("Stream resumed: waiting for motion to stop"); - // Will block until the rx thread send the abort - if let Err(e) = stream_camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", stream_camname)) { - // When stream errors set the thread error and abort the motion - *stream_error.lock().unwrap() = Some(e); - break; - } - debug!("Stream paused: waiting for motion to start"); - } else { - // While stream is aborted we send the blank image - if let Err(e) = outputs.write_last_iframe().with_context(|| format!("Failed to write no motion image for {}", stream_camname)) { - *stream_error.lock().unwrap() = Some(e); - break; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - } - streamthread_stream_abort.abort(); - streamthread_motion_abort.abort(); - println!("Stream stopped"); - }); - }).unwrap(); - - // Should be the only one holding the thread_error so unwrap it - // all the way to an actual error - Err(Arc::try_unwrap(thread_error).unwrap().into_inner().unwrap().unwrap()) + // Should be the only one holding the thread_error so unwrap it + // all the way to an actual error + Err(Arc::try_unwrap(thread_error).unwrap().into_inner().unwrap().unwrap()) + } + None => { + // No motion config + // + // Just stream always + camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", camera_config.name)) + } + } })().map_err(|e| CameraErr{ connected, diff --git a/src/rtsp/motion.rs b/src/rtsp/motion.rs index 920fd1e..7b66da4 100644 --- a/src/rtsp/motion.rs +++ b/src/rtsp/motion.rs @@ -1,19 +1,24 @@ use super::abort::AbortHandle; use crossbeam_channel::{unbounded, Receiver, Sender}; use neolink_core::bc_protocol::{MotionOutput, MotionOutputError, MotionStatus}; +use std::time::Duration; pub(crate) struct MotionStream { tx: Sender, live: AbortHandle, + timeout: Duration, + cooloff: Option, } impl MotionStream { - pub(crate) fn new() -> (Self, Receiver) { + pub(crate) fn new(timeout_seconds: f64) -> (Self, Receiver) { let (tx, rx) = unbounded(); ( Self { tx, live: AbortHandle::new(), + timeout: Duration::from_secs_f64(timeout_seconds), + cooloff: None, }, rx, ) @@ -26,7 +31,36 @@ impl MotionStream { impl MotionOutput for MotionStream { fn motion_recv(&mut self, motion_status: MotionStatus) -> MotionOutputError { - let _ = self.tx.send(motion_status); + match motion_status { + MotionStatus::Start => { + if let Some(handle) = &self.cooloff { + handle.abort(); + self.cooloff = None; + } + let _ = self.tx.send(motion_status); + } + MotionStatus::Stop => { + // Abort any current one + if let Some(handle) = &self.cooloff { + handle.abort(); + self.cooloff = None; + } + let aborter = AbortHandle::new(); + let thread_duration = self.timeout; + let thread_tx = self.tx.clone(); + let thread_message = MotionStatus::Stop; + std::thread::spawn(move || { + std::thread::sleep(thread_duration); + if aborter.is_live() { + let _ = thread_tx.send(thread_message); + } + }); + } + _ => { + let _ = self.tx.send(motion_status); + } + } + Ok(self.live.is_live()) } } From 982d7aa199e6f9c2e2e720ebccdf41e844d075a5 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sat, 2 Oct 2021 14:35:00 +0700 Subject: [PATCH 03/11] Using a sharable struct to track stream state --- crates/core/src/bc_protocol/connection/mod.rs | 2 +- src/rtsp/abort.rs | 4 - src/rtsp/config.rs | 7 ++ src/rtsp/gst.rs | 86 +++------------- src/rtsp/gst/maybe_app_src.rs | 79 +++++++++++++++ src/rtsp/gst/maybe_outputselect.rs | 25 +++++ src/rtsp/gst/maybe_typefind.rs | 25 +++++ src/rtsp/mod.rs | 72 ++++---------- src/rtsp/motion.rs | 39 +++----- src/rtsp/state.rs | 98 +++++++++++++++++++ 10 files changed, 281 insertions(+), 156 deletions(-) create mode 100644 src/rtsp/gst/maybe_app_src.rs create mode 100644 src/rtsp/gst/maybe_outputselect.rs create mode 100644 src/rtsp/gst/maybe_typefind.rs create mode 100644 src/rtsp/state.rs diff --git a/crates/core/src/bc_protocol/connection/mod.rs b/crates/core/src/bc_protocol/connection/mod.rs index 504ae23..0d247d6 100644 --- a/crates/core/src/bc_protocol/connection/mod.rs +++ b/crates/core/src/bc_protocol/connection/mod.rs @@ -46,7 +46,7 @@ pub enum Error { #[error(display = "Deserialization error")] Deserialization(#[error(source)] bc::de::Error), - #[error(display = "BcMedia Deserialization error")] + #[error(display = "BcMedia Deserialization error (conn)")] BcMediaDeserialization(#[error(source)] bcmedia::de::Error), #[error(display = "Serialization error")] diff --git a/src/rtsp/abort.rs b/src/rtsp/abort.rs index 4dab182..1940bb0 100644 --- a/src/rtsp/abort.rs +++ b/src/rtsp/abort.rs @@ -19,10 +19,6 @@ impl AbortHandle { self.live.store(false, Ordering::Relaxed); } - pub(crate) fn reset(&self) { - self.live.store(true, Ordering::Relaxed); - } - pub(crate) fn is_live(&self) -> bool { self.live.load(Ordering::Relaxed) } diff --git a/src/rtsp/config.rs b/src/rtsp/config.rs index ee07e63..f7057b4 100644 --- a/src/rtsp/config.rs +++ b/src/rtsp/config.rs @@ -76,6 +76,9 @@ pub(crate) struct CameraConfig { #[validate] pub(crate) motion: Option, + + #[serde(default = "default_on_client")] + pub(crate) on_client: bool, } #[derive(Debug, Deserialize, Validate, Clone)] @@ -122,6 +125,10 @@ fn default_motion_timeout() -> f64 { 1. } +fn default_on_client() -> bool { + false +} + pub(crate) static RESERVED_NAMES: &[&str] = &["anyone", "anonymous"]; fn validate_username(name: &str) -> Result<(), ValidationError> { if name.trim().is_empty() { diff --git a/src/rtsp/gst.rs b/src/rtsp/gst.rs index a5db706..6116829 100644 --- a/src/rtsp/gst.rs +++ b/src/rtsp/gst.rs @@ -1,13 +1,16 @@ //! This module provides an "RtspServer" abstraction that allows consumers of its API to feed it //! data using an ordinary std::io::Write interface. +mod maybe_app_src; + pub(crate) use self::maybe_app_src::MaybeAppSrc; + +use super::state::States; // use super::adpcm::adpcm_to_pcm; // use super::errors::Error; use gstreamer::prelude::Cast; use gstreamer::{Bin, Structure}; use gstreamer_app::AppSrc; //use gstreamer_rtsp::RTSPLowerTrans; -use super::abort::AbortHandle; use anyhow::anyhow; use gstreamer_rtsp::RTSPAuthMethod; pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate}; @@ -41,7 +44,7 @@ pub(crate) struct GstOutputs { video_format: Option, audio_format: Option, factory: RTSPMediaFactory, - live: AbortHandle, + state: Option, last_iframe: Option>, } @@ -75,7 +78,9 @@ impl StreamOutput for GstOutputs { self.last_iframe = Some(payload.data); // Only stop on an iframe so we have the // last frame to show - should_continue = self.live.is_live(); + if let Some(state) = self.state.as_ref() { + should_continue = state.should_stream() + } } BcMedia::Pframe(payload) => { let video_type = match payload.video_type { @@ -110,15 +115,16 @@ impl GstOutputs { video_format: None, audio_format: None, factory: RTSPMediaFactory::new(), - live: AbortHandle::new(), last_iframe: Default::default(), + state: Default::default(), }; result.apply_format(); result } - pub(crate) fn get_abort_handle(&self) -> AbortHandle { - self.live.clone() + pub(crate) fn set_state(&mut self, state: States) { + self.vidsrc.state = Some(state.clone()); + self.state = Some(state) } pub(crate) fn has_last_iframe(&self) -> bool { @@ -343,71 +349,3 @@ impl RtspServer { main_loop.run(); } } - -mod maybe_app_src { - use super::*; - use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; - - /// A Write implementation around AppSrc that also allows delaying the creation of the AppSrc - /// until later, discarding written data until the AppSrc is provided. - pub(crate) struct MaybeAppSrc { - rx: Receiver, - app_src: Option, - } - - impl MaybeAppSrc { - /// Creates a MaybeAppSrc. Also returns a Sender that you must use to provide an AppSrc as - /// soon as one is available. When it is received, the MaybeAppSrc will start pushing data - /// into the AppSrc when write() is called. - pub(crate) fn new_with_tx() -> (Self, SyncSender) { - let (tx, rx) = sync_channel(3); // The sender should not send very often - (MaybeAppSrc { rx, app_src: None }, tx) - } - - /// Flushes data to Gstreamer on a problem communicating with the underlying video source. - pub(crate) fn on_stream_error(&mut self) { - if let Some(src) = self.try_get_src() { - // Ignore "errors" from Gstreamer such as FLUSHING, which are not really errors. - let _ = src.end_of_stream(); - } - } - - /// Attempts to retrieve the AppSrc that should be passed in by the caller of new_with_tx - /// at some point after this struct has been created. At that point, we swap over to - /// owning the AppSrc directly. This function handles either case and returns the AppSrc, - /// or None if the caller has not yet sent one. - fn try_get_src(&mut self) -> Option<&AppSrc> { - while let Some(src) = self.rx.try_recv().ok() { - self.app_src = Some(src); - } - self.app_src.as_ref() - } - } - - impl Write for MaybeAppSrc { - fn write(&mut self, buf: &[u8]) -> io::Result { - // If we have no AppSrc yet, throw away the data and claim that it was written - let app_src = match self.try_get_src() { - Some(src) => src, - None => return Ok(buf.len()), - }; - - let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); - { - let gst_buf_mut = gst_buf.get_mut().unwrap(); - let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); - gst_buf_data.copy_from_slice(buf); - } - - let res = app_src.push_buffer(gst_buf); //.map_err(|e| io::Error::new(io::ErrorKind::Other, Box::new(e)))?; - if res.is_err() { - self.app_src = None; - } - Ok(buf.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } -} diff --git a/src/rtsp/gst/maybe_app_src.rs b/src/rtsp/gst/maybe_app_src.rs new file mode 100644 index 0000000..9a7f3d2 --- /dev/null +++ b/src/rtsp/gst/maybe_app_src.rs @@ -0,0 +1,79 @@ +use super::*; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; + +/// A Write implementation around AppSrc that also allows delaying the creation of the AppSrc +/// until later, discarding written data until the AppSrc is provided. +pub(crate) struct MaybeAppSrc { + rx: Receiver, + app_src: Option, + pub(super) state: Option, +} + +impl MaybeAppSrc { + /// Creates a MaybeAppSrc. Also returns a Sender that you must use to provide an AppSrc as + /// soon as one is available. When it is received, the MaybeAppSrc will start pushing data + /// into the AppSrc when write() is called. + pub(crate) fn new_with_tx() -> (Self, SyncSender) { + let (tx, rx) = sync_channel(3); // The sender should not send very often + ( + MaybeAppSrc { + rx, + app_src: None, + state: Default::default(), + }, + tx, + ) + } + + /// Flushes data to Gstreamer on a problem communicating with the underlying video source. + pub(crate) fn on_stream_error(&mut self) { + if let Some(src) = self.try_get_src() { + // Ignore "errors" from Gstreamer such as FLUSHING, which are not really errors. + let _ = src.end_of_stream(); + } + } + + /// Attempts to retrieve the AppSrc that should be passed in by the caller of new_with_tx + /// at some point after this struct has been created. At that point, we swap over to + /// owning the AppSrc directly. This function handles either case and returns the AppSrc, + /// or None if the caller has not yet sent one. + fn try_get_src(&mut self) -> Option<&AppSrc> { + while let Some(src) = self.rx.try_recv().ok() { + self.app_src = Some(src); + if let Some(state) = self.state.as_ref() { + state.set_client_connected(true); + } + } + self.app_src.as_ref() + } +} + +impl Write for MaybeAppSrc { + fn write(&mut self, buf: &[u8]) -> io::Result { + // If we have no AppSrc yet, throw away the data and claim that it was written + let app_src = match self.try_get_src() { + Some(src) => src, + None => return Ok(buf.len()), + }; + + let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); + let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(buf); + } + + let res = app_src.push_buffer(gst_buf); //.map_err(|e| io::Error::new(io::ErrorKind::Other, Box::new(e)))?; + if res.is_err() { + self.app_src = None; + if let Some(state) = self.state.as_ref() { + state.set_client_connected(false); + } + } + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/src/rtsp/gst/maybe_outputselect.rs b/src/rtsp/gst/maybe_outputselect.rs new file mode 100644 index 0000000..ab44cb6 --- /dev/null +++ b/src/rtsp/gst/maybe_outputselect.rs @@ -0,0 +1,25 @@ +use gstreamer::Element; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; + +pub(crate) struct MaybeOutputSelect { + rx: Receiver, + typefind: Option, +} + +impl MaybeOutputSelect { + pub(crate) fn new_with_tx() -> (Self, SyncSender) { + let (tx, rx) = sync_channel(3); // The sender should not send very often + (MaybeOutputSelect { rx, typefind: None }, tx) + } + + pub(crate) fn try_get_src(&mut self) -> Option<&Element> { + while let Some(src) = self.rx.try_recv().ok() { + self.typefind = Some(src); + } + self.typefind.as_ref() + } + + pub(crate) fn clear(&mut self) { + self.typefind = None; + } +} diff --git a/src/rtsp/gst/maybe_typefind.rs b/src/rtsp/gst/maybe_typefind.rs new file mode 100644 index 0000000..ca32b97 --- /dev/null +++ b/src/rtsp/gst/maybe_typefind.rs @@ -0,0 +1,25 @@ +use gstreamer::Element; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; + +pub(crate) struct MaybeTypeFind { + rx: Receiver, + typefind: Option, +} + +impl MaybeTypeFind { + pub(crate) fn new_with_tx() -> (Self, SyncSender) { + let (tx, rx) = sync_channel(3); // The sender should not send very often + (MaybeTypeFind { rx, typefind: None }, tx) + } + + pub(crate) fn try_get_src(&mut self) -> Option<&Element> { + while let Some(src) = self.rx.try_recv().ok() { + self.typefind = Some(src); + } + self.typefind.as_ref() + } + + pub(crate) fn clear(&mut self) { + self.typefind = None; + } +} diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index b893ed7..5bde26f 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -30,27 +30,26 @@ /// use anyhow::{Context, Result}; use log::*; -use neolink_core::bc_protocol::{BcCamera, MotionStatus, Stream}; +use neolink_core::bc_protocol::{BcCamera, Stream}; use std::collections::HashSet; use std::fs; use std::sync::{Arc, Mutex}; use std::time::Duration; use validator::Validate; -// mod adpcm; -// The command line parameters for this subcommand +mod abort; mod cmdline; mod config; -// The gstreamer implementation -mod abort; mod gst; mod motion; +mod state; use crate::utils::AddressOrUid; pub(crate) use cmdline::Opt; use config::{CameraConfig, Config, UserConfig}; use gst::{GstOutputs, RtspServer, TlsAuthenticationMode}; use motion::MotionStream; +use state::States; /// Entry point for the rtsp subcommand /// @@ -282,24 +281,18 @@ fn camera_main( camera_config.name, stream_display_name ); - // Used to get the error from the thread - let thread_error: Arc>> = Default::default(); - match &camera_config.motion { Some(motion_config) => { + // Used to get the error from the thread + let thread_error: Arc>> = Default::default(); + + // Used to abort all sub threads if any of them request closure + // and control the motion/client state info + let state = States::new_from_camera_config(camera_config); + outputs.set_state(state.clone()); + crossbeam::scope(|s| { - let (mut motion, rx) = MotionStream::new(motion_config.timeout); - - - let motionthread_motion_abort = motion.get_abort_handle(); - let motionthread_stream_abort = outputs.get_abort_handle(); - let rxthread_motion_abort = motion.get_abort_handle(); - let rxthread_stream_abort = outputs.get_abort_handle(); - let streamthread_stream_abort = outputs.get_abort_handle(); - let streamthread_motion_abort = motion.get_abort_handle(); - - // Start in the aborted state because motion defaults to still - streamthread_stream_abort.abort(); + let mut motion = MotionStream::new(motion_config.timeout, state.clone()); let arc_camera = Arc::new(camera); @@ -307,6 +300,7 @@ fn camera_main( let motion_camera = arc_camera.clone(); let motion_camname = camera_config.name.to_string(); let motion_error = thread_error.clone(); + let motion_state = state.clone(); s.spawn(move |_| { if let Err(e) = motion_camera.listen_on_motion(&mut motion).with_context(|| format!("Error while reading motion {}", motion_camname)) { // When motion errors set the thread error @@ -314,49 +308,26 @@ fn camera_main( } // Always abort the streams/motion when motion stops regardless of why // so that this thread can reset the camera - motionthread_motion_abort.abort(); - motionthread_stream_abort.abort(); + motion_state.abort(); debug!("Motion stopped"); }); - // Motion rx -> stream thread - s.spawn(move |_| { - // Stops when motion is stopped - // which happens when either motion or stream errors - while rxthread_motion_abort.is_live() { - let msg = rx.try_recv(); - if let Ok(msg) = msg { - match msg { - MotionStatus::Start => { - debug!("Motion start"); - rxthread_stream_abort.reset(); - } - MotionStatus::Stop => { - debug!("Motion stop"); - rxthread_stream_abort.abort(); - } - _ => {} - } - } - } - println!("Motion RX Stopped"); - }); - // Stream thread let stream_camera = arc_camera; let stream_camname = camera_config.name.to_string(); let stream_error = thread_error.clone(); + let stream_state = state.clone(); s.spawn(move |_| { - loop { - if streamthread_stream_abort.is_live() || ! outputs.has_last_iframe() { - debug!("Stream resumed: waiting for motion to stop"); + while stream_state.is_live() { + if stream_state.should_stream() || ! outputs.has_last_iframe() { + debug!("Stream resumed"); // Will block until the rx thread send the abort if let Err(e) = stream_camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", stream_camname)) { // When stream errors set the thread error and abort the motion *stream_error.lock().unwrap() = Some(e); break; } - debug!("Stream paused: waiting for motion to start"); + debug!("Stream paused"); } else { // While stream is aborted we send the blank image if let Err(e) = outputs.write_last_iframe().with_context(|| format!("Failed to write no motion image for {}", stream_camname)) { @@ -366,8 +337,7 @@ fn camera_main( std::thread::sleep(std::time::Duration::from_millis(50)); } } - streamthread_stream_abort.abort(); - streamthread_motion_abort.abort(); + stream_state.abort(); println!("Stream stopped"); }); }).unwrap(); diff --git a/src/rtsp/motion.rs b/src/rtsp/motion.rs index 7b66da4..12bf35d 100644 --- a/src/rtsp/motion.rs +++ b/src/rtsp/motion.rs @@ -1,31 +1,21 @@ use super::abort::AbortHandle; -use crossbeam_channel::{unbounded, Receiver, Sender}; +use super::state::States; use neolink_core::bc_protocol::{MotionOutput, MotionOutputError, MotionStatus}; use std::time::Duration; pub(crate) struct MotionStream { - tx: Sender, - live: AbortHandle, + state: States, timeout: Duration, cooloff: Option, } impl MotionStream { - pub(crate) fn new(timeout_seconds: f64) -> (Self, Receiver) { - let (tx, rx) = unbounded(); - ( - Self { - tx, - live: AbortHandle::new(), - timeout: Duration::from_secs_f64(timeout_seconds), - cooloff: None, - }, - rx, - ) - } - - pub(crate) fn get_abort_handle(&self) -> AbortHandle { - self.live.clone() + pub(crate) fn new(timeout_seconds: f64, state: States) -> Self { + Self { + state, + timeout: Duration::from_secs_f64(timeout_seconds), + cooloff: None, + } } } @@ -37,7 +27,7 @@ impl MotionOutput for MotionStream { handle.abort(); self.cooloff = None; } - let _ = self.tx.send(motion_status); + self.state.set_motion_detected(true); } MotionStatus::Stop => { // Abort any current one @@ -47,20 +37,17 @@ impl MotionOutput for MotionStream { } let aborter = AbortHandle::new(); let thread_duration = self.timeout; - let thread_tx = self.tx.clone(); - let thread_message = MotionStatus::Stop; + let thread_state = self.state.clone(); std::thread::spawn(move || { std::thread::sleep(thread_duration); if aborter.is_live() { - let _ = thread_tx.send(thread_message); + thread_state.set_motion_detected(false); } }); } - _ => { - let _ = self.tx.send(motion_status); - } + _ => {} } - Ok(self.live.is_live()) + Ok(self.state.is_live()) } } diff --git a/src/rtsp/state.rs b/src/rtsp/state.rs new file mode 100644 index 0000000..6bc462d --- /dev/null +++ b/src/rtsp/state.rs @@ -0,0 +1,98 @@ +//! Used to handle the stream states +//! +//! When should stream is true then neolink +//! pulls the video from the camera. +//! +//! Otherwise it should use a fallback image +//! +use super::CameraConfig; +use log::*; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, +}; + +enum MotionState { + Motion, + Still, + Ignore, +} + +enum RtspState { + Connected, + Disconnected, + Ignore, +} + +#[derive(Clone)] +pub(crate) struct States { + motion: Arc>, + client: Arc>, + live: Arc, +} + +impl States { + pub(crate) fn new(motion_controlled: bool, client_controlled: bool) -> Self { + let motion = match motion_controlled { + true => MotionState::Still, + false => MotionState::Ignore, + }; + let client = match client_controlled { + true => RtspState::Disconnected, + false => RtspState::Ignore, + }; + + Self { + motion: Arc::new(Mutex::new(motion)), + client: Arc::new(Mutex::new(client)), + live: Arc::new(AtomicBool::new(true)), + } + } + + pub(crate) fn new_from_camera_config(camera_config: &CameraConfig) -> Self { + let motion_controlled = camera_config.motion.is_some(); + let client_controlled = camera_config.on_client; + + Self::new(motion_controlled, client_controlled) + } + + pub(crate) fn should_stream(&self) -> bool { + if let MotionState::Still = *self.motion.lock().unwrap() { + return false; + } + + !matches!(*self.client.lock().unwrap(), RtspState::Disconnected) + } + + pub(crate) fn set_client_connected(&self, value: bool) { + let mut client = self.client.lock().unwrap(); + if let RtspState::Ignore = *client { + return; + } + debug!("Client state: {}", value); + *client = match value { + true => RtspState::Connected, + false => RtspState::Disconnected, + } + } + + pub(crate) fn set_motion_detected(&self, value: bool) { + let mut motion = self.motion.lock().unwrap(); + if let MotionState::Ignore = *motion { + return; + } + debug!("Motion state: {}", value); + *motion = match value { + true => MotionState::Motion, + false => MotionState::Still, + } + } + + pub(crate) fn abort(&self) { + self.live.store(false, Ordering::Relaxed); + } + + pub(crate) fn is_live(&self) -> bool { + self.live.load(Ordering::Relaxed) + } +} From 7ec17c0cc5c198e504dd6992b66435c2b47af328 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 3 Oct 2021 14:50:13 +0700 Subject: [PATCH 04/11] Ensure camera accepts stream start command --- crates/core/src/bc_protocol/stream.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/core/src/bc_protocol/stream.rs b/crates/core/src/bc_protocol/stream.rs index 316c9e9..2bf3d5d 100644 --- a/crates/core/src/bc_protocol/stream.rs +++ b/crates/core/src/bc_protocol/stream.rs @@ -120,6 +120,18 @@ impl BcCamera { sub_video.send(start_video)?; + let msg = sub_video.rx.recv_timeout(RX_TIMEOUT)?; + if let BcMeta { + response_code: 200, .. + } = msg.meta + { + } else { + return Err(Error::UnintelligibleReply { + reply: msg, + why: "The camera did not accept the stream start command.", + }); + } + let mut media_sub = BinarySubscriber::from_bc_sub(&sub_video); let result; From 80d28440b376a8cd17afed7ef08c963a3811fa44 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 3 Oct 2021 14:50:27 +0700 Subject: [PATCH 05/11] Remove unused error enum --- crates/core/src/bc_protocol/errors.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/core/src/bc_protocol/errors.rs b/crates/core/src/bc_protocol/errors.rs index ef79712..1e9d009 100644 --- a/crates/core/src/bc_protocol/errors.rs +++ b/crates/core/src/bc_protocol/errors.rs @@ -5,10 +5,6 @@ use err_derive::Error; #[derive(Debug, Error)] #[allow(clippy::large_enum_variant)] pub enum Error { - /// Error raised when IO fails such as when the connection is lost - #[error(display = "Communication error")] - Communication(#[error(source)] std::io::Error), - /// Error raised during deserlization #[error(display = "Deserialization error")] Deserialization(#[error(source)] super::bc::de::Error), From 8aa8345d875e0c599f3daf99f36acdfdc931a678 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Sun, 3 Oct 2021 14:50:45 +0700 Subject: [PATCH 06/11] Use input select for when stream is paused --- src/rtsp/gst.rs | 104 +++++++++++++++++++++++++----- src/rtsp/gst/maybe_inputselect.rs | 55 ++++++++++++++++ src/rtsp/mod.rs | 13 +++- 3 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 src/rtsp/gst/maybe_inputselect.rs diff --git a/src/rtsp/gst.rs b/src/rtsp/gst.rs index 6116829..cfae21b 100644 --- a/src/rtsp/gst.rs +++ b/src/rtsp/gst.rs @@ -1,8 +1,10 @@ //! This module provides an "RtspServer" abstraction that allows consumers of its API to feed it //! data using an ordinary std::io::Write interface. mod maybe_app_src; +mod maybe_inputselect; pub(crate) use self::maybe_app_src::MaybeAppSrc; +pub(crate) use self::maybe_inputselect::MaybeInputSelect; use super::state::States; // use super::adpcm::adpcm_to_pcm; @@ -23,7 +25,7 @@ use gstreamer_rtsp_server::{ }; use log::*; use neolink_core::{ - bc_protocol::{StreamOutput, StreamOutputError}, + bc_protocol::{Error, StreamOutput, StreamOutputError}, bcmedia::model::*, }; use std::collections::HashSet; @@ -38,9 +40,16 @@ pub(crate) struct RtspServer { server: GstRTSPServer, } +#[derive(Eq, PartialEq, Debug)] +pub(crate) enum InputSources { + Cam, + TestSrc, +} + pub(crate) struct GstOutputs { pub(crate) audsrc: MaybeAppSrc, pub(crate) vidsrc: MaybeAppSrc, + vid_inputselect: MaybeInputSelect, video_format: Option, audio_format: Option, factory: RTSPMediaFactory, @@ -67,6 +76,12 @@ enum StreamFormat { impl StreamOutput for GstOutputs { fn stream_recv(&mut self, media: BcMedia) -> StreamOutputError { let mut should_continue = true; + + // Ensure stream is on cam mode + self.set_input_source(InputSources::Cam).map_err(|e| { + Error::OtherString(format!("Cannot set stream input to camera source {:?}", e)) + })?; + match media { BcMedia::Iframe(payload) => { let video_type = match payload.video_type { @@ -74,7 +89,9 @@ impl StreamOutput for GstOutputs { VideoType::H265 => StreamFormat::H265, }; self.set_format(Some(video_type)); - self.vidsrc.write_all(&payload.data)?; + self.vidsrc.write_all(&payload.data).map_err(|e| { + Error::OtherString(format!("Cannot write IFrame to vidsrc {:?}", e)) + })?; self.last_iframe = Some(payload.data); // Only stop on an iframe so we have the // last frame to show @@ -88,15 +105,21 @@ impl StreamOutput for GstOutputs { VideoType::H265 => StreamFormat::H265, }; self.set_format(Some(video_type)); - self.vidsrc.write_all(&payload.data)?; + self.vidsrc.write_all(&payload.data).map_err(|e| { + Error::OtherString(format!("Cannot write PFrame to vidsrc {:?}", e)) + })?; } BcMedia::Aac(payload) => { self.set_format(Some(StreamFormat::Aac)); - self.audsrc.write_all(&payload.data)?; + self.audsrc.write_all(&payload.data).map_err(|e| { + Error::OtherString(format!("Cannot write AAC to audsrc {:?}", e)) + })?; } BcMedia::Adpcm(payload) => { self.set_format(Some(StreamFormat::Adpcm(payload.data.len() as u16))); - self.audsrc.write_all(&payload.data)?; + self.audsrc.write_all(&payload.data).map_err(|e| { + Error::OtherString(format!("Cannot write ADPCM to audsrc {:?}", e)) + })?; } _ => { //Ignore other BcMedia like InfoV1 and InfoV2 @@ -108,10 +131,15 @@ impl StreamOutput for GstOutputs { } impl GstOutputs { - pub(crate) fn from_appsrcs(vidsrc: MaybeAppSrc, audsrc: MaybeAppSrc) -> GstOutputs { + pub(crate) fn from_appsrcs( + vidsrc: MaybeAppSrc, + audsrc: MaybeAppSrc, + vid_inputselect: MaybeInputSelect, + ) -> GstOutputs { let result = GstOutputs { vidsrc, audsrc, + vid_inputselect, video_format: None, audio_format: None, factory: RTSPMediaFactory::new(), @@ -127,6 +155,13 @@ impl GstOutputs { self.state = Some(state) } + pub(crate) fn set_input_source(&mut self, input_source: InputSources) -> AnyResult<()> { + match &input_source { + InputSources::Cam => self.vid_inputselect.set_input(0), + InputSources::TestSrc => self.vid_inputselect.set_input(1), + } + } + pub(crate) fn has_last_iframe(&self) -> bool { self.last_iframe.is_some() } @@ -160,14 +195,30 @@ impl GstOutputs { } fn apply_format(&self) { - let launch_vid = match self.video_format { + let launch_vid_select = match self.video_format { + Some(StreamFormat::H264) => "! rtph264pay name=pay0", + Some(StreamFormat::H265) => "! rtph265pay name=pay0", + _ => "! fakesink", + }; + + let launch_app_src = match self.video_format { Some(StreamFormat::H264) => { - "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h264parse ! rtph264pay name=pay0" + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h264parse" } Some(StreamFormat::H265) => { - "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h265parse ! rtph265pay name=pay0" + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h265parse" } - _ => "! fakesink", + _ => "", + }; + + let launch_alt_source = match self.video_format { + Some(StreamFormat::H264) => { + "videotestsrc ! x264enc ! video/x-h264,width=1920,height=1080,framerate=25/1 ! h264parse" + } + Some(StreamFormat::H265) => { + "videotestsrc ! x265enc ! video/x-h265,width=1920,height=1080,framerate=25/1 ! h265parse" + } + _ => "", }; let launch_aud = match self.audio_format { @@ -178,10 +229,25 @@ impl GstOutputs { let launch_str = &vec![ "( ", - "appsrc name=vidsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks - launch_vid, - "appsrc name=audsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks - &launch_aud, + // Video out pipe + "(", + "input-selector name=vid_inputselect", + launch_vid_select, + ")", + // Camera vid source + "(", + "appsrc name=vidsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks + launch_app_src, + "! vid_inputselect.sink_0", + ")", + // Test vid source + "(", + launch_alt_source, + "! vid_inputselect.sink_1", + ")", + // Audio pipe + "appsrc name=audsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", + &launch_aud, ")" ] .join(" "); @@ -221,7 +287,10 @@ impl RtspServer { let (maybe_app_src, tx) = MaybeAppSrc::new_with_tx(); let (maybe_app_src_aud, tx_aud) = MaybeAppSrc::new_with_tx(); - let outputs = GstOutputs::from_appsrcs(maybe_app_src, maybe_app_src_aud); + let (maybe_vid_inputselect, tx_vid_inputselect) = MaybeInputSelect::new_with_tx(); + + let outputs = + GstOutputs::from_appsrcs(maybe_app_src, maybe_app_src_aud, maybe_vid_inputselect); let factory = &outputs.factory; @@ -256,6 +325,11 @@ impl RtspServer { .dynamic_cast::() .expect("Source element is expected to be an appsrc!"); let _ = tx_aud.send(app_src_aud); // Receiver may be dropped, don't panic if so + + let maybe_vid_inputselect = bin + .by_name_recurse_up("vid_inputselect") + .expect("vid_inputselect must be present in created bin"); + let _ = tx_vid_inputselect.send(maybe_vid_inputselect); // Receiver may be dropped, don't panic if so }); for path in paths { diff --git a/src/rtsp/gst/maybe_inputselect.rs b/src/rtsp/gst/maybe_inputselect.rs new file mode 100644 index 0000000..c8d2e49 --- /dev/null +++ b/src/rtsp/gst/maybe_inputselect.rs @@ -0,0 +1,55 @@ +use anyhow::{anyhow, Error as Anyhow}; +use gstreamer::prelude::ElementExt; +use gstreamer::prelude::ObjectExt; +use gstreamer::{Element, Pad}; +use log::*; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; + +pub(crate) struct MaybeInputSelect { + rx: Receiver, + typefind: Option, +} + +impl MaybeInputSelect { + pub(crate) fn new_with_tx() -> (Self, SyncSender) { + let (tx, rx) = sync_channel(3); // The sender should not send very often + (MaybeInputSelect { rx, typefind: None }, tx) + } + + pub(crate) fn try_get_src(&mut self) -> Option<&Element> { + while let Some(src) = self.rx.try_recv().ok() { + self.typefind = Some(src); + } + self.typefind.as_ref() + } + + pub(crate) fn set_input(&mut self, path_num: u32) -> Result<(), Anyhow> { + if let Some(element) = self.try_get_src() { + let new_pad = element + .static_pad(&format!("sink_{}", path_num)) + .ok_or_else(|| anyhow!("Unable to set input pad"))?; + + if let Ok(Ok(active_pad)) = element.property("active-pad").map(|e| e.get::<'_, Pad>()) { + if active_pad != new_pad { + debug!("Pad need changing"); + if let Err(e) = element.set_property("active-pad", new_pad) { + debug!("Element is invalid: {:?}", e); + self.clear(); + } else { + debug!("Pad chanaged to {}", path_num); + } + } else { + debug!("Pad already set to {}", path_num); + } + } else { + debug!("Element is invalid"); + self.clear(); + } + } + Ok(()) + } + + pub(crate) fn clear(&mut self) { + self.typefind = None; + } +} diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 5bde26f..6a53a72 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -47,7 +47,7 @@ mod state; use crate::utils::AddressOrUid; pub(crate) use cmdline::Opt; use config::{CameraConfig, Config, UserConfig}; -use gst::{GstOutputs, RtspServer, TlsAuthenticationMode}; +use gst::{GstOutputs, InputSources, RtspServer, TlsAuthenticationMode}; use motion::MotionStream; use state::States; @@ -320,15 +320,24 @@ fn camera_main( s.spawn(move |_| { while stream_state.is_live() { if stream_state.should_stream() || ! outputs.has_last_iframe() { + if let Err(e) = outputs.set_input_source(InputSources::Cam) { + *stream_error.lock().unwrap() = Some(e); + break; + } debug!("Stream resumed"); // Will block until the rx thread send the abort if let Err(e) = stream_camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", stream_camname)) { + error!("Stream error {:?}", e); // When stream errors set the thread error and abort the motion *stream_error.lock().unwrap() = Some(e); break; } debug!("Stream paused"); } else { + if let Err(e) = outputs.set_input_source(InputSources::TestSrc) { + *stream_error.lock().unwrap() = Some(e); + break; + } // While stream is aborted we send the blank image if let Err(e) = outputs.write_last_iframe().with_context(|| format!("Failed to write no motion image for {}", stream_camname)) { *stream_error.lock().unwrap() = Some(e); @@ -338,7 +347,7 @@ fn camera_main( } } stream_state.abort(); - println!("Stream stopped"); + debug!("Stream stopped"); }); }).unwrap(); From 880c06bce9c358a47749c9633aa2c7e78cb6dd49 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 6 Oct 2021 16:07:53 +0700 Subject: [PATCH 07/11] Limit keep alive messages --- .../core/src/bc_protocol/connection/bcconn.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/core/src/bc_protocol/connection/bcconn.rs b/crates/core/src/bc_protocol/connection/bcconn.rs index 4f606f2..ef50b53 100644 --- a/crates/core/src/bc_protocol/connection/bcconn.rs +++ b/crates/core/src/bc_protocol/connection/bcconn.rs @@ -11,7 +11,7 @@ use std::net::{SocketAddr, TcpStream}; use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::{atomic::AtomicBool, atomic::Ordering, Arc, Mutex}; use std::thread::JoinHandle; -use std::time::Duration; +use std::time::{Duration, Instant}; /// A shareable connection to a camera. Handles serialization of messages. To send/receive, call /// .[subscribe()] with a message ID. You can use the BcSubscription to send or receive only @@ -46,6 +46,8 @@ impl BcConnection { let keep_alive_encryption_protocol = connections_encryption_protocol.clone(); let mut context = BcContext::new(connections_encryption_protocol); let mut result; + let mut last_keep_alive = Instant::now(); + let keep_alive_time = Duration::from_millis(500); loop { result = Self::poll(&mut context, &conn, &mut subs); if poll_abort_rx.load(Ordering::Relaxed) { @@ -61,11 +63,14 @@ impl BcConnection { break; } // Send a udp keep alive if set - if let Ok(lock) = connections_keep_alive_msg.try_lock() { - if let Some(keep_alive_msg) = lock.as_ref() { - let _ = keep_alive_msg - .serialize(&conn, &keep_alive_encryption_protocol.lock().unwrap()); - let _ = conn.flush(); + if last_keep_alive.elapsed() > keep_alive_time { + last_keep_alive = Instant::now(); + if let Ok(lock) = connections_keep_alive_msg.try_lock() { + if let Some(keep_alive_msg) = lock.as_ref() { + let _ = keep_alive_msg + .serialize(&conn, &keep_alive_encryption_protocol.lock().unwrap()); + let _ = conn.flush(); + } } } } From 35c53fba90dbb5b5b2616ec3de2a9aa0b6aa6238 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 6 Oct 2021 16:08:10 +0700 Subject: [PATCH 08/11] Handle case of invalid active-pad --- src/rtsp/gst/maybe_inputselect.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/rtsp/gst/maybe_inputselect.rs b/src/rtsp/gst/maybe_inputselect.rs index c8d2e49..6dc508f 100644 --- a/src/rtsp/gst/maybe_inputselect.rs +++ b/src/rtsp/gst/maybe_inputselect.rs @@ -29,21 +29,31 @@ impl MaybeInputSelect { .static_pad(&format!("sink_{}", path_num)) .ok_or_else(|| anyhow!("Unable to set input pad"))?; - if let Ok(Ok(active_pad)) = element.property("active-pad").map(|e| e.get::<'_, Pad>()) { - if active_pad != new_pad { - debug!("Pad need changing"); + match element.property("active-pad").map(|e| e.get::<'_, Pad>()) { + Ok(Ok(active_pad)) => { + if active_pad != new_pad { + debug!("Pad need changing"); + if let Err(e) = element.set_property("active-pad", new_pad) { + debug!("Element is invalid: {:?}", e); + self.clear(); + } else { + debug!("Pad chanaged to {}", path_num); + } + } + } + Ok(Err(e)) => { + debug!("Pad need changing: Active pad invalid: {:?}", e); if let Err(e) = element.set_property("active-pad", new_pad) { debug!("Element is invalid: {:?}", e); self.clear(); } else { debug!("Pad chanaged to {}", path_num); } - } else { - debug!("Pad already set to {}", path_num); } - } else { - debug!("Element is invalid"); - self.clear(); + Err(e) => { + debug!("Element is invalid: Err({:?})", e); + self.clear(); + } } } Ok(()) From f1baaf7458a6a8e14386561f1febf3f7feeaa5b6 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Thu, 7 Oct 2021 15:16:42 +0700 Subject: [PATCH 09/11] Add a still mode to paused streams --- src/rtsp/config.rs | 46 ++++++++++++--- src/rtsp/gst.rs | 30 +++++++++- src/rtsp/mod.rs | 141 ++++++++++++++++++++++----------------------- src/rtsp/state.rs | 6 +- 4 files changed, 137 insertions(+), 86 deletions(-) diff --git a/src/rtsp/config.rs b/src/rtsp/config.rs index f7057b4..a98b500 100644 --- a/src/rtsp/config.rs +++ b/src/rtsp/config.rs @@ -10,6 +10,7 @@ lazy_static! { static ref RE_STREAM_SRC: Regex = Regex::new(r"^(mainStream|subStream|externStream|both|all)$").unwrap(); static ref RE_TLS_CLIENT_AUTH: Regex = Regex::new(r"^(none|request|require)$").unwrap(); + static ref RE_PAUSE_MODE: Regex = Regex::new(r"^(black|still|test)$").unwrap(); } #[derive(Debug, Deserialize, Validate, Clone)] @@ -75,10 +76,8 @@ pub(crate) struct CameraConfig { pub(crate) channel_id: u8, #[validate] - pub(crate) motion: Option, - - #[serde(default = "default_on_client")] - pub(crate) on_client: bool, + #[serde(default = "default_pause")] + pub(crate) pause: PauseConfig, } #[derive(Debug, Deserialize, Validate, Clone)] @@ -92,9 +91,23 @@ pub(crate) struct UserConfig { } #[derive(Debug, Deserialize, Validate, Clone)] -pub(crate) struct MotionConfig { - #[serde(default = "default_motion_timeout", alias = "timeout_seconds")] - pub(crate) timeout: f64, +pub(crate) struct PauseConfig { + #[serde(default = "default_on_motion")] + pub(crate) on_motion: bool, + + #[serde(default = "default_on_disconnect")] + pub(crate) on_disconnect: bool, + + #[serde(default = "default_motion_timeout", alias = "timeout")] + pub(crate) motion_timeout: f64, + + #[serde(default = "default_pause_mode")] + #[validate(regex( + path = "RE_PAUSE_MODE", + message = "Incorrect pause mode", + code = "mode" + ))] + pub(crate) mode: String, } fn default_bind_addr() -> String { @@ -125,10 +138,27 @@ fn default_motion_timeout() -> f64 { 1. } -fn default_on_client() -> bool { +fn default_on_disconnect() -> bool { false } +fn default_on_motion() -> bool { + false +} + +fn default_pause_mode() -> String { + "still".to_string() +} + +fn default_pause() -> PauseConfig { + PauseConfig { + on_motion: default_on_motion(), + on_disconnect: default_on_disconnect(), + motion_timeout: default_motion_timeout(), + mode: default_pause_mode(), + } +} + pub(crate) static RESERVED_NAMES: &[&str] = &["anyone", "anonymous"]; fn validate_username(name: &str) -> Result<(), ValidationError> { if name.trim().is_empty() { diff --git a/src/rtsp/gst.rs b/src/rtsp/gst.rs index cfae21b..a63cb34 100644 --- a/src/rtsp/gst.rs +++ b/src/rtsp/gst.rs @@ -44,6 +44,8 @@ pub(crate) struct RtspServer { pub(crate) enum InputSources { Cam, TestSrc, + Still, + Black, } pub(crate) struct GstOutputs { @@ -159,6 +161,8 @@ impl GstOutputs { match &input_source { InputSources::Cam => self.vid_inputselect.set_input(0), InputSources::TestSrc => self.vid_inputselect.set_input(1), + InputSources::Still => self.vid_inputselect.set_input(2), + InputSources::Black => self.vid_inputselect.set_input(3), } } @@ -213,10 +217,20 @@ impl GstOutputs { let launch_alt_source = match self.video_format { Some(StreamFormat::H264) => { - "videotestsrc ! x264enc ! video/x-h264,width=1920,height=1080,framerate=25/1 ! h264parse" + "videotestsrc ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! x264enc ! video/x-h264,width=896,height=512,framerate=25/1 ! h264parse" } Some(StreamFormat::H265) => { - "videotestsrc ! x265enc ! video/x-h265,width=1920,height=1080,framerate=25/1 ! h265parse" + "videotestsrc ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! x265enc ! video/x-h265,width=896,height=512,framerate=25/1 ! h265parse" + } + _ => "", + }; + + let launch_imgfreeze_source = match self.video_format { + Some(StreamFormat::H264) => { + "avdec_h264 ! imagefreeze allow-replace=true is-live=true ! x264enc ! video/x-h264,width=896,height=512,framerate=25/1 ! h264parse" + } + Some(StreamFormat::H265) => { + "decodebin ! imagefreeze allow-replace=true is-live=true ! x265enc ! video/x-h265,width=896,height=512,framerate=25/1 ! h265parse" } _ => "", }; @@ -238,13 +252,23 @@ impl GstOutputs { "(", "appsrc name=vidsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks launch_app_src, - "! vid_inputselect.sink_0", + "! tee name=vid_src_tee", + "(", + "vid_src_tee. ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! vid_inputselect.sink_0", + ")", + ")", // Test vid source "(", launch_alt_source, "! vid_inputselect.sink_1", ")", + // Image freeze + "(", + "vid_src_tee. ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 !", + launch_imgfreeze_source, + " ! vid_inputselect.sink_2", + ")", // Audio pipe "appsrc name=audsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", &launch_aud, diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 6a53a72..2796574 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -281,86 +281,83 @@ fn camera_main( camera_config.name, stream_display_name ); - match &camera_config.motion { - Some(motion_config) => { - // Used to get the error from the thread - let thread_error: Arc>> = Default::default(); + if ! camera_config.pause.on_motion && ! camera_config.pause.on_disconnect { + // Stream is always on + camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", camera_config.name)) + } else { + let pause_config = &camera_config.pause; + // Used to get the error from the thread + let thread_error: Arc>> = Default::default(); - // Used to abort all sub threads if any of them request closure - // and control the motion/client state info - let state = States::new_from_camera_config(camera_config); - outputs.set_state(state.clone()); + // Used to abort all sub threads if any of them request closure + // and control the motion/client state info + let state = States::new_from_camera_config(camera_config); + outputs.set_state(state.clone()); - crossbeam::scope(|s| { - let mut motion = MotionStream::new(motion_config.timeout, state.clone()); + crossbeam::scope(|s| { + let mut motion = MotionStream::new(pause_config.motion_timeout, state.clone()); - let arc_camera = Arc::new(camera); + let arc_camera = Arc::new(camera); - // Motion detection thread - let motion_camera = arc_camera.clone(); - let motion_camname = camera_config.name.to_string(); - let motion_error = thread_error.clone(); - let motion_state = state.clone(); - s.spawn(move |_| { - if let Err(e) = motion_camera.listen_on_motion(&mut motion).with_context(|| format!("Error while reading motion {}", motion_camname)) { - // When motion errors set the thread error - *motion_error.lock().unwrap() = Some(e); - } - // Always abort the streams/motion when motion stops regardless of why - // so that this thread can reset the camera - motion_state.abort(); - debug!("Motion stopped"); - }); + // Motion detection thread + let motion_camera = arc_camera.clone(); + let motion_camname = camera_config.name.to_string(); + let motion_error = thread_error.clone(); + let motion_state = state.clone(); + s.spawn(move |_| { + if let Err(e) = motion_camera.listen_on_motion(&mut motion).with_context(|| format!("Error while reading motion {}", motion_camname)) { + // When motion errors set the thread error + *motion_error.lock().unwrap() = Some(e); + } + // Always abort the streams/motion when motion stops regardless of why + // so that this thread can reset the camera + motion_state.abort(); + debug!("Motion stopped"); + }); - // Stream thread - let stream_camera = arc_camera; - let stream_camname = camera_config.name.to_string(); - let stream_error = thread_error.clone(); - let stream_state = state.clone(); - s.spawn(move |_| { - while stream_state.is_live() { - if stream_state.should_stream() || ! outputs.has_last_iframe() { - if let Err(e) = outputs.set_input_source(InputSources::Cam) { - *stream_error.lock().unwrap() = Some(e); - break; - } - debug!("Stream resumed"); - // Will block until the rx thread send the abort - if let Err(e) = stream_camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", stream_camname)) { - error!("Stream error {:?}", e); - // When stream errors set the thread error and abort the motion - *stream_error.lock().unwrap() = Some(e); - break; - } - debug!("Stream paused"); - } else { - if let Err(e) = outputs.set_input_source(InputSources::TestSrc) { - *stream_error.lock().unwrap() = Some(e); - break; - } - // While stream is aborted we send the blank image - if let Err(e) = outputs.write_last_iframe().with_context(|| format!("Failed to write no motion image for {}", stream_camname)) { - *stream_error.lock().unwrap() = Some(e); - break; - } - std::thread::sleep(std::time::Duration::from_millis(50)); + // Stream thread + let stream_camera = arc_camera; + let stream_camname = camera_config.name.to_string(); + let stream_error = thread_error.clone(); + let stream_state = state.clone(); + s.spawn(move |_| { + while stream_state.is_live() { + if stream_state.should_stream() || ! outputs.has_last_iframe() { + if let Err(e) = outputs.set_input_source(InputSources::Cam) { + *stream_error.lock().unwrap() = Some(e); + break; } + debug!("Stream resumed"); + // Will block until the rx thread send the abort + if let Err(e) = stream_camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", stream_camname)) { + error!("Stream error {:?}", e); + // When stream errors set the thread error and abort the motion + *stream_error.lock().unwrap() = Some(e); + break; + } + debug!("Stream paused"); + } else { + let input_source = match pause_config.mode.as_str() { + "test" => {InputSources::TestSrc}, + "still" => {InputSources::Still}, + "black" => {InputSources::Black}, + _ => {unreachable!()} + }; + if let Err(e) = outputs.set_input_source(input_source) { + *stream_error.lock().unwrap() = Some(e); + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); } - stream_state.abort(); - debug!("Stream stopped"); - }); - }).unwrap(); + } + stream_state.abort(); + debug!("Stream stopped"); + }); + }).unwrap(); - // Should be the only one holding the thread_error so unwrap it - // all the way to an actual error - Err(Arc::try_unwrap(thread_error).unwrap().into_inner().unwrap().unwrap()) - } - None => { - // No motion config - // - // Just stream always - camera.start_video(outputs, stream_name).with_context(|| format!("Error while streaming {}", camera_config.name)) - } + // Should be the only one holding the thread_error so unwrap it + // all the way to an actual error + Err(Arc::try_unwrap(thread_error).unwrap().into_inner().unwrap().unwrap()) } })().map_err(|e| CameraErr{ diff --git a/src/rtsp/state.rs b/src/rtsp/state.rs index 6bc462d..ad9373d 100644 --- a/src/rtsp/state.rs +++ b/src/rtsp/state.rs @@ -50,8 +50,8 @@ impl States { } pub(crate) fn new_from_camera_config(camera_config: &CameraConfig) -> Self { - let motion_controlled = camera_config.motion.is_some(); - let client_controlled = camera_config.on_client; + let motion_controlled = camera_config.pause.on_motion; + let client_controlled = camera_config.pause.on_disconnect; Self::new(motion_controlled, client_controlled) } @@ -85,7 +85,7 @@ impl States { *motion = match value { true => MotionState::Motion, false => MotionState::Still, - } + }; } pub(crate) fn abort(&self) { From dea3cbdb3998c04edb77f6684322630505474d9f Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Thu, 7 Oct 2021 17:35:16 +0700 Subject: [PATCH 10/11] Make paused sources more customisable Add audio paused source --- src/rtsp/config.rs | 4 +- src/rtsp/gst.rs | 175 ++++++++++++++++++++++++++++++--------------- src/rtsp/mod.rs | 25 ++++--- 3 files changed, 137 insertions(+), 67 deletions(-) diff --git a/src/rtsp/config.rs b/src/rtsp/config.rs index a98b500..21ea58d 100644 --- a/src/rtsp/config.rs +++ b/src/rtsp/config.rs @@ -10,7 +10,7 @@ lazy_static! { static ref RE_STREAM_SRC: Regex = Regex::new(r"^(mainStream|subStream|externStream|both|all)$").unwrap(); static ref RE_TLS_CLIENT_AUTH: Regex = Regex::new(r"^(none|request|require)$").unwrap(); - static ref RE_PAUSE_MODE: Regex = Regex::new(r"^(black|still|test)$").unwrap(); + static ref RE_PAUSE_MODE: Regex = Regex::new(r"^(black|still|test|none)$").unwrap(); } #[derive(Debug, Deserialize, Validate, Clone)] @@ -147,7 +147,7 @@ fn default_on_motion() -> bool { } fn default_pause_mode() -> String { - "still".to_string() + "none".to_string() } fn default_pause() -> PauseConfig { diff --git a/src/rtsp/gst.rs b/src/rtsp/gst.rs index a63cb34..3f6d0de 100644 --- a/src/rtsp/gst.rs +++ b/src/rtsp/gst.rs @@ -41,21 +41,29 @@ pub(crate) struct RtspServer { } #[derive(Eq, PartialEq, Debug)] -pub(crate) enum InputSources { - Cam, +pub(crate) enum InputMode { + Live, + Paused, +} + +#[derive(Eq, PartialEq, Debug)] +pub(crate) enum PausedSources { TestSrc, Still, Black, + None, } pub(crate) struct GstOutputs { pub(crate) audsrc: MaybeAppSrc, pub(crate) vidsrc: MaybeAppSrc, vid_inputselect: MaybeInputSelect, + aud_inputselect: MaybeInputSelect, video_format: Option, audio_format: Option, factory: RTSPMediaFactory, state: Option, + when_paused: PausedSources, last_iframe: Option>, } @@ -80,7 +88,7 @@ impl StreamOutput for GstOutputs { let mut should_continue = true; // Ensure stream is on cam mode - self.set_input_source(InputSources::Cam).map_err(|e| { + self.set_input_source(InputMode::Live).map_err(|e| { Error::OtherString(format!("Cannot set stream input to camera source {:?}", e)) })?; @@ -137,13 +145,16 @@ impl GstOutputs { vidsrc: MaybeAppSrc, audsrc: MaybeAppSrc, vid_inputselect: MaybeInputSelect, + aud_inputselect: MaybeInputSelect, ) -> GstOutputs { let result = GstOutputs { vidsrc, audsrc, vid_inputselect, + aud_inputselect, video_format: None, audio_format: None, + when_paused: PausedSources::None, factory: RTSPMediaFactory::new(), last_iframe: Default::default(), state: Default::default(), @@ -157,13 +168,25 @@ impl GstOutputs { self.state = Some(state) } - pub(crate) fn set_input_source(&mut self, input_source: InputSources) -> AnyResult<()> { - match &input_source { - InputSources::Cam => self.vid_inputselect.set_input(0), - InputSources::TestSrc => self.vid_inputselect.set_input(1), - InputSources::Still => self.vid_inputselect.set_input(2), - InputSources::Black => self.vid_inputselect.set_input(3), + pub(crate) fn set_input_source(&mut self, input_source: InputMode) -> AnyResult<()> { + if self.when_paused != PausedSources::None { + match &input_source { + InputMode::Live => { + self.vid_inputselect.set_input(0)?; + self.aud_inputselect.set_input(0)?; + } + InputMode::Paused => { + self.vid_inputselect.set_input(1)?; + self.aud_inputselect.set_input(1)?; + } + } } + Ok(()) + } + + pub(crate) fn set_paused_source(&mut self, paused_source: PausedSources) { + self.when_paused = paused_source; + self.apply_format(); } pub(crate) fn has_last_iframe(&self) -> bool { @@ -199,48 +222,66 @@ impl GstOutputs { } fn apply_format(&self) { - let launch_vid_select = match self.video_format { - Some(StreamFormat::H264) => "! rtph264pay name=pay0", - Some(StreamFormat::H265) => "! rtph265pay name=pay0", + let launch_vid_select = match self.when_paused { + PausedSources::None => match self.video_format { + Some(StreamFormat::H264) => "! rtph264pay name=pay0", + Some(StreamFormat::H265) => "! rtph265pay name=pay0", + _ => "! fakesink", + }, + _ => match self.video_format { + Some(StreamFormat::H264) => "! x264enc ! rtph264pay name=pay0", + Some(StreamFormat::H265) => "! x265enc ! rtph265pay name=pay0", + _ => "! fakesink", + }, + }; + + let launch_app_src = match self.when_paused { + PausedSources::None => { + match self.video_format { + Some(StreamFormat::H264) => { + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h264parse" + } + Some(StreamFormat::H265) => { + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h265parse" + } + _ => "", + } + } + _ => { + match self.video_format { + Some(StreamFormat::H264) => { + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h264parse ! avdec_h264" + } + Some(StreamFormat::H265) => { + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h265parse ! avdec_h265" + } + _ => "", + } + } + }; + + let launch_alt_source = match self.when_paused { + PausedSources::TestSrc => "videotestsrc ! video/x-raw,width=896,height=512,framerate=25/1", + PausedSources::Black => "videotestsrc pattern=black ! video/x-raw,width=896,height=512,framerate=25/1", + PausedSources::Still => "vid_src_tee. ! imagefreeze allow-replace=true is-live=true ! video/x-raw,framerate=25/1", + PausedSources::None => "", + }; + + let launch_aud_select = match self.audio_format { + Some(StreamFormat::Adpcm(_)) | Some(StreamFormat::Aac) => { + "! audioconvert ! rtpL16pay name=pay1" + } _ => "! fakesink", }; - let launch_app_src = match self.video_format { - Some(StreamFormat::H264) => { - "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h264parse" - } - Some(StreamFormat::H265) => { - "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h265parse" - } - _ => "", - }; - - let launch_alt_source = match self.video_format { - Some(StreamFormat::H264) => { - "videotestsrc ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! x264enc ! video/x-h264,width=896,height=512,framerate=25/1 ! h264parse" - } - Some(StreamFormat::H265) => { - "videotestsrc ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! x265enc ! video/x-h265,width=896,height=512,framerate=25/1 ! h265parse" - } - _ => "", - }; - - let launch_imgfreeze_source = match self.video_format { - Some(StreamFormat::H264) => { - "avdec_h264 ! imagefreeze allow-replace=true is-live=true ! x264enc ! video/x-h264,width=896,height=512,framerate=25/1 ! h264parse" - } - Some(StreamFormat::H265) => { - "decodebin ! imagefreeze allow-replace=true is-live=true ! x265enc ! video/x-h265,width=896,height=512,framerate=25/1 ! h265parse" - } - _ => "", - }; - let launch_aud = match self.audio_format { - Some(StreamFormat::Adpcm(block_size)) => format!("caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! adpcmdec ! audioconvert ! rtpL16pay name=pay1", block_size), // DVI4 is converted to pcm in the appsrc - Some(StreamFormat::Aac) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! aacparse ! decodebin ! audioconvert ! rtpL16pay name=pay1 name=pay1".to_string(), - _ => "! fakesink".to_string(), + Some(StreamFormat::Adpcm(block_size)) => format!("caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! adpcmdec", block_size), // DVI4 is converted to pcm in the appsrc + Some(StreamFormat::Aac) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! aacparse ! decodebin".to_string(), + _ => "".to_string(), }; + let launch_aud_alt = "audiotestsrc wave=silence"; + let launch_str = &vec![ "( ", // Video out pipe @@ -252,26 +293,38 @@ impl GstOutputs { "(", "appsrc name=vidsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks launch_app_src, + // Pipe it though a tee so that the image freeze can grab it "! tee name=vid_src_tee", "(", "vid_src_tee. ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! vid_inputselect.sink_0", ")", ")", - // Test vid source + // Alternaive vid source + // + // Used during paused mode "(", launch_alt_source, - "! vid_inputselect.sink_1", + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! vid_inputselect.sink_1", ")", // Image freeze - "(", - "vid_src_tee. ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 !", - launch_imgfreeze_source, - " ! vid_inputselect.sink_2", - ")", // Audio pipe - "appsrc name=audsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", - &launch_aud, + // Audio out pipe + "(", + "input-selector name=aud_inputselect", + launch_aud_select, + ")", + // Camera aud source + "(", + "appsrc name=audsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", + &launch_aud, + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! aud_inputselect.sink_0", + ")", + // Camera aud alt source + "(", + launch_aud_alt, + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! aud_inputselect.sink_1", + ")", ")" ] .join(" "); @@ -312,9 +365,14 @@ impl RtspServer { let (maybe_app_src_aud, tx_aud) = MaybeAppSrc::new_with_tx(); let (maybe_vid_inputselect, tx_vid_inputselect) = MaybeInputSelect::new_with_tx(); + let (maybe_aud_inputselect, tx_aud_inputselect) = MaybeInputSelect::new_with_tx(); - let outputs = - GstOutputs::from_appsrcs(maybe_app_src, maybe_app_src_aud, maybe_vid_inputselect); + let outputs = GstOutputs::from_appsrcs( + maybe_app_src, + maybe_app_src_aud, + maybe_vid_inputselect, + maybe_aud_inputselect, + ); let factory = &outputs.factory; @@ -354,6 +412,11 @@ impl RtspServer { .by_name_recurse_up("vid_inputselect") .expect("vid_inputselect must be present in created bin"); let _ = tx_vid_inputselect.send(maybe_vid_inputselect); // Receiver may be dropped, don't panic if so + + let maybe_aud_inputselect = bin + .by_name_recurse_up("aud_inputselect") + .expect("aud_inputselect must be present in created bin"); + let _ = tx_aud_inputselect.send(maybe_aud_inputselect); // Receiver may be dropped, don't panic if so }); for path in paths { diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 2796574..bc5503a 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -47,7 +47,7 @@ mod state; use crate::utils::AddressOrUid; pub(crate) use cmdline::Opt; use config::{CameraConfig, Config, UserConfig}; -use gst::{GstOutputs, InputSources, RtspServer, TlsAuthenticationMode}; +use gst::{GstOutputs, InputMode, PausedSources, RtspServer, TlsAuthenticationMode}; use motion::MotionStream; use state::States; @@ -293,6 +293,14 @@ fn camera_main( // and control the motion/client state info let state = States::new_from_camera_config(camera_config); outputs.set_state(state.clone()); + let paused_source = match pause_config.mode.as_str() { + "test" => {PausedSources::TestSrc}, + "still" => {PausedSources::Still}, + "black" => {PausedSources::Black}, + "none" => {PausedSources::None}, + _ => {unreachable!()} + }; + outputs.set_paused_source(paused_source); crossbeam::scope(|s| { let mut motion = MotionStream::new(pause_config.motion_timeout, state.clone()); @@ -323,7 +331,7 @@ fn camera_main( s.spawn(move |_| { while stream_state.is_live() { if stream_state.should_stream() || ! outputs.has_last_iframe() { - if let Err(e) = outputs.set_input_source(InputSources::Cam) { + if let Err(e) = outputs.set_input_source(InputMode::Live) { *stream_error.lock().unwrap() = Some(e); break; } @@ -337,13 +345,12 @@ fn camera_main( } debug!("Stream paused"); } else { - let input_source = match pause_config.mode.as_str() { - "test" => {InputSources::TestSrc}, - "still" => {InputSources::Still}, - "black" => {InputSources::Black}, - _ => {unreachable!()} - }; - if let Err(e) = outputs.set_input_source(input_source) { + if let Err(e) = outputs.set_input_source(InputMode::Paused) { + *stream_error.lock().unwrap() = Some(e); + break; + } + // While stream is aborted we send the last iframe + if let Err(e) = outputs.write_last_iframe().with_context(|| format!("Failed to write no motion image for {}", stream_camname)) { *stream_error.lock().unwrap() = Some(e); break; } From 1c218b358727711e8c359c0d46cc77efcde7934e Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Fri, 8 Oct 2021 10:46:19 +0700 Subject: [PATCH 11/11] Add some notes to the paused inputs --- src/rtsp/gst.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/rtsp/gst.rs b/src/rtsp/gst.rs index 3f6d0de..b693dfb 100644 --- a/src/rtsp/gst.rs +++ b/src/rtsp/gst.rs @@ -169,6 +169,9 @@ impl GstOutputs { } pub(crate) fn set_input_source(&mut self, input_source: InputMode) -> AnyResult<()> { + // PausedSources::None is exceptional in that it dosen't swap the stream + // to an alterntive source. It just repeats the last buffer meaning not re-encoding + // at the cost some clients not handeling the repeating buffers well if self.when_paused != PausedSources::None { match &input_source { InputMode::Live => { @@ -180,6 +183,17 @@ impl GstOutputs { self.aud_inputselect.set_input(1)?; } } + } else { + // For PausedSources::None we still want to use the silence as we don't want to + // repeat the audio buffers + match &input_source { + InputMode::Live => { + self.aud_inputselect.set_input(0)?; + } + InputMode::Paused => { + self.aud_inputselect.set_input(1)?; + } + } } Ok(()) } @@ -222,6 +236,10 @@ impl GstOutputs { } fn apply_format(&self) { + // This is the final sink prior to rtsp + // - In the case of PausedSources::None we just pass through without encoding + // - In the case of all other pause sources we need to recencode in order + // to get smooth streams when the stream is paused/resumed let launch_vid_select = match self.when_paused { PausedSources::None => match self.video_format { Some(StreamFormat::H264) => "! rtph264pay name=pay0", @@ -235,6 +253,10 @@ impl GstOutputs { }, }; + // This is the part that deals with input from camera + // - In the case of PausedSources::None it just passes the stream + // - In the case of other Paused sources we decode it so that we can manipulate + // the stream smoothly and swap the source on demand let launch_app_src = match self.when_paused { PausedSources::None => { match self.video_format { @@ -260,6 +282,8 @@ impl GstOutputs { } }; + // This controls the alternaive source used when in paused state + // PausedSources::None is the exception in that it is never swapped to this state let launch_alt_source = match self.when_paused { PausedSources::TestSrc => "videotestsrc ! video/x-raw,width=896,height=512,framerate=25/1", PausedSources::Black => "videotestsrc pattern=black ! video/x-raw,width=896,height=512,framerate=25/1", @@ -267,6 +291,7 @@ impl GstOutputs { PausedSources::None => "", }; + // This is the final pipeline for the audio prior to rtsp let launch_aud_select = match self.audio_format { Some(StreamFormat::Adpcm(_)) | Some(StreamFormat::Aac) => { "! audioconvert ! rtpL16pay name=pay1" @@ -274,12 +299,15 @@ impl GstOutputs { _ => "! fakesink", }; + // This is the audio source from the camera. We decode to xraw always as some clients + // like blue iris only support raw audio let launch_aud = match self.audio_format { Some(StreamFormat::Adpcm(block_size)) => format!("caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! adpcmdec", block_size), // DVI4 is converted to pcm in the appsrc Some(StreamFormat::Aac) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! aacparse ! decodebin".to_string(), _ => "".to_string(), }; + // The alternaive audio source is silence let launch_aud_alt = "audiotestsrc wave=silence"; let launch_str = &vec![