diff --git a/Cargo.lock b/Cargo.lock index 81bc9aa..2e25e62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,17 +109,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - [[package]] name = "async-trait" version = "0.1.72" @@ -349,15 +338,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" -[[package]] -name = "concurrent-queue" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "console-api" version = "0.5.0" @@ -565,12 +545,6 @@ dependencies = [ "libc", ] -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "fastrand" version = "2.0.0" @@ -1264,12 +1238,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "is_sorted" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357376465c37db3372ef6a00585d336ed3d0f11d4345eef77ebcb05865392b21" - [[package]] name = "itertools" version = "0.10.5" @@ -1279,15 +1247,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.9" @@ -1454,7 +1413,6 @@ name = "neolink" version = "0.5.18" dependencies = [ "anyhow", - "async-channel", "base64 0.21.2", "byte-slice-cast", "clap", @@ -1467,8 +1425,6 @@ dependencies = [ "gstreamer-rtsp", "gstreamer-rtsp-server", "heck", - "is_sorted", - "itertools 0.11.0", "lazy_static", "log", "neolink_core", @@ -1476,7 +1432,6 @@ dependencies = [ "rumqttc", "serde", "serde_json", - "time", "tokio", "tokio-stream", "tokio-util", @@ -1767,7 +1722,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "syn 1.0.109", diff --git a/Cargo.toml b/Cargo.toml index ebefeee..ee470df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,6 @@ members = [ [dependencies] anyhow = "1.0.70" -async-channel = "1.9.0" base64 = "0.21.2" byte-slice-cast = "1.2.2" clap = { version = "4.2.2", features = ["derive", "cargo"] } @@ -26,8 +25,6 @@ gstreamer-app = "0.20.0" gstreamer-rtsp = "0.20.0" gstreamer-rtsp-server = { version = "0.20.3", features = ["v1_16"] } heck = "0.4.1" -is_sorted = "0.1.1" -itertools = "0.11.0" lazy_static = "1.4.0" log = { version = "0.4.17", features = [ "release_max_level_debug" ] } neolink_core = { path = "crates/core", version = "0.5.18" } @@ -35,7 +32,6 @@ regex = "1.7.3" rumqttc = "0.22.0" serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.96" -time = "0.3.20" tokio = { version = "1.27.0", features = ["rt-multi-thread", "macros", "io-util", "tracing"] } tokio-stream = "0.1.12" tokio-util = { version = "0.7.7", features = ["full", "tracing"] } diff --git a/crates/core/src/bc_protocol.rs b/crates/core/src/bc_protocol.rs index aae5902..56929ba 100644 --- a/crates/core/src/bc_protocol.rs +++ b/crates/core/src/bc_protocol.rs @@ -259,6 +259,7 @@ impl BcCamera { }, if allow_remote => Ok(v), Ok(v) = async { let uid_map = uid.clone(); + tokio::time::sleep(tokio::time::Duration::from_millis(250)).await; info!("{}: Trying map discovery", options.name); let result = discovery.map(®_result).await; match result { @@ -276,6 +277,7 @@ impl BcCamera { }, if allow_map => Ok(v), Ok(v) = async { let uid_relay = uid.clone(); + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; info!("{}: Trying relay discovery", options.name); let result = discovery.relay(®_result).await; match result { diff --git a/crates/core/src/bc_protocol/connection/bcconn.rs b/crates/core/src/bc_protocol/connection/bcconn.rs index c9f4382..f77dba6 100644 --- a/crates/core/src/bc_protocol/connection/bcconn.rs +++ b/crates/core/src/bc_protocol/connection/bcconn.rs @@ -59,6 +59,7 @@ impl BcConnection { let thread_cancel = cancel.clone(); let handle = tokio::task::spawn_blocking(move || { let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() .build() .unwrap(); let result = runtime.block_on(async move { diff --git a/crates/core/src/bc_protocol/connection/discovery.rs b/crates/core/src/bc_protocol/connection/discovery.rs index c97c41d..88e6e18 100644 --- a/crates/core/src/bc_protocol/connection/discovery.rs +++ b/crates/core/src/bc_protocol/connection/discovery.rs @@ -26,7 +26,7 @@ use tokio::{ net::UdpSocket, sync::{ mpsc::{channel, Receiver, Sender}, - RwLock, + RwLock, Semaphore, }, task::JoinSet, time::{interval, timeout, Duration}, @@ -83,14 +83,18 @@ lazy_static! { ]; /// Maximum wait for a reply static ref MAXIMUM_WAIT: Duration = Duration::from_secs(15); + /// Wait for tcp connections + static ref TCP_WAIT: Duration = Duration::from_secs(4); /// How long to wait before resending static ref RESEND_WAIT: Duration = Duration::from_millis(500); + } type Subscriber = Arc>>>>; type Handlers = Arc>>>>; type ArcFramedSocket = UdpFramed>; pub(crate) struct Discoverer { + semaphore: Arc, socket: Arc, handle: RwLock>, writer: Sender>, @@ -172,6 +176,7 @@ impl Discoverer { }); Ok(Discoverer { + semaphore: Arc::new(Semaphore::new(1)), socket, handle: RwLock::new(set), writer: sinker, @@ -619,6 +624,12 @@ impl Discoverer { }, }; + let permit = self + .semaphore + .clone() + .acquire_owned() + .await + .map_err(|_| Error::Other("Discovery already complete"))?; // Send and await confirm self.retry_send(msg, addr, |bc, _| { trace!("msg: {:?}", &bc); @@ -675,6 +686,8 @@ impl Discoverer { self.keep_alive_device(local_tid, &result).await; + self.semaphore.close(); + drop(permit); Ok(result) } @@ -736,6 +749,12 @@ impl Discoverer { }, }; + let permit = self + .semaphore + .clone() + .acquire_owned() + .await + .map_err(|_| Error::Other("Discovery already complete"))?; // Send and await confirm self.retry_send(msg, addr, |bc, _| { trace!("msg: {:?}", &bc); @@ -792,6 +811,8 @@ impl Discoverer { self.keep_alive_device(local_tid, &result).await; + self.semaphore.close(); + drop(permit); Ok(result) } @@ -838,6 +859,12 @@ impl Discoverer { camera_id: local_did, }; + let permit = self + .semaphore + .clone() + .acquire_owned() + .await + .map_err(|_| Error::Other("Discovery already complete"))?; // Confirm local to register let msg = UdpDiscovery { tid: 0, @@ -857,6 +884,8 @@ impl Discoverer { self.keep_alive_device(tid, &result).await; + self.semaphore.close(); + drop(permit); Ok(result) } @@ -880,6 +909,12 @@ impl Discoverer { }, }; + let permit = self + .semaphore + .clone() + .acquire_owned() + .await + .map_err(|_| Error::Other("Discovery already complete"))?; let (final_addr, local_did) = self .retry_send(msg, relay_addr, |bc, addr| match bc { UdpDiscovery { @@ -933,6 +968,8 @@ impl Discoverer { self.keep_alive_device(tid, &result).await; // self.keep_alive_relay(tid, &result).await; + self.semaphore.close(); + drop(permit); Ok(result) } @@ -1048,11 +1085,8 @@ impl Discovery { pub(crate) async fn check_tcp(&self, addr: SocketAddr, channel_id: u8) -> Result<()> { let username = "admin"; let password = Some("123456"); - let mut tcp_source = timeout( - *MAXIMUM_WAIT, - TcpSource::new(addr, username, password, false), - ) - .await??; + let mut tcp_source = + timeout(*TCP_WAIT, TcpSource::new(addr, username, password, false)).await??; let md5_username = md5_string(username, Md5Trunc::ZeroLast); let md5_password = password @@ -1076,7 +1110,7 @@ impl Discovery { }) .await?; - let _bc: Bc = timeout(*MAXIMUM_WAIT, tcp_source.next()) + let _bc: Bc = timeout(*TCP_WAIT, tcp_source.next()) .await? .ok_or(Error::CannotInitCamera)??; // Successful recv should mean a Bc packet if not then deser will fail Ok(()) diff --git a/crates/core/src/bc_protocol/connection/udpsource.rs b/crates/core/src/bc_protocol/connection/udpsource.rs index 54ff3b0..2913a6a 100644 --- a/crates/core/src/bc_protocol/connection/udpsource.rs +++ b/crates/core/src/bc_protocol/connection/udpsource.rs @@ -322,6 +322,7 @@ impl UdpPayloadInner { let thread_camera_addr = camera_addr; tokio::task::spawn_blocking(move || { let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() .build() .unwrap(); runtime.block_on(async move { @@ -350,6 +351,7 @@ impl UdpPayloadInner { let mut ack_socket_in_tx = PollSender::new(socket_in_tx.clone()); tokio::task::spawn_blocking(move || { let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() .build() .unwrap(); runtime.block_on(async move { @@ -388,6 +390,7 @@ impl UdpPayloadInner { let mut socket_out_tx = PollSender::new(socket_out_tx); tokio::task::spawn_blocking(move || { let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() .build() .unwrap(); @@ -591,6 +594,7 @@ impl UdpPayloadSource { let thread_cancel_token = cancel_token.clone(); let handle = tokio::task::spawn_blocking(move || { let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() .build() .unwrap(); diff --git a/crates/core/src/bc_protocol/stream.rs b/crates/core/src/bc_protocol/stream.rs index 7f4d8b7..06070a7 100644 --- a/crates/core/src/bc_protocol/stream.rs +++ b/crates/core/src/bc_protocol/stream.rs @@ -22,14 +22,13 @@ pub enum StreamKind { Extern, } -impl StreamKind { - pub fn to_string(&self) -> String { +impl std::fmt::Display for StreamKind { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - StreamKind::Main => "mainStream", - StreamKind::Sub => "subStream", - StreamKind::Extern => "externStream", + StreamKind::Main => write!(f, "mainStream"), + StreamKind::Sub => write!(f, "subStream"), + StreamKind::Extern => write!(f, "externStream"), } - .to_string() } } diff --git a/crates/core/src/bc_protocol/stream_info.rs b/crates/core/src/bc_protocol/stream_info.rs new file mode 100644 index 0000000..900c4c2 --- /dev/null +++ b/crates/core/src/bc_protocol/stream_info.rs @@ -0,0 +1,50 @@ +use super::{BcCamera, Error, Result}; +use crate::bc::{model::*, xml::*}; + +impl BcCamera { + /// Get the [StreamInfoList] xml which contains the supported camera streams + pub async fn get_stream_info(&self) -> Result { + let connection = self.get_connection(); + let msg_num = self.new_message_num(); + let mut sub_get = connection + .subscribe(MSG_ID_STREAM_INFO_LIST, msg_num) + .await?; + let get = Bc { + meta: BcMeta { + msg_id: MSG_ID_STREAM_INFO_LIST, + channel_id: self.channel_id, + msg_num, + response_code: 0, + stream_type: 0, + class: 0x6414, + }, + body: BcBody::ModernMsg(ModernMsg { + extension: None, + payload: None, + }), + }; + + sub_get.send(get).await?; + let msg = sub_get.recv().await?; + if msg.meta.response_code != 200 { + return Err(Error::CameraServiceUnavaliable); + } + + if let BcBody::ModernMsg(ModernMsg { + payload: + Some(BcPayloads::BcXml(BcXml { + stream_info_list: Some(data), + .. + })), + .. + }) = msg.body + { + Ok(data) + } else { + Err(Error::UnintelligibleReply { + reply: std::sync::Arc::new(Box::new(msg)), + why: "Expected StreamInfoList xml but it was not recieved", + }) + } + } +} diff --git a/src/common/instance.rs b/src/common/instance.rs index 5663453..e2c0432 100644 --- a/src/common/instance.rs +++ b/src/common/instance.rs @@ -130,6 +130,7 @@ impl NeoInstance { Ok(instance_rx.await?) } + #[allow(dead_code)] pub(crate) async fn low_stream(&self) -> Result> { let (instance_tx, instance_rx) = oneshot(); self.camera_control @@ -138,6 +139,7 @@ impl NeoInstance { Ok(instance_rx.await?) } + #[allow(dead_code)] pub(crate) async fn high_stream(&self) -> Result> { let (instance_tx, instance_rx) = oneshot(); self.camera_control @@ -146,6 +148,7 @@ impl NeoInstance { Ok(instance_rx.await?) } + #[allow(dead_code)] pub(crate) async fn streams(&self) -> Result> { let (instance_tx, instance_rx) = oneshot(); self.camera_control diff --git a/src/common/neocam.rs b/src/common/neocam.rs index 6efb380..da8afb7 100644 --- a/src/common/neocam.rs +++ b/src/common/neocam.rs @@ -11,13 +11,14 @@ use tokio::sync::{ oneshot::{channel as oneshot, Sender as OneshotSender}, watch::{channel as watch, Receiver as WatchReceiver, Sender as WatchSender}, }; -use tokio_stream::wrappers::{BroadcastStream, ReceiverStream}; +use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::CancellationToken; use super::{NeoCamStreamThread, NeoCamThread, NeoInstance, StreamInstance, StreamRequest}; use crate::{config::CameraConfig, AnyResult, Result}; use neolink_core::bc_protocol::{BcCamera, StreamKind}; +#[allow(dead_code)] pub(crate) enum NeoCamCommand { HangUp, Instance(OneshotSender>), diff --git a/src/common/streamthread.rs b/src/common/streamthread.rs index f56d3dc..f0ad782 100644 --- a/src/common/streamthread.rs +++ b/src/common/streamthread.rs @@ -161,10 +161,10 @@ impl NeoCamStreamThread { sender } => { let config = self.instance.config().await?.borrow_and_update().clone(); - let streams = config.stream.to_stream_kinds(); + let streams = config.stream.as_stream_kinds(); for stream in streams.iter().copied() { if let Entry::Vacant(vac) = self.streams.entry(stream) { - let (tx, rx) = broadcast(1000); + let (tx, _) = broadcast(1000); vac.insert( StreamData::new(tx, stream, self.instance.subscribe().await?, config.strict) .await?, @@ -207,6 +207,7 @@ impl Drop for NeoCamStreamThread { /// The kind of stream we want a async broadcast of pub(crate) enum StreamRequest { + #[allow(dead_code)] /// Get a currently loaded stream Get { name: StreamKind, diff --git a/src/config.rs b/src/config.rs index 2a30e2b..a22a40b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -91,7 +91,7 @@ pub(crate) enum StreamConfig { } impl StreamConfig { - pub(crate) fn to_stream_kinds(&self) -> Vec { + pub(crate) fn as_stream_kinds(&self) -> Vec { match self { StreamConfig::All => { vec![StreamKind::Main, StreamKind::Extern, StreamKind::Sub] @@ -176,13 +176,6 @@ pub(crate) struct CameraConfig { #[serde(default = "default_buffer_size", alias = "size", alias = "buffer")] pub(crate) buffer_size: usize, - #[serde( - default = "default_smoothing", - alias = "smoothing", - alias = "stretching" - )] - pub(crate) use_smoothing: bool, - #[serde(default = "default_true", alias = "splash")] pub(crate) use_splash: bool, diff --git a/src/image/mod.rs b/src/image/mod.rs index 8299044..d83a5fd 100644 --- a/src/image/mod.rs +++ b/src/image/mod.rs @@ -39,7 +39,7 @@ pub(crate) async fn main(opt: Opt, config: Config, reactor: NeoReactor) -> Resul let camera = reactor.get_or_insert(config.clone()).await?; if opt.use_stream { - let mut stream_data = camera + let stream_data = camera .stream(StreamKind::Main) .await .context("Failed to start video")?; @@ -47,7 +47,8 @@ pub(crate) async fn main(opt: Opt, config: Config, reactor: NeoReactor) -> Resul // Get one iframe at the start while also getting the the video type let buf; let vid_type; - let mut stream = BroadcastStream::new(stream_data.stream.resubscribe()); + let mut stream = BroadcastStream::new(stream_data.stream.resubscribe()) + .filter(|f| futures::future::ready(f.is_ok())); // Filter to ignore lagged loop { if let Some(Ok(BcMedia::Iframe(frame))) = stream.next().await { vid_type = frame.video_type; diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 0053b63..9e87730 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -4,19 +4,13 @@ //! expect issues use super::AnyResult; -use crate::common::{AudFormat, NeoReactor, StreamInstance, VidFormat}; -use anyhow::{anyhow, Context}; -use async_channel::{Receiver as AsyncReceiver, Sender as AsyncSender}; -use futures::stream::StreamExt; use gstreamer::glib::object_subclass; use gstreamer::glib::subclass::types::ObjectSubclass; +use gstreamer::Element; use gstreamer::{ glib::{self, Object}, - prelude::*, - ClockTime, Structure, + Structure, }; -use gstreamer::{Bin, Caps, Element, ElementFactory}; -use gstreamer_app::{AppSrc, AppSrcCallbacks, AppStreamType}; use gstreamer_rtsp::RTSPUrl; use gstreamer_rtsp_server::prelude::*; use gstreamer_rtsp_server::subclass::prelude::*; @@ -24,21 +18,9 @@ use gstreamer_rtsp_server::RTSPMediaFactory; use gstreamer_rtsp_server::{RTSPSuspendMode, RTSPTransportMode}; use gstreamer_rtsp_server::{RTSP_PERM_MEDIA_FACTORY_ACCESS, RTSP_PERM_MEDIA_FACTORY_CONSTRUCT}; use log::*; -use neolink_core::{bc_protocol::StreamKind, bcmedia::model::*}; use std::collections::HashSet; -use std::convert::TryInto; use std::sync::Arc; -use tokio::{ - sync::{ - mpsc::{channel as mpsc, Sender as MpscSender}, - oneshot::{channel as oneshot, Receiver as OneshotReceiver, Sender as OneshotSender}, - Mutex, RwLock, - }, - task::JoinSet, - time::Duration, -}; -use tokio_stream::wrappers::ReceiverStream; -use tokio_util::sync::CancellationToken; +use tokio::sync::Mutex; glib::wrapper! { /// The wrapped RTSPMediaFactory @@ -112,6 +94,7 @@ unsafe impl Send for NeoMediaFactory {} unsafe impl Sync for NeoMediaFactory {} pub(crate) struct NeoMediaFactoryImpl { + #[allow(clippy::type_complexity)] call_back: Arc AnyResult> + Send + Sync>>>>, } @@ -154,47 +137,3 @@ impl ObjectSubclass for NeoMediaFactoryImpl { type Type = super::NeoMediaFactory; type ParentType = RTSPMediaFactory; } - -// Convenice funcion to make an element or provide a message -// about what plugin is missing -fn make_element(kind: &str, name: &str) -> AnyResult { - ElementFactory::make_with_name(kind, Some(name)).with_context(|| { - let plugin = match kind { - "appsrc" => "app (gst-plugins-base)", - "audioconvert" => "audioconvert (gst-plugins-base)", - "adpcmdec" => "Required for audio", - "h264parse" => "videoparsersbad (gst-plugins-bad)", - "h265parse" => "videoparsersbad (gst-plugins-bad)", - "rtph264pay" => "rtp (gst-plugins-good)", - "rtph265pay" => "rtp (gst-plugins-good)", - "aacparse" => "audioparsers (gst-plugins-good)", - "rtpL16pay" => "rtp (gst-plugins-good)", - "x264enc" => "x264 (gst-plugins-ugly)", - "x265enc" => "x265 (gst-plugins-bad)", - "avdec_h264" => "libav (gst-libav)", - "avdec_h265" => "libav (gst-libav)", - "videotestsrc" => "videotestsrc (gst-plugins-base)", - "imagefreeze" => "imagefreeze (gst-plugins-good)", - "audiotestsrc" => "audiotestsrc (gst-plugins-base)", - "decodebin" => "playback (gst-plugins-good)", - _ => "Unknown", - }; - format!( - "Missing required gstreamer plugin `{}` for `{}` element", - plugin, kind - ) - }) -} - -fn make_queue(name: &str) -> AnyResult { - let queue = make_element("queue", name)?; - queue.set_property_from_str("leaky", "downstream"); - queue.set_property("max-size-bytes", 0u32); - queue.set_property("max-size-buffers", 0u32); - queue.set_property( - "max-size-time", - std::convert::TryInto::::try_into(tokio::time::Duration::from_secs(5).as_nanos()) - .unwrap_or(0), - ); - Ok(queue) -} diff --git a/src/rtsp/gst/server.rs b/src/rtsp/gst/server.rs index 322bd77..f7154dd 100644 --- a/src/rtsp/gst/server.rs +++ b/src/rtsp/gst/server.rs @@ -3,26 +3,22 @@ //! We are now messing with gstreamer glib objects //! expect issues -use super::{factory::*, AnyResult}; +use super::AnyResult; use crate::config::*; -use anyhow::{anyhow, Context}; +use anyhow::Context; use gstreamer::glib::{self, object_subclass, subclass::types::ObjectSubclass, MainLoop, Object}; use gstreamer_rtsp::RTSPAuthMethod; use gstreamer_rtsp_server::{ gio::{TlsAuthenticationMode, TlsCertificate}, prelude::*, subclass::prelude::*, - RTSPAuth, RTSPFilterResult, RTSPServer, RTSPToken, RTSP_TOKEN_MEDIA_FACTORY_ROLE, + RTSPAuth, RTSPServer, RTSPToken, RTSP_TOKEN_MEDIA_FACTORY_ROLE, }; use log::*; -use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, - fs, - sync::Arc, -}; +use std::{fs, sync::Arc}; use tokio::{ - sync::{mpsc::Sender, RwLock}, + sync::RwLock, task::JoinSet, time::{timeout, Duration}, }; @@ -45,14 +41,6 @@ impl NeoRtspServer { Ok(factory) } - pub(crate) async fn add_permitted_roles, U: AsRef>( - &self, - tag: T, - permitted_users: &HashSet, - ) -> AnyResult<()> { - self.imp().add_permitted_roles(tag, permitted_users).await - } - pub(crate) async fn run(&self, bind_addr: &str, bind_port: u16) -> AnyResult<()> { let server = self; server.set_address(bind_addr); @@ -104,14 +92,8 @@ impl NeoRtspServer { unsafe impl Send for NeoRtspServer {} unsafe impl Sync for NeoRtspServer {} -struct FactoryData { - factory: NeoMediaFactory, - paths: HashSet, -} - #[derive(Default)] pub(crate) struct NeoRtspServerImpl { - medias: RwLock>, threads: RwLock>>, main_loop: RwLock>>, } @@ -127,20 +109,6 @@ impl ObjectSubclass for NeoRtspServerImpl { } impl NeoRtspServerImpl { - pub(crate) async fn add_permitted_roles, U: AsRef>( - &self, - tag: T, - permitted_users: &HashSet, - ) -> AnyResult<()> { - let tag: String = tag.into(); - if let Some(media) = self.medias.write().await.get_mut(&tag) { - media.factory.add_permitted_roles(permitted_users); - Ok(()) - } else { - Err(anyhow!("No media with tag {} to add users to", &tag)) - } - } - pub(crate) fn set_credentials(&self, credentials: &[(&str, &str)]) -> AnyResult<()> { let auth = self.obj().auth().unwrap_or_else(RTSPAuth::new); auth.set_supported_methods(RTSPAuthMethod::Basic); diff --git a/src/rtsp/mod.rs b/src/rtsp/mod.rs index 4b5100d..38b8c93 100644 --- a/src/rtsp/mod.rs +++ b/src/rtsp/mod.rs @@ -53,29 +53,35 @@ // - `"none"`: Resends the last iframe the camera. This does not reencode at all. **Most use cases should use this one as it has the least effort on the cpu and gives what you would expect** // use anyhow::{anyhow, Context, Result}; -use futures::stream::FuturesUnordered; +use gstreamer::{Bin, Caps, ClockTime, Element, ElementFactory}; +use gstreamer_app::{AppSrc, AppSrcCallbacks, AppStreamType}; use gstreamer_rtsp_server::prelude::*; use log::*; -use neolink_core::bc_protocol::{BcCamera, StreamKind}; +use neolink_core::{bc_protocol::StreamKind, bcmedia::model::*}; use std::collections::HashSet; use std::sync::Arc; -use tokio::time::{interval, Duration}; -use tokio_stream::{wrappers::IntervalStream, StreamExt}; +use tokio::{ + sync::{ + broadcast::channel as broadcast, + mpsc::{channel as mpsc, Sender as MpscSender}, + }, + task::JoinSet, + time::{sleep, Duration}, +}; +use tokio_stream::{wrappers::BroadcastStream, Stream, StreamExt}; use tokio_util::sync::CancellationToken; mod cmdline; mod gst; -mod spring; use crate::{ - common::{NeoInstance, NeoReactor, VidFormat}, + common::{AudFormat, NeoInstance, NeoReactor, StreamInstance, VidFormat}, rtsp::gst::NeoMediaFactory, }; use super::config::Config; pub(crate) use cmdline::Opt; use gst::NeoRtspServer; -pub(crate) use spring::*; type AnyResult = anyhow::Result; @@ -101,102 +107,23 @@ pub(crate) async fn main(_opt: Opt, mut config: Config, reactor: NeoReactor) -> let global_cancel = CancellationToken::new(); - let mut set = tokio::task::JoinSet::new(); - for mut camera in cameras.drain(..) { - let stream_info = camera - .run_task(|cam| Box::pin(async move { Ok(cam.get_stream_info().await?) })) - .await?; - - let supported_streams = { - stream_info - .stream_infos - .iter() - .flat_map(|stream_info| stream_info.encode_tables.clone()) - .flat_map(|encode| match encode.name.as_str() { - "mainStream" => Some(StreamKind::Main), - "subStream" => Some(StreamKind::Sub), - "externStream" => Some(StreamKind::Extern), - new_stream_name => { - log::debug!("New stream name {}", new_stream_name); - None - } - }) - .collect::>() - }; - + let mut set = JoinSet::new(); + for camera in cameras.drain(..) { let thread_global_cancel = global_cancel.clone(); let thread_rtsp = rtsp.clone(); - set.spawn(async move { - tokio::select!( - _ = thread_global_cancel.cancelled() => { - AnyResult::Ok(()) - }, - v = async { - let mut camera_config = camera.config().await?.clone(); - loop { - let prev_stream_config = camera_config.borrow_and_update().stream.clone(); - let active_streams = prev_stream_config.to_stream_kinds().drain(..).collect::>(); - - // This select is for changes to camera_config.stream - break tokio::select!{ - v = camera_config.wait_for(|config| config.stream != prev_stream_config) => { - if let Err(e) = v { - AnyResult::Err(e.into()) - } else { - // config.stream changed restart - continue; - } - }, - v = async { - // This select handles enabling the right stream - tokio::select! { - v = async { - let mut stream_instance = camera.stream(StreamKind::Main).await?; - loop { - // Wait for a valid stream format to be detected - stream_instance.config.wait_for(|config| !matches!(config.vid_format, VidFormat::None) && config.resolution[0] > 0 && config.resolution[1] > 0).await?; - let stream = stream_instance.stream.resubscribe(); - let stream_name = match stream_instance.name { - StreamKind::Main => "main", - StreamKind::Sub => "sub", - StreamKind::Extern => "extern", - } - .to_string(); - // This select ensures is for the streams config - break tokio::select!{ - _ = stream_instance.config.changed() => { - // If stream config changes we reload the stream - continue; - }, - v = async { - // Finally ready to create the factory and connect the stream - let mounts = thread_rtsp - .mount_points() - .ok_or(anyhow!("RTSP server lacks mount point"))?; - let name = camera.config().await?.borrow().name.clone(); - let path = format!("/{}/{}", name, stream_name); - log::info!("Path: {}", path); - let factory = NeoMediaFactory::new_with_callback(move |element| { - let stream_data = &stream; - Ok(Some(element)) - }).await?; - factory.add_permitted_roles(&["anonymous"].into()); - mounts.add_factory(&path, factory); - AnyResult::Ok(()) - } => v, - }; - } - }, if active_streams.contains(&StreamKind::Main) && supported_streams.contains(&StreamKind::Main) => v, - else => { - futures::pending!(); - AnyResult::Ok(()) - } - } - } => v, - }; - } - } => v, - ) + set.spawn_blocking(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + tokio::select!( + _ = thread_global_cancel.cancelled() => { + AnyResult::Ok(()) + }, + v = camera_main(camera, &thread_rtsp) => v, + ) + }) }); } info!( @@ -214,296 +141,716 @@ pub(crate) async fn main(_opt: Opt, mut config: Config, reactor: NeoReactor) -> match &joined { Err(_) | Ok(Err(_)) => { // Panicked or error in task + // Cancel all and await terminate + global_cancel.cancel(); rtsp.quit().await?; } Ok(Ok(_)) => { // All good } } - joined?? } Ok(()) } -enum CameraFailureKind { - Fatal(anyhow::Error), - Retry(anyhow::Error), -} +async fn camera_main(camera: NeoInstance, rtsp: &NeoRtspServer) -> Result<()> { + let stream_info = camera + .run_task(|cam| Box::pin(async move { Ok(cam.get_stream_info().await?) })) + .await?; -async fn camera_main(camera: NeoInstance, rtsp: NeoRtspServer) -> Result<(), CameraFailureKind> { - // Connect - let config = camera - .config() - .await - .map_err(CameraFailureKind::Fatal)? - .borrow_and_update() - .clone(); - let name = config.name; + let supported_streams = { + stream_info + .stream_infos + .iter() + .flat_map(|stream_info| stream_info.encode_tables.clone()) + .flat_map(|encode| match encode.name.as_str() { + "mainStream" => Some(StreamKind::Main), + "subStream" => Some(StreamKind::Sub), + "externStream" => Some(StreamKind::Extern), + new_stream_name => { + log::debug!("New stream name {}", new_stream_name); + None + } + }) + .collect::>() + }; - // Clear all buffers present - // Uncomment to clear buffers. This is now handlled in the buffer itself, - // instead of clearing it restamps it whenever there is a jump in the - // timestamps of >1s - // - // tags.iter() - // .map(|tag| rtsp_thread.clear_buffer(tag)) - // .collect::>() - // .collect::>() - // .await; + let mut camera_config = camera.config().await?.clone(); + loop { + let prev_stream_config = camera_config.borrow_and_update().stream; + let active_streams = prev_stream_config + .as_stream_kinds() + .drain(..) + .collect::>(); - // Start pulling data from the camera - // let mut streaming = loggedin - // .stream() - // .await - // .with_context(|| format!("{}: Could not start stream", name)) - // .map_err(CameraFailureKind::Retry)?; + // This select is for changes to camera_config.stream + break tokio::select! { + v = camera_config.wait_for(|config| config.stream != prev_stream_config) => { + if let Err(e) = v { + AnyResult::Err(e.into()) + } else { + // config.stream changed restart + continue; + } + }, + v = async { + // This select handles enabling the right stream + tokio::select! { + v = async { + let name = camera.config().await?.borrow().name.clone(); + stream_main(name, camera.stream(StreamKind::Main).await?, rtsp).await + }, if active_streams.contains(&StreamKind::Main) && supported_streams.contains(&StreamKind::Main) => v, + v = async { + let name = camera.config().await?.borrow().name.clone(); + stream_main(name, camera.stream(StreamKind::Sub).await?, rtsp).await + }, if active_streams.contains(&StreamKind::Sub) && supported_streams.contains(&StreamKind::Sub) => v, + v = async { + let name = camera.config().await?.borrow().name.clone(); + stream_main(name, camera.stream(StreamKind::Extern).await?, rtsp).await + }, if active_streams.contains(&StreamKind::Extern) && supported_streams.contains(&StreamKind::Extern) => v, + else => { + // all disabled just wait here until config is changed + futures::pending!(); + AnyResult::Ok(()) + } + } + } => v, + }; + }?; - // // Wait for buffers to be prepared - // tokio::select! { - // v = async { - // let mut waiter = tokio::time::interval(Duration::from_micros(500)); - // loop { - // waiter.tick().await; - // if tags - // .iter() - // .map(|tag| rtsp_thread.buffer_ready(tag)) - // .collect::>() - // .all(|f| f.unwrap_or(false)) - // .await - // { - // break; - // } - // } - // Ok(()) - // } => v, - // // Or for stream to error - // v = streaming.join() => {v}, - // } - // .with_context(|| format!("{}: Error while waiting for buffers", name)) - // .map_err(CameraFailureKind::Retry)?; - - // tags.iter() - // .map(|tag| rtsp_thread.jump_to_live(tag)) - // .collect::>() - // .collect::>() - // .await; - - // // Clear "stream not ready" media to try and force a reconnect - // // This shoud stop them from watching the "Stream Not Ready" thing - // debug!("Clearing not ready clients"); - // tags.iter() - // .map(|tag| rtsp_thread.clear_session_notready(tag)) - // .collect::>() - // .collect::>() - // .await; - // log::info!("{}: Buffers prepared", name); - - // let mut active_tags = streaming.shared.get_streams().clone(); - // let mut motion_pause = false; - // loop { - // // Wait for error or reason to pause - // let change = tokio::select! { - // v = async { - // // Wait for error - // streaming.join().await - // }, if ! active_tags.is_empty() => { - // info!("{}: Join Pause", name); - // Ok(StreamChange::StreamError(v)) - // }, - // // Send pings - // v = async { - // let camera = streaming.get_camera(); - // let mut interval = IntervalStream::new(interval(Duration::from_secs(5))); - // while let Some(_update) = interval.next().await { - // if camera.ping().await.is_err() { - // break; - // } - // } - // futures::pending!(); // Never actually finish, has to be aborted - // Ok(()) - // } => Ok(StreamChange::StreamError(v)), - // v = await_change( - // streaming.get_camera(), - // &streaming.shared, - // &rtsp_thread, - // &active_tags, - // motion_pause, - // &name, - // ), if streaming.shared.get_config().pause.on_motion || streaming.shared.get_config().pause.on_disconnect => { - // v.with_context(|| format!("{}: Error updating pause state", name)) - // .map_err(CameraFailureKind::Retry) - // } - // }?; - - // match change { - // StreamChange::StreamError(res) => { - // res.map_err(CameraFailureKind::Retry)?; - // } - // StreamChange::MotionStart => { - // motion_pause = false; - // let inactive_streams = streaming - // .shared - // .get_streams() - // .iter() - // .filter(|i| !active_tags.contains(i)) - // .copied() - // .collect::>(); - - // if streaming.shared.get_config().pause.on_disconnect { - // // Pause on client is also on - // // - // // Only resume clients with active connections - // for stream in inactive_streams.iter() { - // if rtsp_thread - // .get_number_of_clients(streaming.shared.get_tag_for_stream(stream)) - // .await - // .map(|n| n > 0) - // .unwrap_or(false) - // { - // streaming - // .start_stream(*stream) - // .await - // .map_err(CameraFailureKind::Retry)?; - // active_tags.insert(*stream); - // rtsp_thread - // .resume(streaming.shared.get_tag_for_stream(stream)) - // .await - // .map_err(CameraFailureKind::Retry)?; - // } - // } - // } else { - // // Pause on client is not on - // // - // // Resume all - // for stream in inactive_streams.iter() { - // streaming - // .start_stream(*stream) - // .await - // .map_err(CameraFailureKind::Retry)?; - // active_tags.insert(*stream); - // rtsp_thread - // .resume(streaming.shared.get_tag_for_stream(stream)) - // .await - // .map_err(CameraFailureKind::Retry)?; - // } - // } - // } - // StreamChange::MotionStop => { - // motion_pause = true; - // // Clear all streams - // for stream in active_tags.drain() { - // rtsp_thread - // .pause(streaming.shared.get_tag_for_stream(&stream)) - // .await - // .map_err(CameraFailureKind::Retry)?; - // streaming - // .stop_stream(stream) - // .await - // .map_err(CameraFailureKind::Retry)?; - // } - // } - // StreamChange::ClientStart(stream) => { - // if !streaming.shared.get_config().pause.on_motion || !motion_pause { - // streaming - // .start_stream(stream) - // .await - // .map_err(CameraFailureKind::Retry)?; - // active_tags.insert(stream); - // rtsp_thread - // .resume(streaming.shared.get_tag_for_stream(&stream)) - // .await - // .map_err(CameraFailureKind::Retry)?; - // } - // } - // StreamChange::ClientStop(stream) => { - // if !streaming.shared.get_config().pause.on_motion || !motion_pause { - // rtsp_thread - // .pause(streaming.shared.get_tag_for_stream(&stream)) - // .await - // .map_err(CameraFailureKind::Retry)?; - // streaming - // .stop_stream(stream) - // .await - // .map_err(CameraFailureKind::Retry)?; - // active_tags.remove(&stream); - // } - // } - // } - // } Ok(()) } -enum StreamChange { - StreamError(Result<()>), - MotionStart, - MotionStop, - ClientStart(StreamKind), - ClientStop(StreamKind), +#[derive(Debug, Clone)] +enum StreamData { + Media { data: Vec, ts: Duration }, + Seek { ts: Duration, reply: MpscSender<()> }, } -// async fn await_change( -// camera: &BcCamera, -// shared: &Shared, -// rtsp_thread: &NeoRtspServer, -// active_tags: &HashSet, -// motion_pause: bool, -// name: &str, -// ) -> Result { -// tokio::select! { -// v = async { -// // Wait for motion stop -// let mut motion = camera.listen_on_motion().await?; -// motion.await_stop(Duration::from_secs_f64(shared.get_config().pause.motion_timeout)).await -// }, if !motion_pause && shared.get_config().pause.on_motion => { -// info!("{}: Motion Pause", name); -// v.map_err(|e| anyhow!("Error while processing motion messages: {:?}", e))?; -// Ok(StreamChange::MotionStop) -// }, -// v = async { -// // Wait for client to disconnect -// let mut inter = tokio::time::interval(tokio::time::Duration::from_secs_f32(0.01)); -// loop { -// inter.tick().await; -// for tag in active_tags.iter() { -// if rtsp_thread.get_number_of_clients(shared.get_tag_for_stream(tag)).await.map(|n| n == 0).unwrap_or(true) { -// return Result::<_,anyhow::Error>::Ok(StreamChange::ClientStop(*tag)) -// } -// } -// } -// }, if shared.get_config().pause.on_disconnect => { -// if let Ok(StreamChange::ClientStop(tag)) = v.as_ref() { -// info!("{}: Client Pause:: {:?}", name, tag); -// } -// v -// }, -// v = async { -// // Wait for motion start -// let mut motion = camera.listen_on_motion().await?; -// motion.await_start(Duration::ZERO).await -// }, if motion_pause && shared.get_config().pause.on_motion => { -// info!("{}: Motion Resume", name); -// v.with_context(|| "Error while processing motion messages")?; -// Ok(StreamChange::MotionStart) -// }, -// v = async { -// // Wait for client to connect -// let mut inter = tokio::time::interval(tokio::time::Duration::from_secs_f32(0.01)); -// let inactive_tags = shared.get_streams().iter().filter(|i| !active_tags.contains(i)).collect::>(); -// trace!("inactive_tags: {:?}", inactive_tags); -// loop { -// inter.tick().await; -// for tag in inactive_tags.iter() { -// if rtsp_thread.get_number_of_clients(shared.get_tag_for_stream(tag)).await.map(|n| n > 0).unwrap_or(false) { -// return Result::<_,anyhow::Error>::Ok(StreamChange::ClientStart(**tag)) -// } -// } -// } -// }, if shared.get_config().pause.on_disconnect => { -// if let Ok(StreamChange::ClientStart(tag)) = v.as_ref() { -// info!("{}: Client Resume:: {:?}", name, tag); -// } -// v -// }, -// }.with_context(|| format!("{}: Error while streaming", name)) -// } +async fn stream_main( + name: String, + mut stream_instance: StreamInstance, + rtsp: &NeoRtspServer, +) -> Result<()> { + loop { + // Wait for a valid stream format to be detected + stream_instance + .config + .wait_for(|config| { + !matches!(config.vid_format, VidFormat::None) + && config.resolution[0] > 0 + && config.resolution[1] > 0 + }) + .await?; + let stream = stream_instance.stream.resubscribe(); + let stream_name = match stream_instance.name { + StreamKind::Main => "main", + StreamKind::Sub => "sub", + StreamKind::Extern => "extern", + } + .to_string(); + let thread_stream_config = stream_instance.config.clone(); + // This select ensures is for the streams config + break tokio::select! { + _ = stream_instance.config.changed() => { + // If stream config changes we reload the stream + continue; + }, + v = async { + // Finally ready to create the factory and connect the stream + let mounts = rtsp + .mount_points() + .ok_or(anyhow!("RTSP server lacks mount point"))?; + let path = format!("/{}/{}", name, stream_name); + log::debug!("Path: {}", path); + let stream_config = thread_stream_config.clone(); + // Create the factory + let (client_tx, mut client_rx) = mpsc(100); + let factory = { + NeoMediaFactory::new_with_callback(move |element| { + clear_bin(&element)?; + let config = stream_config.borrow().clone(); + let (vid_seek_tx, vid_seek_rx) = mpsc(10); + let (aud_seek_tx, aud_seek_rx) = mpsc(10); + let vid = match config.vid_format { + VidFormat::None => { + build_unknown(&element)?; + AnyResult::Ok(None) + }, + VidFormat::H264 => { + let app = build_h264(&element)?; + app.set_callbacks( + AppSrcCallbacks::builder() + .seek_data(move |_, seek_pos| { + log::info!("seek_pos: {seek_pos:?}"); + let (reply_tx, mut reply_rx) = mpsc(1); + vid_seek_tx.blocking_send( + StreamData::Seek{ + ts: Duration::from_micros(seek_pos), + reply: reply_tx, + }).is_ok() && reply_rx.blocking_recv().is_some() + }) + .build(), + ); + AnyResult::Ok(Some(app)) + }, + VidFormat::H265 => { + let app = build_h265(&element)?; + app.set_callbacks( + AppSrcCallbacks::builder() + .seek_data(move |_, seek_pos| { + log::info!("seek_pos: {seek_pos:?}"); + let (reply_tx, mut reply_rx) = mpsc(1); + vid_seek_tx.blocking_send( + StreamData::Seek{ + ts: Duration::from_micros(seek_pos), + reply: reply_tx, + }).is_ok() && reply_rx.blocking_recv().is_some() + }) + .build(), + ); + AnyResult::Ok(Some(app)) + }, + }?; + let aud = if matches!(config.vid_format, VidFormat::None) { + None + } else { + match config.aud_format { + AudFormat::None => { + AnyResult::Ok(None) + }, + AudFormat::Aac => { + let app = build_aac(&element)?; + app.set_callbacks( + AppSrcCallbacks::builder() + .seek_data(move |_, seek_pos| { + log::info!("seek_pos: {seek_pos:?}"); + let (reply_tx, mut reply_rx) = mpsc(1); + aud_seek_tx.blocking_send( + StreamData::Seek{ + ts: Duration::from_micros(seek_pos), + reply: reply_tx, + }).is_ok() && reply_rx.blocking_recv().is_some() + }) + .build(), + ); + AnyResult::Ok(Some(app)) + }, + AudFormat::Adpcm(block_size) => { + let app = build_adpcm(&element, block_size)?; + app.set_callbacks( + AppSrcCallbacks::builder() + .seek_data(move |_, seek_pos| { + log::info!("seek_pos: {seek_pos:?}"); + let (reply_tx, mut reply_rx) = mpsc(1); + aud_seek_tx.blocking_send( + StreamData::Seek{ + ts: Duration::from_micros(seek_pos), + reply: reply_tx, + }).is_ok() && reply_rx.blocking_recv().is_some() + }) + .build(), + ); + AnyResult::Ok(Some(app)) + }, + }? + }; + + client_tx.blocking_send((vid, aud, vid_seek_rx, aud_seek_rx))?; + Ok(Some(element)) + }).await + }?; + + factory.add_permitted_roles(&["anonymous"].into()); + mounts.add_factory(&path, factory); + + // Done now sit on the receiver and push through the vid to the app sources + let stream_cancel = CancellationToken::new(); + let mut set = JoinSet::new(); + while let Some((vid, aud, mut vid_seek, mut aud_seek)) = client_rx.recv().await { + log::trace!("New media"); + // New media created + + // This thread handles splitting the data into two streams + let mut thread_stream = BroadcastStream::new(stream.resubscribe()).filter(|f| f.is_ok()); // Filter to ignore lagged + // Use broadcast because of its overflow protection + let (vid_data_tx, vid_data_rx) = broadcast(100); + let (aud_data_tx, aud_data_rx) = broadcast(100); + let thread_stream_cancel = stream_cancel.clone(); + let thread_vid = vid.clone(); + let thread_aud = aud.clone(); + let thread_vid_data_tx = vid_data_tx.clone(); + let thread_aud_data_tx = aud_data_tx.clone(); + set.spawn(async move { + let r = tokio::select! { + _ = thread_stream_cancel.cancelled() => { + AnyResult::Ok(()) + }, + v = async { + let mut prev_vid_ms: u32 = 0; + // let prev_aud_ms: u32 = 0; + let mut vid_time: u64 = 0; + // let aud_time: u64 = 0; + while let Some(Ok(data)) = thread_stream.next().await { + log::trace!("Media Frame Recieved"); + match data { + BcMedia::Iframe(BcMediaIframe{data, microseconds, ..}) | BcMedia::Pframe(BcMediaPframe{data, microseconds,..}) => { + if let Some(app) = thread_vid.as_ref() { + check_live(app)?; // Stop if appsrc is dropped + let delta = if prev_vid_ms > microseconds { + 0 + } else { + microseconds - prev_vid_ms + }; + prev_vid_ms = microseconds; + vid_time = vid_time.wrapping_add(delta as u64); + thread_vid_data_tx.send( + StreamData::Media{ + data, + ts: Duration::from_micros(vid_time) + })?; + log::trace!("Sent Vid Frame"); + } + } + BcMedia::Aac(BcMediaAac{data, ..}) | BcMedia::Adpcm(BcMediaAdpcm{data,..}) => { + if let Some(app) = thread_aud.as_ref() { + check_live(app)?; // Stop if appsrc is dropped + thread_aud_data_tx.send( + StreamData::Media{ + data, + ts: Duration::from_micros(vid_time) + })?; + log::trace!("Sent Aud Frame"); + } + } + _ => {}, + } + } + AnyResult::Ok(()) + } => v, + }; + log::trace!("Stream Thread End: {:?}", r); + if let Some(app) = thread_vid.as_ref() { + let _ = app.end_of_stream(); + } + if let Some(app) = thread_aud.as_ref() { + let _ = app.end_of_stream(); + } + r + }); + + + // Handles the seek commands + set.spawn(async move { + while let Some(data) = vid_seek.recv().await { + vid_data_tx.send(data)?; + } + log::trace!("Stream Vid Seek End"); + AnyResult::Ok(()) + }); + set.spawn(async move { + while let Some(data) = aud_seek.recv().await { + aud_data_tx.send(data)?; + } + log::trace!("Stream Aud Seek End"); + AnyResult::Ok(()) + }); + + // Handles the sending the video data into gstreamer + let thread_stream_cancel = stream_cancel.clone(); + let vid_data_rx = BroadcastStream::new(vid_data_rx).filter(|f| f.is_ok()); // Filter to ignore lagged + set.spawn(async move { + let r = tokio::select! { + _ = thread_stream_cancel.cancelled() => { + AnyResult::Ok(()) + }, + v = handle_data(vid.as_ref(), vid_data_rx) => v, + }; + log::trace!("Vid Thread End: {:?}", r); + r + }); + + // Handles the audio data + let thread_stream_cancel = stream_cancel.clone(); + let aud_data_rx = BroadcastStream::new(aud_data_rx).filter(|f| f.is_ok()); // Filter to ignore lagged + set.spawn(async move { + let r = tokio::select! { + _ = thread_stream_cancel.cancelled() => { + AnyResult::Ok(()) + }, + v = handle_data(aud.as_ref(), aud_data_rx) => v, + }; + log::trace!("Aud Thread End: {:?}", r); + r + }); + } + // At this point the factory has been destroyed + // Cancel any remaining threads that are trying to send data + // Although it should be finished already when the appsrcs are dropped + stream_cancel.cancel(); + while set.join_next().await.is_some() { + + } + log::trace!("Stream done"); + AnyResult::Ok(()) + } => v, + }; + } +} + +async fn handle_data> + Unpin, E>( + app: Option<&AppSrc>, + mut data_rx: T, +) -> Result<()> { + if let Some(app) = app { + let mut last_ft: Option = None; + let mut ft = Duration::ZERO; + let mut last_rt: Option = None; + let mut rt = Duration::ZERO; + while let Some(Ok(data)) = data_rx.next().await { + check_live(app)?; // Stop if appsrc is dropped + // Cache the last runtime + match data { + StreamData::Seek { ts, reply } => { + rt = ts; + last_rt = None; + let _ = reply.send(()).await; + } + StreamData::Media { data, ts: ft_i } => { + log::info!("Frame recieved with ts: {ft_i:?}"); + // Update rt + if let Some(rt_i) = get_runtime(app) { + if let Some(last_rt) = last_rt { + let delta_rt = rt_i - last_rt; + rt += delta_rt; + } + last_rt = Some(rt_i); + } + // Update ft + if let Some(last_ft) = last_ft { + let delta_ft = ft_i - last_ft; + ft += delta_ft; + } + last_ft = Some(ft_i); + + // Sync ft to rt if > 150ms difference + const MAX_DELTA_T: Duration = Duration::from_millis(150); + let delta_t = if rt > ft { rt - ft } else { ft - rt }; + if delta_t > MAX_DELTA_T { + ft = rt; + } + + // Send frame to app src on another thread + let thread_app = app.clone(); + let thread_rt = rt; + let thread_ft = ft; + tokio::task::spawn(async move { + check_live(&thread_app)?; // Stop if appsrc is dropped + if ft > (rt + Duration::from_millis(50)) { + let delta = ft - rt; + sleep(delta).await; + } + let buf = { + let mut gst_buf = gstreamer::Buffer::with_size(data.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); + log::info!("Setting PTS: {thread_ft:?}, Runtime: {thread_rt:?}"); + let time = ClockTime::from_useconds(thread_ft.as_micros() as u64); + gst_buf_mut.set_dts(time); + gst_buf_mut.set_pts(time); + let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(data.as_slice()); + } + gst_buf + }; + + log::trace!("Pushing Data"); + let appsrc = thread_app.clone(); + tokio::task::spawn_blocking(move || { + appsrc + .push_buffer(buf.copy()) + .map(|_| ()) + .map_err(|_| anyhow!("Could not push buffer to appsrc")) + }) + .await??; + log::trace!(" Pushed Data"); + AnyResult::Ok(()) + }); + } + } + } + } + Ok(()) +} + +fn check_live(app: &AppSrc) -> Result<()> { + app.pads() + .iter() + .all(|pad| pad.is_linked()) + .then_some(()) + .ok_or(anyhow!("App source is closed")) +} + +fn get_runtime(app: &AppSrc) -> Option { + if let Some(clock) = app.clock() { + if let Some(time) = clock.time() { + if let Some(base_time) = app.base_time() { + let runtime = time.saturating_sub(base_time); + return Some(Duration::from_micros(runtime.useconds())); + } + } + } + None +} + +fn clear_bin(bin: &Element) -> Result<()> { + let bin = bin + .clone() + .dynamic_cast::() + .map_err(|_| anyhow!("Media source's element should be a bin"))?; + // Clear the autogenerated ones + log::debug!("Clearing old elements"); + for element in bin.iterate_elements().into_iter().flatten() { + bin.remove(&element)?; + } + + Ok(()) +} +fn build_unknown(bin: &Element) -> Result<()> { + let bin = bin + .clone() + .dynamic_cast::() + .map_err(|_| anyhow!("Media source's element should be a bin"))?; + log::debug!("Building Unknown Pipeline"); + let source = make_element("videotestsrc", "testvidsrc")?; + source.set_property_from_str("pattern", "snow"); + source.set_property("num-buffers", 500i32); // Send buffers then EOS + let queue = make_queue("queue0")?; + + let overlay = make_element("textoverlay", "overlay")?; + overlay.set_property("text", "Stream not Ready"); + overlay.set_property_from_str("valignment", "top"); + overlay.set_property_from_str("halignment", "left"); + overlay.set_property("font-desc", "Sans, 16"); + let encoder = make_element("jpegenc", "encoder")?; + let payload = make_element("rtpjpegpay", "pay0")?; + + bin.add_many(&[&source, &queue, &overlay, &encoder, &payload])?; + source.link_filtered( + &queue, + &Caps::builder("video/x-raw") + .field("format", "YUY2") + .field("width", 896i32) + .field("height", 512i32) + .field("framerate", gstreamer::Fraction::new(25, 1)) + .build(), + )?; + Element::link_many(&[&queue, &overlay, &encoder, &payload])?; + + Ok(()) +} + +fn build_h264(bin: &Element) -> Result { + let bin = bin + .clone() + .dynamic_cast::() + .map_err(|_| anyhow!("Media source's element should be a bin"))?; + log::debug!("Building H264 Pipeline"); + let source = make_element("appsrc", "vidsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(50000000u64); // 50MB + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Seekable); + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; + let queue = make_queue("source_queue")?; + let parser = make_element("h264parse", "parser")?; + let payload = make_element("rtph264pay", "pay0")?; + bin.add_many(&[&source, &queue, &parser, &payload])?; + Element::link_many(&[&source, &queue, &parser, &payload])?; + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + Ok(source) +} + +fn build_h265(bin: &Element) -> Result { + let bin = bin + .clone() + .dynamic_cast::() + .map_err(|_| anyhow!("Media source's element should be a bin"))?; + log::debug!("Building H265 Pipeline"); + let source = make_element("appsrc", "vidsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Seekable); + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; + let queue = make_queue("source_queue")?; + let parser = make_element("h265parse", "parser")?; + let payload = make_element("rtph265pay", "pay0")?; + bin.add_many(&[&source, &queue, &parser, &payload])?; + Element::link_many(&[&source, &queue, &parser, &payload])?; + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + Ok(source) +} + +fn build_aac(bin: &Element) -> Result { + let bin = bin + .clone() + .dynamic_cast::() + .map_err(|_| anyhow!("Media source's element should be a bin"))?; + log::debug!("Building Aac pipeline"); + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Seekable); + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; + + let queue = make_queue("audqueue")?; + let parser = make_element("aacparse", "audparser")?; + let decoder = match make_element("faad", "auddecoder_faad") { + Ok(ele) => Ok(ele), + Err(_) => make_element("avdec_aac", "auddecoder_avdec_aac"), + }?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; + + bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + Ok(source) +} + +fn build_adpcm(bin: &Element, block_size: u32) -> Result { + let bin = bin + .clone() + .dynamic_cast::() + .map_err(|_| anyhow!("Media source's element should be a bin"))?; + log::debug!("Building Adpcm pipeline"); + // Original command line + // caps=audio/x-adpcm,layout=dvi,block_align={},channels=1,rate=8000 + // ! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 + // ! adpcmdec + // ! audioconvert + // ! rtpL16pay name=pay1 + + let source = make_element("appsrc", "audsrc")? + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast to appsrc."))?; + source.set_is_live(true); + source.set_block(false); + source.set_property("emit-signals", false); + source.set_max_bytes(52428800); + source.set_do_timestamp(false); + source.set_stream_type(AppStreamType::Seekable); + + source.set_caps(Some( + &Caps::builder("audio/x-adpcm") + .field("layout", "div") + .field("block_align", block_size as i32) + .field("channels", 1i32) + .field("rate", 8000i32) + .build(), + )); + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot cast back"))?; + + let queue = make_queue("audqueue")?; + let decoder = make_element("decodebin", "auddecoder")?; + let encoder = make_element("audioconvert", "audencoder")?; + let payload = make_element("rtpL16pay", "pay1")?; + + bin.add_many(&[&source, &queue, &decoder, &encoder, &payload])?; + Element::link_many(&[&source, &queue, &decoder])?; + Element::link_many(&[&encoder, &payload])?; + decoder.connect_pad_added(move |_element, pad| { + debug!("Linking encoder to decoder: {:?}", pad.caps()); + let sink_pad = encoder + .static_pad("sink") + .expect("Encoder is missing its pad"); + pad.link(&sink_pad) + .expect("Failed to link ADPCM decoder to encoder"); + }); + + let source = source + .dynamic_cast::() + .map_err(|_| anyhow!("Cannot convert appsrc"))?; + Ok(source) +} + +// Convenice funcion to make an element or provide a message +// about what plugin is missing +fn make_element(kind: &str, name: &str) -> AnyResult { + ElementFactory::make_with_name(kind, Some(name)).with_context(|| { + let plugin = match kind { + "appsrc" => "app (gst-plugins-base)", + "audioconvert" => "audioconvert (gst-plugins-base)", + "adpcmdec" => "Required for audio", + "h264parse" => "videoparsersbad (gst-plugins-bad)", + "h265parse" => "videoparsersbad (gst-plugins-bad)", + "rtph264pay" => "rtp (gst-plugins-good)", + "rtph265pay" => "rtp (gst-plugins-good)", + "aacparse" => "audioparsers (gst-plugins-good)", + "rtpL16pay" => "rtp (gst-plugins-good)", + "x264enc" => "x264 (gst-plugins-ugly)", + "x265enc" => "x265 (gst-plugins-bad)", + "avdec_h264" => "libav (gst-libav)", + "avdec_h265" => "libav (gst-libav)", + "videotestsrc" => "videotestsrc (gst-plugins-base)", + "imagefreeze" => "imagefreeze (gst-plugins-good)", + "audiotestsrc" => "audiotestsrc (gst-plugins-base)", + "decodebin" => "playback (gst-plugins-good)", + _ => "Unknown", + }; + format!( + "Missing required gstreamer plugin `{}` for `{}` element", + plugin, kind + ) + }) +} +fn make_queue(name: &str) -> AnyResult { + let queue = make_element("queue", name)?; + queue.set_property_from_str("leaky", "downstream"); + queue.set_property("max-size-bytes", 0u32); + queue.set_property("max-size-buffers", 0u32); + queue.set_property( + "max-size-time", + std::convert::TryInto::::try_into(tokio::time::Duration::from_secs(5).as_nanos()) + .unwrap_or(0), + ); + Ok(queue) +} diff --git a/src/rtsp/spring.rs b/src/rtsp/spring.rs deleted file mode 100644 index 5cf4de8..0000000 --- a/src/rtsp/spring.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! A critically damped spring -use tokio::time::Instant; - -#[derive(Debug)] -pub(crate) struct Spring { - last_update: Instant, - target: f64, - value: f64, - velocity: f64, - halflife: f64, -} - -impl Spring { - pub(crate) fn new(value: f64, target: f64, halflife: f64) -> Self { - Self { - value, - target, - halflife, - last_update: Instant::now(), - velocity: 0.0f64, - } - } - - pub(crate) async fn update(&mut self) { - let now = Instant::now(); - let dt = now - self.last_update; - self.last_update = now; - spring_update( - &mut self.value, - self.target, - &mut self.velocity, - self.halflife, - dt.as_secs_f64(), - ); - } - - // pub(crate) fn value(&self) -> f64 { - // self.value - // } - - // pub(crate) fn value_u32(&self) -> u32 { - // self.value.max(std::u32::MIN as f64).floor().min(std::u32::MAX as f64) as u32 - // } - pub(crate) fn value_i64(&self) -> i64 { - self.value - .max(std::i64::MIN as f64) - .floor() - .min(std::i64::MAX as f64) as i64 - } - pub(crate) fn value_u64(&self) -> u64 { - self.value - .max(std::u64::MIN as f64) - .floor() - .min(std::u64::MAX as f64) as u64 - } - - #[allow(dead_code)] - pub(crate) fn set_target(&mut self, target: f64) { - self.target = target; - } - - #[allow(dead_code)] // Used in debug print - pub(crate) fn get_target(&self) -> f64 { - self.target - } - - #[allow(dead_code)] - pub(crate) fn mod_value(&mut self, delta: f64) { - self.target += delta; - self.value += delta; - } - - pub(crate) fn reset_to(&mut self, target: f64) { - self.target = target; - self.value = target; - self.velocity = 0.0; - } -} - -const EPS: f64 = 1e-5; -fn halflife_to_damping(halflife: f64) -> f64 { - (4.0 * std::f64::consts::LN_2) / (halflife + EPS) -} - -// fn damping_to_halflife(damping: f32) -> f32 { -// (4.0f32 * std::f32::consts::LN_2) / (damping + EPS) -// } -fn fast_negexp(x: f64) -> f64 { - 1.0 / (1.0 + x + 0.48 * x * x + 0.235 * x * x * x) -} - -fn spring_update(value: &mut f64, target: f64, velocity: &mut f64, halflife: f64, dt: f64) { - if halflife.abs() < 1e-3 { - *value = target; - *velocity = 0.0; - } else { - let y = halflife_to_damping(halflife) / 2.0; - let j0 = *value - target; - let j1 = *velocity + j0 * y; - let eydt = fast_negexp(y * dt); - - *value = eydt * (j0 + j1 * dt) + target; - *velocity = eydt * (*velocity - j1 * y * dt); - } -}