Experimental skip frames

This commit is contained in:
QuantumEntangledAndy committed 2023-10-06 15:02:46 +07:00
1 parent 6daf1f5b3d
commit beb9fed919
12 files changed
+283 -70

No files matched your search

Generated
+72 -18
View File
@@ -118,6 +118,28 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9"
[[package]]
name = "async-stream"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
dependencies = [
"async-stream-impl",
"futures-core",
"pin-project-lite",
]
[[package]]
name = "async-stream-impl"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.28",
]
[[package]]
name = "async-trait"
version = "0.1.72"
@@ -371,29 +393,30 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "console-api"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e"
checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787"
dependencies = [
"prost",
"prost-types",
"futures-core",
"prost 0.12.1",
"prost-types 0.12.1",
"tonic",
"tracing-core",
]
[[package]]
name = "console-subscriber"
version = "0.1.10"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb"
checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e"
dependencies = [
"console-api",
"crossbeam-channel",
"crossbeam-utils",
"futures",
"futures-task",
"hdrhistogram",
"humantime",
"prost-types",
"prost-types 0.12.1",
"serde",
"serde_json",
"thread_local",
@@ -688,7 +711,7 @@ dependencies = [
"base64 0.21.2",
"ece",
"log",
"prost",
"prost 0.11.9",
"prost-build",
"reqwest",
"serde",
@@ -2007,7 +2030,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
dependencies = [
"bytes",
"prost-derive",
"prost-derive 0.11.9",
]
[[package]]
name = "prost"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d"
dependencies = [
"bytes",
"prost-derive 0.12.1",
]
[[package]]
@@ -2024,8 +2057,8 @@ dependencies = [
"multimap",
"petgraph",
"prettyplease",
"prost",
"prost-types",
"prost 0.11.9",
"prost-types 0.11.9",
"regex",
"syn 1.0.109",
"tempfile",
@@ -2045,13 +2078,35 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "prost-derive"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32"
dependencies = [
"anyhow",
"itertools 0.11.0",
"proc-macro2",
"quote",
"syn 2.0.28",
]
[[package]]
name = "prost-types"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
dependencies = [
"prost",
"prost 0.11.9",
]
[[package]]
name = "prost-types"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf"
dependencies = [
"prost 0.12.1",
]
[[package]]
@@ -2970,16 +3025,15 @@ dependencies = [
[[package]]
name = "tonic"
version = "0.9.2"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e"
dependencies = [
"async-stream",
"async-trait",
"axum",
"base64 0.21.2",
"bytes",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
@@ -2987,7 +3041,7 @@ dependencies = [
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost",
"prost 0.12.1",
"tokio",
"tokio-stream",
"tower",
+1 -1
View File
@@ -16,7 +16,7 @@ anyhow = "1.0.70"
base64 = "0.21.2"
byte-slice-cast = "1.2.2"
clap = { version = "4.2.2", features = ["derive", "cargo"] }
console-subscriber = "0.1.8"
console-subscriber = "0.2.0"
crossbeam-channel = "0.5.8"
dirs = "5.0.1"
env_logger = "0.10.0"
+2
View File
@@ -41,6 +41,8 @@ pub const MSG_ID_GET_GENERAL: u32 = 104;
pub const MSG_ID_SET_GENERAL: u32 = 105;
/// Snapshot to get a jpeg image
pub const MSG_ID_SNAP: u32 = 109;
/// Used to grab the UID
pub const MSG_ID_UID: u32 = 114;
/// Used to pass the token and client ID for push notifications
pub const MSG_ID_PUSH_INFO: u32 = 124;
/// StreamInfoList messages have this ID
+14 -1
View File
@@ -103,9 +103,12 @@ pub struct BcXml {
/// Recieved AND send for the snap message
#[yaserde(rename = "Snap")]
pub snap: Option<Snap>,
/// Thre list of streams and their configuration
/// The list of streams and their configuration
#[yaserde(rename = "StreamInfoList")]
pub stream_info_list: Option<StreamInfoList>,
/// Thre list of streams and their configuration
#[yaserde(rename = "Uid")]
pub uid: Option<Uid>,
}
impl BcXml {
@@ -826,6 +829,16 @@ pub struct StreamResolution {
pub height: u32,
}
/// Uid xml
#[derive(PartialEq, Eq, Default, Debug, YaDeserialize, YaSerialize)]
pub struct Uid {
/// XML Version
#[yaserde(attribute)]
pub version: String,
/// This the UID of the camera
pub uid: String,
}
/// Convience function to return the xml version used throughout the library
pub fn xml_ver() -> String {
"1.1".to_string()
+1
View File
@@ -35,6 +35,7 @@ mod stream;
mod stream_info;
mod talk;
mod time;
mod uid;
mod version;
pub(crate) use connection::*;
+52
View File
@@ -0,0 +1,52 @@
use super::{BcCamera, Error, Result};
use crate::bc::{model::*, xml::*};
impl BcCamera {
/// Get the [Uid] xml which contains the uid of the camera
pub async fn get_uid(&self) -> Result<Uid> {
let connection = self.get_connection();
let msg_num = self.new_message_num();
let mut sub_get = connection.subscribe(MSG_ID_UID, msg_num).await?;
let get = Bc {
meta: BcMeta {
msg_id: MSG_ID_UID,
channel_id: self.channel_id,
msg_num,
response_code: 0,
stream_type: 0,
class: 0x6414,
},
body: BcBody::ModernMsg(ModernMsg {
extension: None,
payload: None,
}),
};
sub_get.send(get).await?;
let msg = sub_get.recv().await?;
if msg.meta.response_code != 200 {
return Err(Error::CameraServiceUnavaliable);
}
if let BcBody::ModernMsg(ModernMsg {
payload:
Some(BcPayloads::BcXml(BcXml {
uid: Some(uid_xml), ..
})),
..
}) = msg.body
{
Ok(uid_xml)
} else {
Err(Error::UnintelligibleReply {
reply: std::sync::Arc::new(Box::new(msg)),
why: "Expected Uid xml but it was not recieved",
})
}
}
/// Get the UID
pub async fn uid(&self) -> Result<String> {
Ok(self.get_uid().await?.uid)
}
}
+30 -3
View File
@@ -8,12 +8,13 @@ use anyhow::anyhow;
use futures::TryFutureExt;
use std::sync::{Arc, Weak};
use tokio::sync::{
mpsc::Sender as MpscSender, oneshot::channel as oneshot, watch::Receiver as WatchReceiver,
mpsc::Sender as MpscSender, oneshot::channel as oneshot, watch::channel as watch,
watch::Receiver as WatchReceiver,
};
use tokio_util::sync::CancellationToken;
use super::{MdState, NeoCamCommand, NeoCamThreadState, Permit, PushNoti, StreamInstance};
use crate::{config::CameraConfig, Result};
use crate::{config::CameraConfig, AnyResult, Result};
use neolink_core::bc_protocol::{BcCamera, StreamKind};
/// This instance is the primary interface used throughout the app
@@ -221,11 +222,37 @@ impl NeoInstance {
}
pub(crate) async fn push_notifications(&self) -> Result<WatchReceiver<Option<PushNoti>>> {
let uid = self
.run_task(|cam| Box::pin(async move { Ok(cam.uid().await?) }))
.await?;
let (instance_tx, instance_rx) = oneshot();
self.camera_control
.send(NeoCamCommand::PushNoti(instance_tx))
.await?;
Ok(instance_rx.await?)
let mut source_watch = instance_rx.await?;
let (fwatch_tx, fwatch_rx) = watch(None);
tokio::task::spawn(async move {
loop {
match source_watch
.wait_for(|i| {
i.as_ref()
.is_some_and(|i| i.message.contains(&format!("\"{uid}\"")))
})
.await
{
Ok(pn) => {
let _ = fwatch_tx.send_replace(pn.clone());
}
Err(e) => {
break Err(e);
}
}
}?;
AnyResult::Ok(())
});
Ok(fwatch_rx)
}
pub(crate) async fn motion(&self) -> Result<WatchReceiver<MdState>> {
+13 -10
View File
@@ -21,8 +21,7 @@ use tokio_util::sync::CancellationToken;
use super::{
MdRequest, MdState, NeoCamMdThread, NeoCamStreamThread, NeoCamThread, NeoCamThreadState,
NeoInstance, Permit, PnRequest, PushNoti, PushNotiThread, StreamInstance, StreamRequest,
UseCounter,
NeoInstance, Permit, PnRequest, PushNoti, StreamInstance, StreamRequest, UseCounter,
};
use crate::{config::CameraConfig, AnyResult, Result};
use neolink_core::bc_protocol::{BcCamera, StreamKind};
@@ -53,13 +52,15 @@ pub(crate) struct NeoCam {
}
impl NeoCam {
pub(crate) async fn new(config: CameraConfig) -> Result<NeoCam> {
pub(crate) async fn new(
config: CameraConfig,
pn_request_tx: MpscSender<PnRequest>,
) -> Result<NeoCam> {
let (commander_tx, commander_rx) = mpsc(100);
let (watch_config_tx, watch_config_rx) = watch(config.clone());
let (camera_watch_tx, camera_watch_rx) = watch(Weak::new());
let (stream_request_tx, stream_request_rx) = mpsc(100);
let (md_request_tx, md_request_rx) = mpsc(100);
let (pn_request_tx, mut pn_request_rx) = mpsc(100);
let (state_tx, state_rx) = watch(NeoCamThreadState::Connected);
let set = JoinSet::new();
@@ -83,6 +84,7 @@ impl NeoCam {
let strict = config.strict;
let thread_commander_tx = commander_tx.clone();
let thread_watch_config_rx = watch_config_rx.clone();
let thread_pn_request_tx = pn_request_tx.clone();
me.set.spawn(async move {
let thread_cancel = sender_cancel.clone();
let res = tokio::select! {
@@ -167,7 +169,7 @@ impl NeoCam {
let _ = sender.send(users.create_activated().await?);
}
NeoCamCommand::PushNoti(sender) => {
pn_request_tx.send(
thread_pn_request_tx.send(
PnRequest::Get {
sender,
}
@@ -276,6 +278,7 @@ impl NeoCam {
// Handles push notifications
let pn_root_instance = instance.subscribe().await?;
let pn_cancel = me.cancel.clone();
let thread_pn_request_tx = pn_request_tx.clone();
me.set.spawn(async move {
tokio::select!{
_ = pn_cancel.cancelled() => {
@@ -286,15 +289,15 @@ impl NeoCam {
loop {
// Wait for the green light
config_rx.wait_for(|config| config.push_notifications).await?;
// Activate it
let pn_instance = pn_root_instance.subscribe().await?;
let mut push_notifier = PushNotiThread::new(pn_instance).await?;
let (tx, rx) = oneshot();
thread_pn_request_tx.send(PnRequest::Activate{sender: tx, instance: pn_instance}).await?;
rx.await??;
let pn_permit_instance = pn_root_instance.subscribe().await?;
let r = tokio::select! {
// This thread handles the push notfications
v = push_notifier.run(&mut pn_request_rx) => {
v
},
// Push notification permits
v = async {
let mut prev_noti = None;
+21 -26
View File
@@ -20,7 +20,6 @@ use crate::AnyResult;
pub(crate) struct PushNotiThread {
pn_watcher: Arc<WatchSender<Option<PushNoti>>>,
instance: NeoInstance,
}
// The push notification
@@ -34,15 +33,18 @@ pub(crate) enum PnRequest {
Get {
sender: OneshotSender<WatchReceiver<Option<PushNoti>>>,
},
Activate {
instance: NeoInstance,
sender: OneshotSender<AnyResult<()>>,
},
}
impl PushNotiThread {
pub(crate) async fn new(instance: NeoInstance) -> AnyResult<Self> {
pub(crate) async fn new() -> AnyResult<Self> {
let (pn_watcher, _) = watch(None);
Ok(PushNotiThread {
pn_watcher: Arc::new(pn_watcher),
instance,
})
}
@@ -97,32 +99,10 @@ impl PushNotiThread {
// Send registration.fcm_token to the server to allow it to send push messages to you.
log::debug!("registration.fcm_token: {}", registration.fcm_token);
let md5ed = md5::compute(format!("WHY_REOLINK_{:?}", registration.fcm_token));
let uid = format!("{:X}", md5ed);
let fcm_token = registration.fcm_token.clone();
log::debug!("push notification UID: {}", uid);
if let Err(e) = self
.instance
.run_task(|camera| {
let uid = uid.clone();
let fcm_token = registration.fcm_token.clone();
Box::pin(async move {
camera.send_pushinfo_android(&fcm_token, &uid).await?;
AnyResult::Ok(())
})
})
.await
{
match e.downcast::<neolink_core::Error>() {
Ok(neolink_core::Error::UnintelligibleReply { .. }) => {
log::warn!("Push notitfications not supported for this camera");
futures::future::pending().await
}
_ => {
continue;
} // Retry
};
}
log::debug!("Push notification Listening");
let thread_pn_watcher = self.pn_watcher.clone();
@@ -167,6 +147,21 @@ impl PushNotiThread {
PnRequest::Get{sender} => {
let _ = sender.send(self.pn_watcher.subscribe());
}
PnRequest::Activate{instance, sender} => {
let uid = uid.clone();
let fcm_token = fcm_token.clone();
tokio::task::spawn(async move {
let r = instance.run_task(|camera| {
let fcm_token = fcm_token.clone();
let uid = uid.clone();
Box::pin(async move {
camera.send_pushinfo_android(&fcm_token, &uid).await?;
AnyResult::Ok(())
})
}).await;
let _ = sender.send(r);
});
}
}
}
} => {
+23 -2
View File
@@ -12,11 +12,12 @@ use tokio::{
watch::{channel as watch, Receiver as WatchReceiver},
},
task::JoinSet,
time::{sleep, Duration},
};
use tokio_util::sync::CancellationToken;
use super::{NeoCam, NeoInstance};
use crate::{config::Config, AnyResult, Result};
use crate::{common::PushNotiThread, config::Config, AnyResult, Result};
#[allow(clippy::large_enum_variant)]
enum NeoReactorCommand {
@@ -37,6 +38,7 @@ pub(crate) struct NeoReactor {
impl NeoReactor {
pub(crate) async fn new(config: Config) -> Self {
let (commad_tx, mut command_rx) = mpsc(100);
let (push_noti, mut pn_rx) = mpsc(10);
let cancel = CancellationToken::new();
let (config_tx, _) = watch(config);
let mut set = JoinSet::new();
@@ -71,7 +73,7 @@ impl NeoReactor {
log::debug!("Inserting new insance");
let current_config: Config = (*config_tx.borrow()).clone();
if let Some(config) = current_config.cameras.iter().find(|cam| cam.name == name).cloned() {
let cam = NeoCam::new(config).await?;
let cam = NeoCam::new(config, push_noti.clone()).await?;
log::debug!("New instance created");
Result::Ok(Some(
vac.insert(
@@ -112,6 +114,25 @@ impl NeoReactor {
r
});
// Push notification client
let cancel1 = cancel.clone();
set.spawn(async move {
tokio::select! {
_ = cancel1.cancelled() => AnyResult::Ok(()),
v = async {
let mut pn = PushNotiThread::new().await?;
loop {
let r = pn.run(&mut pn_rx).await;
if r.is_err() {
log::debug!("Issue with push notifier: {r:?}");
sleep(Duration::from_secs(5)).await;
}
}
} => v,
}
});
Self {
cancel,
commander: commad_tx,
+5
View File
@@ -422,6 +422,8 @@ impl StreamData {
let watchdog_tx = watchdog_tx.clone();
log::debug!("Running Stream Instance Task");
Box::pin(async move {
// use std::io::Write;
// let mut file = std::fs::File::create("reference.h264")?;
let mut recieved_iframe = false;
let mut aud_keyframe = false;
let res = async {
@@ -469,6 +471,7 @@ impl StreamData {
false
}
});
// let _ = file.write(&frame.data);
}
BcMedia::Pframe(frame) => {
stream_config.send_if_modified(|state| {
@@ -483,6 +486,7 @@ impl StreamData {
false
}
});
// let _ = file.write(&frame.data);
},
BcMedia::Aac(_) => {
stream_config.send_if_modified(|state| {
@@ -529,6 +533,7 @@ impl StreamData {
},
BcMedia::Pframe(BcMediaPframe{data, microseconds,..}) if recieved_iframe => {
prev_ts = Duration::from_micros(microseconds as u64);
// log::debug!("data: {data:02X?}");
let d = StampedData{
keyframe: false,
data: Arc::new(data),
+49 -9
View File
@@ -801,17 +801,56 @@ async fn stream_run(
}
// Send new
while let Some(data) = vidstream.next().await {
if let Ok(data) = data {
thread_vid_data_tx.send(
StreamData::Media {
data: data.data,
ts: data.ts
let mut hold_buffer = vec![];
let mut last_hold = None;
loop {
let timeout = sleep(Duration::from_millis(1000/25));
let r = tokio::select!{
v = vidstream.next() => {
if let Some(data) = v {
if let Ok(data) = data {
if !data.keyframe {
hold_buffer.push(data);
} else {
// Send all held pframes
last_hold = Some(data.clone());
for held in hold_buffer.drain(..) {
thread_vid_data_tx.send(
StreamData::Media {
data: held.data,
ts: held.ts
}
)?;
}
// Send the new keyframe
thread_vid_data_tx.send(
StreamData::Media {
data: data.data,
ts: data.ts
}
)?;
}
}
}
)?;
AnyResult::Ok(())
}
_ = timeout, if last_hold.is_some() => {
log::debug!("Send Skip");
let data = last_hold.clone().unwrap();
thread_vid_data_tx.send(
StreamData::Media {
data: data.data,
ts: data.ts
}
)?;
AnyResult::Ok(())
}
};
if r.is_err() {
break r;
}
// Ignore broadcast lag errors
}
}?;
AnyResult::Ok(())
} => v
};
@@ -1356,6 +1395,7 @@ fn build_h264(bin: &Element, stream_config: &StreamConfig) -> Result<AppSrc> {
.map_err(|_| anyhow!("Cannot cast back"))?;
let queue = make_queue("source_queue", buffer_size)?;
let parser = make_element("h264parse", "parser")?;
parser.set_property("config-interval", -1i32); // Send SPS and PPS every Iframe to imporove joining frame late
let payload = make_element("rtph264pay", "pay0")?;
bin.add_many([&source, &queue, &parser, &payload])?;
Element::link_many([&source, &queue, &parser, &payload])?;