Track more error types for connect restarts

This commit is contained in:
QuantumEntangledAndy committed 2023-09-28 12:12:32 +07:00
1 parent 969bcb8956
commit c14cdcccc2
10 files changed
+85 -94

No files matched your search

Generated
-38
View File
@@ -1521,18 +1521,6 @@ version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
[[package]]
name = "local-ip-address"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fefe707432eb6bd4704b3dacfc87aab269d56667ad05dcd6869534e8890e767"
dependencies = [
"libc",
"neli",
"thiserror",
"windows-sys",
]
[[package]]
name = "lock_api"
version = "0.4.10"
@@ -1648,31 +1636,6 @@ dependencies = [
"tempfile",
]
[[package]]
name = "neli"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1100229e06604150b3becd61a4965d5c70f3be1759544ea7274166f4be41ef43"
dependencies = [
"byteorder",
"libc",
"log",
"neli-proc-macros",
]
[[package]]
name = "neli-proc-macros"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c168194d373b1e134786274020dae7fc5513d565ea2ebb9bc9ff17ffb69106d4"
dependencies = [
"either",
"proc-macro2",
"quote",
"serde",
"syn 1.0.109",
]
[[package]]
name = "neolink"
version = "0.6.2"
@@ -1728,7 +1691,6 @@ dependencies = [
"get_if_addrs",
"indoc",
"lazy_static",
"local-ip-address",
"log",
"md5",
"nom",
+1 -1
View File
@@ -62,7 +62,7 @@ fn platform_cfg() {
fn platform_cfg() {
let gstreamer_dir = env::var_os("GSTREAMER_1_0_ROOT_MACOSX")
.and_then(|x| x.into_string().ok())
.unwrap_or_else(|| r#"/Library/Frameworks/GStreamer.framework/Versions/1.0"#.to_string());
.unwrap_or_else(|| r"/Library/Frameworks/GStreamer.framework/Versions/1.0".to_string());
println!(r"cargo:rustc-link-search=native={}/lib", gstreamer_dir);
println!(r"cargo:rustc-link-arg=-Wl,-rpath,{}/lib", gstreamer_dir);
-1
View File
@@ -18,7 +18,6 @@ err-derive = "0.3.1"
futures = "0.3.28"
get_if_addrs = "0.5.3"
lazy_static = "1.4.0"
local-ip-address = "0.5.5"
log = "0.4.17"
md5 = "0.7.0"
nom = { version = "7.1.3", features = ["alloc"] }
@@ -14,7 +14,6 @@ use futures::{
stream::{FuturesUnordered, Stream, StreamExt},
};
use lazy_static::lazy_static;
use local_ip_address::local_ip;
use log::*;
use rand::{seq::SliceRandom, thread_rng, Rng};
use std::collections::{btree_map::Entry, BTreeMap, HashSet};
@@ -461,7 +460,7 @@ impl Discoverer {
lookup: &UidLookupResults,
) -> Result<RegisterResult> {
let tid = generate_tid();
let local_addr = SocketAddr::new(local_ip()?, self.local_addr().port());
let local_addr = SocketAddr::new(get_local_ip()?, self.local_addr().port());
let local_ip = local_addr.ip();
let local_port = local_addr.port();
let local_family = if local_addr.ip().is_ipv4() { 4 } else { 6 };
@@ -1278,6 +1277,14 @@ impl Discovery {
}
}
fn get_local_ip() -> Result<std::net::IpAddr> {
get_if_addrs::get_if_addrs()?
.iter()
.find(|i| !i.is_loopback() && matches!(i.addr, get_if_addrs::IfAddr::V4(_)))
.map(|iface| Ok(iface.ip()))
.unwrap_or_else(|| Err(Error::Other("No Local Ip Address Found")))
}
fn get_broadcasts(ports: &[u16]) -> Result<Vec<SocketAddr>> {
let mut broadcasts = vec![Ipv4Addr::BROADCAST];
for iface in get_if_addrs::get_if_addrs()?.iter() {
+3 -13
View File
@@ -48,11 +48,11 @@ pub enum Error {
DroppedConnection,
/// Raised when a connection is dropped during a tokio mpsc TryRecv event
#[error(display = "Dropped connection")]
#[error(display = "Dropped connection (TryRecv)")]
DroppedConnectionTry(#[error(source)] tokio::sync::mpsc::error::TryRecvError),
/// Raised when a connection is dropped during a TryRecv event
#[error(display = "Dropped connection")]
#[error(display = "Dropped connection (Broadcast TryRecv)")]
BroadcastDroppedConnectionTry(#[error(source)] tokio::sync::broadcast::error::TryRecvError),
/// Raised when a connection is dropped during a TryRecv event
@@ -68,7 +68,7 @@ pub enum Error {
TimeoutError(#[error(source)] tokio::time::error::Error),
/// Raised when connection is dropped because the timeout is reach
#[error(display = "Dropped connection")]
#[error(display = "Dropped connection (Timeout)")]
TimeoutDisconnected,
/// Raised when a camera cannot be connected to ay any of the given addresses
@@ -143,10 +143,6 @@ pub enum Error {
#[error(display = "Cannot perform lookup with this camera against reolink servers")]
NoDev,
/// Raised when cannot determine local IP address
#[error(display = "Local IP address is unknown: {}", _0)]
LocalIpError(#[error(source)] std::sync::Arc<local_ip_address::Error>),
/// Raised when a discovery fails to be accepted by the register
#[error(display = "Register refuses to accept us")]
RegisterError,
@@ -220,12 +216,6 @@ impl From<cookie_factory::GenError> for Error {
}
}
impl From<local_ip_address::Error> for Error {
fn from(k: local_ip_address::Error) -> Self {
Error::LocalIpError(std::sync::Arc::new(k))
}
}
impl From<tokio::task::JoinError> for Error {
fn from(k: tokio::task::JoinError) -> Self {
Error::JoinError(std::sync::Arc::new(k))
+1 -1
View File
@@ -123,7 +123,7 @@ impl NeoInstance {
Err(e) => {
match e.downcast::<neolink_core::Error>() {
// Retry is a None
Ok(neolink_core::Error::DroppedConnection) => {
Ok(neolink_core::Error::DroppedConnection) | Ok(neolink_core::Error::TimeoutDisconnected) => {
camera = None;
None
},
+37 -13
View File
@@ -176,7 +176,10 @@ impl NeoCam {
}
log::debug!("Control thread Senders dropped");
Ok(())
} => v
} => {
log::debug!("Camera Control Thread Ended; {:?}", v);
v
}
};
log::debug!("Control thread terminated");
res
@@ -200,7 +203,11 @@ impl NeoCam {
me.cancel.clone(),
)
.await;
me.set.spawn(async move { cam_thread.run().await });
me.set.spawn(async move {
let v = cam_thread.run().await;
log::debug!("Camera MAIN thread ended; {:?}", v);
v
});
// This thread maintains the streams
let stream_instance = instance.subscribe().await?;
@@ -209,7 +216,10 @@ impl NeoCam {
me.set.spawn(async move {
tokio::select! {
_ = stream_cancel.cancelled() => AnyResult::Ok(()),
v = stream_thread.run() => v,
v = stream_thread.run() => {
log::debug!("Camera Stream thread ended; {:?}", v);
v
},
}
});
@@ -220,7 +230,10 @@ impl NeoCam {
me.set.spawn(async move {
tokio::select! {
_ = md_cancel.cancelled() => AnyResult::Ok(()),
v = md_thread.run() => v,
v = md_thread.run() => {
log::debug!("MD thread ended; {:?}", v);
v
},
}
});
@@ -278,7 +291,9 @@ impl NeoCam {
let pn_permit_instance = pn_root_instance.subscribe().await?;
let r = tokio::select! {
// This thread handles the push notfications
v = push_notifier.run(&mut pn_request_rx) => v,
v = push_notifier.run(&mut pn_request_rx) => {
v
},
// Push notification permits
v = async {
let mut prev_noti = None;
@@ -301,7 +316,10 @@ impl NeoCam {
}
}?;
AnyResult::Ok(())
} => v,
} => {
log::debug!("Push notification thread ended; {:?}", v);
v
},
}
});
@@ -314,20 +332,23 @@ impl NeoCam {
AnyResult::Ok(())
},
v = async {
let mut md = md_permit_instance.motion().await?;
let mut md = md_permit_instance.motion().await.with_context(|| "Unable to acquire motion watcher")?;
loop{
md.wait_for(|md| matches!(md, MdState::Start(_))).await?;
let _permit = Some(md_permit_instance.permit().await?);
md.wait_for(|md| matches!(md, MdState::Stop(_))).await?;
md.wait_for(|md| matches!(md, MdState::Start(_))).await.with_context(|| "MD Watcher lost")?;
let _permit = md_permit_instance.permit().await.with_context(|| "Unuable to acquire motion permit")?;
md.wait_for(|md| matches!(md, MdState::Stop(_))).await.with_context(|| "MD Watcher lost")?;
// Try waiting for 30s
// If in those 30s we get motion then return to
// loop early to reaquire the permit
tokio::select!{
_ = sleep(Duration::from_secs(30)) => {},
_ = md.wait_for(|md| matches!(md, MdState::Start(_))) => {},
v = md.wait_for(|md| matches!(md, MdState::Start(_))) => {v.with_context(|| "MD Watcher lost")?;},
}
}
} => v,
} => {
log::debug!("MD Wake Up thread ended; {:?}", v);
v
},
}
});
@@ -382,7 +403,10 @@ impl NeoCam {
}
}?;
AnyResult::Ok(())
} => v,
} => {
log::debug!("Idle Disconnect thread ended; {:?}", v);
v
},
}
});
+23 -16
View File
@@ -3,6 +3,7 @@
//! as is, which comes fromt the json structure
//!
use anyhow::Context;
use fcm_push_listener::*;
use std::{fs, sync::Arc};
use tokio::{
@@ -72,7 +73,8 @@ impl PushNotiThread {
log::debug!("Registering new push notification token");
match fcm_push_listener::register(sender_id).await {
Ok(registration) => {
let new_token = toml::to_string(&registration)?;
let new_token = toml::to_string(&registration)
.with_context(|| "Unable to serialise fcm token")?;
if let Some(Err(e)) = token_path
.as_ref()
.map(|token_path| fs::write(token_path, &new_token))
@@ -99,7 +101,8 @@ impl PushNotiThread {
let md5ed = md5::compute(format!("WHY_REOLINK_{:?}", registration.fcm_token));
let uid = format!("{:X}", md5ed);
log::debug!("push notification UID: {}", uid);
self.instance
if let Err(e) = self
.instance
.run_task(|camera| {
let uid = uid.clone();
let fcm_token = registration.fcm_token.clone();
@@ -108,7 +111,18 @@ impl PushNotiThread {
AnyResult::Ok(())
})
})
.await?;
.await
{
match e.downcast::<neolink_core::Error>() {
Ok(neolink_core::Error::UnintelligibleReply { .. }) => {
log::warn!("Push notitfications not supported for this camera");
futures::future::pending().await
}
_ => {
continue;
} // Retry
};
}
log::debug!("Push notification Listening");
let thread_pn_watcher = self.pn_watcher.clone();
@@ -134,22 +148,17 @@ impl PushNotiThread {
fs::write(token_path, "")
);
log::debug!("Error on push notification listener: {:?}. Clearing token", e);
AnyResult::Ok(()) // Allow to restart
},
Http(e) if e.is_request() || e.is_connect() || e.is_timeout() => {
log::debug!("Error on push notification listener: {:?}", e);
AnyResult::Ok(()) // Allow to restart
}
_ => {
log::warn!("Error on push notification listener: {:?}", e);
// Err(e.into()) // Propegate error so it breaks
// Wait forever since it will not work
// we just leave the push notificaitons as disabled
futures::future::pending().await
log::debug!("Error on push notification listener: {:?}", e);
// This sort of error can be a network error
// Wait for a little longer
sleep(Duration::from_secs(30)).await;
}
}
} else {
AnyResult::Ok(())
}
} => v,
v = async {
@@ -160,14 +169,12 @@ impl PushNotiThread {
}
}
}
AnyResult::Ok(())
} => {
// These are critical errors
// break the loop and return
log::debug!("Push Notification thread ended {:?}", v);
break v;
break AnyResult::Ok(());
},
}?;
};
}
}
}
+5 -3
View File
@@ -18,7 +18,7 @@ use tokio::{
watch::{channel as watch, Receiver as WatchReceiver, Sender as WatchSender},
},
task::JoinHandle,
time::{timeout, Duration},
time::{sleep, timeout, Duration},
};
use tokio_util::sync::CancellationToken;
@@ -341,7 +341,8 @@ impl StreamData {
.bitrate_table
.get(encode.default_bitrate as usize)
.copied()
.unwrap_or(encode.default_bitrate),
.unwrap_or(encode.default_bitrate)
* 1024,
))
} else {
Ok(([0, 0], 0))
@@ -402,11 +403,12 @@ impl StreamData {
v = async {
watchdog_rx.recv().await; // Wait forever for the first feed
loop {
let check_timeout = timeout(Duration::from_secs(3), watchdog_rx.recv()).await;
let check_timeout = timeout(Duration::from_secs(10), watchdog_rx.recv()).await;
if let Err(_)| Ok(None) = check_timeout {
// Timeout
// Reply with Ok to trigger the restart
log::debug!("Watchdog kicking the stream");
sleep(Duration::from_secs(1)).await;
break AnyResult::Ok(());
}
}
+6 -6
View File
@@ -1189,7 +1189,7 @@ fn build_unknown(bin: &Element) -> Result<()> {
let source = make_element("videotestsrc", "testvidsrc")?;
source.set_property_from_str("pattern", "snow");
source.set_property("num-buffers", 500i32); // Send buffers then EOS
let queue = make_queue("queue0", buffer_size)?;
let queue = make_queue("queue0", 1024 * 1024 * 4)?;
let overlay = make_element("textoverlay", "overlay")?;
overlay.set_property("text", "Stream not Ready");
@@ -1229,7 +1229,7 @@ fn build_h264(bin: &Element, stream_config: &StreamConfig) -> Result<AppSrc> {
source.set_is_live(false);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(buffer_size);
source.set_max_bytes(buffer_size as u64);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
@@ -1262,7 +1262,7 @@ fn build_h265(bin: &Element, stream_config: &StreamConfig) -> Result<AppSrc> {
source.set_is_live(false);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(buffer_size);
source.set_max_bytes(buffer_size as u64);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
@@ -1296,7 +1296,7 @@ fn build_aac(bin: &Element, stream_config: &StreamConfig) -> Result<AppSrc> {
source.set_is_live(false);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(buffer_size);
source.set_max_bytes(buffer_size as u64);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
@@ -1343,7 +1343,7 @@ fn build_adpcm(bin: &Element, block_size: u32, stream_config: &StreamConfig) ->
source.set_is_live(false);
source.set_block(false);
source.set_property("emit-signals", false);
source.set_max_bytes(buffer_size);
source.set_max_bytes(buffer_size as u64);
source.set_do_timestamp(false);
source.set_stream_type(AppStreamType::Seekable);
@@ -1416,7 +1416,7 @@ fn make_element(kind: &str, name: &str) -> AnyResult<Element> {
}
fn make_queue(name: &str, buffer_size: u32) -> AnyResult<Element> {
let queue = make_element("queue", &format!("queue1_{}", name))?;
queue.set_property("max-size-bytes", buffer - size);
queue.set_property("max-size-bytes", buffer_size);
// queue.set_property("max-size-buffers", 0u32);
// queue.set_property(
// "max-size-time",