Start Neoreactor

This is centralised reconnect managment that allows async retries over reconnects of the camera
This commit is contained in:
Andrew committed 2023-08-25 09:54:20 +07:00
1 parent 0768bcf959
commit b200288b38
16 files changed
+529 -50

No files matched your search

Generated
+1
View File
@@ -1452,6 +1452,7 @@ dependencies = [
"time",
"tokio",
"tokio-stream",
"tokio-util",
"toml",
"validator",
"validator_derive",
+1
View File
@@ -37,6 +37,7 @@ serde_json = "1.0.96"
time = "0.3.20"
tokio = { version = "1.27.0", features = ["rt-multi-thread", "macros", "io-util", "tracing"] }
tokio-stream = "0.1.12"
tokio-util = { version = "0.7.7", features = ["full", "tracing"] }
toml = "0.7.3"
validator = "0.16.0"
validator_derive = "0.16.0"
+9 -7
View File
@@ -8,6 +8,7 @@ use std::{
sync::atomic::{AtomicBool, AtomicU16, Ordering},
};
use tokio::sync::RwLock;
use tokio_util::sync::CancellationToken;
use Md5Trunc::*;
@@ -19,6 +20,7 @@ mod errors;
mod floodlight_status;
mod keepalive;
mod ledstate;
mod link;
mod login;
mod logout;
mod motion;
@@ -65,6 +67,8 @@ pub struct BcCamera {
// Certain commands such as logout require the username/pass in plain text.... why....???
credentials: Credentials,
abilities: RwLock<HashMap<String, ReadKind>>,
#[allow(dead_code)]
cancel: CancellationToken,
}
/// Options used to construct a camera
@@ -345,6 +349,7 @@ impl BcCamera {
logged_in: AtomicBool::new(false),
credentials: Credentials::new(username, passwd),
abilities: Default::default(),
cancel: CancellationToken::new(),
};
me.keepalive().await?;
Ok(me)
@@ -409,13 +414,10 @@ impl BcCamera {
self.connection.join().await
}
/// Disconnect from the camera. This is done by dropping the reciever
/// so further communications are not possible.
pub async fn disconnect(self) -> Result<()> {
let connection = Arc::try_unwrap(self.connection)
.map_err(|_| Error::Other("Connection still active"))?;
connection.disconnect().await?;
/// Disconnect from the camera. This is done by sending cancel to
/// all threads then waiting for the join
pub async fn shutdown(&self) -> Result<()> {
self.connection.shutdown().await?;
Ok(())
}
}
@@ -10,7 +10,7 @@ use std::sync::Arc;
use tokio::sync::mpsc::{channel, Sender};
use tokio::task::yield_now;
use tokio_stream::wrappers::ReceiverStream;
use tokio_util::sync::PollSender;
use tokio_util::sync::{CancellationToken, PollSender};
use tokio::{sync::RwLock, task::JoinSet};
@@ -39,11 +39,13 @@ pub struct BcConnection {
sink: Sender<Result<Bc>>,
poll_commander: Sender<PollCommand>,
rx_thread: RwLock<JoinSet<Result<()>>>,
cancel: CancellationToken,
}
impl BcConnection {
pub async fn new(mut sink: BcConnSink, mut source: BcConnSource) -> Result<BcConnection> {
let (sinker, sinker_rx) = channel::<Result<Bc>>(100);
let cancel = CancellationToken::new();
let (poll_commander, poll_commanded) = channel(200);
let mut poller = Poller {
@@ -54,39 +56,65 @@ impl BcConnection {
let mut rx_thread = JoinSet::<Result<()>>::new();
let thread_poll_commander = poll_commander.clone();
let thread_cancel = cancel.clone();
let handle = tokio::task::spawn_blocking(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let result = runtime.block_on(async move {
let mut sender = PollSender::new(thread_poll_commander);
while let Some(bc) = source.next().await {
sender.send(PollCommand::Bc(Box::new(bc))).await?;
tokio::select! {
_ = thread_cancel.cancelled() => {
Result::Ok(())
},
v = async {
let mut sender = PollSender::new(thread_poll_commander);
while let Some(bc) = source.next().await {
sender.send(PollCommand::Bc(Box::new(bc))).await?;
}
Result::Ok(())
} => v
}
Result::Ok(())
});
log::trace!("PollSender Bc {:?}", result);
result
});
let thread_cancel = cancel.clone();
rx_thread.spawn(async move {
handle.await??;
Ok(())
});
rx_thread.spawn(async move {
let mut stream = ReceiverStream::new(sinker_rx);
while let Some(packet) = stream.next().await {
sink.send(packet?).await?;
tokio::select! {
_ = thread_cancel.cancelled() => Result::Ok(()),
v = async {
handle.await??;
Ok(())
} => v
}
Ok(())
});
let thread_cancel = cancel.clone();
rx_thread.spawn(async move {
loop {
if let n @ Err(_) = poller.run().await {
trace!("Polling has ended");
return n;
}
tokio::select! {
_ = thread_cancel.cancelled() => Result::Ok(()),
v = async {
let mut stream = ReceiverStream::new(sinker_rx);
while let Some(packet) = stream.next().await {
sink.send(packet?).await?;
}
Ok(())
} => v
}
});
let thread_cancel = cancel.clone();
rx_thread.spawn(async move {
tokio::select! {
_ = thread_cancel.cancelled() => Result::Ok(()),
v = async {
loop {
if let n @ Err(_) = poller.run().await {
trace!("Polling has ended");
return n;
}
}
}=> v
}
});
@@ -94,6 +122,7 @@ impl BcConnection {
sink: sinker,
poll_commander,
rx_thread: RwLock::new(rx_thread),
cancel,
})
}
@@ -165,16 +194,21 @@ impl BcConnection {
Ok(())
}
pub async fn disconnect(mut self) -> Result<()> {
self.poll_commander.send(PollCommand::Disconnect).await?;
drop(self.sink);
drop(self.poll_commander);
let locked_threads = self.rx_thread.get_mut();
pub async fn shutdown(&self) -> Result<()> {
let _ = self.poll_commander.send(PollCommand::Disconnect).await;
self.cancel.cancel();
let mut locked_threads = self.rx_thread.write().await;
while locked_threads.join_next().await.is_some() {}
Ok(())
}
}
impl Drop for BcConnection {
fn drop(&mut self) {
self.cancel.cancel();
}
}
enum PollCommand {
Bc(Box<Result<Bc>>),
AddHandler(u32, Arc<MsgHandler>),
+1 -1
View File
@@ -8,7 +8,7 @@ impl BcCamera {
pub async fn get_linktype(&self) -> Result<LinkType> {
let connection = self.get_connection();
let msg_num = self.new_message_num();
let mut sub_get = connection.subscribe(msg_num).await?;
let mut sub_get = connection.subscribe(MSG_ID_PING, msg_num).await?;
let get = Bc {
meta: BcMeta {
msg_id: MSG_ID_PING,
+1 -1
View File
@@ -36,7 +36,7 @@ pub(crate) async fn main(opt: Opt, config: Config) -> Result<()> {
println!("{}", ser);
let _ = camera.logout().await;
camera.disconnect().await?;
camera.shutdown().await?;
Ok(())
}
+1
View File
@@ -0,0 +1 @@
pub(crate) mod neocam;
+411
View File
@@ -0,0 +1,411 @@
//! This is the common code for creating a camera instance
//!
//! Features:
//! Shared stream BC delivery
//! Common restart code
//! Clonable interface to share amongst threads
use crate::{config::CameraConfig, utils::connect_and_login, Result};
use neolink_core::bc_protocol::BcCamera;
use anyhow::anyhow;
use futures::stream::StreamExt;
use std::collections::{hash_map::Entry, HashMap};
use std::sync::{Arc, Weak};
use tokio::sync::{
mpsc::{channel as mpsc, Sender as MpscSender, WeakSender as MpscWeakSender},
oneshot::{channel as oneshot, Sender as OneshotSender},
watch::{channel as watch, Receiver as WatchReceiver, Sender as WatchSender},
};
use tokio::time::{interval, sleep, Duration, Instant};
use tokio_stream::wrappers::ReceiverStream;
use tokio_util::sync::CancellationToken;
enum NeoCamCommand {
HangUp,
Instance(OneshotSender<Result<NeoInstance>>),
}
/// The underlying camera binding
struct NeoCam {
config: WatchReceiver<CameraConfig>,
cancel: CancellationToken,
camera_watch: WatchSender<Weak<BcCamera>>,
commander: ReceiverStream<NeoCamCommand>,
sender: MpscWeakSender<NeoCamCommand>,
}
impl NeoCam {
async fn init(config: CameraConfig) -> Result<(WatchSender<CameraConfig>, NeoInstance)> {
let (watch_config_tx, watch_config_rx) = watch(config);
let (camera_watch_tx, _) = watch(Weak::new());
let (commander_tx, commander_rx) = mpsc(100);
let mut me = Self {
config: watch_config_rx,
cancel: CancellationToken::new(),
camera_watch: camera_watch_tx,
commander: ReceiverStream::new(commander_rx),
sender: commander_tx.downgrade(),
};
tokio::task::spawn(async move { me.run().await });
let (instance_tx, instance_rx) = oneshot();
commander_tx
.send(NeoCamCommand::Instance(instance_tx))
.await?;
Ok((watch_config_tx, instance_rx.await??))
}
async fn run_camera(&mut self, config: &CameraConfig) -> Result<()> {
let camera = Arc::new(connect_and_login(config).await?);
self.camera_watch.send_replace(Arc::downgrade(&camera));
let cancel = self.cancel.clone();
let cancel_check = self.cancel.clone();
// Now we wait for a disconnect
tokio::select! {
_ = cancel_check.cancelled() => {
Ok(())
}
v = camera.join() => v,
v = async {
let mut interval = interval(Duration::from_secs(5));
loop {
interval.tick().await;
match camera.get_linktype().await {
Ok(_) => continue,
Err(neolink_core::Error::UnintelligibleReply { .. }) => {
// Camera does not support pings just wait forever
futures::pending!();
},
Err(e) => return Err(e),
}
}
} => v,
v = async {
while let Some(command) = self.commander.next().await {
match command {
NeoCamCommand::HangUp => {
cancel.cancel();
return Ok(());
}
NeoCamCommand::Instance(result) => {
let instance = NeoInstance::new(self);
let _ = result.send(instance);
}
}
}
Ok(())
} => v
}?;
let _ = camera.logout().await;
let _ = camera.shutdown().await;
Ok(())
}
// Will run and attempt to maintain the connection
// while also delivering messages
async fn run(&mut self) -> Result<()> {
const MAX_BACKOFF: Duration = Duration::from_secs(5);
const MIN_BACKOFF: Duration = Duration::from_millis(50);
let mut backoff = MIN_BACKOFF;
loop {
let mut config_rec = self.config.clone();
let config = config_rec.borrow_and_update().clone();
let now = Instant::now();
let res = tokio::select! {
Ok(_) = config_rec.changed() => {
None
}
v = self.run_camera(&config) => {
Some(v)
}
};
self.camera_watch.send_replace(Weak::new());
if res.is_none() {
// If None go back and reload NOW
//
// This occurs if there was a config change
continue;
}
// Else we see if the result actually was
let result = res.unwrap();
if Instant::now() - now > Duration::from_secs(60) {
// Command ran long enough to be considered a success
backoff = MIN_BACKOFF;
}
if backoff > MAX_BACKOFF {
backoff = MAX_BACKOFF;
}
match result {
Ok(()) => {
// Normal shutdown
self.cancel.cancel();
return Ok(());
}
Err(e) => {
// An error
// Check if it is non-retry
let e_inner = e.downcast_ref::<neolink_core::Error>();
match e_inner {
Some(neolink_core::Error::CameraLoginFail) => {
// Fatal
log::error!("Login credentials were not accepted");
self.cancel.cancel();
return Err(e);
}
_ => {
// Non fatal
log::warn!("Connection Lost: {:?}", e);
log::info!("Attempt reconnect in {:?}", backoff);
sleep(backoff).await;
backoff *= 2;
}
}
}
}
}
}
}
impl Drop for NeoCam {
fn drop(&mut self) {
self.cancel.cancel();
}
}
/// The sharable instance
///
/// This communicaes with the [`NeoCam`] over channels
///
/// The camera watch is used as an event to be triggered
/// whenever the camera is lost/updated
pub(crate) struct NeoInstance {
camera_watch: tokio::sync::watch::Receiver<Weak<BcCamera>>,
camera_control: MpscSender<NeoCamCommand>,
cancel: CancellationToken,
}
impl NeoInstance {
fn new(cam: &NeoCam) -> Result<Self> {
Ok(Self {
camera_watch: cam.camera_watch.subscribe(),
camera_control: cam
.sender
.upgrade()
.ok_or_else(|| anyhow!("Camera is shutting down"))?,
cancel: cam.cancel.clone(),
})
}
/// Create a new instance to the same camera
pub(crate) async fn subscribe(&self) -> Result<Self> {
let (instance_tx, instance_rx) = oneshot();
self.camera_control
.send(NeoCamCommand::Instance(instance_tx))
.await?;
instance_rx.await?
}
pub(crate) async fn shutdown(&self) {
let _ = self.camera_control.send(NeoCamCommand::HangUp).await;
self.cancel.cancelled().await
}
/// This is a helpful convience function
///
/// Given an async task it will:
/// - Run the task with a reference to a BcCamera
/// - If the camera instance is changed: Rerun the task with the new instance
/// - If the camera returns a retryable error, wait for camera instance to change then rerun
/// - else return the result of the function
pub(crate) async fn run_task<F, T>(&self, task: F) -> Result<T>
where
F: for<'a> Fn(
&'a BcCamera,
) -> std::pin::Pin<Box<dyn futures::Future<Output = Result<T>> + 'a>>,
{
let mut camera_watch = self.camera_watch.clone();
let mut camera = camera_watch.borrow_and_update().upgrade();
loop {
let res = tokio::select! {
_ = self.cancel.cancelled() => {
Some(Err(anyhow!("Camera is disconnecting")))
}
v = camera_watch.changed() => {
// Camera value has changed!
// update and try again
if v.is_ok() {
camera = camera_watch.borrow_and_update().upgrade();
None
} else {
Some(Err(anyhow!("Camera is disconnecting")))
}
},
Some(v) = async {
if let Some(cam) = camera.clone() {
let cam_ref = cam.as_ref();
Some(task(cam_ref).await)
} else {
None
}
}, if camera.is_some() => {
match v {
// Ok means we are done
Ok(v) => Some(Ok(v)),
// If error we check for retryable errors
Err(e) => {
match e.downcast::<neolink_core::Error>() {
// Retry is a None
Ok(neolink_core::Error::DroppedConnection) => {
camera = None;
None
},
Ok(e) => Some(Err(e.into())),
Err(e) => Some(Err(e)),
}
}
}
},
};
if let Some(res) = res {
return res;
}
}
}
}
#[allow(clippy::large_enum_variant)]
enum NeoReactorCommand {
HangUp,
Get(String, OneshotSender<Result<NeoInstance>>),
GetOrInsert(String, OneshotSender<Result<NeoInstance>>, CameraConfig),
}
struct NeoReactorData {
instance: NeoInstance,
config_sender: WatchSender<CameraConfig>,
}
/// Reactor handles the collection of cameras
#[derive(Clone)]
pub(crate) struct NeoReactor {
cancel: CancellationToken,
commander: MpscSender<NeoReactorCommand>,
}
impl NeoReactor {
pub(crate) async fn new() -> Self {
let (commad_tx, mut command_rx) = mpsc(100);
let me = Self {
cancel: CancellationToken::new(),
commander: commad_tx,
};
let cancel = me.cancel.clone();
let cancel2 = me.cancel.clone();
tokio::task::spawn(async move {
let mut instances: HashMap<String, NeoReactorData> = Default::default();
tokio::select! {
_ = cancel.cancelled() => {
for instance in instances.values() {
instance.instance.shutdown().await;
}
Ok(())
},
v = async {
while let Some(command) = command_rx.recv().await {
match command {
NeoReactorCommand::HangUp => {
for instance in instances.values() {
instance.instance.shutdown().await;
}
cancel2.cancel();
return Result::<(), anyhow::Error>::Ok(());
}
NeoReactorCommand::Get(name, sender) => {
let new = instances
.get(&name)
.ok_or_else(|| anyhow!("Camera not found"))
.map(|data| data.instance.subscribe())?
.await;
let _ = sender.send(new);
}
NeoReactorCommand::GetOrInsert(name, sender, config) => {
let new = match instances.entry(name) {
Entry::Occupied(occ) => occ.get().instance.subscribe().await,
Entry::Vacant(vac) => {
let (config_sender, instance) = NeoCam::init(config).await?;
vac.insert(NeoReactorData {
instance,
config_sender,
})
.instance
.subscribe()
.await
}
};
let _ = sender.send(new);
}
}
}
Ok(())
} => v,
}
});
me
}
#[allow(dead_code)]
pub(crate) async fn get(&self, name: &str) -> Result<NeoInstance> {
let (sender_tx, sender_rx) = oneshot();
self.commander
.send(NeoReactorCommand::Get(name.to_string(), sender_tx))
.await?;
sender_rx.await?
}
pub(crate) async fn get_or_insert(
&self,
name: &str,
config: CameraConfig,
) -> Result<NeoInstance> {
let (sender_tx, sender_rx) = oneshot();
self.commander
.send(NeoReactorCommand::GetOrInsert(
name.to_string(),
sender_tx,
config,
))
.await?;
sender_rx.await?
}
pub(crate) async fn shutdown(&self) {
let _ = self.commander.send(NeoReactorCommand::HangUp).await;
self.cancel.cancelled().await;
}
}
impl Drop for NeoReactor {
fn drop(&mut self) {
self.cancel.cancel();
}
}
+13
View File
@@ -48,6 +48,19 @@ pub(crate) struct Config {
pub(crate) users: Vec<UserConfig>,
}
impl Config {
pub(crate) fn get_camera_config(&self, name: &str) -> Result<&CameraConfig, anyhow::Error> {
self.cameras
.iter()
.filter(|c| c.enabled)
.find(|cam| cam.name == name)
.ok_or(anyhow::anyhow!(
"Camera with name `{}` not found in the config",
name
))
}
}
#[derive(Debug, Deserialize, Validate, Clone)]
#[validate(schema(function = "validate_camera_config"))]
pub(crate) struct CameraConfig {
+1 -1
View File
@@ -94,7 +94,7 @@ pub(crate) async fn main(opt: Opt, config: Config) -> Result<()> {
}
let _ = camera.logout().await;
camera.disconnect().await?;
camera.shutdown().await?;
Ok(())
}
+7 -1
View File
@@ -31,6 +31,7 @@ use validator::Validate;
mod battery;
mod cmdline;
mod common;
mod config;
mod image;
mod mqtt;
@@ -43,6 +44,7 @@ mod talk;
mod utils;
use cmdline::{Command, Opt};
use common::neocam::NeoReactor;
use config::Config;
use console_subscriber as _;
@@ -84,6 +86,8 @@ async fn main() -> Result<()> {
tokio_console_enable();
}
let neo_reactor = NeoReactor::new().await;
match opt.cmd {
None => {
warn!(
@@ -102,7 +106,7 @@ async fn main() -> Result<()> {
reboot::main(opts, config).await?;
}
Some(Command::Pir(opts)) => {
pir::main(opts, config).await?;
pir::main(opts, config, neo_reactor.clone()).await?;
}
Some(Command::Ptz(opts)) => {
ptz::main(opts, config).await?;
@@ -121,5 +125,7 @@ async fn main() -> Result<()> {
}
}
neo_reactor.shutdown().await;
Ok(())
}
+21 -11
View File
@@ -18,25 +18,36 @@ use anyhow::{Context, Result};
mod cmdline;
use super::config::Config;
use crate::utils::find_and_connect;
use crate::common::neocam::NeoReactor;
pub(crate) use cmdline::Opt;
/// Entry point for the pir subcommand
///
/// Opt is the command line options
pub(crate) async fn main(opt: Opt, config: Config) -> Result<()> {
let camera = find_and_connect(&config, &opt.camera).await?;
pub(crate) async fn main(opt: Opt, config: Config, reactor: NeoReactor) -> Result<()> {
let config = config.get_camera_config(&opt.camera)?;
let camera = reactor.get_or_insert(&opt.camera, config.clone()).await?;
if let Some(on) = opt.on {
camera
.pir_set(on)
.await
.context("Unable to set camera PIR state")?;
.run_task(|cam| {
Box::pin(async move {
cam.pir_set(on)
.await
.context("Unable to set camera PIR state")
})
})
.await?;
} else {
let pir_state = camera
.get_pirstate()
.await
.context("Unable to get camera PIR state")?;
.run_task(|cam| {
Box::pin(async move {
cam.get_pirstate()
.await
.context("Unable to get camera PIR state")
})
})
.await?;
let pir_ser = String::from_utf8(
yaserde::ser::serialize_with_writer(&pir_state, vec![], &Default::default())
.expect("Should Ser the struct"),
@@ -45,8 +56,7 @@ pub(crate) async fn main(opt: Opt, config: Config) -> Result<()> {
println!("{}", pir_ser);
}
let _ = camera.logout().await;
camera.disconnect().await?;
camera.shutdown().await;
Ok(())
}
+1 -1
View File
@@ -92,7 +92,7 @@ pub(crate) async fn main(opt: Opt, config: Config) -> Result<()> {
};
let _ = camera.logout().await;
camera.disconnect().await?;
camera.shutdown().await?;
Ok(())
}
+1 -1
View File
@@ -30,7 +30,7 @@ pub(crate) async fn main(opt: Opt, config: Config) -> Result<()> {
.await
.context("Could not send reboot command to the camera")?;
camera.disconnect().await?;
camera.shutdown().await?;
Ok(())
}
+1 -1
View File
@@ -35,7 +35,7 @@ pub(crate) async fn main(opt: Opt, config: Config) -> Result<()> {
.context("Unable to set camera light state")?;
let _ = camera.logout().await;
camera.disconnect().await?;
camera.shutdown().await?;
Ok(())
}
+1 -1
View File
@@ -87,7 +87,7 @@ pub(crate) async fn main(opt: Opt, config: Config) -> Result<()> {
.context("Talk stream ended early")?;
let _ = camera.logout().await;
camera.disconnect().await?;
camera.shutdown().await?;
Ok(())
}