Rework error retries in the tasks

This commit is contained in:
QuantumEntangledAndy committed 2023-10-10 11:28:43 +07:00
1 parent 6e8c744025
commit 6058d95b3a
2 files changed
+47 -47

No files matched your search

+19 -13
View File
@@ -1,11 +1,12 @@
use anyhow::Context;
use std::sync::{Arc, Weak};
use tokio::{
sync::watch::{Receiver as WatchReceiver, Sender as WatchSender},
time::{interval, sleep, Duration, Instant},
time::{interval, sleep, timeout, Duration, Instant},
};
use tokio_util::sync::CancellationToken;
use crate::{config::CameraConfig, utils::connect_and_login, Result};
use crate::{config::CameraConfig, utils::connect_and_login, AnyResult};
use neolink_core::bc_protocol::BcCamera;
#[derive(Eq, PartialEq, Copy, Clone)]
@@ -35,7 +36,7 @@ impl NeoCamThread {
camera_watch: camera_watch_tx,
}
}
async fn run_camera(&mut self, config: &CameraConfig) -> Result<()> {
async fn run_camera(&mut self, config: &CameraConfig) -> AnyResult<()> {
let camera = Arc::new(connect_and_login(config).await?);
update_camera_time(&camera, &config.name, config.update_time).await?;
@@ -47,23 +48,28 @@ impl NeoCamThread {
tokio::select! {
_ = cancel_check.cancelled() => {
log::debug!("Camera Cancelled");
Ok(())
AnyResult::Ok(())
}
v = camera.join() => {
log::debug!("Camera Join: {:?}", v);
v
v?;
Ok(())
},
v = async {
let mut interval = interval(Duration::from_secs(5));
loop {
interval.tick().await;
match camera.get_linktype().await {
Ok(_) => continue,
Err(neolink_core::Error::UnintelligibleReply { .. }) => {
match timeout(Duration::from_secs(15), camera.get_linktype()).await {
Ok(Ok(_)) => continue,
Ok(Err(neolink_core::Error::UnintelligibleReply { .. })) => {
// Camera does not support pings just wait forever
futures::future::pending().await
},
Err(e) => return Err(e),
Ok(Err(e)) => return Err(e.into()),
Err(e) => {
// Timeout
return AnyResult::Err(e.into()).with_context(|| "Timed out waiting for camera ping reply");
}
}
}
} => v,
@@ -79,7 +85,7 @@ impl NeoCamThread {
//
// A watch sender is used to send the new camera
// whenever it changes
pub(crate) async fn run(&mut self) -> Result<()> {
pub(crate) async fn run(&mut self) -> AnyResult<()> {
const MAX_BACKOFF: Duration = Duration::from_secs(5);
const MIN_BACKOFF: Duration = Duration::from_millis(50);
@@ -116,10 +122,10 @@ impl NeoCamThread {
continue;
}
// Else we see if the result actually was
// Else we see what the result actually was
let result = res.unwrap();
if Instant::now() - now > Duration::from_secs(60) {
if now.elapsed() > Duration::from_secs(60) {
// Command ran long enough to be considered a success
backoff = MIN_BACKOFF;
}
@@ -167,7 +173,7 @@ impl Drop for NeoCamThread {
}
}
async fn update_camera_time(camera: &BcCamera, name: &str, update_time: bool) -> Result<()> {
async fn update_camera_time(camera: &BcCamera, name: &str, update_time: bool) -> AnyResult<()> {
let cam_time = camera.get_time().await?;
let mut update = false;
if let Some(time) = cam_time {
+28 -34
View File
@@ -4,7 +4,7 @@
//!
//! The camera watch is used as an event to be triggered
//! whenever the camera is lost/updated
use anyhow::anyhow;
use anyhow::{anyhow, Context};
use futures::TryFutureExt;
use std::sync::{Arc, Weak};
use tokio::sync::{
@@ -93,49 +93,50 @@ impl NeoInstance {
-> std::pin::Pin<Box<dyn futures::Future<Output = Result<T>> + Send + 'a>>,
{
let mut camera_watch = self.camera_watch.clone();
let mut camera = camera_watch.borrow_and_update().upgrade();
let mut camera = None;
loop {
let res = tokio::select! {
camera = camera_watch
.wait_for(|new_cam| {
let curr_as_weak = camera.as_ref().map(Arc::downgrade).unwrap_or_default();
!Weak::ptr_eq(new_cam, &curr_as_weak)
})
.map_ok(|new_cam| new_cam.upgrade())
.await
.with_context(|| "Camera is disconnecting")?;
break tokio::select! {
_ = self.cancel.cancelled() => {
Some(Err(anyhow!("Camera is disconnecting")))
Err(anyhow!("Camera is disconnecting"))
}
v = camera_watch.wait_for(|new_cam| !Weak::ptr_eq(new_cam, &camera.as_ref().map(Arc::downgrade).unwrap_or_default())).map_ok(|new_cam| new_cam.upgrade()) => {
_ = camera_watch.wait_for(|new_cam| !Weak::ptr_eq(new_cam, &camera.as_ref().map(Arc::downgrade).unwrap_or_default())).map_ok(|new_cam| new_cam.upgrade()) => {
// Camera value has changed!
// update and try again
if let Ok(new_cam) = v {
camera = new_cam;
None
} else {
Some(Err(anyhow!("Camera is disconnecting")))
}
// Go back and see how it changed
continue;
},
Some(v) = async {
v = async {
if let Some(cam) = camera.clone() {
let cam_ref = cam.as_ref();
Some(task(cam_ref).await)
task(cam_ref).await
} else {
None
unreachable!()
}
}, if camera.is_some() => {
match v {
// Ok means we are done
Ok(v) => Some(Ok(v)),
Ok(v) => Ok(v),
// If error we check for retryable errors
Err(e) => {
match e.downcast::<neolink_core::Error>() {
// Retry is a None
Ok(neolink_core::Error::DroppedConnection) | Ok(neolink_core::Error::TimeoutDisconnected) => {
camera = None;
None
continue;
},
Ok(neolink_core::Error::Io(e)) => {
log::debug!("Std IO Error");
use std::io::ErrorKind::*;
if let ConnectionReset | ConnectionAborted | BrokenPipe | TimedOut = e.kind() {
// Resetable IO
camera = None;
None
continue;
} else {
// Check if the inner error is the Other type and then the discomnect
let is_dropped = e.get_ref().is_some_and(|e| {
@@ -146,14 +147,13 @@ impl NeoInstance {
});
if is_dropped {
// Retry is a None
camera = None;
None
continue;
} else {
Some(Err(e.into()))
Err(e.into())
}
}
}
Ok(e) => Some(Err(e.into())),
Ok(e) => Err(e.into()),
Err(e) => {
// Check if it is an io error
log::debug!("Other Error: {:?}", e);
@@ -164,8 +164,7 @@ impl NeoInstance {
use std::io::ErrorKind::*;
if let ConnectionReset | ConnectionAborted | BrokenPipe | TimedOut = e.kind() {
// Resetable IO
camera = None;
None
continue;
} else {
let is_dropped = e.get_ref().is_some_and(|e| {
log::debug!("Std IO Error: Inner: {:?}", e);
@@ -175,16 +174,15 @@ impl NeoInstance {
});
if is_dropped {
// Retry is a None
camera = None;
None
continue;
} else {
Some(Err(e.into()))
Err(e.into())
}
}
},
Err(e) => {
log::debug!("Other Error: {:?}", e);
Some(Err(e))
Err(e)
}
}
},
@@ -193,10 +191,6 @@ impl NeoInstance {
}
},
};
if let Some(res) = res {
return res;
}
}
}