Implement exponential backoff/retry in case of errors from camera

This commit is contained in:
George Hilliard committed 2020-04-29 23:59:19 -05:00
1 parent 38ca3f262e
commit 1a373e1acd
3 files changed
+179 -6

No files matched your search

Generated
+157 -4
View File
@@ -38,12 +38,28 @@ dependencies = [
"winapi",
]
[[package]]
name = "autocfg"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
[[package]]
name = "autocfg"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
[[package]]
name = "backoff"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66483433ebf577e03c6546df761778e4cc40b00e5a1ea7ed850733ffb555d71f"
dependencies = [
"instant",
"rand",
]
[[package]]
name = "bitflags"
version = "1.2.1"
@@ -71,6 +87,15 @@ dependencies = [
"vec_map",
]
[[package]]
name = "cloudabi"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
dependencies = [
"bitflags",
]
[[package]]
name = "cookie-factory"
version = "0.3.1"
@@ -83,7 +108,7 @@ version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg",
"autocfg 1.0.0",
"cfg-if",
"lazy_static",
]
@@ -115,6 +140,12 @@ dependencies = [
"synstructure",
]
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "futures-channel"
version = "0.3.4"
@@ -601,6 +632,12 @@ dependencies = [
"unindent",
]
[[package]]
name = "instant"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7152d2aed88aa566e7a342250f21ba2222c1ae230ad577499dbfa3c18475b80"
[[package]]
name = "lazy_static"
version = "1.4.0"
@@ -658,6 +695,7 @@ checksum = "0419348c027fa7be448d2ae7ea0e4e04c2334c31dc4e74ab29f00a2a7ca69204"
name = "neolink"
version = "0.1.0"
dependencies = [
"backoff",
"cookie-factory",
"crossbeam-utils",
"env_logger",
@@ -703,7 +741,7 @@ version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba"
dependencies = [
"autocfg",
"autocfg 1.0.0",
"num-traits",
]
@@ -713,7 +751,7 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
dependencies = [
"autocfg",
"autocfg 1.0.0",
"num-integer",
"num-traits",
]
@@ -724,7 +762,7 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096"
dependencies = [
"autocfg",
"autocfg 1.0.0",
]
[[package]]
@@ -823,6 +861,121 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
dependencies = [
"autocfg 0.1.7",
"libc",
"rand_chacha",
"rand_core 0.4.2",
"rand_hc",
"rand_isaac",
"rand_jitter",
"rand_os",
"rand_pcg",
"rand_xorshift",
"winapi",
]
[[package]]
name = "rand_chacha"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
dependencies = [
"autocfg 0.1.7",
"rand_core 0.3.1",
]
[[package]]
name = "rand_core"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
dependencies = [
"rand_core 0.4.2",
]
[[package]]
name = "rand_core"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]]
name = "rand_hc"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rand_isaac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rand_jitter"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
dependencies = [
"libc",
"rand_core 0.4.2",
"winapi",
]
[[package]]
name = "rand_os"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
dependencies = [
"cloudabi",
"fuchsia-cprng",
"libc",
"rand_core 0.4.2",
"rdrand",
"winapi",
]
[[package]]
name = "rand_pcg"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
dependencies = [
"autocfg 0.1.7",
"rand_core 0.4.2",
]
[[package]]
name = "rand_xorshift"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rdrand"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "regex"
version = "1.3.7"
+1
View File
@@ -7,6 +7,7 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
backoff = "0.1"
cookie-factory = "0.3"
crossbeam-utils = "0.7"
env_logger = "*"
+21 -2
View File
@@ -6,6 +6,7 @@ mod config;
mod cmdline;
mod gst;
use backoff::{ExponentialBackoff, Operation};
use bc_protocol::BcCamera;
use config::{Config, CameraConfig};
use cmdline::Opt;
@@ -13,6 +14,7 @@ use crossbeam_utils::thread;
use err_derive::Error;
use gst::RtspServer;
use std::fs;
use std::time::Duration;
use std::io::Write;
use structopt::StructOpt;
@@ -37,7 +39,7 @@ fn main() -> Result<(), Error> {
s.spawn(move |_| {
// TODO handle these errors
let mut output = rtsp.add_stream(&camera.name).unwrap(); // TODO
camera_main(&camera, &mut output)
camera_loop(&camera, &mut output)
});
}
@@ -47,7 +49,24 @@ fn main() -> Result<(), Error> {
Ok(())
}
fn camera_main(camera_config: &CameraConfig, output: &mut dyn Write) -> Result<(), Error> {
fn camera_loop(camera_config: &CameraConfig, output: &mut dyn Write) -> Result<(), Error> {
let mut backoff = ExponentialBackoff {
initial_interval: Duration::from_secs(5),
max_interval: Duration::from_secs(60),
..Default::default()
};
let mut op = || {
camera_main(camera_config, output)
};
op.retry(&mut backoff).map_err(|backoff_err| -> Error {
match backoff_err {
backoff::Error::Transient(e) |
backoff::Error::Permanent(e) => e.into()
}
})
}
fn camera_main(camera_config: &CameraConfig, output: &mut dyn Write) -> Result<(), backoff::Error<bc_protocol::Error>> {
let mut camera = BcCamera::new_with_addr(camera_config.camera_addr)?;
println!("{}: Connecting to camera at {}", camera_config.name, camera_config.camera_addr);