Add a low level socket reconnect

This commit is contained in:
QuantumEntangledAndy committed 2023-09-14 12:30:12 +07:00
1 parent e996279e8a
commit 1f5db9f0b1
4 files changed
+210 -93

No files matched your search

@@ -32,6 +32,7 @@ use tokio::{
time::{interval, timeout, Duration},
};
use tokio_stream::wrappers::ReceiverStream;
use tokio_util::sync::CancellationToken;
use tokio_util::udp::UdpFramed;
#[derive(Debug, Clone)]
@@ -96,11 +97,12 @@ type ArcFramedSocket = UdpFramed<BcUdpCodex, Arc<UdpSocket>>;
pub(crate) struct Discoverer {
semaphore: Arc<Semaphore>,
socket: Arc<UdpSocket>,
handle: RwLock<JoinSet<()>>,
handle: RwLock<JoinSet<Result<()>>>,
writer: Sender<Result<(BcUdp, SocketAddr)>>,
subsribers: Subscriber,
handlers: Handlers,
local_addr: SocketAddr,
cancel: CancellationToken,
}
fn valid_ip(ip: &str) -> bool {
@@ -120,6 +122,7 @@ impl Discoverer {
let socket = Arc::new(connect().await?);
let local_addr = socket.local_addr()?;
let inner: ArcFramedSocket = UdpFramed::new(socket.clone(), BcUdpCodex::new());
let cancel = CancellationToken::new();
let (mut writer, mut reader) = inner.split();
let mut set = JoinSet::new();
@@ -128,50 +131,63 @@ impl Discoverer {
let thread_subscriber = subsribers.clone();
let thread_handlers = handlers.clone();
let thread_cancel = cancel.clone();
set.spawn(async move {
loop {
tokio::task::yield_now().await;
match reader.next().await {
Some(Ok((BcUdp::Discovery(bcudp), addr))) => {
let tid = bcudp.tid;
let mut needs_removal = false;
if let (Some(sender), true) =
(thread_subscriber.read().await.get(&tid), tid > 0)
{
if sender.send(Ok((bcudp, addr))).await.is_err() {
needs_removal = true;
tokio::select! {
_ = thread_cancel.cancelled() => Result::Ok(()),
v = async {
loop {
tokio::task::yield_now().await;
match reader.next().await {
Some(Ok((BcUdp::Discovery(bcudp), addr))) => {
let tid = bcudp.tid;
let mut needs_removal = false;
if let (Some(sender), true) =
(thread_subscriber.read().await.get(&tid), tid > 0)
{
if sender.send(Ok((bcudp, addr))).await.is_err() {
needs_removal = true;
}
} else {
for sender in thread_handlers.write().await.iter_mut() {
let _ = sender.send(Ok((bcudp.clone(), addr))).await;
}
}
if needs_removal {
thread_subscriber.write().await.remove(&tid);
}
}
} else {
for sender in thread_handlers.write().await.iter_mut() {
let _ = sender.send(Ok((bcudp.clone(), addr))).await;
Some(Ok(bcudp)) => {
// Only discovery packets should be possible atm
trace!("Got non Discovery during discovery: {:?}", bcudp);
}
}
if needs_removal {
thread_subscriber.write().await.remove(&tid);
Some(Err(e)) => {
let mut locked_sub = thread_subscriber.write().await;
for (_, sub) in locked_sub.iter() {
let _ = sub.send(Err(e.clone())).await;
}
locked_sub.clear();
break Result::Ok(());
}
None => break Result::Ok(()),
}
}
Some(Ok(bcudp)) => {
// Only discovery packets should be possible atm
trace!("Got non Discovery during discovery: {:?}", bcudp);
}
Some(Err(e)) => {
let mut locked_sub = thread_subscriber.write().await;
for (_, sub) in locked_sub.iter() {
let _ = sub.send(Err(e.clone())).await;
}
locked_sub.clear();
break;
}
None => break,
}
}=> v,
}
});
let (sinker, sinker_rx) = channel::<Result<(BcUdp, SocketAddr)>>(100);
let thread_cancel = cancel.clone();
set.spawn(async move {
let mut sinker_rx = ReceiverStream::new(sinker_rx);
while let Some(Ok(packet)) = sinker_rx.next().await {
let _ = writer.send(packet).await;
tokio::select! {
_ = thread_cancel.cancelled() => Result::Ok(()),
v = async {
let mut sinker_rx = ReceiverStream::new(sinker_rx);
while let Some(Ok(packet)) = sinker_rx.next().await {
let _ = writer.send(packet).await;
}
Result::Ok(())
} => v,
}
});
@@ -183,6 +199,7 @@ impl Discoverer {
subsribers,
handlers,
local_addr,
cancel,
})
}
@@ -979,23 +996,29 @@ impl Discoverer {
let addr = connect_result.addr;
let mut sender = ArcFramedSocket::new(self.socket.clone(), BcUdpCodex::new());
let mut interval = interval(Duration::from_secs(1));
let thread_cancel = self.cancel.clone();
self.handle.write().await.spawn(async move {
loop {
tokio::task::yield_now().await;
interval.tick().await;
let msg = BcUdp::Discovery(UdpDiscovery {
tid,
payload: UdpXml {
c2d_hb: Some(C2dHb {
cid: client_id,
did: camera_id,
}),
..Default::default()
},
});
if sender.send((msg, addr)).await.is_err() {
return;
}
tokio::select! {
_ = thread_cancel.cancelled() => Result::Ok(()),
v = async {
loop {
tokio::task::yield_now().await;
interval.tick().await;
let msg = BcUdp::Discovery(UdpDiscovery {
tid,
payload: UdpXml {
c2d_hb: Some(C2dHb {
cid: client_id,
did: camera_id,
}),
..Default::default()
},
});
if sender.send((msg, addr)).await.is_err() {
break Result::Ok(());
}
}
} => v,
}
});
}
@@ -1008,29 +1031,41 @@ impl Discoverer {
let addr = connect_result.addr;
let mut sender = ArcFramedSocket::new(self.socket.clone(), BcUdpCodex::new());
let mut interval = interval(Duration::from_secs(1));
let thread_cancel = self.cancel.clone();
self.handle.write().await.spawn(async move {
loop {
tokio::task::yield_now().await;
interval.tick().await;
let msg = BcUdp::Discovery(UdpDiscovery {
tid,
payload: UdpXml {
c2r_hb: Some(C2rHb {
sid,
cid: client_id,
did: camera_id,
}),
..Default::default()
},
});
if sender.send((msg, addr)).await.is_err() {
return;
}
tokio::select! {
_ = thread_cancel.cancelled() => Result::Ok(()),
v = async {
loop {
tokio::task::yield_now().await;
interval.tick().await;
let msg = BcUdp::Discovery(UdpDiscovery {
tid,
payload: UdpXml {
c2r_hb: Some(C2rHb {
sid,
cid: client_id,
did: camera_id,
}),
..Default::default()
},
});
if sender.send((msg, addr)).await.is_err() {
break Result::Ok(());
}
}
} => v,
}
});
}
}
impl Drop for Discoverer {
fn drop(&mut self) {
self.cancel.cancel();
}
}
pub(crate) struct Discovery {
discoverer: Discoverer,
client_id: i32,
@@ -2,14 +2,14 @@ use super::DiscoveryResult;
use crate::bc::codex::BcCodex;
use crate::bc::model::*;
use crate::bcudp::codex::BcUdpCodex;
use crate::bcudp::model::*;
use crate::bcudp::{model::*, xml::*};
use crate::{Credentials, Error, Result};
use delegate::delegate;
use futures::{
sink::{Sink, SinkExt},
stream::{IntoAsyncRead, Stream, StreamExt, TryStreamExt},
};
use rand::{seq::SliceRandom, thread_rng};
use rand::{seq::SliceRandom, thread_rng, Rng};
use std::collections::BTreeMap;
use std::io::{Error as IoError, ErrorKind, Result as IoResult};
use std::net::SocketAddr;
@@ -317,7 +317,7 @@ impl UdpPayloadInner {
let (socket_out_tx, socket_out_rx) = channel::<(BcUdp, SocketAddr)>(1000);
// let (mut socket_tx, mut socket_rx) = inner.split();
// Send to socket
// Send/Recv on the socket
let send_cancel = cancel.clone();
let mut socket_in_rx = ReceiverStream::new(socket_in_rx);
let thread_camera_addr = camera_addr;
@@ -342,6 +342,7 @@ impl UdpPayloadInner {
Err(Error::DroppedConnection)
}
packet = inner.next() => {
log::trace!("Cam->App");
let packet = packet.ok_or(Error::DroppedConnection)??;
recv_timeout.as_mut().reset(Instant::now() + Duration::from_secs(TIME_OUT));
// let packet = socket_rx.next().await.ok_or(Error::DroppedConnection)??;
@@ -350,7 +351,21 @@ impl UdpPayloadInner {
},
packet = socket_in_rx.next() => {
let packet = packet.ok_or(Error::DroppedConnection)?;
inner.send((packet, thread_camera_addr)).await?;
match tokio::time::timeout(tokio::time::Duration::from_millis(200), inner.send((packet, thread_camera_addr))).await {
Ok(written) => {
written?;
}
Err(_) => {
// Socket is (maybe) broken
// Seems to happen with network reconnects like over
// a lossy cellular network
log::info!("Reconnect: Due to socket failure");
let stream = Arc::new(connect().await?);
inner = BcUdpSource::new_from_socket(stream, inner.addr).await?;
}
}
log::trace!("Send Packet");
continue;
}
}
@@ -408,6 +423,39 @@ impl UdpPayloadInner {
})
});
// Queue up Hb packets
let thread_client_id = client_id;
let thread_camera_id = camera_id;
let thread_sender = socket_in_tx.clone();
let mut thread_interval = interval(Duration::from_secs(1));
let thread_cancel = cancel.clone();
tokio::task::spawn(async move {
tokio::select! {
_ = thread_cancel.cancelled() => Result::Ok(()),
v = async {
loop {
thread_interval.tick().await;
let msg = BcUdp::Discovery(UdpDiscovery {
tid: {
let mut rng = thread_rng();
(rng.gen::<u8>()) as u32
},
payload: UdpXml {
c2d_hb: Some(C2dHb {
cid: thread_client_id,
did: thread_camera_id,
}),
..Default::default()
},
});
if thread_sender.send(msg).await.is_err() {
break Result::Ok(());
}
}
} => v,
}
});
Self {
camera_addr,
ack_tx: PollSender::new(ack_tx),
+3 -3
View File
@@ -378,20 +378,20 @@ pub(crate) struct StreamData {
users: UseCounter,
}
#[derive(Eq, PartialEq, Clone)]
#[derive(Eq, PartialEq, Clone, Debug)]
pub(crate) enum VidFormat {
None,
H264,
H265,
}
#[derive(Eq, PartialEq, Clone)]
#[derive(Eq, PartialEq, Clone, Debug)]
pub(crate) enum AudFormat {
None,
Aac,
Adpcm(u32),
}
#[derive(Clone)]
#[derive(Clone, Eq, PartialEq, Debug)]
pub(crate) struct StreamConfig {
pub(crate) resolution: [u32; 2],
pub(crate) vid_format: VidFormat,
+52 -18
View File
@@ -403,9 +403,14 @@ async fn stream_main(
paths: &[String],
) -> Result<()> {
let mut camera_config = camera.config().await?.clone();
let name = camera_config.borrow().name.clone();
let mut curr_pause;
loop {
log::trace!("{}: Activating Stream", &name);
stream_instance.activate().await?;
// Wait for a valid stream format to be detected
log::trace!("{}: Waiting for Valid Stream", &name);
stream_instance
.config
.wait_for(|config| {
@@ -415,31 +420,49 @@ async fn stream_main(
})
.await?;
stream_instance.activate().await?;
// After vid give it 1s to look for audio
// Ignore timeout but check err
if let Ok(v) = tokio::time::timeout(
Duration::from_secs(1),
stream_instance.config.wait_for(|config| {
!matches!(config.vid_format, VidFormat::None)
&& !matches!(config.aud_format, AudFormat::None)
&& config.resolution[0] > 0
&& config.resolution[1] > 0
}),
)
.await
{
v?;
}
curr_pause = camera_config.borrow().pause.clone();
let vid = stream_instance.vid.resubscribe();
let aud = stream_instance.aud.resubscribe();
let thread_stream_config = stream_instance.config.clone();
let mut thread_stream_config2 = stream_instance.config.clone();
let last_stream_config = stream_instance.config.borrow().clone();
let mut thread_stream_config = stream_instance.config.clone();
let mut set = JoinSet::<AnyResult<()>>::new();
log::trace!("{}: Creating Client Counters", &name);
// Handles the on off of the stream with the client pause
let client_counter = UseCounter::new().await;
let client_count = client_counter.create_deactivated().await?;
if curr_pause.on_disconnect {
log::trace!("{}: Enabling Client Pause", &name);
// Take over activation
let mut client_activator = stream_instance.activator_handle().await;
client_activator.activate().await?;
stream_instance.deactivate().await?;
let client_count = client_counter.create_deactivated().await?;
let thread_name = name.clone();
set.spawn(async move {
loop {
log::trace!("Activating Client");
log::info!("{}: Activating Client", thread_name);
client_activator.activate().await?;
client_count.dropped_users().await?;
log::trace!("Pausing Client");
log::info!("{}: Pausing Client", thread_name);
client_activator.deactivate().await?;
client_count.aquired_users().await?;
}
@@ -448,17 +471,19 @@ async fn stream_main(
// Handles on motion pausing
if curr_pause.on_motion {
log::trace!("{}: Activating Motion Pause", &name);
// Take over activation
let mut client_activator = stream_instance.activator_handle().await;
stream_instance.deactivate().await?;
let mut motion = camera.motion().await?;
let delta = Duration::from_secs_f64(curr_pause.motion_timeout);
let thread_name = name.clone();
set.spawn(async move {
loop {
log::trace!("Activating Motion");
log::info!("{}: Enabling Motion", thread_name);
client_activator.activate().await?;
motion.wait_for(|md| matches!(md, crate::common::MdState::Stop(_))).await?;
log::trace!("Pausing Motion");
log::info!("{}: Pausing Motion", thread_name);
client_activator.deactivate().await?;
motion.wait_for(|md| matches!(md, crate::common::MdState::Start(n) if (*n - Instant::now())>delta)).await?;
}
@@ -466,25 +491,33 @@ async fn stream_main(
}
// This select is for the streams config updates
log::trace!("{}: Stream Activated", &name);
break tokio::select! {
_ = thread_stream_config2.changed() => {
v = thread_stream_config.wait_for(|new_conf| new_conf != &last_stream_config) => {
let v = v?;
// If stream config changes we reload the stream
log::info!("{}: Stream Configuration Changed. Reloading Streams", &name);
log::trace!(" From {:?} to {:?}", last_stream_config, v.clone());
continue;
},
_ = camera_config.wait_for(|new_conf| new_conf.pause != curr_pause ) => {
v = camera_config.wait_for(|new_conf| new_conf.pause != curr_pause ) => {
v?;
// If pause config changes restart
log::info!("{}: Pause Configuration Changed. Reloading Streams", &name);
continue;
},
v = stream_run(vid, aud, rtsp, thread_stream_config.borrow().clone(), users, paths, client_count) => v,
v = stream_run(&name, vid, aud, rtsp, &last_stream_config, users, paths, client_count) => v,
};
}
}
#[allow(clippy::too_many_arguments)] // Seriously clippy needs to be done
async fn stream_run(
name: &str,
vidstream: BroadcastReceiver<StampedData>,
audstream: BroadcastReceiver<StampedData>,
rtsp: &NeoRtspServer,
stream_config: StreamConfig,
stream_config: &StreamConfig,
users: &HashSet<String>,
paths: &[String],
client_count: CountUses,
@@ -494,7 +527,7 @@ async fn stream_run(
.mount_points()
.ok_or(anyhow!("RTSP server lacks mount point"))?;
// Create the factory
let (factory, mut client_rx) = make_factory(&stream_config).await?;
let (factory, mut client_rx) = make_factory(stream_config).await?;
factory.add_permitted_roles(users);
@@ -502,6 +535,7 @@ async fn stream_run(
log::debug!("Path: {}", path);
mounts.add_factory(path, factory.clone());
}
log::info!("{}: Avaliable at {:?}", name, paths);
let stream_cancel = CancellationToken::new();
let mut set = JoinSet::new();
@@ -673,7 +707,7 @@ async fn make_factory(
app.set_callbacks(
AppSrcCallbacks::builder()
.seek_data(move |_, seek_pos| {
log::info!("seek_pos: {seek_pos:?}");
log::trace!("seek_pos: {seek_pos:?}");
let (reply_tx, mut reply_rx) = mpsc(1);
vid_seek_tx
.blocking_send(StreamData::Seek {
@@ -692,7 +726,7 @@ async fn make_factory(
app.set_callbacks(
AppSrcCallbacks::builder()
.seek_data(move |_, seek_pos| {
log::info!("seek_pos: {seek_pos:?}");
log::trace!("seek_pos: {seek_pos:?}");
let (reply_tx, mut reply_rx) = mpsc(1);
vid_seek_tx
.blocking_send(StreamData::Seek {
@@ -717,7 +751,7 @@ async fn make_factory(
app.set_callbacks(
AppSrcCallbacks::builder()
.seek_data(move |_, seek_pos| {
log::info!("seek_pos: {seek_pos:?}");
log::trace!("seek_pos: {seek_pos:?}");
let (reply_tx, mut reply_rx) = mpsc(1);
aud_seek_tx
.blocking_send(StreamData::Seek {
@@ -736,7 +770,7 @@ async fn make_factory(
app.set_callbacks(
AppSrcCallbacks::builder()
.seek_data(move |_, seek_pos| {
log::info!("seek_pos: {seek_pos:?}");
log::trace!("seek_pos: {seek_pos:?}");
let (reply_tx, mut reply_rx) = mpsc(1);
aud_seek_tx
.blocking_send(StreamData::Seek {
@@ -790,7 +824,7 @@ async fn handle_data<T: Stream<Item = Result<StreamData, E>> + Unpin, E>(
let _ = reply.send(()).await;
}
StreamData::Media { data, ts: ft_i } => {
log::info!("Frame recieved with ts: {ft_i:?}");
log::trace!("Frame recieved with ts: {ft_i:?}");
// Update rt
if let Some(rt_i) = get_runtime(app) {
if let Some(last_rt) = last_rt {
@@ -827,7 +861,7 @@ async fn handle_data<T: Stream<Item = Result<StreamData, E>> + Unpin, E>(
let mut gst_buf = gstreamer::Buffer::with_size(data.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
log::info!("Setting PTS: {thread_ft:?}, Runtime: {thread_rt:?}");
log::trace!("Setting PTS: {thread_ft:?}, Runtime: {thread_rt:?}");
let time = ClockTime::from_useconds(thread_ft.as_micros() as u64);
gst_buf_mut.set_dts(time);
gst_buf_mut.set_pts(time);