Rework thread creation to reduce memory stress

This commit is contained in:
QuantumEntangledAndy committed 2023-09-25 15:30:31 +07:00
1 parent d9c8caeff8
commit da66b1b2ff
22 files changed
+456 -472

No files matched your search

Generated
+21
View File
@@ -1432,6 +1432,7 @@ dependencies = [
"rumqttc",
"serde",
"serde_json",
"tikv-jemallocator",
"tokio",
"tokio-stream",
"tokio-util",
@@ -2279,6 +2280,26 @@ dependencies = [
"once_cell",
]
[[package]]
name = "tikv-jemalloc-sys"
version = "0.5.4+5.3.0-patched"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "tikv-jemallocator"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca"
dependencies = [
"libc",
"tikv-jemalloc-sys",
]
[[package]]
name = "time"
version = "0.3.25"
+3
View File
@@ -39,3 +39,6 @@ toml = "0.7.3"
validator = "0.16.0"
validator_derive = "0.16.0"
yaserde = "0.8.0"
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = "0.5"
-1
View File
@@ -45,7 +45,6 @@ impl Encoder<Bc> for BcCodex {
n => *n,
};
let buf = item.serialize(buf, &enc_protocol)?;
dst.reserve(buf.len());
dst.extend_from_slice(buf.as_slice());
Ok(())
}
-2
View File
@@ -15,7 +15,6 @@ impl Bc {
/// Returns Ok(deserialized data, the amount of data consumed)
/// Can then use this as the amount that should be remove from a buffer
pub(crate) fn deserialize(context: &BcContext, buf: &mut BytesMut) -> Result<Bc, Error> {
const TYPICAL_HEADER: usize = 24;
let parser = BcParser { context };
let (result, amount) = match consumed(parser)(buf) {
Ok((_, (parsed_buff, result))) => Ok((result, parsed_buff.len())),
@@ -23,7 +22,6 @@ impl Bc {
}?;
buf.advance(amount);
buf.reserve(amount + TYPICAL_HEADER); // Preallocate for future buffer calls
Ok(result)
}
}
@@ -10,7 +10,7 @@ use std::sync::Arc;
use tokio::sync::mpsc::{channel, Sender};
use tokio::task::yield_now;
use tokio_stream::wrappers::ReceiverStream;
use tokio_util::sync::{CancellationToken, PollSender};
use tokio_util::sync::CancellationToken;
use tokio::{sync::RwLock, task::JoinSet};
@@ -57,27 +57,19 @@ impl BcConnection {
let mut rx_thread = JoinSet::<Result<()>>::new();
let thread_poll_commander = poll_commander.clone();
let thread_cancel = cancel.clone();
rx_thread.spawn_blocking(move || {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
let result = runtime.block_on(async move {
tokio::select! {
_ = thread_cancel.cancelled() => {
Result::Ok(())
},
v = async {
let mut sender = PollSender::new(thread_poll_commander);
while let Some(bc) = source.next().await {
sender.send(PollCommand::Bc(Box::new(bc))).await?;
}
Result::Ok(())
} => v
}
});
log::trace!("PollSender Bc {:?}", result);
result
rx_thread.spawn(async move {
tokio::select! {
_ = thread_cancel.cancelled() => {
Result::Ok(())
},
v = async {
let sender = thread_poll_commander;
while let Some(bc) = source.next().await {
sender.send(PollCommand::Bc(Box::new(bc))).await?;
}
Result::Ok(())
} => v
}
});
let thread_cancel = cancel.clone();
@@ -199,13 +191,17 @@ impl Drop for BcConnection {
fn drop(&mut self) {
log::trace!("Drop BcConnection");
self.cancel.cancel();
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
let _ = self.shutdown().await;
Result::Ok(())
});
let poll_commander = self.poll_commander.clone();
let _gt = tokio::runtime::Handle::current().enter();
let mut threads = std::mem::take(&mut self.rx_thread);
tokio::task::spawn(async move {
let _ = poll_commander.send(PollCommand::Disconnect).await;
log::debug!("BcConnection::shutdown Cancel");
let locked_threads = threads.get_mut();
while locked_threads.join_next().await.is_some() {}
log::trace!("Dropped BcConnection");
});
log::trace!("Dropped BcConnection");
}
}
@@ -1064,12 +1064,9 @@ impl Drop for Discoverer {
fn drop(&mut self) {
log::trace!("Drop Discoverer");
self.cancel.cancel();
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
while self.handle.get_mut().join_next().await.is_some() {}
Result::Ok(())
});
});
let _gt = tokio::runtime::Handle::current().enter();
let mut handle = std::mem::take(&mut self.handle);
tokio::task::spawn(async move { while handle.get_mut().join_next().await.is_some() {} });
log::trace!("Dropped Discoverer");
}
}
@@ -16,9 +16,10 @@ use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tokio::sync::mpsc::channel;
use tokio::{
net::UdpSocket,
sync::mpsc::channel,
task::JoinSet,
time::{interval, sleep, Duration, Instant, Interval},
};
@@ -33,8 +34,9 @@ use tokio_util::{
const MTU: usize = 1350;
const UDPDATA_HEADER_SIZE: usize = 20;
pub(crate) type InnerFramed = Framed<Compat<IntoAsyncRead<UdpPayloadSource>>, BcCodex>;
pub(crate) struct UdpSource {
inner: Framed<Compat<IntoAsyncRead<UdpPayloadSource>>, BcCodex>,
inner: Pin<Box<InnerFramed>>,
}
impl UdpSource {
@@ -94,7 +96,9 @@ impl UdpSource {
};
let framed = Framed::new(async_read, codex);
Ok(Self { inner: framed })
Ok(Self {
inner: Box::pin(framed),
})
}
// pub(crate) async fn send(&mut self, bc: Bc) -> Result<()> {
@@ -113,13 +117,13 @@ impl Stream for UdpSource {
type Item = std::result::Result<<BcCodex as Decoder>::Item, <BcCodex as Decoder>::Error>;
delegate! {
to Pin::new(&mut self.inner) {
to self.inner.as_mut() {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>>;
}
}
delegate! {
to self.inner {
to self.inner.as_ref().get_ref() {
fn size_hint(&self) -> (usize, Option<usize>);
}
}
@@ -129,7 +133,7 @@ impl Sink<Bc> for UdpSource {
type Error = <BcCodex as Encoder<Bc>>::Error;
delegate! {
to Pin::new(&mut self.inner) {
to self.inner.as_mut() {
fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>>;
fn start_send(mut self: Pin<&mut Self>, item: Bc) -> std::result::Result<(), Self::Error>;
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>>;
@@ -139,7 +143,7 @@ impl Sink<Bc> for UdpSource {
}
pub(crate) struct BcUdpSource {
inner: UdpFramed<BcUdpCodex, Arc<UdpSocket>>,
inner: Pin<Box<UdpFramed<BcUdpCodex, Arc<UdpSocket>>>>,
addr: SocketAddr,
}
@@ -158,7 +162,7 @@ impl BcUdpSource {
pub(crate) async fn new_from_socket(stream: Arc<UdpSocket>, addr: SocketAddr) -> Result<Self> {
Ok(Self {
inner: UdpFramed::new(stream, BcUdpCodex::new()),
inner: Box::pin(UdpFramed::new(stream, BcUdpCodex::new())),
addr,
})
}
@@ -176,13 +180,13 @@ impl Stream for BcUdpSource {
type Item = Result<(BcUdp, SocketAddr)>;
delegate! {
to Pin::new(&mut self.inner) {
to self.inner.as_mut() {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>>;
}
}
delegate! {
to self.inner {
to self.inner.as_ref().get_ref() {
fn size_hint(&self) -> (usize, Option<usize>);
}
}
@@ -192,7 +196,7 @@ impl Sink<(BcUdp, SocketAddr)> for BcUdpSource {
type Error = Error;
delegate! {
to Pin::new(&mut self.inner) {
to self.inner.as_mut() {
fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>>;
fn start_send(mut self: Pin<&mut Self>, item: (BcUdp, SocketAddr)) -> std::result::Result<(), Self::Error>;
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>>;
@@ -262,7 +266,7 @@ impl AckLatency {
}
pub(crate) struct UdpPayloadSource {
inner_stream: ReceiverStream<IoResult<Vec<u8>>>,
inner_stream: Pin<Box<ReceiverStream<IoResult<Vec<u8>>>>>,
inner_sink: PollSender<Vec<u8>>,
set: JoinSet<Result<()>>,
cancel_token: CancellationToken,
@@ -272,13 +276,12 @@ impl Drop for UdpPayloadSource {
fn drop(&mut self) {
log::trace!("Drop UdpPayloadSource");
self.cancel_token.cancel();
tokio::task::block_in_place(|| {
let _ = tokio::runtime::Handle::current().block_on(async {
while self.set.join_next().await.is_some() {}
Result::Ok(())
});
let _gt = tokio::runtime::Handle::current().enter();
let mut set = std::mem::take(&mut self.set);
tokio::task::spawn(async move {
while set.join_next().await.is_some() {}
log::trace!("Dropped UdpPayloadSource");
});
log::trace!("Dropped UdpPayloadSource");
}
}
@@ -322,8 +325,8 @@ impl UdpPayloadInner {
// In order to achieve this we use dedicated threads for ACK
// and the socket
let (socket_in_tx, socket_in_rx) = channel::<BcUdp>(1000);
let (socket_out_tx, socket_out_rx) = channel::<(BcUdp, SocketAddr)>(1000);
let (socket_in_tx, socket_in_rx) = channel::<BcUdp>(100);
let (socket_out_tx, socket_out_rx) = channel::<(BcUdp, SocketAddr)>(100);
// let (mut socket_tx, mut socket_rx) = inner.split();
// Send/Recv on the socket
@@ -335,122 +338,110 @@ impl UdpPayloadInner {
let thread_camera_id = camera_id;
const TIME_OUT: u64 = 10;
let mut recv_timeout = Box::pin(sleep(Duration::from_secs(TIME_OUT)));
set.spawn_blocking(move || {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
runtime.block_on(async move {
let result = tokio::select! {
_ = send_cancel.cancelled() => {
Result::Ok(())
},
v = async {
loop {
break tokio::select!{
_ = recv_timeout.as_mut() => {
log::trace!("DroppedConnection: Timeout");
Err(Error::DroppedConnection)
}
packet = inner.next() => {
log::trace!("Cam->App");
let packet = packet.ok_or(Error::DroppedConnection)??;
recv_timeout.as_mut().reset(Instant::now() + Duration::from_secs(TIME_OUT));
// let packet = socket_rx.next().await.ok_or(Error::DroppedConnection)??;
socket_out_tx.send(packet).await?;
continue;
},
packet = socket_in_rx.next() => {
let packet = packet.ok_or(Error::DroppedConnection)?;
match tokio::time::timeout(tokio::time::Duration::from_millis(250), inner.send((packet, thread_camera_addr))).await {
Ok(written) => {
written?;
}
Err(_) => {
// Socket is (maybe) broken
// Seems to happen with network reconnects like over
// a lossy cellular network
log::debug!("Quick reconnect: Due to socket timeout");
let stream = Arc::new(connect_try_port(inner.inner.get_ref().local_addr()?.port()).await?);
inner = BcUdpSource::new_from_socket(stream, inner.addr).await?;
// Inform the camera that we are the same client
//
// At least I think that is what this is for.
// Might also have to do this for the relay but not sure
let msg = BcUdp::Discovery(UdpDiscovery {
tid: {
let mut rng = thread_rng();
(rng.gen::<u8>()) as u32
},
payload: UdpXml {
c2d_hb: Some(C2dHb {
cid: thread_client_id,
did: thread_camera_id,
}),
..Default::default()
},
});
let _ = tokio::time::timeout(Duration::from_millis(250), inner.send((msg, thread_camera_addr))).await;
}
}
log::trace!("Send Packet");
continue;
}
set.spawn(async move {
let result = tokio::select! {
_ = send_cancel.cancelled() => {
Result::Ok(())
},
v = async {
loop {
break tokio::select!{
_ = recv_timeout.as_mut() => {
log::trace!("DroppedConnection: Timeout");
Err(Error::DroppedConnection)
}
}?;
Ok(())
} => v,
};
log::debug!("UdpPayloadInner::new SendToSocket Cancel");
send_cancel.cancel();
result
})
packet = inner.next() => {
log::trace!("Cam->App");
let packet = packet.ok_or(Error::DroppedConnection)??;
recv_timeout.as_mut().reset(Instant::now() + Duration::from_secs(TIME_OUT));
// let packet = socket_rx.next().await.ok_or(Error::DroppedConnection)??;
socket_out_tx.send(packet).await?;
continue;
},
packet = socket_in_rx.next() => {
let packet = packet.ok_or(Error::DroppedConnection)?;
match tokio::time::timeout(tokio::time::Duration::from_millis(250), inner.send((packet, thread_camera_addr))).await {
Ok(written) => {
written?;
}
Err(_) => {
// Socket is (maybe) broken
// Seems to happen with network reconnects like over
// a lossy cellular network
log::debug!("Quick reconnect: Due to socket timeout");
let stream = Arc::new(connect_try_port(inner.inner.get_ref().local_addr()?.port()).await?);
inner = BcUdpSource::new_from_socket(stream, inner.addr).await?;
// Inform the camera that we are the same client
//
// At least I think that is what this is for.
// Might also have to do this for the relay but not sure
let msg = BcUdp::Discovery(UdpDiscovery {
tid: {
let mut rng = thread_rng();
(rng.gen::<u8>()) as u32
},
payload: UdpXml {
c2d_hb: Some(C2dHb {
cid: thread_client_id,
did: thread_camera_id,
}),
..Default::default()
},
});
let _ = tokio::time::timeout(Duration::from_millis(250), inner.send((msg, thread_camera_addr))).await;
}
}
log::trace!("Send Packet");
continue;
}
}
}?;
Ok(())
} => v,
};
log::debug!("UdpPayloadInner::new SendToSocket Cancel");
send_cancel.cancel();
result
});
// Queue up ack packets
let ack_cancel = cancel.clone();
let mut ack_interval = interval(Duration::from_millis(10)); // Offical Client does ack every 10ms
ack_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let (ack_tx, mut ack_rx) = channel(1000);
let (ack_tx, mut ack_rx) = channel(100);
let thread_camera_id = camera_id;
let ack_socket_in_tx = socket_in_tx.clone();
set.spawn_blocking(move || {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
runtime.block_on(async move {
tokio::select! {
_ = ack_cancel.cancelled() => {
Result::Ok(())
},
v = async {
let mut ack_packet = UdpAck::empty(thread_camera_id);
loop {
tokio::select! {
v = ack_rx.recv() => {
// Update the ACK packet
if let Some(v) = v {
ack_packet = v;
Ok(())
} else {
log::trace!("ack_rx.recv() Error::DroppedConnection");
Err(Error::DroppedConnection)
}
},
_ = ack_interval.tick() => {
// Send an ack packet
log::trace!("send ack");
ack_socket_in_tx.send(BcUdp::Ack(ack_packet.clone())).await?;
set.spawn(async move {
tokio::select! {
_ = ack_cancel.cancelled() => {
Result::Ok(())
},
v = async {
let mut ack_packet = UdpAck::empty(thread_camera_id);
loop {
tokio::select! {
v = ack_rx.recv() => {
// Update the ACK packet
if let Some(v) = v {
ack_packet = v;
Ok(())
} else {
log::trace!("ack_rx.recv() Error::DroppedConnection");
Err(Error::DroppedConnection)
}
}?;
}
} => v,
}
})
},
_ = ack_interval.tick() => {
// Send an ack packet
log::trace!("send ack");
ack_socket_in_tx.send(BcUdp::Ack(ack_packet.clone())).await?;
Ok(())
}
}?;
}
} => v,
}
});
// Queue up Hb packets
@@ -513,7 +504,7 @@ impl UdpPayloadInner {
for (_, resend) in self.sent.iter() {
self.socket_in.feed(BcUdp::Data(resend.clone())).await?;
}
self.ack_tx.send(self.build_send_ack()).await?; // Ensure we update the ack packet sometimes too
self.ack_tx.feed(self.build_send_ack()).await?; // Ensure we update the ack packet sometimes too
Result::Ok(())
},
v = self.thread_sink.next() => {
@@ -559,7 +550,7 @@ impl UdpPayloadInner {
if packet_id >= self.packets_want {
// error!("packets_want: {}", this.packets_want);
self.recieved.insert(packet_id, data.payload);
self.ack_tx.send(self.build_send_ack()).await?;
self.ack_tx.feed(self.build_send_ack()).await?;
}
}
},
@@ -573,10 +564,12 @@ impl UdpPayloadInner {
while let Some(payload) = self.recieved.remove(&self.packets_want) {
log::trace!(" + {}", self.packets_want);
self.packets_want += 1;
self.thread_stream.send(Ok(payload)).await?;
self.thread_stream.feed(Ok(payload)).await?;
}
log::trace!("Flush");
self.socket_in.flush().await?;
self.thread_stream.flush().await?;
self.ack_tx.flush().await?;
log::trace!("Flushed");
Ok(())
}
@@ -632,31 +625,24 @@ impl UdpPayloadInner {
}
self.ack_latency.feed_ack();
}
async fn shutdown(&mut self) -> Result<()> {
self.cancel.cancel();
while self.set.join_next().await.is_some() {}
Ok(())
}
}
impl Drop for UdpPayloadInner {
fn drop(&mut self) {
log::trace!("Drop UdpPayloadInner");
self.cancel.cancel();
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
self.shutdown().await?;
Result::Ok(())
});
let _gt = tokio::runtime::Handle::current().enter();
let mut set = std::mem::take(&mut self.set);
tokio::task::spawn(async move {
while set.join_next().await.is_some() {}
log::trace!("Dropped UdpPayloadInner");
});
log::trace!("Dropped UdpPayloadInner");
}
}
impl UdpPayloadSource {
async fn new(inner: BcUdpSource, client_id: i32, camera_id: i32) -> Self {
let (inner_sink, thread_sink) = channel(1000);
let (thread_stream, inner_stream) = channel(1000);
let (inner_sink, thread_sink) = channel(100);
let (thread_stream, inner_stream) = channel(100);
let mut payload_inner = UdpPayloadInner::new(
inner,
@@ -669,45 +655,38 @@ impl UdpPayloadSource {
let thread_cancel_token = cancel_token.clone();
let mut set = JoinSet::new();
set.spawn_blocking(move || {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
runtime.block_on(async move {
tokio::select! {
v = async {
loop {
if payload_inner.thread_stream.is_closed() {
log::trace!("payload_inner.thread_stream.is_closed");
payload_inner.thread_sink.close();
return Err(Error::DroppedConnection);
}
log::trace!("Calling inner");
let res = payload_inner.run().await;
log::trace!("Called inner: {:?}", res);
match res {
Ok(()) => {}
Err(e) => {
log::trace!("UDP Error. Connection will Drop: {:?}", e);
// Pass error up
let _ = payload_inner
.thread_stream
.send(Err(IoError::new(ErrorKind::Other, e.clone())))
.await;
return Result::<()>::Err(e);
}
set.spawn(async move {
tokio::select! {
v = async {
loop {
if payload_inner.thread_stream.is_closed() {
log::trace!("payload_inner.thread_stream.is_closed");
payload_inner.thread_sink.close();
return Err(Error::DroppedConnection);
}
log::trace!("Calling inner");
let res = payload_inner.run().await;
log::trace!("Called inner: {:?}", res);
match res {
Ok(()) => {}
Err(e) => {
log::trace!("UDP Error. Connection will Drop: {:?}", e);
// Pass error up
let _ = payload_inner
.thread_stream
.send(Err(IoError::new(ErrorKind::Other, e.clone())))
.await;
return Result::<()>::Err(e);
}
}
} => v,
_ = thread_cancel_token.cancelled() => Ok(()),
}
})
}
} => v,
_ = thread_cancel_token.cancelled() => Ok(()),
}
});
UdpPayloadSource {
inner_stream: ReceiverStream::new(inner_stream),
inner_stream: Box::pin(ReceiverStream::new(inner_stream)),
inner_sink: PollSender::new(inner_sink),
set,
cancel_token,
@@ -719,13 +698,13 @@ impl Stream for UdpPayloadSource {
type Item = IoResult<Vec<u8>>;
delegate! {
to Pin::new(&mut self.inner_stream) {
to self.inner_stream.as_mut() {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>>;
}
}
delegate! {
to self.inner_stream {
to self.inner_stream.as_ref().get_ref() {
fn size_hint(&self) -> (usize, Option<usize>);
}
}
@@ -793,8 +772,11 @@ impl futures::AsyncWrite for UdpPayloadSource {
/// Helper to create a UdpStream
async fn connect() -> Result<UdpSocket> {
let mut ports: Vec<u16> = (53500..54000).collect();
let mut rng = thread_rng();
ports.shuffle(&mut rng);
{
let mut rng = thread_rng();
ports.shuffle(&mut rng);
drop(rng); // Do not hold RNG over an await
}
let addrs: Vec<_> = ports
.iter()
@@ -807,8 +789,11 @@ async fn connect() -> Result<UdpSocket> {
async fn connect_try_port(port: u16) -> Result<UdpSocket> {
let mut ports: Vec<u16> = (53500..54000).collect();
let mut rng = thread_rng();
ports.shuffle(&mut rng);
{
let mut rng = thread_rng();
ports.shuffle(&mut rng);
drop(rng); // Do not hold RNG over an await
}
let addrs: Vec<_> = vec![port]
.iter()
+5 -6
View File
@@ -286,12 +286,11 @@ impl Drop for MotionData {
fn drop(&mut self) {
log::trace!("Drop MotionData");
self.cancel.cancel();
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
while self.handle.join_next().await.is_some() {}
Result::Ok(())
});
let mut handle = std::mem::take(&mut self.handle);
let _gt = tokio::runtime::Handle::current().enter();
tokio::task::spawn(async move {
while handle.join_next().await.is_some() {}
log::trace!("Dropped MotionData");
});
log::trace!("Dropped MotionData");
}
}
+6 -4
View File
@@ -89,11 +89,13 @@ impl StreamData {
impl Drop for StreamData {
fn drop(&mut self) {
log::trace!("Drop StreamData");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let _ = self.shutdown().await;
self.abort_handle.cancel();
if let Some(handle) = self.handle.take() {
let _gt = tokio::runtime::Handle::current().enter();
tokio::task::spawn(async move {
let _ = handle.await;
});
});
}
log::trace!("Dropped MotionData");
}
}
-1
View File
@@ -30,7 +30,6 @@ impl Encoder<BcMedia> for BcMediaCodex {
fn encode(&mut self, item: BcMedia, dst: &mut BytesMut) -> Result<()> {
let buf: Vec<u8> = Default::default();
let buf = item.serialize(buf)?;
dst.reserve(buf.len());
dst.extend_from_slice(buf.as_slice());
Ok(())
}
-2
View File
@@ -11,13 +11,11 @@ const PAD_SIZE: u32 = 8;
impl BcMedia {
pub(crate) fn deserialize(buf: &mut BytesMut) -> Result<BcMedia, Error> {
const TYPICAL_HEADER: usize = 25;
let (result, len) = match consumed(bcmedia)(buf) {
Ok((_, (parsed_buff, result))) => Ok((result, parsed_buff.len())),
Err(e) => Err(e),
}?;
buf.advance(len);
buf.reserve(len + TYPICAL_HEADER); // Preallocate for future buffer calls
Ok(result)
}
}
-1
View File
@@ -24,7 +24,6 @@ impl Encoder<BcUdp> for BcUdpCodex {
log::trace!("Encoding: {item:?}");
let buf: Vec<u8> = Default::default();
let buf = item.serialize(buf)?;
dst.reserve(buf.len());
dst.extend_from_slice(buf.as_slice());
log::trace!(" Encoding: Done: {}", buf.len());
Ok(())
+26 -16
View File
@@ -231,12 +231,15 @@ impl NeoInstance {
pub(crate) fn drop_command<F>(self, task: F, timeout: tokio::time::Duration) -> DropRunTask<F>
where
F: for<'a> Fn(
&'a BcCamera,
)
-> std::pin::Pin<Box<dyn futures::Future<Output = Result<()>> + Send + 'a>>,
&'a BcCamera,
)
-> std::pin::Pin<Box<dyn futures::Future<Output = Result<()>> + Send + 'a>>
+ Send
+ Sync
+ 'static,
{
DropRunTask {
instance: self,
instance: Some(self),
command: Some(Box::new(task)),
timeout,
}
@@ -247,10 +250,14 @@ impl NeoInstance {
pub(crate) struct DropRunTask<F>
where
F: for<'a> Fn(
&'a BcCamera,
) -> std::pin::Pin<Box<dyn futures::Future<Output = Result<()>> + Send + 'a>>,
&'a BcCamera,
)
-> std::pin::Pin<Box<dyn futures::Future<Output = Result<()>> + Send + 'a>>
+ Send
+ Sync
+ 'static,
{
instance: NeoInstance,
instance: Option<NeoInstance>,
command: Option<Box<F>>,
timeout: tokio::time::Duration,
}
@@ -258,17 +265,20 @@ where
impl<F> Drop for DropRunTask<F>
where
F: for<'a> Fn(
&'a BcCamera,
) -> std::pin::Pin<Box<dyn futures::Future<Output = Result<()>> + Send + 'a>>,
&'a BcCamera,
)
-> std::pin::Pin<Box<dyn futures::Future<Output = Result<()>> + Send + 'a>>
+ Send
+ Sync
+ 'static,
{
fn drop(&mut self) {
if let Some(command) = self.command.take() {
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
tokio::time::timeout(self.timeout, self.instance.run_passive_task(*command))
.await??;
crate::AnyResult::Ok(())
});
if let (Some(command), Some(instance)) = (self.command.take(), self.instance.take()) {
let _gt = tokio::runtime::Handle::current().enter();
let timeout = self.timeout;
tokio::task::spawn(async move {
tokio::time::timeout(timeout, instance.run_passive_task(*command)).await??;
crate::AnyResult::Ok(())
});
}
}
+7 -12
View File
@@ -286,23 +286,18 @@ impl NeoCam {
self.config_watch.send_replace(config);
Ok(())
}
async fn shutdown(&mut self) -> AnyResult<()> {
let _ = self.commander.send(NeoCamCommand::HangUp).await;
self.set.shutdown().await;
AnyResult::Ok(())
}
}
impl Drop for NeoCam {
fn drop(&mut self) {
log::trace!("Drop NeoCam");
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
let _ = self.shutdown().await;
AnyResult::Ok(())
});
let mut set = std::mem::take(&mut self.set);
let commander = self.commander.clone();
let _gt = tokio::runtime::Handle::current().enter();
tokio::task::spawn(async move {
let _ = commander.send(NeoCamCommand::HangUp).await;
while set.join_next().await.is_some() {}
log::trace!("Dropped NeoCam");
});
log::trace!("Dropped NeoCam");
}
}
+6 -7
View File
@@ -156,14 +156,13 @@ impl Drop for NeoReactor {
if let Ok(mut set) = Arc::try_unwrap(set) {
log::trace!("Drop NeoReactor");
self.cancel.cancel();
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
let _ = self.commander.send(NeoReactorCommand::HangUp).await;
while set.join_next().await.is_some() {}
AnyResult::Ok(())
});
let commander = self.commander.clone();
let _gt = tokio::runtime::Handle::current().enter();
tokio::task::spawn(async move {
let _ = commander.send(NeoReactorCommand::HangUp).await;
while set.join_next().await.is_some() {}
log::trace!("Dropped NeoReactor");
});
log::trace!("Dropped NeoReactor");
}
}
}
+6 -18
View File
@@ -300,16 +300,6 @@ impl StreamInstance {
}
}
impl Drop for StreamInstance {
fn drop(&mut self) {
tokio::task::block_in_place(move || {
tokio::runtime::Handle::current().block_on(async move {
let _ = self.in_use.deactivate().await;
});
});
}
}
impl StreamData {
async fn new(name: StreamKind, instance: NeoInstance, strict: bool) -> Result<Self> {
let (vid, _) = broadcast::<StampedData>(100);
@@ -589,14 +579,12 @@ impl Drop for StreamData {
fn drop(&mut self) {
log::trace!("Drop StreamData");
self.cancel.cancel();
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
if let Some(h) = self.handle.take() {
let _ = h.await;
}
AnyResult::Ok(())
if let Some(h) = self.handle.take() {
let _gt = tokio::runtime::Handle::current().enter();
tokio::task::spawn(async move {
let _ = h.await;
log::trace!("Dropped StreamData");
});
});
log::trace!("Dropped StreamData");
}
}
}
+9 -10
View File
@@ -72,12 +72,10 @@ impl Drop for UseCounter {
fn drop(&mut self) {
log::trace!("Drop UseCounter");
self.cancel.cancel();
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
while self.set.join_next().await.is_some() {}
AnyResult::Ok(())
});
});
let mut set = std::mem::take(&mut self.set);
let _gt = tokio::runtime::Handle::current().enter();
tokio::task::spawn(async move { while set.join_next().await.is_some() {} });
log::trace!("Dropped UseCounter");
}
}
@@ -148,10 +146,11 @@ impl Drop for Permit {
fn drop(&mut self) {
if self.is_active {
self.is_active = false;
tokio::task::block_in_place(move || {
tokio::runtime::Handle::current().block_on(async move {
let _ = self.notifier.send(self.is_active).await;
});
let _gt = tokio::runtime::Handle::current().enter();
let notifier = self.notifier.clone();
let is_active = self.is_active;
tokio::task::spawn(async move {
let _ = notifier.send(is_active).await;
});
}
}
+5 -6
View File
@@ -62,13 +62,12 @@ impl Drop for GstSender {
fn drop(&mut self) {
log::trace!("Drop GstSender");
self.cancel.cancel();
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
while self.set.join_next().await.is_some() {}
AnyResult::Ok(())
});
let _gt = tokio::runtime::Handle::current().enter();
let mut set = std::mem::take(&mut self.set);
tokio::task::spawn(async move {
while set.join_next().await.is_some() {}
log::trace!("Dropped GstSender");
});
log::trace!("Dropped GstSender");
}
}
+8
View File
@@ -22,6 +22,13 @@
//!
//! Neolink source code is available online at <https://github.com/thirtythreeforty/neolink>
//!
#[cfg(not(target_env = "msvc"))]
use tikv_jemallocator::Jemalloc;
#[cfg(not(target_env = "msvc"))]
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;
use anyhow::{Context, Result};
use clap::Parser;
use env_logger::Env;
@@ -47,6 +54,7 @@ use cmdline::{Command, Opt};
use common::NeoReactor;
use config::Config;
use console_subscriber as _;
pub(crate) type AnyResult<T> = Result<T, anyhow::Error>;
#[cfg(tokio_unstable)]
+25 -31
View File
@@ -132,34 +132,28 @@ pub(crate) async fn main(_: Opt, reactor: NeoReactor) -> Result<()> {
let thread_reactor2 = thread_reactor.clone();
let mqtt_instance = thread_instance.subscribe(name).await?;
let name = name.clone();
set.spawn_blocking(|| {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
runtime.block_on(async move {
loop {
let camera = thread_reactor2.get(&name).await?;
let mqtt_instance = mqtt_instance.resubscribe().await?;
let r = tokio::select!{
_ = thread_global_cancel.cancelled() => {
AnyResult::Ok(())
},
_ = local_cancel.cancelled() => {
AnyResult::Ok(())
},
v = listen_on_camera(camera, mqtt_instance) => {
v
},
};
if let Ok(()) = &r {
break r
} else {
log::debug!("listen_on_camera stopped: {:?}", r);
continue;
}
set.spawn(async move {
loop {
let camera = thread_reactor2.get(&name).await?;
let mqtt_instance = mqtt_instance.resubscribe().await?;
let r = tokio::select!{
_ = thread_global_cancel.cancelled() => {
AnyResult::Ok(())
},
_ = local_cancel.cancelled() => {
AnyResult::Ok(())
},
v = listen_on_camera(camera, mqtt_instance) => {
v
},
};
if let Ok(()) = &r {
break r
} else {
log::debug!("listen_on_camera stopped: {:?}", r);
continue;
}
})
}
}) ;
}
}
@@ -476,7 +470,7 @@ async fn listen_on_camera(camera: NeoInstance, mqtt_instance: MqttInstance) -> R
// Handle the battery publish
v = async {
let mut wait = IntervalStream::new({
let mut i = interval(Duration::from_millis(config.preview_update));
let mut i = interval(Duration::from_millis(config.battery_update));
i.set_missed_tick_behavior(MissedTickBehavior::Skip);
i
});
@@ -766,7 +760,7 @@ async fn handle_mqtt_message(
if let (Some(seconds), Some(bc_direction)) = (seconds, bc_direction) {
// On drop send the stop command again just to make sure it stops
let _drop_command = camera.clone().drop_command(
|cam| {
move |cam| {
Box::pin(async move {
cam.send_ptz(BcDirection::Stop, speed).await?;
AnyResult::Ok(())
@@ -1073,9 +1067,9 @@ async fn handle_mqtt_message(
}
Ok(bytes) => {
if let Err(e) = mqtt
.send_message("status/preview", BASE64.encode(image).as_str(), true)
.send_message("status/preview", BASE64.encode(bytes).as_str(), true)
.await
.with_context(|| format!("{}: Failed to publish preview", camera_name))
.with_context(|| "Failed to publish preview")
{
error!("Failed to send preview: {e:?}");
"FAIL"
+94 -83
View File
@@ -93,19 +93,21 @@ impl Mqtt {
impl Drop for Mqtt {
fn drop(&mut self) {
log::trace!("Drop MQTT");
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
let (tx, rx) = oneshot();
let _ = self.outgoing_tx.send(MqttRequest::HangUp(tx)).await;
let _ = rx.await;
let outgoing_tx = self.outgoing_tx.clone();
let cancel = self.cancel.clone();
let mut set = std::mem::take(&mut self.set);
log::debug!("Mqtt::drop Cancel");
self.cancel.cancel();
while self.set.join_next().await.is_some() {}
AnyResult::Ok(())
});
let _gt = tokio::runtime::Handle::current().enter();
tokio::task::spawn(async move {
let (tx, rx) = oneshot();
let _ = outgoing_tx.send(MqttRequest::HangUp(tx)).await;
let _ = rx.await;
log::debug!("Mqtt::drop Cancel");
cancel.cancel();
while set.join_next().await.is_some() {}
log::trace!("Dropped MQTT");
});
log::trace!("Dropped MQTT");
}
}
@@ -172,8 +174,6 @@ impl<'a> MqttBackend<'a> {
let (client, mut connection) = AsyncClient::new(mqttoptions, 100);
let mut set = JoinSet::new();
let client = Arc::new(client);
let send_client = client.clone();
send_client
@@ -185,6 +185,8 @@ impl<'a> MqttBackend<'a> {
)
.await?;
log::debug!("MQTT Published Startup");
let loop_cancel = CancellationToken::new();
let _drop_guard = loop_cancel.clone().drop_guard();
loop {
let r = tokio::select! {
v = self.outgoing_rx.recv() => {
@@ -194,64 +196,72 @@ impl<'a> MqttBackend<'a> {
let outgoing_tx = self.outgoing_tx.clone();
let incomming_tx = self.incomming_tx.clone();
let send_client = send_client.clone();
set.spawn(async move {
match msg {
MqttRequest::Send(msg, tx) => {
let v = send_client.publish(
msg.topic.clone(),
QoS::AtLeastOnce,
false,
msg.message.clone(),
).await;
match &v {
Ok(()) => {
let _ = tx.send(Ok(()));
},
Err(rumqttc::ClientError::Request(_)) | Err(rumqttc::ClientError::TryRequest(_)) => {
// Requeue it
outgoing_tx.send(MqttRequest::Send(msg, tx)).await?;
let cancel = self.cancel.clone();
let thread_cancel = loop_cancel.clone();
tokio::task::spawn(async move {
tokio::select!{
_ = cancel.cancelled() => AnyResult::Ok(()),
_ = thread_cancel.cancelled() => AnyResult::Ok(()),
v = async {
match msg {
MqttRequest::Send(msg, tx) => {
let v = send_client.publish(
msg.topic.clone(),
QoS::AtLeastOnce,
false,
(*msg.message).clone(),
).await;
match &v {
Ok(()) => {
let _ = tx.send(Ok(()));
},
Err(rumqttc::ClientError::Request(_)) | Err(rumqttc::ClientError::TryRequest(_)) => {
// Requeue it
outgoing_tx.send(MqttRequest::Send(msg, tx)).await?;
}
};
v?;
}
};
v?;
}
MqttRequest::SendRetained(msg, tx) => {
let v = send_client.publish(
msg.topic.clone(),
QoS::AtLeastOnce,
true,
msg.message.clone(),
).await;
match &v {
Ok(()) => {
let _ = tx.send(Ok(()));
},
Err(rumqttc::ClientError::Request(_)) | Err(rumqttc::ClientError::TryRequest(_)) => {
// Requeue it
outgoing_tx.send(MqttRequest::Send(msg, tx)).await?;
MqttRequest::SendRetained(msg, tx) => {
let v = send_client.publish(
msg.topic.clone(),
QoS::AtLeastOnce,
true,
(*msg.message).clone(),
).await;
match &v {
Ok(()) => {
let _ = tx.send(Ok(()));
},
Err(rumqttc::ClientError::Request(_)) | Err(rumqttc::ClientError::TryRequest(_)) => {
// Requeue it
outgoing_tx.send(MqttRequest::Send(msg, tx)).await?;
}
};
v?;
}
};
v?;
}
MqttRequest::HangUp(reply) => {
send_client.publish(
"neolink/status".to_string(),
QoS::AtLeastOnce,
true,
"disconnected".to_string(),
).await?;
let _ = reply.send(());
return Err(anyhow!("Disconneting"));
}
MqttRequest::Subscribe(name, reply) => {
let instance = MqttInstance {
name,
incomming_rx: BroadcastStream::new(incomming_tx.subscribe()),
outgoing_tx: outgoing_tx.clone(),
};
let _ = reply.send(Ok(instance));
}
MqttRequest::HangUp(reply) => {
send_client.publish(
"neolink/status".to_string(),
QoS::AtLeastOnce,
true,
"disconnected".to_string(),
).await?;
let _ = reply.send(());
return Err(anyhow!("Disconneting"));
}
MqttRequest::Subscribe(name, reply) => {
let instance = MqttInstance {
name,
incomming_rx: BroadcastStream::new(incomming_tx.subscribe()),
outgoing_tx: outgoing_tx.clone(),
};
let _ = reply.send(Ok(instance));
}
}
AnyResult::Ok(())
} => v,
}
AnyResult::Ok(())
});
AnyResult::Ok(())
@@ -262,9 +272,11 @@ impl<'a> MqttBackend<'a> {
let client = client.clone();
let incomming_tx = self.incomming_tx.clone();
let cancel = self.cancel.clone();
set.spawn(async move {
let thread_cancel = loop_cancel.clone();
tokio::task::spawn(async move {
tokio::select!{
_ = cancel.cancelled() => AnyResult::Ok(()),
_ = thread_cancel.cancelled() => AnyResult::Ok(()),
v = async {
match notification {
Event::Incoming(Incoming::ConnAck(connected)) => {
@@ -292,8 +304,8 @@ impl<'a> MqttBackend<'a> {
let _ = incomming_tx
.send(MqttReply {
topic: sub_topic.to_string(),
message: String::from_utf8_lossy(published_message.payload.as_ref())
.into_owned(),
message: Arc::new(String::from_utf8_lossy(published_message.payload.as_ref())
.into_owned()),
});
}
}
@@ -370,7 +382,7 @@ impl MqttInstance {
.send(MqttRequest::SendRetained(
MqttReply {
topic: topics.join("/"),
message: message.to_string(),
message: Arc::new(message.to_string()),
},
tx,
))
@@ -382,7 +394,7 @@ impl MqttInstance {
.send(MqttRequest::Send(
MqttReply {
topic: topics.join("/"),
message: message.to_string(),
message: Arc::new(message.to_string()),
},
tx,
))
@@ -439,7 +451,7 @@ impl MqttInstance {
message: &str,
) -> AnyResult<DropSender> {
Ok(DropSender {
instance: self.resubscribe().await?,
instance: Some(self.resubscribe().await?),
topic: topic.to_string(),
message: message.to_string(),
})
@@ -449,7 +461,7 @@ impl MqttInstance {
#[derive(Clone, Debug)]
pub(crate) struct MqttReply {
pub(crate) topic: String,
pub(crate) message: String,
pub(crate) message: Arc<String>, // Messages can be long so avoid costly clones with an arc
}
impl MqttReply {
@@ -474,19 +486,18 @@ enum MqttRequest {
}
pub(crate) struct DropSender {
instance: MqttInstance,
instance: Option<MqttInstance>,
topic: String,
message: String,
}
impl Drop for DropSender {
fn drop(&mut self) {
tokio::task::block_in_place(move || {
let _ = tokio::runtime::Handle::current().block_on(async move {
self.instance
.send_message(&self.topic, &self.message, true)
.await
});
});
if let Some(instance) = self.instance.take() {
let _gt = tokio::runtime::Handle::current().enter();
let topic = self.topic.clone();
let message = self.message.clone();
tokio::task::spawn(async move { instance.send_message(&topic, &message, true).await });
}
}
}
+41 -56
View File
@@ -175,23 +175,17 @@ pub(crate) async fn main(_opt: Opt, reactor: NeoReactor) -> Result<()> {
let thread_rtsp2 = thread_rtsp.clone();
let thread_reactor2 = thread_reactor.clone();
let name = name.clone();
set.spawn_blocking(|| {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
runtime.block_on(async move {
let camera = thread_reactor2.get(&name).await?;
tokio::select!(
_ = thread_global_cancel.cancelled() => {
AnyResult::Ok(())
},
_ = local_cancel.cancelled() => {
AnyResult::Ok(())
},
v = camera_main(camera, &thread_rtsp2) => v,
)
})
set.spawn(async move {
let camera = thread_reactor2.get(&name).await?;
tokio::select!(
_ = thread_global_cancel.cancelled() => {
AnyResult::Ok(())
},
_ = local_cancel.cancelled() => {
AnyResult::Ok(())
},
v = camera_main(camera, &thread_rtsp2) => v,
)
}) ;
}
}
@@ -991,11 +985,24 @@ async fn handle_data<T: Stream<Item = Result<StreamData, E>> + Unpin, E>(
mut data_rx: T,
) -> Result<()> {
if let Some(app) = app {
let mut set = JoinSet::<AnyResult<()>>::new();
let mut last_ft: Option<Duration> = None;
let mut ft = Duration::ZERO;
let mut last_rt: Option<Duration> = None;
let mut rt = Duration::ZERO;
let (gst_data_tx, mut gst_data_rx) = mpsc(100);
let appsrc = app.clone();
// push_buffer can block so we need to do this on a dedicated thread
// however it is expensive to call spawn_blocking too often
// so we call it once and push via a channel
tokio::task::spawn_blocking(move || {
while let Some(buf) = gst_data_rx.blocking_recv() {
appsrc
.push_buffer(buf)
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))?;
}
AnyResult::Ok(())
});
while let Some(Ok(data)) = data_rx.next().await {
check_live(app)?; // Stop if appsrc is dropped
// Cache the last runtime
@@ -1029,44 +1036,21 @@ async fn handle_data<T: Stream<Item = Result<StreamData, E>> + Unpin, E>(
ft = rt;
}
// Send frame to app src on another thread
let thread_app = app.clone();
let thread_rt = rt;
let thread_ft = ft;
set.spawn(async move {
check_live(&thread_app)?; // Stop if appsrc is dropped
if ft > (rt + Duration::from_millis(50)) {
let delta = ft - rt;
sleep(delta).await;
let buf = {
let mut gst_buf = gstreamer::Buffer::with_size(data.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
log::trace!("Setting PTS: {ft:?}, Runtime: {rt:?}");
let time = ClockTime::from_useconds(ft.as_micros() as u64);
gst_buf_mut.set_dts(time);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(data.as_slice());
}
let buf = {
let mut gst_buf = gstreamer::Buffer::with_size(data.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
log::trace!("Setting PTS: {thread_ft:?}, Runtime: {thread_rt:?}");
let time = ClockTime::from_useconds(thread_ft.as_micros() as u64);
gst_buf_mut.set_dts(time);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(data.as_slice());
}
gst_buf
};
gst_buf
};
log::trace!("Pushing Data");
let appsrc = thread_app.clone();
let mut inner_set = JoinSet::new();
inner_set.spawn_blocking(move || {
appsrc
.push_buffer(buf.copy())
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
});
inner_set.join_next().await.unwrap()??;
while inner_set.join_next().await.is_some() {}
log::trace!(" Pushed Data");
AnyResult::Ok(())
});
gst_data_tx.send(buf).await?;
}
}
}
@@ -1107,6 +1091,7 @@ fn clear_bin(bin: &Element) -> Result<()> {
Ok(())
}
fn build_unknown(bin: &Element) -> Result<()> {
let bin = bin
.clone()
@@ -1346,8 +1331,8 @@ fn make_queue(name: &str) -> AnyResult<Element> {
// Ok(queue)
let queue = make_element("queue2", name)?;
queue.set_property("max-size-bytes", 0u32);
queue.set_property("max-size-buffers", 0u32);
// queue.set_property("max-size-bytes", 0u32);
// queue.set_property("max-size-buffers", 0u32);
queue.set_property(
"max-size-time",
std::convert::TryInto::<u64>::try_into(tokio::time::Duration::from_secs(5).as_nanos())