Video streams using new method

This commit is contained in:
QuantumEntangledAndy authored and Andrew committed 2023-04-05 18:19:29 +07:00
1 parent 8b825cc5fa
commit ceff4c2495
7 files changed
+342 -211

No files matched your search

+19 -17
View File
@@ -48,26 +48,28 @@ impl<'a> Stream for BcPayloadStream<'a> {
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<IoResult<Vec<u8>>>> {
loop {
match Pin::new(&mut self.rx).poll_next(cx) {
Poll::Ready(Some(Ok(bc))) => match bc {
Bc {
body:
BcBody::ModernMsg(ModernMsg {
payload: Some(BcPayloads::Binary(data)),
..
}),
match Pin::new(&mut self.rx).poll_next(cx) {
Poll::Ready(Some(Ok(Bc {
body:
BcBody::ModernMsg(ModernMsg {
payload: Some(BcPayloads::Binary(data)),
..
} => return Poll::Ready(Some(Ok(data))),
_ => continue,
},
Poll::Ready(Some(Err(e))) => {
return Poll::Ready(Some(Err(IoError::new(ErrorKind::Other, e))))
}
Poll::Ready(None) => return Poll::Ready(None),
Poll::Pending => return Poll::Pending,
}),
..
}))) => {
return Poll::Ready(Some(Ok(data)));
}
Poll::Ready(Some(Ok(_bc))) => {
// trace!("Got other BC in payload stream");
}
Poll::Ready(Some(Err(e))) => {
return Poll::Ready(Some(Err(IoError::new(ErrorKind::Other, e))))
}
Poll::Ready(None) => return Poll::Ready(None),
Poll::Pending => return Poll::Pending,
}
cx.waker().wake_by_ref();
Poll::Pending
}
}
@@ -163,7 +163,7 @@ impl BcUdpSource {
recieved: Default::default(),
state: State::Normal,
send_buffer: Default::default(),
ack_interval: interval(Duration::from_millis(10)), // Offical Client does ack every 10ms
ack_interval: interval(Duration::from_millis(100)), // Offical Client does ack every 10ms
resend_interval: interval(Duration::from_millis(500)), // Offical Client does resend every 500ms
}
}
@@ -226,7 +226,11 @@ pub(crate) struct UdpPayloadSource {
impl UdpPayloadSource {
fn build_send_ack(&self) -> UdpAck {
if self.packets_want > 0 {
let start: u32 = self.packets_want - 1;
let mut start: u32 = self.packets_want - 1;
while self.recieved.contains_key(&start) {
// Happens if we have recieved but not consumed yet
start += 1;
}
let missing_ids = if let Some(end) = self.recieved.keys().max() {
let mut vec = vec![];
// From last contiguous packet to last recieved packet
@@ -257,8 +261,8 @@ impl UdpPayloadSource {
fn handle_ack(&mut self, ack: UdpAck) {
let start = ack.packet_id;
// -1 is the special case of no messages recieved
if start != 0xFFFFFFFF {
if start != 0xffffffff {
// -1 means havent got anything yet
self.sent.retain(|&k, _| k > start);
for (idx, &value) in ack.payload.iter().enumerate() {
@@ -291,98 +295,120 @@ impl Stream for UdpPayloadSource {
type Item = IoResult<Vec<u8>>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
trace!("UdpPayloadSource.poll_next");
let camera_addr = self.inner.addr;
let mut this = self.get_mut();
loop {
match this.state {
State::Normal => {
trace!("UDPSource.State: Normal");
// Data ready to go
if let Some(payload) = this.recieved.remove(&this.packets_want) {
trace!("UDPSource.Data: PreReady: {}", this.packets_want);
this.packets_want += 1;
let ack = BcUdp::Ack(this.build_send_ack());
this.send_buffer.push_back(ack);
return Poll::Ready(Some(Ok(payload)));
}
// Handle resend events
this.maintanence(cx);
// Normal behaviors
match this.inner.poll_next_unpin(cx) {
Poll::Ready(Some(Ok((
BcUdp::Data(UdpData {
connection_id,
packet_id,
payload,
}),
addr,
)))) if connection_id == this.client_id
&& addr == camera_addr
&& packet_id >= this.packets_want =>
{
if packet_id == this.packets_want {
trace!("UDPSource.RecievedPacker: NewData: {}", this.packets_want);
this.packets_want += 1;
let ack = BcUdp::Ack(this.build_send_ack());
this.send_buffer.push_back(ack);
return Poll::Ready(Some(Ok(payload)));
} else {
trace!("UDPSource.RecievedPacker: OtherData");
this.recieved.insert(packet_id, payload);
}
}
Poll::Ready(Some(Ok((
BcUdp::Ack(ack @ UdpAck { connection_id, .. }),
addr,
)))) if connection_id == this.client_id && addr == camera_addr => {
trace!("UDPSource.RecievedPacket: Ack");
this.handle_ack(ack);
// Rather then immediatly flush wait for the next call
// this.state = State::Flushing;
}
Poll::Ready(Some(Err(e))) => {
trace!("UDPSource.RecievedPacker: Error");
return Poll::Ready(Some(Err(IoError::new(ErrorKind::Other, e))));
}
Poll::Ready(None) => {
trace!("UDPSource.RecievedPacker: Empty");
return Poll::Ready(None);
}
Poll::Pending => {
return Poll::Pending;
}
Poll::Ready(Some(Ok((bcudp, addr)))) => {
match this.state {
State::Normal => {
trace!("UdpPayloadSource.State: Normal");
// Data ready to go
if let Some(payload) = this.recieved.remove(&this.packets_want) {
trace!("UdpPayloadSource.Data: PreReady: {}", this.packets_want);
this.packets_want += 1;
// error!("packets_want: {}", this.packets_want);
let ack = BcUdp::Ack(this.build_send_ack());
this.send_buffer.push_back(ack);
return Poll::Ready(Some(Ok(payload)));
}
// Handle resend events
this.maintanence(cx);
// Normal behaviors
match this.inner.poll_next_unpin(cx) {
Poll::Ready(Some(Ok((
BcUdp::Data(UdpData {
connection_id,
packet_id,
payload,
}),
addr,
)))) if connection_id == this.client_id
&& addr == camera_addr
&& packet_id >= this.packets_want =>
{
if packet_id == this.packets_want {
trace!(
"UDPSource.RecievedPacker: UnexpectedData {:?} from {}",
bcudp,
addr
"UdpPayloadSource.RecievedPacker: NewData: {}",
this.packets_want
);
} // _ => {
// trace!("UDPSource.RecievedPacker: Other?");
// // Repeat/unintersting packet
// }
}
}
State::Flushing => {
trace!("UDPSource.State: Flushing");
match this.poll_flush_unpin(cx) {
Poll::Ready(Ok(())) => {
this.state = State::Normal;
}
Poll::Ready(Err(e)) => {
return Poll::Ready(Some(Err(e)));
}
Poll::Pending => {
return Poll::Pending;
this.packets_want += 1;
// error!("packets_want: {}", this.packets_want);
let ack = BcUdp::Ack(this.build_send_ack());
this.send_buffer.push_back(ack);
return Poll::Ready(Some(Ok(payload)));
} else {
trace!(
"UdpPayloadSource.RecievedPacker: Future Data: {}",
packet_id
);
this.recieved.insert(packet_id, payload);
}
}
}
State::Closed => {
trace!("UDPSource.State: Closed");
return Poll::Ready(Some(Err(IoError::from(ErrorKind::ConnectionAborted))));
Poll::Ready(Some(Ok((
BcUdp::Data(UdpData {
connection_id,
packet_id,
..
}),
addr,
)))) if connection_id == this.client_id
&& addr == camera_addr
&& packet_id < this.packets_want =>
{
trace!("UdpPayloadSource.RecievedPacket: Old data: {}", packet_id);
}
Poll::Ready(Some(Ok((
BcUdp::Ack(ack @ UdpAck { connection_id, .. }),
addr,
)))) if connection_id == this.client_id && addr == camera_addr => {
trace!("UdpPayloadSource.RecievedPacket: Ack: {:?}", ack);
this.handle_ack(ack);
// Rather then immediatly flush wait for the next call
// this.state = State::Flushing;
}
Poll::Ready(Some(Err(e))) => {
trace!("UdpPayloadSource.RecievedPacker: Error");
return Poll::Ready(Some(Err(IoError::new(ErrorKind::Other, e))));
}
Poll::Ready(None) => {
trace!("UdpPayloadSource.RecievedPacker: Empty");
return Poll::Ready(None);
}
Poll::Pending => {
return Poll::Pending;
}
Poll::Ready(Some(Ok((bcudp, addr)))) => {
trace!(
"UdpPayloadSource.RecievedPacker: UnexpectedData {:?} from {}",
bcudp,
addr
);
} // _ => {
// trace!("UDPSource.RecievedPacker: Other?");
// // Repeat/unintersting packet
// }
}
}
State::Flushing => {
trace!("UdpPayloadSource.State: Flushing");
match this.poll_flush_unpin(cx) {
Poll::Ready(Ok(())) => {
this.state = State::Normal;
}
Poll::Ready(Err(e)) => {
return Poll::Ready(Some(Err(e)));
}
Poll::Pending => {
return Poll::Pending;
}
}
}
State::Closed => {
trace!("UdpPayloadSource.State: Closed");
return Poll::Ready(Some(Err(IoError::from(ErrorKind::ConnectionAborted))));
}
}
cx.waker().wake_by_ref();
Poll::Pending
}
}
@@ -434,43 +460,43 @@ impl Sink<Vec<u8>> for UdpPayloadSource {
cx: &mut Context<'_>,
) -> Poll<std::result::Result<(), Self::Error>> {
let this = self.get_mut();
loop {
match this
.inner
.poll_flush_unpin(cx)
.map_err(|e| IoError::new(ErrorKind::Other, e))
{
Poll::Ready(Ok(())) => {
if let Some(next) = this.send_buffer.pop_front() {
match this
.inner
.poll_ready_unpin(cx)
.map_err(|e| IoError::new(ErrorKind::Other, e))
{
Poll::Ready(Ok(())) => {
if let BcUdp::Data(data) = &next {
let id = data.packet_id;
this.sent.insert(id, data.clone());
}
match this
.inner
.poll_flush_unpin(cx)
.map_err(|e| IoError::new(ErrorKind::Other, e))
{
Poll::Ready(Ok(())) => {
if let Some(next) = this.send_buffer.pop_front() {
match this
.inner
.poll_ready_unpin(cx)
.map_err(|e| IoError::new(ErrorKind::Other, e))
{
Poll::Ready(Ok(())) => {
if let BcUdp::Data(data) = &next {
let id = data.packet_id;
this.sent.insert(id, data.clone());
}
let addr = this.inner.addr;
Pin::new(&mut this.inner)
.start_send((next, addr))
.map_err(|e| IoError::new(ErrorKind::Other, e))?;
}
poll => {
return poll;
}
let addr = this.inner.addr;
Pin::new(&mut this.inner)
.start_send((next, addr))
.map_err(|e| IoError::new(ErrorKind::Other, e))?;
}
poll => {
return poll;
}
} else {
return Poll::Ready(Ok(()));
}
}
poll => {
return poll;
} else {
return Poll::Ready(Ok(()));
}
}
poll => {
return poll;
}
}
cx.waker().wake_by_ref();
Poll::Pending
}
fn poll_close(
self: Pin<&mut Self>,
+3
View File
@@ -58,6 +58,9 @@ impl Decoder for BcMediaCodex {
} else if src.is_empty() {
return Ok(None);
} else {
if amount_skipped == 0 {
trace!("Error in stream attempting to restore: {:?}", e);
}
src.advance(1);
amount_skipped += 1;
continue;
+10 -1
View File
@@ -106,8 +106,15 @@ pub(crate) struct NeoMediaFactoryImpl {
threads: JoinSet<AnyResult<()>>,
}
impl Drop for NeoMediaFactoryImpl {
fn drop(&mut self) {
log::info!("Dopping NeoMediaFactoryImpl");
}
}
impl Default for NeoMediaFactoryImpl {
fn default() -> Self {
warn!("Constructing Factor Impl");
let (datasender, datarx) = channel(3);
let (appsender, rx_appsender) = channel(3);
let shared: Arc<NeoMediaShared> = Default::default();
@@ -187,8 +194,10 @@ impl NeoMediaFactoryImpl {
for element in bin.iterate_elements().into_iter().flatten() {
bin.remove(&element)?;
}
debug!("self.shared owners: {}", Arc::strong_count(&self.shared));
// Now contruct the actual ones
match self.shared.vid_format.load(Ordering::Relaxed) {
match VidFormats::from(self.shared.vid_format.load(Ordering::Relaxed)) {
VidFormats::H265 => {
debug!("Building H265 Pipeline");
let source = make_element("appsrc", "vidsrc")?
+108 -66
View File
@@ -1,31 +1,46 @@
//! The component that handles passing BcMedia into
//! gstreamer media stream
use gstreamer_app::AppSrc;
use anyhow::anyhow;
use futures::stream::StreamExt;
use gstreamer::ClockTime;
use gstreamer_app::AppSrc;
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
use log::*;
use neolink_core::bcmedia::model::*;
use std::sync::{atomic::Ordering, Arc};
use tokio::time::{interval, Duration, Instant, MissedTickBehavior};
use tokio::time::{interval, Duration, MissedTickBehavior};
use tokio_stream::wrappers::ReceiverStream;
use super::{shared::*, AnyResult};
#[derive(Hash, PartialEq, Eq)]
pub(super) struct ClientData {
pub(super) appsrc: AppSrc,
pub(super) start_time: u64,
}
pub(super) struct NeoMediaSender {
pub(super) data_source: ReceiverStream<BcMedia>,
pub(super) app_source: ReceiverStream<AppSrc>,
pub(super) shared: Arc<NeoMediaShared>,
pub(super) appsrcs: Vec<AppSrc>,
pub(super) appsrcs: std::collections::HashSet<ClientData>,
pub(super) waiting_for_iframe: bool,
}
#[derive(Clone)]
struct StampedData {
ms: u64,
data: Vec<u8>,
}
struct ProcessData<'a> {
live: StampedData,
resend: &'a [StampedData],
}
impl NeoMediaSender {
pub(super) async fn run(&mut self) -> AnyResult<()> {
let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 25.0f32));
let mut resend_pause = interval(Duration::from_secs_f32(1.0f32 / 5.0f32));
resend_pause.set_missed_tick_behavior(MissedTickBehavior::Skip);
let mut last_iframe: Option<(Instant, u64, Vec<u8>)> = None;
let mut buffer: Vec<StampedData> = Default::default();
loop {
tokio::select! {
v = self.data_source.next() => {
@@ -35,15 +50,35 @@ impl NeoMediaSender {
self.inspect_bcmedia(&bc_media)?;
match bc_media {
BcMedia::Iframe(frame) => {
last_iframe = Some((
Instant::now(),
self.shared.microseconds.load(Ordering::Relaxed),
frame.data.clone(),
));
self.process_buffer(&frame.data)?;
buffer.clear();
buffer.push(
StampedData {
ms: frame.microseconds as u64,
data: frame.data.clone(),
}
);
self.process_buffer(&ProcessData {
live: StampedData{
ms: frame.microseconds as u64,
data: frame.data,
},
resend: buffer.as_slice()
})?;
}
BcMedia::Pframe(frame) => {
self.process_buffer(&frame.data)?;
buffer.push(
StampedData {
ms: frame.microseconds as u64,
data: frame.data.clone(),
}
);
self.process_buffer(&ProcessData {
live: StampedData{
ms: frame.microseconds as u64,
data: frame.data,
},
resend: buffer.as_slice()
})?;
}
_ => {}
}
@@ -53,18 +88,21 @@ impl NeoMediaSender {
}
}
v = self.app_source.next() => {
if let Some(app_src) = v {
self.appsrcs.push(app_src);
if let Some(appsrc) = v {
self.appsrcs.insert(ClientData {
appsrc,
start_time: self.shared.microseconds.load(Ordering::Relaxed)
});
} else {
break;
}
},
v = resend_pause.tick() => {
if let Some((time,microseconds, last_iframe)) = last_iframe.as_ref() {
let passed: Duration = v - *time;
let ms = *microseconds + passed.as_micros() as u64;
self.shared.microseconds.store(ms, Ordering::Relaxed);
self.process_buffer(last_iframe)?;
_ = resend_pause.tick() => {
if let Some(live) = buffer.last().cloned() {
self.process_buffer(&ProcessData {
live,
resend: buffer.as_slice()
})?;
}
}
}
@@ -72,58 +110,62 @@ impl NeoMediaSender {
Ok(())
}
fn process_buffer(&mut self, buf: &Vec<u8>) -> AnyResult<()> {
fn process_buffer(&mut self, process_data: &ProcessData) -> AnyResult<()> {
let frame_ms = process_data.live.ms;
self.appsrcs.retain(|data| {
let start_time = data.start_time;
// If frame_ms < our start time then we just joined the stream
// Send all data from the last iframe to catch up
if frame_ms < start_time {
for (idx, resend_buffer) in process_data.resend.iter().enumerate() {
if Self::send_buffer(&data.appsrc, resend_buffer.data.as_slice(), idx as u64, 0)
.is_err()
{
return false; // If fails then this appsrc is dead
}
}
}
Self::send_buffer(
&data.appsrc,
process_data.live.data.as_slice(),
process_data.live.ms,
data.start_time,
)
.is_ok() // If ok retain is true
});
Ok(())
}
fn send_buffer(appsrc: &AppSrc, buf: &[u8], frame_ms: u64, start_time: u64) -> AnyResult<()> {
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let time = ClockTime::from_useconds(self.shared.microseconds.load(Ordering::Relaxed));
let micros = if frame_ms > start_time {
frame_ms - start_time
} else {
start_time
};
let time = ClockTime::from_useconds(micros);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(buf);
}
for appsrc in self.appsrcs.iter() {
appsrc.push_buffer(gst_buf.copy())?;
}
Ok(())
appsrc
.push_buffer(gst_buf.copy())
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
}
fn skip_bcmedia(&mut self, bc_media: &BcMedia) -> AnyResult<bool> {
if self.waiting_for_iframe {
if let BcMedia::Iframe(frame) = bc_media {
let ms = frame.microseconds as u64;
let current_ms = self.shared.microseconds.load(Ordering::Relaxed);
if ms > current_ms {
self.waiting_for_iframe = false;
} else {
debug!(
"Waiting on iframe but it was in the past: got {} but on {}",
ms, current_ms
);
return Ok(true);
}
if let BcMedia::Iframe(_) = bc_media {
self.waiting_for_iframe = false;
} else {
debug!("Waaiting on iframe");
return Ok(true);
}
}
match bc_media {
BcMedia::Iframe(frame) => {
let ms = frame.microseconds as u64;
if ms < self.shared.microseconds.load(Ordering::Relaxed) {
debug!("Got iframe in the past");
return Ok(true);
}
}
BcMedia::Pframe(frame) => {
let ms = frame.microseconds as u64;
if ms < self.shared.microseconds.load(Ordering::Relaxed) {
debug!("Got pframe in the past");
return Ok(true);
}
}
_ => {}
}
Ok(false)
}
@@ -134,12 +176,12 @@ impl NeoMediaSender {
VideoType::H264 => {
self.shared
.vid_format
.store(VidFormats::H264, Ordering::Relaxed);
.store(VidFormats::H264.into(), Ordering::Relaxed);
}
VideoType::H265 => {
self.shared
.vid_format
.store(VidFormats::H265, Ordering::Relaxed);
.store(VidFormats::H265.into(), Ordering::Relaxed);
}
}
self.shared
@@ -151,12 +193,12 @@ impl NeoMediaSender {
VideoType::H264 => {
self.shared
.vid_format
.store(VidFormats::H264, Ordering::Relaxed);
.store(VidFormats::H264.into(), Ordering::Relaxed);
}
VideoType::H265 => {
self.shared
.vid_format
.store(VidFormats::H265, Ordering::Relaxed);
.store(VidFormats::H265.into(), Ordering::Relaxed);
}
}
self.shared
@@ -166,12 +208,12 @@ impl NeoMediaSender {
BcMedia::Aac(_aac) => {
self.shared
.aud_format
.store(AudFormats::Aac, Ordering::Relaxed);
.store(AudFormats::Aac.into(), Ordering::Relaxed);
}
BcMedia::Adpcm(_) => {
self.shared
.aud_format
.store(AudFormats::Adpcm, Ordering::Relaxed);
.store(AudFormats::Adpcm.into(), Ordering::Relaxed);
}
_ => {}
}
+47 -8
View File
@@ -1,11 +1,9 @@
//! Data shared between the various
//! components that manage a media stream
use atomic_enum::atomic_enum;
pub use gstreamer_rtsp_server::gio::{TlsAuthenticationMode, TlsCertificate};
use std::convert::{From, Into};
use std::sync::atomic::AtomicU64;
#[atomic_enum]
#[derive(PartialEq)]
pub(super) enum VidFormats {
Unknown = 0,
@@ -13,7 +11,27 @@ pub(super) enum VidFormats {
H265,
}
#[atomic_enum]
impl From<VidFormats> for u64 {
fn from(value: VidFormats) -> u64 {
match value {
VidFormats::Unknown => 0,
VidFormats::H264 => 1,
VidFormats::H265 => 2,
}
}
}
impl From<u64> for VidFormats {
fn from(value: u64) -> Self {
match value {
0 => VidFormats::Unknown,
1 => VidFormats::H264,
2 => VidFormats::H265,
_ => unreachable!(),
}
}
}
#[derive(PartialEq)]
pub(super) enum AudFormats {
Unknown = 0,
@@ -21,17 +39,38 @@ pub(super) enum AudFormats {
Adpcm,
}
impl From<AudFormats> for u64 {
fn from(value: AudFormats) -> u64 {
match value {
AudFormats::Unknown => 0,
AudFormats::Aac => 1,
AudFormats::Adpcm => 2,
}
}
}
impl From<u64> for AudFormats {
fn from(value: u64) -> Self {
match value {
0 => AudFormats::Unknown,
1 => AudFormats::Aac,
2 => AudFormats::Adpcm,
_ => unreachable!(),
}
}
}
pub(super) struct NeoMediaShared {
pub(super) vid_format: AtomicVidFormats,
pub(super) aud_format: AtomicAudFormats,
pub(super) vid_format: AtomicU64,
pub(super) aud_format: AtomicU64,
pub(super) microseconds: AtomicU64,
}
impl Default for NeoMediaShared {
fn default() -> Self {
Self {
vid_format: AtomicVidFormats::new(VidFormats::Unknown),
aud_format: AtomicAudFormats::new(AudFormats::Unknown),
vid_format: AtomicU64::new(VidFormats::Unknown.into()),
aud_format: AtomicU64::new(AudFormats::Unknown.into()),
microseconds: AtomicU64::new(0),
}
}
+11 -1
View File
@@ -8,7 +8,7 @@ use log::*;
use std::collections::HashMap;
use tokio::task::{self, JoinHandle};
use neolink_core::bc_protocol::StreamKind as Stream;
use neolink_core::{bc_protocol::StreamKind as Stream, bcmedia::model::BcMedia};
use super::{CameraState, Shared};
@@ -54,7 +54,17 @@ impl CameraState for Streaming {
.start_video(stream_thead, 0, strict_thread)
.await?;
while thread_abort_handle.is_live() {
trace!("BcMediaStreamRecv");
let data = stream_data.get_data().await?;
match &data {
Ok(BcMedia::InfoV1(_)) => trace!(" - InfoV1"),
Ok(BcMedia::InfoV2(_)) => trace!(" - InfoV2"),
Ok(BcMedia::Iframe(_)) => trace!(" - Iframe"),
Ok(BcMedia::Pframe(_)) => trace!(" - Pframe"),
Ok(BcMedia::Aac(_)) => trace!(" - Aac"),
Ok(BcMedia::Adpcm(_)) => trace!(" - Adpcm"),
Err(_) => trace!(" - Error"),
}
sender.send(data?).await?;
}
Ok(())