Audio and Format auto detect (#68)
* Add queues to the gstreamer launch * Add audio to the stream * Removing format option in prep for auto detect * Adding format detect from headers * Set format in gstreamer launch depending on video magic bytes * Rename MaybeAppSrcs to reflect new role of managing gst outputs * Auto detect format * Remove all setting of stream format * Re-add missing media_format caused by rebase * Add audio format auto detect too * Use more appropiate adpcm in gst * Combined gst and media_packet format enums into a single enum StreamFormat * Reduce adpcm header size * Disable adpcm output * Add oki adpcm native rust support * Maintain state between packets for adpcm * Use bytes in adpcm stream to determine adpcm decoding state * Clean up adpcm code * Use raw rtp payload for the adpcm stream * Bug fix to treat adpcm and aac seperatly in the match * Remove redundent second clamp of the step index in adpcm. * ADPCM format changes. This is a squashed commit of changes made during live testing of ADPCM format. Summary: Using DVI4 for the ADPCM, with our own decoder written in rust. * Changes to comments and IMA max range in the adpcm decoder * ADPCM magic contains block size use it for chunking the adpcm data * Use error logs rather than asserts in adpcm decode * Rustfmt and clippy styling * Use static slice for adpcm step and changes arrays * Bounds and error checks in ADPCM * Use i/u32 in adpcm * Rust fmted
This commit is contained in:
1 parent
b5312edb90
commit
f7f12c76a1
6 files changed
+462
-95
No files matched your search
+23
-6
@@ -2,6 +2,8 @@ use self::connection::BcConnection;
|
||||
use self::media_packet::{MediaDataKind, MediaDataSubscriber};
|
||||
use crate::bc;
|
||||
use crate::bc::{model::*, xml::*};
|
||||
use crate::gst::GstOutputs;
|
||||
use adpcm::adpcm_to_pcm;
|
||||
use err_derive::Error;
|
||||
use log::*;
|
||||
use std::io::Write;
|
||||
@@ -10,6 +12,7 @@ use std::time::Duration;
|
||||
|
||||
use Md5Trunc::*;
|
||||
|
||||
mod adpcm;
|
||||
mod connection;
|
||||
mod media_packet;
|
||||
mod time;
|
||||
@@ -49,12 +52,12 @@ pub enum Error {
|
||||
#[error(display = "Timeout")]
|
||||
Timeout(#[error(source)] std::sync::mpsc::RecvTimeoutError),
|
||||
|
||||
#[error(display = "Media")]
|
||||
MediaPacket(#[error(source)] self::media_packet::Error),
|
||||
|
||||
#[error(display = "Credential error")]
|
||||
AuthFailed,
|
||||
|
||||
#[error(display = "ADPCM Decoding Error")]
|
||||
AdpcmDecodingError(&'static str),
|
||||
|
||||
#[error(display = "Other error")]
|
||||
Other(&'static str),
|
||||
}
|
||||
@@ -245,7 +248,7 @@ impl BcCamera {
|
||||
|
||||
pub fn start_video(
|
||||
&self,
|
||||
data_out: &mut dyn Write,
|
||||
data_outs: &mut GstOutputs,
|
||||
stream_name: &str,
|
||||
channel_id: u32,
|
||||
) -> Result<Never> {
|
||||
@@ -278,12 +281,26 @@ impl BcCamera {
|
||||
let mut media_sub = MediaDataSubscriber::from_bc_sub(&sub_video);
|
||||
|
||||
loop {
|
||||
let binary_data = media_sub.next_media_packet(RX_TIMEOUT)?;
|
||||
let binary_data = media_sub.next_media_packet()?;
|
||||
// We now have a complete interesting packet. Send it to gst.
|
||||
// Process the packet
|
||||
match binary_data.kind() {
|
||||
MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => {
|
||||
data_out.write_all(binary_data.body())?;
|
||||
let media_format = binary_data.media_format();
|
||||
data_outs.set_format(media_format);
|
||||
data_outs.vidsrc.write_all(binary_data.body())?;
|
||||
}
|
||||
MediaDataKind::AudioDataAac => {
|
||||
let media_format = binary_data.media_format();
|
||||
data_outs.set_format(media_format);
|
||||
data_outs.audsrc.write_all(binary_data.body())?;
|
||||
}
|
||||
MediaDataKind::AudioDataAdpcm => {
|
||||
let media_format = binary_data.media_format();
|
||||
data_outs.set_format(media_format);
|
||||
let adpcm = binary_data.body();
|
||||
let pcm = adpcm_to_pcm(adpcm)?;
|
||||
data_outs.audsrc.write_all(&pcm)?;
|
||||
}
|
||||
_ => {}
|
||||
};
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
/*
|
||||
This is a rust implementation of OKI and DVI/IMA ADPCM.
|
||||
*/
|
||||
use crate::Error;
|
||||
use log::error;
|
||||
use std::convert::TryInto;
|
||||
|
||||
struct AdpcmSetup {
|
||||
max_step_index: u32,
|
||||
steps: &'static [u32],
|
||||
max_sample_size: i32,
|
||||
changes: &'static [i32],
|
||||
}
|
||||
|
||||
impl AdpcmSetup {
|
||||
// Unused, originally we thought BC might be using OKI but it is actually DVI4
|
||||
fn new_oki() -> Self {
|
||||
Self {
|
||||
max_step_index: 48,
|
||||
steps: &[
|
||||
16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97,
|
||||
107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449,
|
||||
494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552,
|
||||
],
|
||||
changes: &[-1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8],
|
||||
max_sample_size: 2048,
|
||||
}
|
||||
}
|
||||
|
||||
// This is IMA format, but it is the same as DVI4 format except in the block header
|
||||
fn new_ima() -> Self {
|
||||
Self {
|
||||
max_step_index: 88,
|
||||
steps: &[
|
||||
7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50,
|
||||
55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279,
|
||||
307, 337, 371, 408, 449, 494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282,
|
||||
1411, 1552, 1707, 1878, 2066, 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871,
|
||||
5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899, 15289, 16818,
|
||||
18500, 20350, 22385, 24623, 27086, 29794, 32767,
|
||||
],
|
||||
changes: &[-1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8],
|
||||
max_sample_size: 32768,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Nibble {
|
||||
// A nibble is a 4bit int
|
||||
data: u8, // This is the raw data for the nibble
|
||||
}
|
||||
|
||||
impl Nibble {
|
||||
// Use u/i32 throughout to ensure that we always have enough
|
||||
// Headroom to do the math without needing `as` casting everywhere
|
||||
fn unsigned(&self) -> u32 {
|
||||
(self.data & 0b00001111) as u32 // Mask first 4 bits it just to be sure its in nibble range
|
||||
}
|
||||
|
||||
fn signed_magnitude(&self) -> u32 {
|
||||
(self.data & 0b00000111) as u32 // Mask of first 3 bits which are the magnitiude bits in signed int
|
||||
}
|
||||
|
||||
fn signed(&self) -> i32 {
|
||||
match self.data & 0b00001000 {
|
||||
// Sign bit is at the 4th bit
|
||||
0b00001000 => -(self.signed_magnitude() as i32),
|
||||
_ => self.signed_magnitude() as i32,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_byte(byte: &u8) -> [Self; 2] {
|
||||
// Two nibbles per byte
|
||||
[
|
||||
Self {
|
||||
data: (byte & 0b11110000) >> 4,
|
||||
},
|
||||
Self {
|
||||
data: byte & 0b00001111,
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
pub fn adpcm_to_pcm(bytes: &[u8]) -> Result<Vec<u8>, Error> {
|
||||
let context = AdpcmSetup::new_ima();
|
||||
|
||||
let mut result: Vec<u8> = vec![]; // Stores the PCM byte array
|
||||
|
||||
// ADPCM is not really a streamable format
|
||||
// Each audio sample requires information on the previous sample
|
||||
// To solve this reolink caches the intermediate variables (step_index and last_output)
|
||||
// Into the block header in the stream. When ever an adpcm packet arrives it starts with
|
||||
// 0x0001 which is the frame time from HISilicon documentation (I think) following by
|
||||
// 0xWW which is half the block size
|
||||
// 0xYY which is the last output
|
||||
// 0xZZ which is the step index of the last output
|
||||
// We must initialise our decoder with this data
|
||||
|
||||
if bytes.len() < 4 {
|
||||
error!("ADPCM data is too short for even the magic.");
|
||||
return Err(Error::AdpcmDecodingError(
|
||||
"ADPCM data is too short for even the magic.",
|
||||
));
|
||||
}
|
||||
|
||||
// Check for valid number of frame type
|
||||
let frame_type_bytes = &bytes[0..2];
|
||||
const FRAME_TYPE_HISILICON: &[u8] = &[0x00, 0x01];
|
||||
if frame_type_bytes != FRAME_TYPE_HISILICON {
|
||||
error!("Unexpected ADPCM frame type: {:x?}", frame_type_bytes);
|
||||
return Err(Error::AdpcmDecodingError("Unexpected ADPCM frame type"));
|
||||
}
|
||||
|
||||
// Check for valid block size
|
||||
let block_size_bytes = &bytes[2..4];
|
||||
let block_size = (u16::from_le_bytes(
|
||||
block_size_bytes
|
||||
.try_into()
|
||||
.expect("slice with incorrect length"),
|
||||
) as u32)
|
||||
* 2; // Block size is stored as 1/2 (don't know why)
|
||||
let full_block_size = block_size + 4; // block_size + magic (2 bytes) + size (2 bytes)
|
||||
if !bytes.len() % full_block_size as usize == 0 {
|
||||
error!("ADPCM Data is not a multiple of the block size");
|
||||
return Err(Error::AdpcmDecodingError(
|
||||
"ADPCM block size does not match data length.",
|
||||
));
|
||||
}
|
||||
|
||||
// Chunk on block size
|
||||
for bytes in bytes.chunks(full_block_size as usize) {
|
||||
// Get predictor state from block header using DVI 4 format.
|
||||
if bytes.len() < 8 {
|
||||
error!("ADPCM Block size is not long enough for header");
|
||||
return Err(Error::AdpcmDecodingError(
|
||||
"ADPCM has insufficent block size",
|
||||
));
|
||||
}
|
||||
let step_output_bytes = &bytes[4..6];
|
||||
let mut last_output = i16::from_le_bytes(
|
||||
step_output_bytes
|
||||
.try_into()
|
||||
.expect("slice with incorrect length"),
|
||||
) as i32;
|
||||
let step_index_bytes = &bytes[6..8];
|
||||
let mut step_index = u16::from_le_bytes(
|
||||
step_index_bytes
|
||||
.try_into()
|
||||
.expect("slice with incorrect length"),
|
||||
) as i32;
|
||||
|
||||
// To avoid casting to u8 <-> u16 <-> u32 and back all the time I just do all maths in u/i32
|
||||
// This gives enough headroom to do all calculations without overflow because adpcm puts artifical
|
||||
// limits on the sample sizes
|
||||
let mut step: u32;
|
||||
|
||||
// The rest is all data to be decoded
|
||||
let data = &bytes[8..];
|
||||
|
||||
for byte in data {
|
||||
let nibbles: [Nibble; 2] = Nibble::from_byte(byte);
|
||||
for nibble in &nibbles {
|
||||
let unibble = nibble.unsigned();
|
||||
|
||||
// Specifications say: Clamp it in max index range 0..context.max_step_index
|
||||
step_index = match step_index {
|
||||
n if n < 0 => 0,
|
||||
n if n > context.max_step_index as i32 => context.max_step_index as i32,
|
||||
n => n,
|
||||
};
|
||||
|
||||
// This is just Eulers approximation with a variable step size
|
||||
// **Adaptive** Differential PCM
|
||||
// Adaptive: because the step size is variable
|
||||
step = context.steps[step_index as usize];
|
||||
|
||||
let raw_sample;
|
||||
/* == Non approxiate version ===
|
||||
// This is the full maths version
|
||||
// We don't use this one as we need to match the way the encoder
|
||||
// works if we want to use the state stored in the header.
|
||||
// I have Left it here as it is easier to understand then the bit shift version below
|
||||
let inibble = nibble.signed();
|
||||
|
||||
// Calculate the delta (which is really what adpcm is all about)
|
||||
// Adaptive **Differential** PCM
|
||||
// Differential: Becuase its all about the difference (gradient)
|
||||
let diff = (step as i32) * (inibble) / 2 + (step as i32) / 8;
|
||||
|
||||
// Eulers approxiation
|
||||
// Sample = Previous_Sample + difference*step_size
|
||||
raw_sample = last_output + diff;
|
||||
*/
|
||||
|
||||
// === Approximate version ==
|
||||
// Approximate form uses bit shift operators.
|
||||
// This is a legacy of the days when mult/divides were expensive
|
||||
// It is also the format used on low end CPUs like cameras
|
||||
let mut diff = step >> 3;
|
||||
if (unibble & 0b0100) == 0b0100 {
|
||||
diff += step;
|
||||
}
|
||||
if (unibble & 0b0010) == 0b0010 {
|
||||
diff += step >> 1;
|
||||
}
|
||||
if (unibble & 0b0001) == 0b0001 {
|
||||
diff += step >> 2;
|
||||
}
|
||||
// Sign test
|
||||
if (unibble & 0b1000) == 0b1000 {
|
||||
raw_sample = last_output - (diff as i32);
|
||||
} else {
|
||||
raw_sample = last_output + (diff as i32);
|
||||
}
|
||||
|
||||
// Specifications say: Clamp it in max sample range -context.max_sample_size..context.max_sample_size
|
||||
let sample = match raw_sample {
|
||||
value if value > context.max_sample_size - 1 => context.max_sample_size - 1,
|
||||
value if value < -context.max_sample_size => -context.max_sample_size,
|
||||
value => value,
|
||||
};
|
||||
|
||||
// PCM is really in i16 range
|
||||
// Some formats e.g. OKI are not in the full PCM range of values
|
||||
// To convert we must scale it to the i16 range
|
||||
// We also cast to i16 at this point ready for the conversion to u8 bytes of the output
|
||||
let scaled_sample = (sample as i32 * (i16::MAX as i32)
|
||||
/ (context.max_sample_size - 1) as i32)
|
||||
as i16;
|
||||
|
||||
// Get the results in bytes
|
||||
result.extend(scaled_sample.to_le_bytes().iter());
|
||||
|
||||
// Increment the step index
|
||||
step_index = step_index as i32 + context.changes[unibble as usize];
|
||||
|
||||
// cache the last_output ready for next run
|
||||
last_output = sample;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
@@ -1,11 +1,11 @@
|
||||
use super::{Error, Result, RX_TIMEOUT};
|
||||
use crate::bc::model::*;
|
||||
use crate::bc_protocol::connection::BcSubscription;
|
||||
use err_derive::Error;
|
||||
use crate::gst::StreamFormat;
|
||||
use log::trace;
|
||||
use log::*;
|
||||
use std::collections::VecDeque;
|
||||
use std::convert::TryInto;
|
||||
use std::time::Duration;
|
||||
|
||||
const INVALID_MEDIA_PACKETS: &[MediaDataKind] = &[MediaDataKind::Unknown];
|
||||
|
||||
@@ -15,14 +15,6 @@ const MAGIC_SIZE: usize = 4;
|
||||
// PAD_SIZE: Media packets use 8 byte padding
|
||||
const PAD_SIZE: usize = 8;
|
||||
|
||||
type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error(display = "Timeout")]
|
||||
Timeout(#[error(source)] std::sync::mpsc::RecvTimeoutError),
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
|
||||
pub enum MediaDataKind {
|
||||
VideoDataIframe,
|
||||
@@ -50,7 +42,7 @@ impl MediaData {
|
||||
MediaDataKind::VideoDataIframe => 32,
|
||||
MediaDataKind::VideoDataPframe => 24,
|
||||
MediaDataKind::AudioDataAac => 8,
|
||||
MediaDataKind::AudioDataAdpcm => 16,
|
||||
MediaDataKind::AudioDataAdpcm => 8,
|
||||
MediaDataKind::InfoData => 32,
|
||||
MediaDataKind::Unknown => 0,
|
||||
}
|
||||
@@ -61,6 +53,39 @@ impl MediaData {
|
||||
MediaData::header_size_from_kind(kind)
|
||||
}
|
||||
|
||||
pub fn header(&self) -> &[u8] {
|
||||
&self.data[0..self.header_size()]
|
||||
}
|
||||
|
||||
pub fn header_dump(&self) {
|
||||
info!("{:?}-hex: {:02?}", self.kind(), self.header());
|
||||
let mut result = vec![];
|
||||
for four in self.header().chunks(4) {
|
||||
result.push(u32::from_le_bytes(four.try_into().unwrap()));
|
||||
}
|
||||
info!("{:?}-32: {:?}", self.kind(), result);
|
||||
let mut result = vec![];
|
||||
for two in self.header().chunks(2) {
|
||||
result.push(u16::from_le_bytes(two.try_into().unwrap()));
|
||||
}
|
||||
info!("{:?}-16: {:?}", self.kind(), result);
|
||||
let mut result = vec![];
|
||||
for one in self.header().chunks(1) {
|
||||
result.push(u8::from_le_bytes(one.try_into().unwrap()));
|
||||
}
|
||||
info!("{:?}-8: {:?}", self.kind(), result);
|
||||
let mut result = vec![];
|
||||
for four in self.header().chunks(4) {
|
||||
result.push(f32::from_le_bytes(four.try_into().unwrap()));
|
||||
}
|
||||
info!("{:?}-f32: {:?}", self.kind(), result);
|
||||
let mut result = vec![];
|
||||
for four in self.header().chunks(4) {
|
||||
result.push(String::from_utf8_lossy(four));
|
||||
}
|
||||
info!("{:?}-utf8: {:?}", self.kind(), result);
|
||||
}
|
||||
|
||||
fn header_size(&self) -> usize {
|
||||
MediaData::header_size_from_raw(&self.data)
|
||||
}
|
||||
@@ -129,7 +154,7 @@ impl MediaData {
|
||||
MAGIC_IFRAME => MediaDataKind::VideoDataIframe,
|
||||
MAGIC_PFRAME => MediaDataKind::VideoDataPframe,
|
||||
_ => {
|
||||
trace!("Unknown magic kind: {:x?}", &magic);
|
||||
//trace!("Unknown magic kind: {:x?}", &magic);
|
||||
MediaDataKind::Unknown
|
||||
}
|
||||
}
|
||||
@@ -138,6 +163,39 @@ impl MediaData {
|
||||
pub fn kind(&self) -> MediaDataKind {
|
||||
MediaData::kind_from_raw(&self.data)
|
||||
}
|
||||
|
||||
pub fn media_format(&self) -> Option<StreamFormat> {
|
||||
let kind = self.kind();
|
||||
match kind {
|
||||
MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => {
|
||||
let stream_type = &self.data[4..8];
|
||||
const H264_STR_UPPER: &[u8] = &[0x48, 0x32, 0x36, 0x34];
|
||||
const H264_STR_LOWER: &[u8] = &[0x68, 0x32, 0x36, 0x34];
|
||||
const H265_STR_UPPER: &[u8] = &[0x48, 0x32, 0x36, 0x35];
|
||||
const H265_STR_LOWER: &[u8] = &[0x68, 0x32, 0x36, 0x35];
|
||||
match stream_type {
|
||||
H264_STR_UPPER | H264_STR_LOWER => Some(StreamFormat::H264), // Offically it should be "H264" not "h264" but covering all cases
|
||||
H265_STR_UPPER | H265_STR_LOWER => Some(StreamFormat::H265),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
MediaDataKind::AudioDataAac => Some(StreamFormat::AAC),
|
||||
MediaDataKind::AudioDataAdpcm => Some(StreamFormat::ADPCM),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn timestamp(&self) -> Option<u64> {
|
||||
let kind = self.kind();
|
||||
match kind {
|
||||
MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => Some(
|
||||
Self::bytes_to_size(&self.data[16..20])
|
||||
.try_into()
|
||||
.expect("usize wont fit into u64"),
|
||||
),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MediaDataSubscriber<'a> {
|
||||
@@ -153,10 +211,10 @@ impl<'a> MediaDataSubscriber<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
fn fill_binary_buffer(&mut self, rx_timeout: Duration) -> Result<()> {
|
||||
fn fill_binary_buffer(&mut self) -> Result<()> {
|
||||
// Loop messages until we get binary add that data and return
|
||||
loop {
|
||||
let msg = self.bc_sub.rx.recv_timeout(rx_timeout)?;
|
||||
let msg = self.bc_sub.rx.recv_timeout(RX_TIMEOUT)?;
|
||||
if let BcBody::ModernMsg(ModernMsg {
|
||||
binary: Some(binary),
|
||||
..
|
||||
@@ -170,11 +228,11 @@ impl<'a> MediaDataSubscriber<'a> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn advance_to_media_packet(&mut self, rx_timeout: Duration) -> Result<()> {
|
||||
fn advance_to_media_packet(&mut self) -> Result<()> {
|
||||
// In the event we get an unknown packet we advance by brute force
|
||||
// reading of bytes to the next valid magic
|
||||
while self.binary_buffer.len() < MAGIC_SIZE {
|
||||
self.fill_binary_buffer(rx_timeout)?;
|
||||
self.fill_binary_buffer()?;
|
||||
}
|
||||
|
||||
// Check the kind, if its invalid use pop a byte and try again
|
||||
@@ -186,7 +244,7 @@ impl<'a> MediaDataSubscriber<'a> {
|
||||
while INVALID_MEDIA_PACKETS.contains(&MediaData::kind_from_raw(&magic)) {
|
||||
self.binary_buffer.pop_front();
|
||||
while self.binary_buffer.len() < MAGIC_SIZE {
|
||||
self.fill_binary_buffer(rx_timeout)?;
|
||||
self.fill_binary_buffer()?;
|
||||
}
|
||||
magic = MediaDataSubscriber::get_first_n_deque(&self.binary_buffer, MAGIC_SIZE);
|
||||
}
|
||||
@@ -213,12 +271,9 @@ impl<'a> MediaDataSubscriber<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_media_packet(
|
||||
&mut self,
|
||||
rx_timeout: Duration,
|
||||
) -> std::result::Result<MediaData, Error> {
|
||||
pub fn next_media_packet(&mut self) -> std::result::Result<MediaData, Error> {
|
||||
// Find the first packet (does nothing if already at one)
|
||||
self.advance_to_media_packet(rx_timeout)?;
|
||||
self.advance_to_media_packet()?;
|
||||
|
||||
// Get the magic bytes (guaranteed by advance_to_media_packet)
|
||||
let magic = MediaDataSubscriber::get_first_n_deque(&self.binary_buffer, MAGIC_SIZE);
|
||||
@@ -226,7 +281,7 @@ impl<'a> MediaDataSubscriber<'a> {
|
||||
// Get enough for the full header
|
||||
let header_size = MediaData::header_size_from_raw(&magic);
|
||||
while self.binary_buffer.len() < header_size {
|
||||
self.fill_binary_buffer(rx_timeout)?;
|
||||
self.fill_binary_buffer()?;
|
||||
}
|
||||
|
||||
// Get enough for the full data + 8 byte buffer
|
||||
@@ -235,7 +290,7 @@ impl<'a> MediaDataSubscriber<'a> {
|
||||
let pad_size = MediaData::pad_size_from_raw(&header);
|
||||
let full_size = header_size + data_size + pad_size;
|
||||
while self.binary_buffer.len() < full_size {
|
||||
self.fill_binary_buffer(rx_timeout)?;
|
||||
self.fill_binary_buffer()?;
|
||||
}
|
||||
|
||||
// Pop the full binary buffer
|
||||
|
||||
+2
-12
@@ -8,7 +8,6 @@ use validator::{Validate, ValidationError};
|
||||
use validator_derive::Validate;
|
||||
|
||||
lazy_static! {
|
||||
static ref RE_STREAM_FORM: Regex = Regex::new(r"^([hH]26[45]|[ \t]*[!].*)$").unwrap();
|
||||
static ref RE_STREAM_SRC: Regex = Regex::new(r"^(mainStream|subStream|both)$").unwrap();
|
||||
static ref RE_TLS_CLIENT_AUTH: Regex = Regex::new(r"^(none|request|require)$").unwrap();
|
||||
}
|
||||
@@ -54,13 +53,8 @@ pub struct CameraConfig {
|
||||
// no longer used, but still here so we can warn users:
|
||||
pub timeout: Option<Duration>,
|
||||
|
||||
#[validate(regex(
|
||||
path = "RE_STREAM_FORM",
|
||||
message = "Incorrect stream format",
|
||||
code = "format"
|
||||
))]
|
||||
#[serde(default = "default_format")]
|
||||
pub format: String,
|
||||
// no longer used, but still here so we can warn users:
|
||||
pub format: Option<String>,
|
||||
|
||||
#[validate(regex(
|
||||
path = "RE_STREAM_SRC",
|
||||
@@ -95,10 +89,6 @@ fn default_bind_port() -> u16 {
|
||||
8554
|
||||
}
|
||||
|
||||
fn default_format() -> String {
|
||||
"h265".to_string()
|
||||
}
|
||||
|
||||
fn default_stream() -> String {
|
||||
"both".to_string()
|
||||
}
|
||||
|
||||
+96
-29
@@ -1,7 +1,6 @@
|
||||
//! This module provides an "RtspServer" abstraction that allows consumers of its API to feed it
|
||||
//! data using an ordinary std::io::Write interface.
|
||||
pub use self::maybe_app_src::MaybeAppSrc;
|
||||
|
||||
use gstreamer::prelude::Cast;
|
||||
use gstreamer::{Bin, Structure};
|
||||
use gstreamer_app::AppSrc;
|
||||
@@ -27,10 +26,82 @@ pub struct RtspServer {
|
||||
server: GstRTSPServer,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
|
||||
pub enum StreamFormat {
|
||||
H264,
|
||||
H265,
|
||||
Custom(String),
|
||||
AAC,
|
||||
ADPCM,
|
||||
}
|
||||
|
||||
pub struct GstOutputs {
|
||||
pub audsrc: MaybeAppSrc,
|
||||
pub vidsrc: MaybeAppSrc,
|
||||
video_format: Option<StreamFormat>,
|
||||
audio_format: Option<StreamFormat>,
|
||||
factory: RTSPMediaFactory,
|
||||
}
|
||||
|
||||
impl GstOutputs {
|
||||
pub fn from_appsrcs(vidsrc: MaybeAppSrc, audsrc: MaybeAppSrc) -> GstOutputs {
|
||||
let result = GstOutputs {
|
||||
vidsrc,
|
||||
audsrc,
|
||||
video_format: None,
|
||||
audio_format: None,
|
||||
factory: RTSPMediaFactory::new(),
|
||||
};
|
||||
result.apply_format();
|
||||
result
|
||||
}
|
||||
|
||||
pub fn set_format(&mut self, format: Option<StreamFormat>) {
|
||||
match format {
|
||||
Some(StreamFormat::H264) | Some(StreamFormat::H265) => {
|
||||
if format != self.video_format {
|
||||
self.video_format = format;
|
||||
self.apply_format();
|
||||
}
|
||||
}
|
||||
Some(StreamFormat::AAC) | Some(StreamFormat::ADPCM) => {
|
||||
if format != self.audio_format {
|
||||
self.audio_format = format;
|
||||
self.apply_format();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_format(&self) {
|
||||
let launch_vid = match self.video_format {
|
||||
Some(StreamFormat::H264) => {
|
||||
"! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h264parse ! rtph264pay name=pay0"
|
||||
}
|
||||
Some(StreamFormat::H265) => {
|
||||
"! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h265parse ! rtph265pay name=pay0"
|
||||
}
|
||||
_ => "! fakesink",
|
||||
};
|
||||
|
||||
let launch_aud = match self.audio_format {
|
||||
Some(StreamFormat::ADPCM) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! rawaudioparse format=pcm pcm-format=s16le sample-rate=8000 num-channels=1 interleaved=true ! audioconvert ! rtpL16pay name=pay1", // DVI4 is converted to pcm in the appsrc
|
||||
Some(StreamFormat::AAC) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! aacparse ! rtpmp4apay name=pay1",
|
||||
_ => "! fakesink",
|
||||
};
|
||||
|
||||
self.factory.set_launch(
|
||||
&vec![
|
||||
"( ",
|
||||
"appsrc name=vidsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks
|
||||
launch_vid,
|
||||
"appsrc name=audsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks
|
||||
launch_aud,
|
||||
")"
|
||||
]
|
||||
.join(" "),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RtspServer {
|
||||
@@ -50,21 +121,23 @@ impl RtspServer {
|
||||
pub fn add_stream(
|
||||
&self,
|
||||
paths: &[&str],
|
||||
stream_format: StreamFormat,
|
||||
permitted_users: &HashSet<&str>,
|
||||
) -> Result<MaybeAppSrc> {
|
||||
) -> Result<GstOutputs> {
|
||||
let mounts = self
|
||||
.server
|
||||
.get_mount_points()
|
||||
.expect("The server should have mountpoints");
|
||||
|
||||
let launch_str = match stream_format {
|
||||
StreamFormat::H264 => "! h264parse ! rtph264pay name=pay0",
|
||||
StreamFormat::H265 => "! h265parse ! rtph265pay name=pay0",
|
||||
StreamFormat::Custom(ref custom_format) => custom_format,
|
||||
};
|
||||
// Create a MaybeAppSrc: Write which we will give the caller. When the backing AppSrc is
|
||||
// created by the factory, fish it out and give it to the waiting MaybeAppSrc via the
|
||||
// channel it provided. This callback may be called more than once by Gstreamer if it is
|
||||
// unhappy with the pipeline, so keep updating the MaybeAppSrc.
|
||||
let (maybe_app_src, tx) = MaybeAppSrc::new_with_tx();
|
||||
let (maybe_app_src_aud, tx_aud) = MaybeAppSrc::new_with_tx();
|
||||
|
||||
let factory = RTSPMediaFactory::new();
|
||||
let outputs = GstOutputs::from_appsrcs(maybe_app_src, maybe_app_src_aud);
|
||||
|
||||
let factory = &outputs.factory;
|
||||
|
||||
debug!(
|
||||
"Permitting {} to access {}",
|
||||
@@ -76,25 +149,10 @@ impl RtspServer {
|
||||
.collect::<String>(),
|
||||
paths.join(", ")
|
||||
);
|
||||
self.add_permitted_roles(&factory, permitted_users);
|
||||
self.add_permitted_roles(factory, permitted_users);
|
||||
|
||||
//factory.set_protocols(RTSPLowerTrans::TCP);
|
||||
factory.set_launch(&format!("{}{}{}{}",
|
||||
"( ",
|
||||
"appsrc name=writesrc is-live=true block=true emit-signals=false max-bytes=0 do-timestamp=true ",
|
||||
launch_str,
|
||||
" )"
|
||||
));
|
||||
factory.set_shared(true);
|
||||
|
||||
// TODO maybe set video format via
|
||||
// https://gitlab.freedesktop.org/gstreamer/gstreamer-rs/-/blob/master/examples/src/bin/appsrc.rs#L66
|
||||
|
||||
// Create a MaybeAppSrc: Write which we will give the caller. When the backing AppSrc is
|
||||
// created by the factory, fish it out and give it to the waiting MaybeAppSrc via the
|
||||
// channel it provided. This callback may be called more than once by Gstreamer if it is
|
||||
// unhappy with the pipeline, so keep updating the MaybeAppSrc.
|
||||
let (maybe_app_src, tx) = MaybeAppSrc::new_with_tx();
|
||||
factory.connect_media_configure(move |_factory, media| {
|
||||
debug!("RTSP: media was configured");
|
||||
let bin = media
|
||||
@@ -103,18 +161,25 @@ impl RtspServer {
|
||||
.dynamic_cast::<Bin>()
|
||||
.expect("Media source's element should be a bin");
|
||||
let app_src = bin
|
||||
.get_by_name_recurse_up("writesrc")
|
||||
.get_by_name_recurse_up("vidsrc")
|
||||
.expect("write_src must be present in created bin")
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.expect("Source element is expected to be an appsrc!");
|
||||
let _ = tx.send(app_src); // Receiver may be dropped, don't panic if so
|
||||
|
||||
let app_src_aud = bin
|
||||
.get_by_name_recurse_up("audsrc")
|
||||
.expect("write_src must be present in created bin")
|
||||
.dynamic_cast::<AppSrc>()
|
||||
.expect("Source element is expected to be an appsrc!");
|
||||
let _ = tx_aud.send(app_src_aud); // Receiver may be dropped, don't panic if so
|
||||
});
|
||||
|
||||
for path in paths {
|
||||
mounts.add_factory(path, &factory);
|
||||
mounts.add_factory(path, factory);
|
||||
}
|
||||
|
||||
Ok(maybe_app_src)
|
||||
Ok(outputs)
|
||||
}
|
||||
|
||||
pub fn add_permitted_roles(&self, factory: &RTSPMediaFactory, permitted_roles: &HashSet<&str>) {
|
||||
@@ -241,12 +306,14 @@ mod maybe_app_src {
|
||||
Some(src) => src,
|
||||
None => return Ok(buf.len()),
|
||||
};
|
||||
|
||||
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
|
||||
{
|
||||
let gst_buf_mut = gst_buf.get_mut().unwrap();
|
||||
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
|
||||
gst_buf_data.copy_from_slice(buf);
|
||||
}
|
||||
|
||||
let res = app_src.push_buffer(gst_buf); //.map_err(|e| io::Error::new(io::ErrorKind::Other, Box::new(e)))?;
|
||||
if res.is_err() {
|
||||
self.app_src = None;
|
||||
|
||||
+18
-24
@@ -3,11 +3,10 @@ use err_derive::Error;
|
||||
use gio::TlsAuthenticationMode;
|
||||
use log::*;
|
||||
use neolink::bc_protocol::BcCamera;
|
||||
use neolink::gst::{MaybeAppSrc, RtspServer, StreamFormat};
|
||||
use neolink::gst::{GstOutputs, RtspServer};
|
||||
use neolink::Never;
|
||||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use structopt::StructOpt;
|
||||
@@ -29,6 +28,8 @@ pub enum Error {
|
||||
IoError(#[error(source)] std::io::Error),
|
||||
#[error(display = "Validation error")]
|
||||
ValidationError(#[error(source)] validator::ValidationErrors),
|
||||
#[error(display = "ADPCM Decoding Error")]
|
||||
AdpcmDecodingError(&'static str),
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Error> {
|
||||
@@ -59,22 +60,14 @@ fn main() -> Result<(), Error> {
|
||||
|
||||
crossbeam::scope(|s| {
|
||||
for camera in config.cameras {
|
||||
let stream_format = match &*camera.format {
|
||||
"h264" | "H264" => StreamFormat::H264,
|
||||
"h265" | "H265" => StreamFormat::H265,
|
||||
custom_format => StreamFormat::Custom(custom_format.to_string()),
|
||||
};
|
||||
|
||||
if camera.format.is_some() {
|
||||
warn!("The format config option of the camera has been removed in favour of auto detection.")
|
||||
}
|
||||
// Let subthreads share the camera object; in principle I think they could share
|
||||
// the object as it sits in the config.cameras block, but I have not figured out the
|
||||
// syntax for that.
|
||||
let arc_cam = Arc::new(camera);
|
||||
|
||||
// The substream always seems to be H264, even on B800 cameras
|
||||
let substream_format = match &*arc_cam.format {
|
||||
"h264" | "H264" | "h265" | "H265" => StreamFormat::H264,
|
||||
custom_format => StreamFormat::Custom(custom_format.to_string()),
|
||||
};
|
||||
let permitted_users =
|
||||
get_permitted_users(config.users.as_slice(), &arc_cam.permitted_users);
|
||||
|
||||
@@ -84,20 +77,20 @@ fn main() -> Result<(), Error> {
|
||||
&*format!("/{}", arc_cam.name),
|
||||
&*format!("/{}/mainStream", arc_cam.name),
|
||||
];
|
||||
let mut output = rtsp
|
||||
.add_stream(paths, stream_format, &permitted_users)
|
||||
let mut outputs = rtsp
|
||||
.add_stream(paths, &permitted_users)
|
||||
.unwrap();
|
||||
let main_camera = arc_cam.clone();
|
||||
s.spawn(move |_| camera_loop(&*main_camera, "mainStream", &mut output, true));
|
||||
s.spawn(move |_| camera_loop(&*main_camera, "mainStream", &mut outputs, true));
|
||||
}
|
||||
if ["both", "subStream"].iter().any(|&e| e == arc_cam.stream) {
|
||||
let paths = &[&*format!("/{}/subStream", arc_cam.name)];
|
||||
let mut output = rtsp
|
||||
.add_stream(paths, substream_format, &permitted_users)
|
||||
let mut outputs = rtsp
|
||||
.add_stream(paths, &permitted_users)
|
||||
.unwrap();
|
||||
let sub_camera = arc_cam.clone();
|
||||
let manage = arc_cam.stream == "subStream";
|
||||
s.spawn(move |_| camera_loop(&*sub_camera, "subStream", &mut output, manage));
|
||||
s.spawn(move |_| camera_loop(&*sub_camera, "subStream", &mut outputs, manage));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,7 +104,7 @@ fn main() -> Result<(), Error> {
|
||||
fn camera_loop(
|
||||
camera_config: &CameraConfig,
|
||||
stream_name: &str,
|
||||
output: &mut MaybeAppSrc,
|
||||
outputs: &mut GstOutputs,
|
||||
manage: bool,
|
||||
) -> Result<Never, Error> {
|
||||
let min_backoff = Duration::from_secs(1);
|
||||
@@ -119,8 +112,9 @@ fn camera_loop(
|
||||
let mut current_backoff = min_backoff;
|
||||
|
||||
loop {
|
||||
let cam_err = camera_main(camera_config, stream_name, output, manage).unwrap_err();
|
||||
output.on_stream_error();
|
||||
let cam_err = camera_main(camera_config, stream_name, outputs, manage).unwrap_err();
|
||||
outputs.vidsrc.on_stream_error();
|
||||
outputs.audsrc.on_stream_error();
|
||||
// Authentication failures are permanent; we retry everything else
|
||||
if cam_err.connected {
|
||||
current_backoff = min_backoff;
|
||||
@@ -199,7 +193,7 @@ fn get_permitted_users<'a>(
|
||||
fn camera_main(
|
||||
camera_config: &CameraConfig,
|
||||
stream_name: &str,
|
||||
output: &mut dyn Write,
|
||||
outputs: &mut GstOutputs,
|
||||
manage: bool,
|
||||
) -> Result<Never, CameraErr> {
|
||||
let mut connected = false;
|
||||
@@ -251,7 +245,7 @@ fn camera_main(
|
||||
"{}: Starting video stream {}",
|
||||
camera_config.name, stream_name
|
||||
);
|
||||
camera.start_video(output, stream_name, camera_config.channel_id)
|
||||
camera.start_video(outputs, stream_name, camera_config.channel_id)
|
||||
})()
|
||||
.map_err(|err| CameraErr { connected, err })
|
||||
}
|
||||
Reference in new issue
Block a user