Audio and Format auto detect (#68)

* Add queues to the gstreamer launch

* Add audio to the stream

* Removing format option in prep for auto detect

* Adding format detect from headers

* Set format in gstreamer launch depending on video magic bytes

* Rename MaybeAppSrcs to reflect new role of managing gst outputs

* Auto detect format

* Remove all setting of stream format

* Re-add missing media_format caused by rebase

* Add audio format auto detect too

* Use more appropiate adpcm in gst

* Combined gst and media_packet format enums into a single enum StreamFormat

* Reduce adpcm header size

* Disable adpcm output

* Add oki adpcm native rust support

* Maintain state between packets for adpcm

* Use bytes in adpcm stream to determine adpcm decoding state

* Clean up adpcm code

* Use raw rtp payload for the adpcm stream

* Bug fix to treat adpcm and aac seperatly in the match

* Remove redundent second clamp of the step index in adpcm.

* ADPCM format changes.

This is a squashed commit of changes made during live testing of ADPCM 
format.

Summary: Using DVI4 for the ADPCM, with our own decoder written in rust.

* Changes to comments and IMA max range in the adpcm decoder

* ADPCM magic contains block size use it for chunking the adpcm data

* Use error logs rather than asserts in adpcm decode

* Rustfmt and clippy styling

* Use static slice for adpcm step and changes arrays

* Bounds and error checks in ADPCM

* Use i/u32 in adpcm

* Rust fmted
This commit is contained in:
Andrew King authored and GitHub committed 2020-11-13 08:45:40 -06:00
1 parent b5312edb90
commit f7f12c76a1
6 files changed
+462 -95

No files matched your search

+23 -6
View File
@@ -2,6 +2,8 @@ use self::connection::BcConnection;
use self::media_packet::{MediaDataKind, MediaDataSubscriber};
use crate::bc;
use crate::bc::{model::*, xml::*};
use crate::gst::GstOutputs;
use adpcm::adpcm_to_pcm;
use err_derive::Error;
use log::*;
use std::io::Write;
@@ -10,6 +12,7 @@ use std::time::Duration;
use Md5Trunc::*;
mod adpcm;
mod connection;
mod media_packet;
mod time;
@@ -49,12 +52,12 @@ pub enum Error {
#[error(display = "Timeout")]
Timeout(#[error(source)] std::sync::mpsc::RecvTimeoutError),
#[error(display = "Media")]
MediaPacket(#[error(source)] self::media_packet::Error),
#[error(display = "Credential error")]
AuthFailed,
#[error(display = "ADPCM Decoding Error")]
AdpcmDecodingError(&'static str),
#[error(display = "Other error")]
Other(&'static str),
}
@@ -245,7 +248,7 @@ impl BcCamera {
pub fn start_video(
&self,
data_out: &mut dyn Write,
data_outs: &mut GstOutputs,
stream_name: &str,
channel_id: u32,
) -> Result<Never> {
@@ -278,12 +281,26 @@ impl BcCamera {
let mut media_sub = MediaDataSubscriber::from_bc_sub(&sub_video);
loop {
let binary_data = media_sub.next_media_packet(RX_TIMEOUT)?;
let binary_data = media_sub.next_media_packet()?;
// We now have a complete interesting packet. Send it to gst.
// Process the packet
match binary_data.kind() {
MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => {
data_out.write_all(binary_data.body())?;
let media_format = binary_data.media_format();
data_outs.set_format(media_format);
data_outs.vidsrc.write_all(binary_data.body())?;
}
MediaDataKind::AudioDataAac => {
let media_format = binary_data.media_format();
data_outs.set_format(media_format);
data_outs.audsrc.write_all(binary_data.body())?;
}
MediaDataKind::AudioDataAdpcm => {
let media_format = binary_data.media_format();
data_outs.set_format(media_format);
let adpcm = binary_data.body();
let pcm = adpcm_to_pcm(adpcm)?;
data_outs.audsrc.write_all(&pcm)?;
}
_ => {}
};
+244
View File
@@ -0,0 +1,244 @@
/*
This is a rust implementation of OKI and DVI/IMA ADPCM.
*/
use crate::Error;
use log::error;
use std::convert::TryInto;
struct AdpcmSetup {
max_step_index: u32,
steps: &'static [u32],
max_sample_size: i32,
changes: &'static [i32],
}
impl AdpcmSetup {
// Unused, originally we thought BC might be using OKI but it is actually DVI4
fn new_oki() -> Self {
Self {
max_step_index: 48,
steps: &[
16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97,
107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449,
494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552,
],
changes: &[-1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8],
max_sample_size: 2048,
}
}
// This is IMA format, but it is the same as DVI4 format except in the block header
fn new_ima() -> Self {
Self {
max_step_index: 88,
steps: &[
7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50,
55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279,
307, 337, 371, 408, 449, 494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282,
1411, 1552, 1707, 1878, 2066, 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871,
5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899, 15289, 16818,
18500, 20350, 22385, 24623, 27086, 29794, 32767,
],
changes: &[-1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8],
max_sample_size: 32768,
}
}
}
struct Nibble {
// A nibble is a 4bit int
data: u8, // This is the raw data for the nibble
}
impl Nibble {
// Use u/i32 throughout to ensure that we always have enough
// Headroom to do the math without needing `as` casting everywhere
fn unsigned(&self) -> u32 {
(self.data & 0b00001111) as u32 // Mask first 4 bits it just to be sure its in nibble range
}
fn signed_magnitude(&self) -> u32 {
(self.data & 0b00000111) as u32 // Mask of first 3 bits which are the magnitiude bits in signed int
}
fn signed(&self) -> i32 {
match self.data & 0b00001000 {
// Sign bit is at the 4th bit
0b00001000 => -(self.signed_magnitude() as i32),
_ => self.signed_magnitude() as i32,
}
}
fn from_byte(byte: &u8) -> [Self; 2] {
// Two nibbles per byte
[
Self {
data: (byte & 0b11110000) >> 4,
},
Self {
data: byte & 0b00001111,
},
]
}
}
pub fn adpcm_to_pcm(bytes: &[u8]) -> Result<Vec<u8>, Error> {
let context = AdpcmSetup::new_ima();
let mut result: Vec<u8> = vec![]; // Stores the PCM byte array
// ADPCM is not really a streamable format
// Each audio sample requires information on the previous sample
// To solve this reolink caches the intermediate variables (step_index and last_output)
// Into the block header in the stream. When ever an adpcm packet arrives it starts with
// 0x0001 which is the frame time from HISilicon documentation (I think) following by
// 0xWW which is half the block size
// 0xYY which is the last output
// 0xZZ which is the step index of the last output
// We must initialise our decoder with this data
if bytes.len() < 4 {
error!("ADPCM data is too short for even the magic.");
return Err(Error::AdpcmDecodingError(
"ADPCM data is too short for even the magic.",
));
}
// Check for valid number of frame type
let frame_type_bytes = &bytes[0..2];
const FRAME_TYPE_HISILICON: &[u8] = &[0x00, 0x01];
if frame_type_bytes != FRAME_TYPE_HISILICON {
error!("Unexpected ADPCM frame type: {:x?}", frame_type_bytes);
return Err(Error::AdpcmDecodingError("Unexpected ADPCM frame type"));
}
// Check for valid block size
let block_size_bytes = &bytes[2..4];
let block_size = (u16::from_le_bytes(
block_size_bytes
.try_into()
.expect("slice with incorrect length"),
) as u32)
* 2; // Block size is stored as 1/2 (don't know why)
let full_block_size = block_size + 4; // block_size + magic (2 bytes) + size (2 bytes)
if !bytes.len() % full_block_size as usize == 0 {
error!("ADPCM Data is not a multiple of the block size");
return Err(Error::AdpcmDecodingError(
"ADPCM block size does not match data length.",
));
}
// Chunk on block size
for bytes in bytes.chunks(full_block_size as usize) {
// Get predictor state from block header using DVI 4 format.
if bytes.len() < 8 {
error!("ADPCM Block size is not long enough for header");
return Err(Error::AdpcmDecodingError(
"ADPCM has insufficent block size",
));
}
let step_output_bytes = &bytes[4..6];
let mut last_output = i16::from_le_bytes(
step_output_bytes
.try_into()
.expect("slice with incorrect length"),
) as i32;
let step_index_bytes = &bytes[6..8];
let mut step_index = u16::from_le_bytes(
step_index_bytes
.try_into()
.expect("slice with incorrect length"),
) as i32;
// To avoid casting to u8 <-> u16 <-> u32 and back all the time I just do all maths in u/i32
// This gives enough headroom to do all calculations without overflow because adpcm puts artifical
// limits on the sample sizes
let mut step: u32;
// The rest is all data to be decoded
let data = &bytes[8..];
for byte in data {
let nibbles: [Nibble; 2] = Nibble::from_byte(byte);
for nibble in &nibbles {
let unibble = nibble.unsigned();
// Specifications say: Clamp it in max index range 0..context.max_step_index
step_index = match step_index {
n if n < 0 => 0,
n if n > context.max_step_index as i32 => context.max_step_index as i32,
n => n,
};
// This is just Eulers approximation with a variable step size
// **Adaptive** Differential PCM
// Adaptive: because the step size is variable
step = context.steps[step_index as usize];
let raw_sample;
/* == Non approxiate version ===
// This is the full maths version
// We don't use this one as we need to match the way the encoder
// works if we want to use the state stored in the header.
// I have Left it here as it is easier to understand then the bit shift version below
let inibble = nibble.signed();
// Calculate the delta (which is really what adpcm is all about)
// Adaptive **Differential** PCM
// Differential: Becuase its all about the difference (gradient)
let diff = (step as i32) * (inibble) / 2 + (step as i32) / 8;
// Eulers approxiation
// Sample = Previous_Sample + difference*step_size
raw_sample = last_output + diff;
*/
// === Approximate version ==
// Approximate form uses bit shift operators.
// This is a legacy of the days when mult/divides were expensive
// It is also the format used on low end CPUs like cameras
let mut diff = step >> 3;
if (unibble & 0b0100) == 0b0100 {
diff += step;
}
if (unibble & 0b0010) == 0b0010 {
diff += step >> 1;
}
if (unibble & 0b0001) == 0b0001 {
diff += step >> 2;
}
// Sign test
if (unibble & 0b1000) == 0b1000 {
raw_sample = last_output - (diff as i32);
} else {
raw_sample = last_output + (diff as i32);
}
// Specifications say: Clamp it in max sample range -context.max_sample_size..context.max_sample_size
let sample = match raw_sample {
value if value > context.max_sample_size - 1 => context.max_sample_size - 1,
value if value < -context.max_sample_size => -context.max_sample_size,
value => value,
};
// PCM is really in i16 range
// Some formats e.g. OKI are not in the full PCM range of values
// To convert we must scale it to the i16 range
// We also cast to i16 at this point ready for the conversion to u8 bytes of the output
let scaled_sample = (sample as i32 * (i16::MAX as i32)
/ (context.max_sample_size - 1) as i32)
as i16;
// Get the results in bytes
result.extend(scaled_sample.to_le_bytes().iter());
// Increment the step index
step_index = step_index as i32 + context.changes[unibble as usize];
// cache the last_output ready for next run
last_output = sample;
}
}
}
Ok(result)
}
+79 -24
View File
@@ -1,11 +1,11 @@
use super::{Error, Result, RX_TIMEOUT};
use crate::bc::model::*;
use crate::bc_protocol::connection::BcSubscription;
use err_derive::Error;
use crate::gst::StreamFormat;
use log::trace;
use log::*;
use std::collections::VecDeque;
use std::convert::TryInto;
use std::time::Duration;
const INVALID_MEDIA_PACKETS: &[MediaDataKind] = &[MediaDataKind::Unknown];
@@ -15,14 +15,6 @@ const MAGIC_SIZE: usize = 4;
// PAD_SIZE: Media packets use 8 byte padding
const PAD_SIZE: usize = 8;
type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, Error)]
pub enum Error {
#[error(display = "Timeout")]
Timeout(#[error(source)] std::sync::mpsc::RecvTimeoutError),
}
#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
pub enum MediaDataKind {
VideoDataIframe,
@@ -50,7 +42,7 @@ impl MediaData {
MediaDataKind::VideoDataIframe => 32,
MediaDataKind::VideoDataPframe => 24,
MediaDataKind::AudioDataAac => 8,
MediaDataKind::AudioDataAdpcm => 16,
MediaDataKind::AudioDataAdpcm => 8,
MediaDataKind::InfoData => 32,
MediaDataKind::Unknown => 0,
}
@@ -61,6 +53,39 @@ impl MediaData {
MediaData::header_size_from_kind(kind)
}
pub fn header(&self) -> &[u8] {
&self.data[0..self.header_size()]
}
pub fn header_dump(&self) {
info!("{:?}-hex: {:02?}", self.kind(), self.header());
let mut result = vec![];
for four in self.header().chunks(4) {
result.push(u32::from_le_bytes(four.try_into().unwrap()));
}
info!("{:?}-32: {:?}", self.kind(), result);
let mut result = vec![];
for two in self.header().chunks(2) {
result.push(u16::from_le_bytes(two.try_into().unwrap()));
}
info!("{:?}-16: {:?}", self.kind(), result);
let mut result = vec![];
for one in self.header().chunks(1) {
result.push(u8::from_le_bytes(one.try_into().unwrap()));
}
info!("{:?}-8: {:?}", self.kind(), result);
let mut result = vec![];
for four in self.header().chunks(4) {
result.push(f32::from_le_bytes(four.try_into().unwrap()));
}
info!("{:?}-f32: {:?}", self.kind(), result);
let mut result = vec![];
for four in self.header().chunks(4) {
result.push(String::from_utf8_lossy(four));
}
info!("{:?}-utf8: {:?}", self.kind(), result);
}
fn header_size(&self) -> usize {
MediaData::header_size_from_raw(&self.data)
}
@@ -129,7 +154,7 @@ impl MediaData {
MAGIC_IFRAME => MediaDataKind::VideoDataIframe,
MAGIC_PFRAME => MediaDataKind::VideoDataPframe,
_ => {
trace!("Unknown magic kind: {:x?}", &magic);
//trace!("Unknown magic kind: {:x?}", &magic);
MediaDataKind::Unknown
}
}
@@ -138,6 +163,39 @@ impl MediaData {
pub fn kind(&self) -> MediaDataKind {
MediaData::kind_from_raw(&self.data)
}
pub fn media_format(&self) -> Option<StreamFormat> {
let kind = self.kind();
match kind {
MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => {
let stream_type = &self.data[4..8];
const H264_STR_UPPER: &[u8] = &[0x48, 0x32, 0x36, 0x34];
const H264_STR_LOWER: &[u8] = &[0x68, 0x32, 0x36, 0x34];
const H265_STR_UPPER: &[u8] = &[0x48, 0x32, 0x36, 0x35];
const H265_STR_LOWER: &[u8] = &[0x68, 0x32, 0x36, 0x35];
match stream_type {
H264_STR_UPPER | H264_STR_LOWER => Some(StreamFormat::H264), // Offically it should be "H264" not "h264" but covering all cases
H265_STR_UPPER | H265_STR_LOWER => Some(StreamFormat::H265),
_ => None,
}
}
MediaDataKind::AudioDataAac => Some(StreamFormat::AAC),
MediaDataKind::AudioDataAdpcm => Some(StreamFormat::ADPCM),
_ => None,
}
}
pub fn timestamp(&self) -> Option<u64> {
let kind = self.kind();
match kind {
MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => Some(
Self::bytes_to_size(&self.data[16..20])
.try_into()
.expect("usize wont fit into u64"),
),
_ => None,
}
}
}
pub struct MediaDataSubscriber<'a> {
@@ -153,10 +211,10 @@ impl<'a> MediaDataSubscriber<'a> {
}
}
fn fill_binary_buffer(&mut self, rx_timeout: Duration) -> Result<()> {
fn fill_binary_buffer(&mut self) -> Result<()> {
// Loop messages until we get binary add that data and return
loop {
let msg = self.bc_sub.rx.recv_timeout(rx_timeout)?;
let msg = self.bc_sub.rx.recv_timeout(RX_TIMEOUT)?;
if let BcBody::ModernMsg(ModernMsg {
binary: Some(binary),
..
@@ -170,11 +228,11 @@ impl<'a> MediaDataSubscriber<'a> {
Ok(())
}
fn advance_to_media_packet(&mut self, rx_timeout: Duration) -> Result<()> {
fn advance_to_media_packet(&mut self) -> Result<()> {
// In the event we get an unknown packet we advance by brute force
// reading of bytes to the next valid magic
while self.binary_buffer.len() < MAGIC_SIZE {
self.fill_binary_buffer(rx_timeout)?;
self.fill_binary_buffer()?;
}
// Check the kind, if its invalid use pop a byte and try again
@@ -186,7 +244,7 @@ impl<'a> MediaDataSubscriber<'a> {
while INVALID_MEDIA_PACKETS.contains(&MediaData::kind_from_raw(&magic)) {
self.binary_buffer.pop_front();
while self.binary_buffer.len() < MAGIC_SIZE {
self.fill_binary_buffer(rx_timeout)?;
self.fill_binary_buffer()?;
}
magic = MediaDataSubscriber::get_first_n_deque(&self.binary_buffer, MAGIC_SIZE);
}
@@ -213,12 +271,9 @@ impl<'a> MediaDataSubscriber<'a> {
}
}
pub fn next_media_packet(
&mut self,
rx_timeout: Duration,
) -> std::result::Result<MediaData, Error> {
pub fn next_media_packet(&mut self) -> std::result::Result<MediaData, Error> {
// Find the first packet (does nothing if already at one)
self.advance_to_media_packet(rx_timeout)?;
self.advance_to_media_packet()?;
// Get the magic bytes (guaranteed by advance_to_media_packet)
let magic = MediaDataSubscriber::get_first_n_deque(&self.binary_buffer, MAGIC_SIZE);
@@ -226,7 +281,7 @@ impl<'a> MediaDataSubscriber<'a> {
// Get enough for the full header
let header_size = MediaData::header_size_from_raw(&magic);
while self.binary_buffer.len() < header_size {
self.fill_binary_buffer(rx_timeout)?;
self.fill_binary_buffer()?;
}
// Get enough for the full data + 8 byte buffer
@@ -235,7 +290,7 @@ impl<'a> MediaDataSubscriber<'a> {
let pad_size = MediaData::pad_size_from_raw(&header);
let full_size = header_size + data_size + pad_size;
while self.binary_buffer.len() < full_size {
self.fill_binary_buffer(rx_timeout)?;
self.fill_binary_buffer()?;
}
// Pop the full binary buffer
+2 -12
View File
@@ -8,7 +8,6 @@ use validator::{Validate, ValidationError};
use validator_derive::Validate;
lazy_static! {
static ref RE_STREAM_FORM: Regex = Regex::new(r"^([hH]26[45]|[ \t]*[!].*)$").unwrap();
static ref RE_STREAM_SRC: Regex = Regex::new(r"^(mainStream|subStream|both)$").unwrap();
static ref RE_TLS_CLIENT_AUTH: Regex = Regex::new(r"^(none|request|require)$").unwrap();
}
@@ -54,13 +53,8 @@ pub struct CameraConfig {
// no longer used, but still here so we can warn users:
pub timeout: Option<Duration>,
#[validate(regex(
path = "RE_STREAM_FORM",
message = "Incorrect stream format",
code = "format"
))]
#[serde(default = "default_format")]
pub format: String,
// no longer used, but still here so we can warn users:
pub format: Option<String>,
#[validate(regex(
path = "RE_STREAM_SRC",
@@ -95,10 +89,6 @@ fn default_bind_port() -> u16 {
8554
}
fn default_format() -> String {
"h265".to_string()
}
fn default_stream() -> String {
"both".to_string()
}
+96 -29
View File
@@ -1,7 +1,6 @@
//! This module provides an "RtspServer" abstraction that allows consumers of its API to feed it
//! data using an ordinary std::io::Write interface.
pub use self::maybe_app_src::MaybeAppSrc;
use gstreamer::prelude::Cast;
use gstreamer::{Bin, Structure};
use gstreamer_app::AppSrc;
@@ -27,10 +26,82 @@ pub struct RtspServer {
server: GstRTSPServer,
}
#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
pub enum StreamFormat {
H264,
H265,
Custom(String),
AAC,
ADPCM,
}
pub struct GstOutputs {
pub audsrc: MaybeAppSrc,
pub vidsrc: MaybeAppSrc,
video_format: Option<StreamFormat>,
audio_format: Option<StreamFormat>,
factory: RTSPMediaFactory,
}
impl GstOutputs {
pub fn from_appsrcs(vidsrc: MaybeAppSrc, audsrc: MaybeAppSrc) -> GstOutputs {
let result = GstOutputs {
vidsrc,
audsrc,
video_format: None,
audio_format: None,
factory: RTSPMediaFactory::new(),
};
result.apply_format();
result
}
pub fn set_format(&mut self, format: Option<StreamFormat>) {
match format {
Some(StreamFormat::H264) | Some(StreamFormat::H265) => {
if format != self.video_format {
self.video_format = format;
self.apply_format();
}
}
Some(StreamFormat::AAC) | Some(StreamFormat::ADPCM) => {
if format != self.audio_format {
self.audio_format = format;
self.apply_format();
}
}
_ => {}
}
}
fn apply_format(&self) {
let launch_vid = match self.video_format {
Some(StreamFormat::H264) => {
"! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h264parse ! rtph264pay name=pay0"
}
Some(StreamFormat::H265) => {
"! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h265parse ! rtph265pay name=pay0"
}
_ => "! fakesink",
};
let launch_aud = match self.audio_format {
Some(StreamFormat::ADPCM) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! rawaudioparse format=pcm pcm-format=s16le sample-rate=8000 num-channels=1 interleaved=true ! audioconvert ! rtpL16pay name=pay1", // DVI4 is converted to pcm in the appsrc
Some(StreamFormat::AAC) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! aacparse ! rtpmp4apay name=pay1",
_ => "! fakesink",
};
self.factory.set_launch(
&vec![
"( ",
"appsrc name=vidsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks
launch_vid,
"appsrc name=audsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks
launch_aud,
")"
]
.join(" "),
);
}
}
impl Default for RtspServer {
@@ -50,21 +121,23 @@ impl RtspServer {
pub fn add_stream(
&self,
paths: &[&str],
stream_format: StreamFormat,
permitted_users: &HashSet<&str>,
) -> Result<MaybeAppSrc> {
) -> Result<GstOutputs> {
let mounts = self
.server
.get_mount_points()
.expect("The server should have mountpoints");
let launch_str = match stream_format {
StreamFormat::H264 => "! h264parse ! rtph264pay name=pay0",
StreamFormat::H265 => "! h265parse ! rtph265pay name=pay0",
StreamFormat::Custom(ref custom_format) => custom_format,
};
// Create a MaybeAppSrc: Write which we will give the caller. When the backing AppSrc is
// created by the factory, fish it out and give it to the waiting MaybeAppSrc via the
// channel it provided. This callback may be called more than once by Gstreamer if it is
// unhappy with the pipeline, so keep updating the MaybeAppSrc.
let (maybe_app_src, tx) = MaybeAppSrc::new_with_tx();
let (maybe_app_src_aud, tx_aud) = MaybeAppSrc::new_with_tx();
let factory = RTSPMediaFactory::new();
let outputs = GstOutputs::from_appsrcs(maybe_app_src, maybe_app_src_aud);
let factory = &outputs.factory;
debug!(
"Permitting {} to access {}",
@@ -76,25 +149,10 @@ impl RtspServer {
.collect::<String>(),
paths.join(", ")
);
self.add_permitted_roles(&factory, permitted_users);
self.add_permitted_roles(factory, permitted_users);
//factory.set_protocols(RTSPLowerTrans::TCP);
factory.set_launch(&format!("{}{}{}{}",
"( ",
"appsrc name=writesrc is-live=true block=true emit-signals=false max-bytes=0 do-timestamp=true ",
launch_str,
" )"
));
factory.set_shared(true);
// TODO maybe set video format via
// https://gitlab.freedesktop.org/gstreamer/gstreamer-rs/-/blob/master/examples/src/bin/appsrc.rs#L66
// Create a MaybeAppSrc: Write which we will give the caller. When the backing AppSrc is
// created by the factory, fish it out and give it to the waiting MaybeAppSrc via the
// channel it provided. This callback may be called more than once by Gstreamer if it is
// unhappy with the pipeline, so keep updating the MaybeAppSrc.
let (maybe_app_src, tx) = MaybeAppSrc::new_with_tx();
factory.connect_media_configure(move |_factory, media| {
debug!("RTSP: media was configured");
let bin = media
@@ -103,18 +161,25 @@ impl RtspServer {
.dynamic_cast::<Bin>()
.expect("Media source's element should be a bin");
let app_src = bin
.get_by_name_recurse_up("writesrc")
.get_by_name_recurse_up("vidsrc")
.expect("write_src must be present in created bin")
.dynamic_cast::<AppSrc>()
.expect("Source element is expected to be an appsrc!");
let _ = tx.send(app_src); // Receiver may be dropped, don't panic if so
let app_src_aud = bin
.get_by_name_recurse_up("audsrc")
.expect("write_src must be present in created bin")
.dynamic_cast::<AppSrc>()
.expect("Source element is expected to be an appsrc!");
let _ = tx_aud.send(app_src_aud); // Receiver may be dropped, don't panic if so
});
for path in paths {
mounts.add_factory(path, &factory);
mounts.add_factory(path, factory);
}
Ok(maybe_app_src)
Ok(outputs)
}
pub fn add_permitted_roles(&self, factory: &RTSPMediaFactory, permitted_roles: &HashSet<&str>) {
@@ -241,12 +306,14 @@ mod maybe_app_src {
Some(src) => src,
None => return Ok(buf.len()),
};
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(buf);
}
let res = app_src.push_buffer(gst_buf); //.map_err(|e| io::Error::new(io::ErrorKind::Other, Box::new(e)))?;
if res.is_err() {
self.app_src = None;
+18 -24
View File
@@ -3,11 +3,10 @@ use err_derive::Error;
use gio::TlsAuthenticationMode;
use log::*;
use neolink::bc_protocol::BcCamera;
use neolink::gst::{MaybeAppSrc, RtspServer, StreamFormat};
use neolink::gst::{GstOutputs, RtspServer};
use neolink::Never;
use std::collections::HashSet;
use std::fs;
use std::io::Write;
use std::sync::Arc;
use std::time::Duration;
use structopt::StructOpt;
@@ -29,6 +28,8 @@ pub enum Error {
IoError(#[error(source)] std::io::Error),
#[error(display = "Validation error")]
ValidationError(#[error(source)] validator::ValidationErrors),
#[error(display = "ADPCM Decoding Error")]
AdpcmDecodingError(&'static str),
}
fn main() -> Result<(), Error> {
@@ -59,22 +60,14 @@ fn main() -> Result<(), Error> {
crossbeam::scope(|s| {
for camera in config.cameras {
let stream_format = match &*camera.format {
"h264" | "H264" => StreamFormat::H264,
"h265" | "H265" => StreamFormat::H265,
custom_format => StreamFormat::Custom(custom_format.to_string()),
};
if camera.format.is_some() {
warn!("The format config option of the camera has been removed in favour of auto detection.")
}
// Let subthreads share the camera object; in principle I think they could share
// the object as it sits in the config.cameras block, but I have not figured out the
// syntax for that.
let arc_cam = Arc::new(camera);
// The substream always seems to be H264, even on B800 cameras
let substream_format = match &*arc_cam.format {
"h264" | "H264" | "h265" | "H265" => StreamFormat::H264,
custom_format => StreamFormat::Custom(custom_format.to_string()),
};
let permitted_users =
get_permitted_users(config.users.as_slice(), &arc_cam.permitted_users);
@@ -84,20 +77,20 @@ fn main() -> Result<(), Error> {
&*format!("/{}", arc_cam.name),
&*format!("/{}/mainStream", arc_cam.name),
];
let mut output = rtsp
.add_stream(paths, stream_format, &permitted_users)
let mut outputs = rtsp
.add_stream(paths, &permitted_users)
.unwrap();
let main_camera = arc_cam.clone();
s.spawn(move |_| camera_loop(&*main_camera, "mainStream", &mut output, true));
s.spawn(move |_| camera_loop(&*main_camera, "mainStream", &mut outputs, true));
}
if ["both", "subStream"].iter().any(|&e| e == arc_cam.stream) {
let paths = &[&*format!("/{}/subStream", arc_cam.name)];
let mut output = rtsp
.add_stream(paths, substream_format, &permitted_users)
let mut outputs = rtsp
.add_stream(paths, &permitted_users)
.unwrap();
let sub_camera = arc_cam.clone();
let manage = arc_cam.stream == "subStream";
s.spawn(move |_| camera_loop(&*sub_camera, "subStream", &mut output, manage));
s.spawn(move |_| camera_loop(&*sub_camera, "subStream", &mut outputs, manage));
}
}
@@ -111,7 +104,7 @@ fn main() -> Result<(), Error> {
fn camera_loop(
camera_config: &CameraConfig,
stream_name: &str,
output: &mut MaybeAppSrc,
outputs: &mut GstOutputs,
manage: bool,
) -> Result<Never, Error> {
let min_backoff = Duration::from_secs(1);
@@ -119,8 +112,9 @@ fn camera_loop(
let mut current_backoff = min_backoff;
loop {
let cam_err = camera_main(camera_config, stream_name, output, manage).unwrap_err();
output.on_stream_error();
let cam_err = camera_main(camera_config, stream_name, outputs, manage).unwrap_err();
outputs.vidsrc.on_stream_error();
outputs.audsrc.on_stream_error();
// Authentication failures are permanent; we retry everything else
if cam_err.connected {
current_backoff = min_backoff;
@@ -199,7 +193,7 @@ fn get_permitted_users<'a>(
fn camera_main(
camera_config: &CameraConfig,
stream_name: &str,
output: &mut dyn Write,
outputs: &mut GstOutputs,
manage: bool,
) -> Result<Never, CameraErr> {
let mut connected = false;
@@ -251,7 +245,7 @@ fn camera_main(
"{}: Starting video stream {}",
camera_config.name, stream_name
);
camera.start_video(output, stream_name, camera_config.channel_id)
camera.start_video(outputs, stream_name, camera_config.channel_id)
})()
.map_err(|err| CameraErr { connected, err })
}