From f7f12c76a1ac59ed7b9919769f6333917aee94aa Mon Sep 17 00:00:00 2001 From: Andrew King Date: Fri, 13 Nov 2020 21:45:40 +0700 Subject: [PATCH] Audio and Format auto detect (#68) * Add queues to the gstreamer launch * Add audio to the stream * Removing format option in prep for auto detect * Adding format detect from headers * Set format in gstreamer launch depending on video magic bytes * Rename MaybeAppSrcs to reflect new role of managing gst outputs * Auto detect format * Remove all setting of stream format * Re-add missing media_format caused by rebase * Add audio format auto detect too * Use more appropiate adpcm in gst * Combined gst and media_packet format enums into a single enum StreamFormat * Reduce adpcm header size * Disable adpcm output * Add oki adpcm native rust support * Maintain state between packets for adpcm * Use bytes in adpcm stream to determine adpcm decoding state * Clean up adpcm code * Use raw rtp payload for the adpcm stream * Bug fix to treat adpcm and aac seperatly in the match * Remove redundent second clamp of the step index in adpcm. * ADPCM format changes. This is a squashed commit of changes made during live testing of ADPCM format. Summary: Using DVI4 for the ADPCM, with our own decoder written in rust. * Changes to comments and IMA max range in the adpcm decoder * ADPCM magic contains block size use it for chunking the adpcm data * Use error logs rather than asserts in adpcm decode * Rustfmt and clippy styling * Use static slice for adpcm step and changes arrays * Bounds and error checks in ADPCM * Use i/u32 in adpcm * Rust fmted --- src/bc_protocol.rs | 29 +++- src/bc_protocol/adpcm.rs | 244 ++++++++++++++++++++++++++++++++ src/bc_protocol/media_packet.rs | 103 ++++++++++---- src/config.rs | 14 +- src/gst.rs | 125 ++++++++++++---- src/main.rs | 42 +++--- 6 files changed, 462 insertions(+), 95 deletions(-) create mode 100644 src/bc_protocol/adpcm.rs diff --git a/src/bc_protocol.rs b/src/bc_protocol.rs index 640f24b..9e0bad2 100644 --- a/src/bc_protocol.rs +++ b/src/bc_protocol.rs @@ -2,6 +2,8 @@ use self::connection::BcConnection; use self::media_packet::{MediaDataKind, MediaDataSubscriber}; use crate::bc; use crate::bc::{model::*, xml::*}; +use crate::gst::GstOutputs; +use adpcm::adpcm_to_pcm; use err_derive::Error; use log::*; use std::io::Write; @@ -10,6 +12,7 @@ use std::time::Duration; use Md5Trunc::*; +mod adpcm; mod connection; mod media_packet; mod time; @@ -49,12 +52,12 @@ pub enum Error { #[error(display = "Timeout")] Timeout(#[error(source)] std::sync::mpsc::RecvTimeoutError), - #[error(display = "Media")] - MediaPacket(#[error(source)] self::media_packet::Error), - #[error(display = "Credential error")] AuthFailed, + #[error(display = "ADPCM Decoding Error")] + AdpcmDecodingError(&'static str), + #[error(display = "Other error")] Other(&'static str), } @@ -245,7 +248,7 @@ impl BcCamera { pub fn start_video( &self, - data_out: &mut dyn Write, + data_outs: &mut GstOutputs, stream_name: &str, channel_id: u32, ) -> Result { @@ -278,12 +281,26 @@ impl BcCamera { let mut media_sub = MediaDataSubscriber::from_bc_sub(&sub_video); loop { - let binary_data = media_sub.next_media_packet(RX_TIMEOUT)?; + let binary_data = media_sub.next_media_packet()?; // We now have a complete interesting packet. Send it to gst. // Process the packet match binary_data.kind() { MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => { - data_out.write_all(binary_data.body())?; + let media_format = binary_data.media_format(); + data_outs.set_format(media_format); + data_outs.vidsrc.write_all(binary_data.body())?; + } + MediaDataKind::AudioDataAac => { + let media_format = binary_data.media_format(); + data_outs.set_format(media_format); + data_outs.audsrc.write_all(binary_data.body())?; + } + MediaDataKind::AudioDataAdpcm => { + let media_format = binary_data.media_format(); + data_outs.set_format(media_format); + let adpcm = binary_data.body(); + let pcm = adpcm_to_pcm(adpcm)?; + data_outs.audsrc.write_all(&pcm)?; } _ => {} }; diff --git a/src/bc_protocol/adpcm.rs b/src/bc_protocol/adpcm.rs new file mode 100644 index 0000000..e8b8793 --- /dev/null +++ b/src/bc_protocol/adpcm.rs @@ -0,0 +1,244 @@ +/* + This is a rust implementation of OKI and DVI/IMA ADPCM. +*/ +use crate::Error; +use log::error; +use std::convert::TryInto; + +struct AdpcmSetup { + max_step_index: u32, + steps: &'static [u32], + max_sample_size: i32, + changes: &'static [i32], +} + +impl AdpcmSetup { + // Unused, originally we thought BC might be using OKI but it is actually DVI4 + fn new_oki() -> Self { + Self { + max_step_index: 48, + steps: &[ + 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97, + 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449, + 494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, + ], + changes: &[-1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8], + max_sample_size: 2048, + } + } + + // This is IMA format, but it is the same as DVI4 format except in the block header + fn new_ima() -> Self { + Self { + max_step_index: 88, + steps: &[ + 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, + 55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, + 307, 337, 371, 408, 449, 494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, + 1411, 1552, 1707, 1878, 2066, 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, + 5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899, 15289, 16818, + 18500, 20350, 22385, 24623, 27086, 29794, 32767, + ], + changes: &[-1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8], + max_sample_size: 32768, + } + } +} + +struct Nibble { + // A nibble is a 4bit int + data: u8, // This is the raw data for the nibble +} + +impl Nibble { + // Use u/i32 throughout to ensure that we always have enough + // Headroom to do the math without needing `as` casting everywhere + fn unsigned(&self) -> u32 { + (self.data & 0b00001111) as u32 // Mask first 4 bits it just to be sure its in nibble range + } + + fn signed_magnitude(&self) -> u32 { + (self.data & 0b00000111) as u32 // Mask of first 3 bits which are the magnitiude bits in signed int + } + + fn signed(&self) -> i32 { + match self.data & 0b00001000 { + // Sign bit is at the 4th bit + 0b00001000 => -(self.signed_magnitude() as i32), + _ => self.signed_magnitude() as i32, + } + } + + fn from_byte(byte: &u8) -> [Self; 2] { + // Two nibbles per byte + [ + Self { + data: (byte & 0b11110000) >> 4, + }, + Self { + data: byte & 0b00001111, + }, + ] + } +} + +pub fn adpcm_to_pcm(bytes: &[u8]) -> Result, Error> { + let context = AdpcmSetup::new_ima(); + + let mut result: Vec = vec![]; // Stores the PCM byte array + + // ADPCM is not really a streamable format + // Each audio sample requires information on the previous sample + // To solve this reolink caches the intermediate variables (step_index and last_output) + // Into the block header in the stream. When ever an adpcm packet arrives it starts with + // 0x0001 which is the frame time from HISilicon documentation (I think) following by + // 0xWW which is half the block size + // 0xYY which is the last output + // 0xZZ which is the step index of the last output + // We must initialise our decoder with this data + + if bytes.len() < 4 { + error!("ADPCM data is too short for even the magic."); + return Err(Error::AdpcmDecodingError( + "ADPCM data is too short for even the magic.", + )); + } + + // Check for valid number of frame type + let frame_type_bytes = &bytes[0..2]; + const FRAME_TYPE_HISILICON: &[u8] = &[0x00, 0x01]; + if frame_type_bytes != FRAME_TYPE_HISILICON { + error!("Unexpected ADPCM frame type: {:x?}", frame_type_bytes); + return Err(Error::AdpcmDecodingError("Unexpected ADPCM frame type")); + } + + // Check for valid block size + let block_size_bytes = &bytes[2..4]; + let block_size = (u16::from_le_bytes( + block_size_bytes + .try_into() + .expect("slice with incorrect length"), + ) as u32) + * 2; // Block size is stored as 1/2 (don't know why) + let full_block_size = block_size + 4; // block_size + magic (2 bytes) + size (2 bytes) + if !bytes.len() % full_block_size as usize == 0 { + error!("ADPCM Data is not a multiple of the block size"); + return Err(Error::AdpcmDecodingError( + "ADPCM block size does not match data length.", + )); + } + + // Chunk on block size + for bytes in bytes.chunks(full_block_size as usize) { + // Get predictor state from block header using DVI 4 format. + if bytes.len() < 8 { + error!("ADPCM Block size is not long enough for header"); + return Err(Error::AdpcmDecodingError( + "ADPCM has insufficent block size", + )); + } + let step_output_bytes = &bytes[4..6]; + let mut last_output = i16::from_le_bytes( + step_output_bytes + .try_into() + .expect("slice with incorrect length"), + ) as i32; + let step_index_bytes = &bytes[6..8]; + let mut step_index = u16::from_le_bytes( + step_index_bytes + .try_into() + .expect("slice with incorrect length"), + ) as i32; + + // To avoid casting to u8 <-> u16 <-> u32 and back all the time I just do all maths in u/i32 + // This gives enough headroom to do all calculations without overflow because adpcm puts artifical + // limits on the sample sizes + let mut step: u32; + + // The rest is all data to be decoded + let data = &bytes[8..]; + + for byte in data { + let nibbles: [Nibble; 2] = Nibble::from_byte(byte); + for nibble in &nibbles { + let unibble = nibble.unsigned(); + + // Specifications say: Clamp it in max index range 0..context.max_step_index + step_index = match step_index { + n if n < 0 => 0, + n if n > context.max_step_index as i32 => context.max_step_index as i32, + n => n, + }; + + // This is just Eulers approximation with a variable step size + // **Adaptive** Differential PCM + // Adaptive: because the step size is variable + step = context.steps[step_index as usize]; + + let raw_sample; + /* == Non approxiate version === + // This is the full maths version + // We don't use this one as we need to match the way the encoder + // works if we want to use the state stored in the header. + // I have Left it here as it is easier to understand then the bit shift version below + let inibble = nibble.signed(); + + // Calculate the delta (which is really what adpcm is all about) + // Adaptive **Differential** PCM + // Differential: Becuase its all about the difference (gradient) + let diff = (step as i32) * (inibble) / 2 + (step as i32) / 8; + + // Eulers approxiation + // Sample = Previous_Sample + difference*step_size + raw_sample = last_output + diff; + */ + + // === Approximate version == + // Approximate form uses bit shift operators. + // This is a legacy of the days when mult/divides were expensive + // It is also the format used on low end CPUs like cameras + let mut diff = step >> 3; + if (unibble & 0b0100) == 0b0100 { + diff += step; + } + if (unibble & 0b0010) == 0b0010 { + diff += step >> 1; + } + if (unibble & 0b0001) == 0b0001 { + diff += step >> 2; + } + // Sign test + if (unibble & 0b1000) == 0b1000 { + raw_sample = last_output - (diff as i32); + } else { + raw_sample = last_output + (diff as i32); + } + + // Specifications say: Clamp it in max sample range -context.max_sample_size..context.max_sample_size + let sample = match raw_sample { + value if value > context.max_sample_size - 1 => context.max_sample_size - 1, + value if value < -context.max_sample_size => -context.max_sample_size, + value => value, + }; + + // PCM is really in i16 range + // Some formats e.g. OKI are not in the full PCM range of values + // To convert we must scale it to the i16 range + // We also cast to i16 at this point ready for the conversion to u8 bytes of the output + let scaled_sample = (sample as i32 * (i16::MAX as i32) + / (context.max_sample_size - 1) as i32) + as i16; + + // Get the results in bytes + result.extend(scaled_sample.to_le_bytes().iter()); + + // Increment the step index + step_index = step_index as i32 + context.changes[unibble as usize]; + + // cache the last_output ready for next run + last_output = sample; + } + } + } + Ok(result) +} diff --git a/src/bc_protocol/media_packet.rs b/src/bc_protocol/media_packet.rs index 3d4689b..b69a1d8 100644 --- a/src/bc_protocol/media_packet.rs +++ b/src/bc_protocol/media_packet.rs @@ -1,11 +1,11 @@ +use super::{Error, Result, RX_TIMEOUT}; use crate::bc::model::*; use crate::bc_protocol::connection::BcSubscription; -use err_derive::Error; +use crate::gst::StreamFormat; use log::trace; use log::*; use std::collections::VecDeque; use std::convert::TryInto; -use std::time::Duration; const INVALID_MEDIA_PACKETS: &[MediaDataKind] = &[MediaDataKind::Unknown]; @@ -15,14 +15,6 @@ const MAGIC_SIZE: usize = 4; // PAD_SIZE: Media packets use 8 byte padding const PAD_SIZE: usize = 8; -type Result = std::result::Result; - -#[derive(Debug, Error)] -pub enum Error { - #[error(display = "Timeout")] - Timeout(#[error(source)] std::sync::mpsc::RecvTimeoutError), -} - #[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)] pub enum MediaDataKind { VideoDataIframe, @@ -50,7 +42,7 @@ impl MediaData { MediaDataKind::VideoDataIframe => 32, MediaDataKind::VideoDataPframe => 24, MediaDataKind::AudioDataAac => 8, - MediaDataKind::AudioDataAdpcm => 16, + MediaDataKind::AudioDataAdpcm => 8, MediaDataKind::InfoData => 32, MediaDataKind::Unknown => 0, } @@ -61,6 +53,39 @@ impl MediaData { MediaData::header_size_from_kind(kind) } + pub fn header(&self) -> &[u8] { + &self.data[0..self.header_size()] + } + + pub fn header_dump(&self) { + info!("{:?}-hex: {:02?}", self.kind(), self.header()); + let mut result = vec![]; + for four in self.header().chunks(4) { + result.push(u32::from_le_bytes(four.try_into().unwrap())); + } + info!("{:?}-32: {:?}", self.kind(), result); + let mut result = vec![]; + for two in self.header().chunks(2) { + result.push(u16::from_le_bytes(two.try_into().unwrap())); + } + info!("{:?}-16: {:?}", self.kind(), result); + let mut result = vec![]; + for one in self.header().chunks(1) { + result.push(u8::from_le_bytes(one.try_into().unwrap())); + } + info!("{:?}-8: {:?}", self.kind(), result); + let mut result = vec![]; + for four in self.header().chunks(4) { + result.push(f32::from_le_bytes(four.try_into().unwrap())); + } + info!("{:?}-f32: {:?}", self.kind(), result); + let mut result = vec![]; + for four in self.header().chunks(4) { + result.push(String::from_utf8_lossy(four)); + } + info!("{:?}-utf8: {:?}", self.kind(), result); + } + fn header_size(&self) -> usize { MediaData::header_size_from_raw(&self.data) } @@ -129,7 +154,7 @@ impl MediaData { MAGIC_IFRAME => MediaDataKind::VideoDataIframe, MAGIC_PFRAME => MediaDataKind::VideoDataPframe, _ => { - trace!("Unknown magic kind: {:x?}", &magic); + //trace!("Unknown magic kind: {:x?}", &magic); MediaDataKind::Unknown } } @@ -138,6 +163,39 @@ impl MediaData { pub fn kind(&self) -> MediaDataKind { MediaData::kind_from_raw(&self.data) } + + pub fn media_format(&self) -> Option { + let kind = self.kind(); + match kind { + MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => { + let stream_type = &self.data[4..8]; + const H264_STR_UPPER: &[u8] = &[0x48, 0x32, 0x36, 0x34]; + const H264_STR_LOWER: &[u8] = &[0x68, 0x32, 0x36, 0x34]; + const H265_STR_UPPER: &[u8] = &[0x48, 0x32, 0x36, 0x35]; + const H265_STR_LOWER: &[u8] = &[0x68, 0x32, 0x36, 0x35]; + match stream_type { + H264_STR_UPPER | H264_STR_LOWER => Some(StreamFormat::H264), // Offically it should be "H264" not "h264" but covering all cases + H265_STR_UPPER | H265_STR_LOWER => Some(StreamFormat::H265), + _ => None, + } + } + MediaDataKind::AudioDataAac => Some(StreamFormat::AAC), + MediaDataKind::AudioDataAdpcm => Some(StreamFormat::ADPCM), + _ => None, + } + } + + pub fn timestamp(&self) -> Option { + let kind = self.kind(); + match kind { + MediaDataKind::VideoDataIframe | MediaDataKind::VideoDataPframe => Some( + Self::bytes_to_size(&self.data[16..20]) + .try_into() + .expect("usize wont fit into u64"), + ), + _ => None, + } + } } pub struct MediaDataSubscriber<'a> { @@ -153,10 +211,10 @@ impl<'a> MediaDataSubscriber<'a> { } } - fn fill_binary_buffer(&mut self, rx_timeout: Duration) -> Result<()> { + fn fill_binary_buffer(&mut self) -> Result<()> { // Loop messages until we get binary add that data and return loop { - let msg = self.bc_sub.rx.recv_timeout(rx_timeout)?; + let msg = self.bc_sub.rx.recv_timeout(RX_TIMEOUT)?; if let BcBody::ModernMsg(ModernMsg { binary: Some(binary), .. @@ -170,11 +228,11 @@ impl<'a> MediaDataSubscriber<'a> { Ok(()) } - fn advance_to_media_packet(&mut self, rx_timeout: Duration) -> Result<()> { + fn advance_to_media_packet(&mut self) -> Result<()> { // In the event we get an unknown packet we advance by brute force // reading of bytes to the next valid magic while self.binary_buffer.len() < MAGIC_SIZE { - self.fill_binary_buffer(rx_timeout)?; + self.fill_binary_buffer()?; } // Check the kind, if its invalid use pop a byte and try again @@ -186,7 +244,7 @@ impl<'a> MediaDataSubscriber<'a> { while INVALID_MEDIA_PACKETS.contains(&MediaData::kind_from_raw(&magic)) { self.binary_buffer.pop_front(); while self.binary_buffer.len() < MAGIC_SIZE { - self.fill_binary_buffer(rx_timeout)?; + self.fill_binary_buffer()?; } magic = MediaDataSubscriber::get_first_n_deque(&self.binary_buffer, MAGIC_SIZE); } @@ -213,12 +271,9 @@ impl<'a> MediaDataSubscriber<'a> { } } - pub fn next_media_packet( - &mut self, - rx_timeout: Duration, - ) -> std::result::Result { + pub fn next_media_packet(&mut self) -> std::result::Result { // Find the first packet (does nothing if already at one) - self.advance_to_media_packet(rx_timeout)?; + self.advance_to_media_packet()?; // Get the magic bytes (guaranteed by advance_to_media_packet) let magic = MediaDataSubscriber::get_first_n_deque(&self.binary_buffer, MAGIC_SIZE); @@ -226,7 +281,7 @@ impl<'a> MediaDataSubscriber<'a> { // Get enough for the full header let header_size = MediaData::header_size_from_raw(&magic); while self.binary_buffer.len() < header_size { - self.fill_binary_buffer(rx_timeout)?; + self.fill_binary_buffer()?; } // Get enough for the full data + 8 byte buffer @@ -235,7 +290,7 @@ impl<'a> MediaDataSubscriber<'a> { let pad_size = MediaData::pad_size_from_raw(&header); let full_size = header_size + data_size + pad_size; while self.binary_buffer.len() < full_size { - self.fill_binary_buffer(rx_timeout)?; + self.fill_binary_buffer()?; } // Pop the full binary buffer diff --git a/src/config.rs b/src/config.rs index 765649a..79b4af5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -8,7 +8,6 @@ use validator::{Validate, ValidationError}; use validator_derive::Validate; lazy_static! { - static ref RE_STREAM_FORM: Regex = Regex::new(r"^([hH]26[45]|[ \t]*[!].*)$").unwrap(); static ref RE_STREAM_SRC: Regex = Regex::new(r"^(mainStream|subStream|both)$").unwrap(); static ref RE_TLS_CLIENT_AUTH: Regex = Regex::new(r"^(none|request|require)$").unwrap(); } @@ -54,13 +53,8 @@ pub struct CameraConfig { // no longer used, but still here so we can warn users: pub timeout: Option, - #[validate(regex( - path = "RE_STREAM_FORM", - message = "Incorrect stream format", - code = "format" - ))] - #[serde(default = "default_format")] - pub format: String, + // no longer used, but still here so we can warn users: + pub format: Option, #[validate(regex( path = "RE_STREAM_SRC", @@ -95,10 +89,6 @@ fn default_bind_port() -> u16 { 8554 } -fn default_format() -> String { - "h265".to_string() -} - fn default_stream() -> String { "both".to_string() } diff --git a/src/gst.rs b/src/gst.rs index 6cec61e..d826c8c 100644 --- a/src/gst.rs +++ b/src/gst.rs @@ -1,7 +1,6 @@ //! This module provides an "RtspServer" abstraction that allows consumers of its API to feed it //! data using an ordinary std::io::Write interface. pub use self::maybe_app_src::MaybeAppSrc; - use gstreamer::prelude::Cast; use gstreamer::{Bin, Structure}; use gstreamer_app::AppSrc; @@ -27,10 +26,82 @@ pub struct RtspServer { server: GstRTSPServer, } +#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)] pub enum StreamFormat { H264, H265, - Custom(String), + AAC, + ADPCM, +} + +pub struct GstOutputs { + pub audsrc: MaybeAppSrc, + pub vidsrc: MaybeAppSrc, + video_format: Option, + audio_format: Option, + factory: RTSPMediaFactory, +} + +impl GstOutputs { + pub fn from_appsrcs(vidsrc: MaybeAppSrc, audsrc: MaybeAppSrc) -> GstOutputs { + let result = GstOutputs { + vidsrc, + audsrc, + video_format: None, + audio_format: None, + factory: RTSPMediaFactory::new(), + }; + result.apply_format(); + result + } + + pub fn set_format(&mut self, format: Option) { + match format { + Some(StreamFormat::H264) | Some(StreamFormat::H265) => { + if format != self.video_format { + self.video_format = format; + self.apply_format(); + } + } + Some(StreamFormat::AAC) | Some(StreamFormat::ADPCM) => { + if format != self.audio_format { + self.audio_format = format; + self.apply_format(); + } + } + _ => {} + } + } + + fn apply_format(&self) { + let launch_vid = match self.video_format { + Some(StreamFormat::H264) => { + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h264parse ! rtph264pay name=pay0" + } + Some(StreamFormat::H265) => { + "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! h265parse ! rtph265pay name=pay0" + } + _ => "! fakesink", + }; + + let launch_aud = match self.audio_format { + Some(StreamFormat::ADPCM) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! rawaudioparse format=pcm pcm-format=s16le sample-rate=8000 num-channels=1 interleaved=true ! audioconvert ! rtpL16pay name=pay1", // DVI4 is converted to pcm in the appsrc + Some(StreamFormat::AAC) => "! queue silent=true max-size-bytes=10485760 min-threshold-bytes=1024 ! aacparse ! rtpmp4apay name=pay1", + _ => "! fakesink", + }; + + self.factory.set_launch( + &vec![ + "( ", + "appsrc name=vidsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks + launch_vid, + "appsrc name=audsrc is-live=true block=true emit-signals=false max-bytes=52428800 do-timestamp=true format=GST_FORMAT_TIME", // 50MB max size so that it won't grow to infinite if the queue blocks + launch_aud, + ")" + ] + .join(" "), + ); + } } impl Default for RtspServer { @@ -50,21 +121,23 @@ impl RtspServer { pub fn add_stream( &self, paths: &[&str], - stream_format: StreamFormat, permitted_users: &HashSet<&str>, - ) -> Result { + ) -> Result { let mounts = self .server .get_mount_points() .expect("The server should have mountpoints"); - let launch_str = match stream_format { - StreamFormat::H264 => "! h264parse ! rtph264pay name=pay0", - StreamFormat::H265 => "! h265parse ! rtph265pay name=pay0", - StreamFormat::Custom(ref custom_format) => custom_format, - }; + // Create a MaybeAppSrc: Write which we will give the caller. When the backing AppSrc is + // created by the factory, fish it out and give it to the waiting MaybeAppSrc via the + // channel it provided. This callback may be called more than once by Gstreamer if it is + // unhappy with the pipeline, so keep updating the MaybeAppSrc. + let (maybe_app_src, tx) = MaybeAppSrc::new_with_tx(); + let (maybe_app_src_aud, tx_aud) = MaybeAppSrc::new_with_tx(); - let factory = RTSPMediaFactory::new(); + let outputs = GstOutputs::from_appsrcs(maybe_app_src, maybe_app_src_aud); + + let factory = &outputs.factory; debug!( "Permitting {} to access {}", @@ -76,25 +149,10 @@ impl RtspServer { .collect::(), paths.join(", ") ); - self.add_permitted_roles(&factory, permitted_users); + self.add_permitted_roles(factory, permitted_users); - //factory.set_protocols(RTSPLowerTrans::TCP); - factory.set_launch(&format!("{}{}{}{}", - "( ", - "appsrc name=writesrc is-live=true block=true emit-signals=false max-bytes=0 do-timestamp=true ", - launch_str, - " )" - )); factory.set_shared(true); - // TODO maybe set video format via - // https://gitlab.freedesktop.org/gstreamer/gstreamer-rs/-/blob/master/examples/src/bin/appsrc.rs#L66 - - // Create a MaybeAppSrc: Write which we will give the caller. When the backing AppSrc is - // created by the factory, fish it out and give it to the waiting MaybeAppSrc via the - // channel it provided. This callback may be called more than once by Gstreamer if it is - // unhappy with the pipeline, so keep updating the MaybeAppSrc. - let (maybe_app_src, tx) = MaybeAppSrc::new_with_tx(); factory.connect_media_configure(move |_factory, media| { debug!("RTSP: media was configured"); let bin = media @@ -103,18 +161,25 @@ impl RtspServer { .dynamic_cast::() .expect("Media source's element should be a bin"); let app_src = bin - .get_by_name_recurse_up("writesrc") + .get_by_name_recurse_up("vidsrc") .expect("write_src must be present in created bin") .dynamic_cast::() .expect("Source element is expected to be an appsrc!"); let _ = tx.send(app_src); // Receiver may be dropped, don't panic if so + + let app_src_aud = bin + .get_by_name_recurse_up("audsrc") + .expect("write_src must be present in created bin") + .dynamic_cast::() + .expect("Source element is expected to be an appsrc!"); + let _ = tx_aud.send(app_src_aud); // Receiver may be dropped, don't panic if so }); for path in paths { - mounts.add_factory(path, &factory); + mounts.add_factory(path, factory); } - Ok(maybe_app_src) + Ok(outputs) } pub fn add_permitted_roles(&self, factory: &RTSPMediaFactory, permitted_roles: &HashSet<&str>) { @@ -241,12 +306,14 @@ mod maybe_app_src { Some(src) => src, None => return Ok(buf.len()), }; + let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); { let gst_buf_mut = gst_buf.get_mut().unwrap(); let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); gst_buf_data.copy_from_slice(buf); } + let res = app_src.push_buffer(gst_buf); //.map_err(|e| io::Error::new(io::ErrorKind::Other, Box::new(e)))?; if res.is_err() { self.app_src = None; diff --git a/src/main.rs b/src/main.rs index 0660aa3..ffba065 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,11 +3,10 @@ use err_derive::Error; use gio::TlsAuthenticationMode; use log::*; use neolink::bc_protocol::BcCamera; -use neolink::gst::{MaybeAppSrc, RtspServer, StreamFormat}; +use neolink::gst::{GstOutputs, RtspServer}; use neolink::Never; use std::collections::HashSet; use std::fs; -use std::io::Write; use std::sync::Arc; use std::time::Duration; use structopt::StructOpt; @@ -29,6 +28,8 @@ pub enum Error { IoError(#[error(source)] std::io::Error), #[error(display = "Validation error")] ValidationError(#[error(source)] validator::ValidationErrors), + #[error(display = "ADPCM Decoding Error")] + AdpcmDecodingError(&'static str), } fn main() -> Result<(), Error> { @@ -59,22 +60,14 @@ fn main() -> Result<(), Error> { crossbeam::scope(|s| { for camera in config.cameras { - let stream_format = match &*camera.format { - "h264" | "H264" => StreamFormat::H264, - "h265" | "H265" => StreamFormat::H265, - custom_format => StreamFormat::Custom(custom_format.to_string()), - }; - + if camera.format.is_some() { + warn!("The format config option of the camera has been removed in favour of auto detection.") + } // Let subthreads share the camera object; in principle I think they could share // the object as it sits in the config.cameras block, but I have not figured out the // syntax for that. let arc_cam = Arc::new(camera); - // The substream always seems to be H264, even on B800 cameras - let substream_format = match &*arc_cam.format { - "h264" | "H264" | "h265" | "H265" => StreamFormat::H264, - custom_format => StreamFormat::Custom(custom_format.to_string()), - }; let permitted_users = get_permitted_users(config.users.as_slice(), &arc_cam.permitted_users); @@ -84,20 +77,20 @@ fn main() -> Result<(), Error> { &*format!("/{}", arc_cam.name), &*format!("/{}/mainStream", arc_cam.name), ]; - let mut output = rtsp - .add_stream(paths, stream_format, &permitted_users) + let mut outputs = rtsp + .add_stream(paths, &permitted_users) .unwrap(); let main_camera = arc_cam.clone(); - s.spawn(move |_| camera_loop(&*main_camera, "mainStream", &mut output, true)); + s.spawn(move |_| camera_loop(&*main_camera, "mainStream", &mut outputs, true)); } if ["both", "subStream"].iter().any(|&e| e == arc_cam.stream) { let paths = &[&*format!("/{}/subStream", arc_cam.name)]; - let mut output = rtsp - .add_stream(paths, substream_format, &permitted_users) + let mut outputs = rtsp + .add_stream(paths, &permitted_users) .unwrap(); let sub_camera = arc_cam.clone(); let manage = arc_cam.stream == "subStream"; - s.spawn(move |_| camera_loop(&*sub_camera, "subStream", &mut output, manage)); + s.spawn(move |_| camera_loop(&*sub_camera, "subStream", &mut outputs, manage)); } } @@ -111,7 +104,7 @@ fn main() -> Result<(), Error> { fn camera_loop( camera_config: &CameraConfig, stream_name: &str, - output: &mut MaybeAppSrc, + outputs: &mut GstOutputs, manage: bool, ) -> Result { let min_backoff = Duration::from_secs(1); @@ -119,8 +112,9 @@ fn camera_loop( let mut current_backoff = min_backoff; loop { - let cam_err = camera_main(camera_config, stream_name, output, manage).unwrap_err(); - output.on_stream_error(); + let cam_err = camera_main(camera_config, stream_name, outputs, manage).unwrap_err(); + outputs.vidsrc.on_stream_error(); + outputs.audsrc.on_stream_error(); // Authentication failures are permanent; we retry everything else if cam_err.connected { current_backoff = min_backoff; @@ -199,7 +193,7 @@ fn get_permitted_users<'a>( fn camera_main( camera_config: &CameraConfig, stream_name: &str, - output: &mut dyn Write, + outputs: &mut GstOutputs, manage: bool, ) -> Result { let mut connected = false; @@ -251,7 +245,7 @@ fn camera_main( "{}: Starting video stream {}", camera_config.name, stream_name ); - camera.start_video(output, stream_name, camera_config.channel_id) + camera.start_video(outputs, stream_name, camera_config.channel_id) })() .map_err(|err| CameraErr { connected, err }) }