From e5bbd631eb2336d1268d974f896027198e4b0ea9 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Wed, 4 Aug 2021 15:28:15 +0700 Subject: [PATCH] Add TalkAbility protocol and update talk to use it. Also add a `talk_stream` method --- crates/core/src/bc/model.rs | 2 + crates/core/src/bc/xml.rs | 49 ++++- crates/core/src/bc_protocol/errors.rs | 4 + crates/core/src/bc_protocol/talk.rs | 292 ++++++++++++++++++++++++-- 4 files changed, 326 insertions(+), 21 deletions(-) diff --git a/crates/core/src/bc/model.rs b/crates/core/src/bc/model.rs index 41c1fa2..18eacd1 100644 --- a/crates/core/src/bc/model.rs +++ b/crates/core/src/bc/model.rs @@ -10,6 +10,8 @@ pub const MSG_ID_LOGIN: u32 = 1; pub const MSG_ID_LOGOUT: u32 = 2; /// Video and Audio Streams messages have this ID pub const MSG_ID_VIDEO: u32 = 3; +/// TalkAbility messages have this ID +pub const MSG_ID_TALKABILITY: u32 = 10; /// Reboot messages have this ID pub const MSG_ID_REBOOT: u32 = 23; /// Version messages have this ID diff --git a/crates/core/src/bc/xml.rs b/crates/core/src/bc/xml.rs index 37cba1b..1aea64a 100644 --- a/crates/core/src/bc/xml.rs +++ b/crates/core/src/bc/xml.rs @@ -63,6 +63,9 @@ pub struct BcXml { /// Sent as part of the TalkConfig to prepare the camera for audio talk-back #[yaserde(rename = "TalkConfig")] pub talk_config: Option, + /// Revieced as part of the TalkAbility request + #[yaserde(rename = "TalkAbility")] + pub talk_ability: Option, } impl BcXml { @@ -319,9 +322,11 @@ pub struct TalkConfig { } /// audioConfig xml -#[derive(PartialEq, Eq, Default, Debug, YaDeserialize, YaSerialize)] +#[derive(PartialEq, Eq, Default, Debug, YaDeserialize, YaSerialize, Clone)] #[yaserde(rename = "audioConfig")] pub struct AudioConfig { + /// Unknown only sent during TalkAbility request from the camera + pub priority: Option, /// Audio type known values are `"adpcm"` /// /// Do not expect camera to support anything else. @@ -340,7 +345,47 @@ pub struct AudioConfig { /// /// Do not expect camera to support anything else #[yaserde(rename = "soundTrack")] - pub soundTrack: String, + pub sound_track: String, +} + +/// TalkAbility xml +#[derive(PartialEq, Eq, Default, Debug, YaDeserialize, YaSerialize)] +pub struct TalkAbility { + /// XML Version + #[yaserde(attribute)] + pub version: String, + /// Duplexes known values `"FDX"` + #[yaserde(rename = "duplexList")] + pub duplex_list: Vec, + /// audioStreamModes known values `"followVideoStream"` + #[yaserde(rename = "audioStreamModeList")] + pub audio_stream_mode_list: Vec, + /// AudioConfigs contans the details of the audio to follow + #[yaserde(rename = "audioConfigList")] + pub audio_config_list: Vec, +} + +/// duplexList xml +#[derive(PartialEq, Eq, Default, Debug, YaDeserialize, YaSerialize)] +pub struct DuplexList { + /// The supported duplex known values are "FBX" + pub duplex: String, +} + +/// audioStreamModeList xml +#[derive(PartialEq, Eq, Default, Debug, YaDeserialize, YaSerialize)] +pub struct AudioStreamModeList { + /// The supported audio stream mode + #[yaserde(rename = "audioStreamMode")] + pub audio_stream_mode: String, +} + +/// audioConfigList xml +#[derive(PartialEq, Eq, Default, Debug, YaDeserialize, YaSerialize)] +pub struct AudioConfigList { + /// The supported audio configs + #[yaserde(rename = "audioConfig")] + pub audio_config: AudioConfig, } /// Convience function to return the xml version used throughout the library diff --git a/crates/core/src/bc_protocol/errors.rs b/crates/core/src/bc_protocol/errors.rs index d0b532d..ef79712 100644 --- a/crates/core/src/bc_protocol/errors.rs +++ b/crates/core/src/bc_protocol/errors.rs @@ -58,6 +58,10 @@ pub enum Error { #[error(display = "Failed to translate camera address")] AddrResolutionError, + /// Raised non adpcm data is sent to the talk command + #[error(display = "Talk data is not ADPCM")] + UnknownTalkEncoding, + /// A generic catch all error #[error(display = "Other error")] Other(&'static str), diff --git a/crates/core/src/bc_protocol/talk.rs b/crates/core/src/bc_protocol/talk.rs index fc34cde..bb8b501 100644 --- a/crates/core/src/bc_protocol/talk.rs +++ b/crates/core/src/bc_protocol/talk.rs @@ -1,39 +1,86 @@ use super::{BcCamera, Error, Result, RX_TIMEOUT}; use crate::{bc::model::*, bc::xml::*, bcmedia::model::*}; +use std::io::{BufRead, Error as IoError, ErrorKind, Read}; +use std::sync::mpsc::Receiver; + +type IoResult = std::result::Result; impl BcCamera { + /// + /// Requests the [`TalkAbility`] xml + /// + pub fn talk_ability(&self) -> Result { + let connection = self + .connection + .as_ref() + .expect("Must be connected to get time"); + let sub_get = connection.subscribe(MSG_ID_TALKABILITY)?; + let get = Bc { + meta: BcMeta { + msg_id: MSG_ID_TALKABILITY, + channel_id: self.channel_id, + msg_num: self.new_message_num(), + response_code: 0, + stream_type: 0, + class: 0x6414, + }, + body: BcBody::ModernMsg(ModernMsg { + extension: Some(Extension { + channel_id: Some(self.channel_id), + ..Default::default() + }), + payload: None, + }), + }; + + sub_get.send(get)?; + let msg = sub_get.rx.recv_timeout(RX_TIMEOUT)?; + + if let BcBody::ModernMsg(ModernMsg { + payload: + Some(BcPayloads::BcXml(BcXml { + talk_ability: Some(talk_ability), + .. + })), + .. + }) = msg.body + { + Ok(talk_ability) + } else { + Err(Error::UnintelligibleReply { + reply: msg, + why: "Expected TalkAbility xml but it was not recieved", + }) + } + } + /// /// Send sound to the camera /// - /// long description + /// The data should be in the format as described in `` xml + /// This method assumes that you have set up the data in the desired format + /// in the `` xml + /// + /// It also checks that it is ADPCM as the code is written to accept only that /// /// # Parameters /// /// * `adpcm` - Data must be adpcm in DVI-4 format /// - /// * `block_size` - is block-align used to encode the adpcm - /// - /// * `sample_rate` - sample rate of the audio + /// * `talk_config` - The talk config that describes the adpcm data /// /// - pub fn talk(&self, adpcm: &[u8], block_size: u16, sample_rate: u16) -> Result<()> { - let connection = self.connection.as_ref().expect("Must be connected to ping"); + pub fn talk(&self, adpcm: &[u8], talk_config: TalkConfig) -> Result<()> { + let connection = self.connection.as_ref().expect("Must be connected"); let sub = connection.subscribe(MSG_ID_TALKCONFIG)?; - let talk_config = TalkConfig { - channel_id: self.channel_id, - duplex: "FDX".to_string(), - audio_stream_mode: "followVideoStream".to_string(), - audio_config: AudioConfig { - audio_type: "adpcm".to_string(), - sample_rate, - sample_precision: 16, - length_per_encoder: block_size * 2, - soundTrack: "mono".to_string(), - }, - ..Default::default() - }; + if &talk_config.audio_config.audio_type != "adpcm" { + return Err(Error::UnknownTalkEncoding); + } + + let block_size = talk_config.audio_config.length_per_encoder / 2; + let sample_rate = talk_config.audio_config.sample_rate; let msg = Bc { meta: BcMeta { @@ -126,4 +173,211 @@ impl BcCamera { Ok(()) } + + /// + /// Send sound to the camera through a channel + /// + /// This is similar to [`talk`] except it uses a channel to receive data + /// + /// The data should be in the format as described in `` xml + /// This method assumes that you have set up the data in the desired format + /// in the `` xml + /// + /// It also checks that it is ADPCM as the code is written to accept only that + /// + /// # Parameters + /// + /// * `adpcm` - Data must be adpcm in DVI-4 format + /// + /// * `talk_config` - The talk config that describes the adpcm data + /// + /// + pub fn talk_stream(&self, rx: Receiver>, talk_config: TalkConfig) -> Result<()> { + let connection = self.connection.as_ref().expect("Must be connected"); + + let sub = connection.subscribe(MSG_ID_TALKCONFIG)?; + + if &talk_config.audio_config.audio_type != "adpcm" { + return Err(Error::UnknownTalkEncoding); + } + + let block_size = talk_config.audio_config.length_per_encoder / 2; + let sample_rate = talk_config.audio_config.sample_rate; + + let msg = Bc { + meta: BcMeta { + msg_id: MSG_ID_TALKCONFIG, + channel_id: self.channel_id, + msg_num: self.new_message_num(), + stream_type: 0, + response_code: 0, + class: 0x6414, + }, + body: BcBody::ModernMsg(ModernMsg { + extension: Some(Extension { + channel_id: Some(self.channel_id), + ..Default::default() + }), + payload: Some(BcPayloads::BcXml(BcXml { + talk_config: Some(talk_config), + ..Default::default() + })), + }), + }; + + sub.send(msg)?; + let msg = sub.rx.recv_timeout(RX_TIMEOUT)?; + + if let BcMeta { + response_code: 200, .. + } = msg.meta + { + } else { + return Err(Error::UnintelligibleReply { + reply: msg, + why: + "The camera did not accept the TalkConfig xml. Audio format is likely incorrect", + }); + } + + let full_block_size = block_size + 4; // Block size + predictor state + let sub = connection.subscribe(MSG_ID_TALK)?; + + const BLOCK_PER_PAYLOAD: usize = 4; + const BLOCK_HEADER_SIZE: usize = 4; + const SAMPLES_PER_BYTE: usize = 2; + + let mut buffered_recv = BufferedStream::from_rx(rx); + + let target_chunks = full_block_size as usize * BLOCK_PER_PAYLOAD; + + let mut payload_bytes = vec![]; + let mut end_of_stream = false; + while !end_of_stream { + while payload_bytes.len() < target_chunks { + let mut buffer = vec![0; target_chunks - payload_bytes.len()]; + if let Ok(read) = buffered_recv.read(&mut buffer) { + payload_bytes.extend(&buffer[..read]); + } else { + // Error should occur if the channel is dropped + // and all bytes are consumed + end_of_stream = true; + } + if end_of_stream { + break; + } + } + + let mut payload = vec![]; + for bytes in payload_bytes.chunks(full_block_size as usize) { + let bcmedia_adpcm = BcMedia::Adpcm(BcMediaAdpcm { + data: bytes.to_vec(), + }); + payload = bcmedia_adpcm.serialize(payload)?; + } + + let adpcm_len = payload_bytes.len(); + payload_bytes = vec![]; + + // There are two samples per byte + // + // To calculate the bytes we subtract the block headers from the len + // + // There is 1 initial sample stored in the block header so we add that in the end + // + let samples_sent = (adpcm_len - BLOCK_HEADER_SIZE * BLOCK_PER_PAYLOAD) + * SAMPLES_PER_BYTE + + BLOCK_PER_PAYLOAD; + + // Time to play the sample in seconds + let play_length = samples_sent as f32 / sample_rate as f32; + + let msg = Bc { + meta: BcMeta { + msg_id: MSG_ID_TALK, + channel_id: self.channel_id, + msg_num: self.new_message_num(), + stream_type: 0, + response_code: 0, + class: 0x6414, + }, + body: BcBody::ModernMsg(ModernMsg { + extension: Some(Extension { + channel_id: Some(self.channel_id), + binary_data: Some(1), + ..Default::default() + }), + payload: Some(BcPayloads::Binary(payload)), + }), + }; + + sub.send(msg)?; + + std::thread::sleep(std::time::Duration::from_secs_f32(play_length * 0.95)); + } + + Ok(()) + } +} + +struct BufferedStream { + rx: Receiver>, + buffer: Vec, + consumed: usize, +} + +impl BufferedStream { + pub fn from_rx(rx: Receiver>) -> BufferedStream { + BufferedStream { + rx, + buffer: vec![], + consumed: 0, + } + } +} + +impl Read for BufferedStream { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + let buffer = self.fill_buf()?; + let amt = std::cmp::min(buf.len(), buffer.len()); + + // First check if the amount of bytes we want to read is small: + // `copy_from_slice` will generally expand to a call to `memcpy`, and + // for a single byte the overhead is significant. + if amt == 1 { + buf[0] = buffer[0]; + } else { + buf[..amt].copy_from_slice(&buffer[..amt]); + } + + self.consume(amt); + + Ok(amt) + } +} + +impl BufRead for BufferedStream { + fn fill_buf(&mut self) -> IoResult<&[u8]> { + const CLEAR_CONSUMED_AT: usize = 1024; + // This is a trade off between caching too much dead memory + // and calling the drain method too often + if self.consumed > CLEAR_CONSUMED_AT { + let _ = self.buffer.drain(0..self.consumed).collect::>(); + self.consumed = 0; + } + while self.buffer.len() <= self.consumed { + let data = self + .rx + .recv() + .map_err(|err| IoError::new(ErrorKind::ConnectionReset, err))?; + self.buffer.extend(data); + } + + Ok(&self.buffer.as_slice()[self.consumed..]) + } + + fn consume(&mut self, amt: usize) { + assert!(self.consumed + amt <= self.buffer.len()); + self.consumed += amt; + } }