Use talk_stream and gstreamer to play other types of audio

This commit is contained in:
QuantumEntangledAndy committed 2021-08-04 15:28:45 +07:00
1 parent e5bbd631eb
commit 892bbc1cd1
4 files changed
+216 -14

No files matched your search

+6 -11
View File
@@ -1,9 +1,10 @@
use std::path::PathBuf;
use structopt::StructOpt;
/// The talk command will send adpcm data to the camera to say
/// The talk command will send audio for the camera to say
///
/// This data should be encoded in adpcm with dvi4 layout
/// This data should be encoded in a way that gstreamer can understand.
/// This should be ok with most common formats.
///
/// `gst-launch` can be used to prepare this data
#[derive(StructOpt, Debug)]
@@ -13,13 +14,7 @@ pub struct Opt {
pub config: PathBuf,
/// The name of the camera to change the lights of. Must be a name in the config
pub camera: String,
/// The path to the adpcm data to send in DVI-4 layout
#[structopt(short, long, parse(from_os_str), name = "media-path")]
pub adpcm_file: PathBuf,
/// The block size used to encode the adpcm data (recommended 512)
#[structopt(short, long)]
pub block_size: u16,
/// The sample rate used to encode the adpcm data (recommended 16000)
#[structopt(short, long)]
pub sample_rate: u16,
/// The path to the audio file.
#[structopt(short, long, parse(from_os_str))]
pub media_path: PathBuf,
}
+22
View File
@@ -1,4 +1,5 @@
use err_derive::Error;
use gstreamer::glib;
/// The main error for the status-light subcommand
#[derive(Debug, Error)]
@@ -17,4 +18,25 @@ pub enum Error {
/// Raised when the config file fails validataion
#[error(display = "Validation error")]
Validation(#[error(source)] validator::ValidationErrors),
/// Raised when gstreamer fails to init
#[error(display = "Gstreamer init failed")]
GstreamerInit(#[error(source)] glib::Error),
/// Raised when a needed part of the gsteamer pipeline is not found
///
/// Usually this means missing pluings
#[error(display = "Gstreamer element not supported. Check your gstreamer plugins")]
GstreamerElement(gstreamer::Element),
/// Raised where there is an issue starting a audio playback in gstraemer
#[error(display = "Gstreamer Unable to play the file")]
GstreamerPlayback(#[error(source)] gstreamer::StateChangeError),
/// Generic gstreamer error
#[error(display = "Gstreamer raised an error")]
Gstreamer {
error: String,
debug: Option<String>,
},
/// Raised when talk back is unsupported
#[error(display = "This camera does not support talkback")]
TalkUnsupported,
}
+150
View File
@@ -0,0 +1,150 @@
use gstreamer::{
element_error, parse_launch, prelude::*, ClockTime, FlowError, FlowSuccess, MessageView,
Pipeline, ResourceError, State,
};
use gstreamer_app::{AppSink, AppSinkCallbacks};
use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
use super::errors::Error;
use byte_slice_cast::*;
pub(super) fn file_input(
filename: &str,
block_align: u16,
sample_rate: u16,
) -> Result<Receiver<Vec<u8>>, Error> {
let pipeline = create_pipeline(filename, block_align, sample_rate)?;
let appsink = get_sink(&pipeline)?;
let (tx, rx) = sync_channel(30);
set_data_channel(&appsink, tx);
std::thread::spawn(move || {
let _ = start_pipeline(pipeline);
});
Ok(rx)
}
fn start_pipeline(pipeline: Pipeline) -> Result<(), Error> {
pipeline.set_state(State::Playing)?;
let bus = pipeline
.bus()
.expect("Pipeline without bus. Shouldn't happen!");
for msg in bus.iter_timed(ClockTime::NONE) {
match msg.view() {
MessageView::Eos(..) => break,
MessageView::Error(err) => {
pipeline.set_state(State::Null)?;
log::warn!(
"{:?}",
Error::Gstreamer {
error: err.error().to_string(),
debug: err.debug(),
}
);
}
_ => (),
}
}
pipeline.set_state(State::Null)?;
Ok(())
}
fn get_sink(pipeline: &Pipeline) -> Result<AppSink, Error> {
let sink = pipeline
.by_name("thesink")
.expect("There shoud be a `thesink`");
sink.dynamic_cast::<AppSink>()
.map_err(Error::GstreamerElement)
}
fn set_data_channel(appsink: &AppSink, tx: SyncSender<Vec<u8>>) {
// Getting data out of the appsink is done by setting callbacks on it.
// The appsink will then call those handlers, as soon as data is available.
appsink.set_callbacks(
AppSinkCallbacks::builder()
// Add a handler to the "new-sample" signal.
.new_sample(move |appsink| {
// Pull the sample in question out of the appsink's buffer.
let sample = appsink.pull_sample().map_err(|_| FlowError::Eos)?;
let buffer = sample.buffer().ok_or_else(|| {
element_error!(
appsink,
ResourceError::Failed,
("Failed to get buffer from appsink")
);
FlowError::Error
})?;
// At this point, buffer is only a reference to an existing memory region somewhere.
// When we want to access its content, we have to map it while requesting the required
// mode of access (read, read/write).
// This type of abstraction is necessary, because the buffer in question might not be
// on the machine's main memory itself, but rather in the GPU's memory.
// So mapping the buffer makes the underlying memory region accessible to us.
// See: https://gstreamer.freedesktop.org/documentation/plugin-development/advanced/allocation.html
let map = buffer.map_readable().map_err(|_| {
element_error!(
appsink,
ResourceError::Failed,
("Failed to map buffer readable")
);
FlowError::Error
})?;
// We know what format the data in the memory region has, since we requested
// it by setting the appsink's caps. So what we do here is interpret the
// memory region we mapped as an array of signed 16 bit integers.
let samples = map.as_slice_of::<u8>().map_err(|_| {
element_error!(
appsink,
ResourceError::Failed,
("Failed to interprete buffer as S16 PCM")
);
FlowError::Error
})?;
// Ready!
let _ = tx.send(samples.to_vec());
Ok(FlowSuccess::Ok)
})
.build(),
);
}
fn create_pipeline(filename: &str, block_align: u16, sample_rate: u16) -> Result<Pipeline, Error> {
gstreamer::init()?;
let launch_str = format!(
"filesrc location={} \
! decodebin \
! audioconvert \
! audioresample \
! audio/x-raw,rate={},channels=1 \
! queue \
! adpcmenc blockalign={} layout=dvi \
! appsink name=thesink",
filename, sample_rate, block_align
);
// log::info!("{}", launch_str);
// Parse the pipeline we want to probe from a static in-line string.
// Here we give our audiotestsrc a name, so we can retrieve that element
// from the resulting pipeline.
let pipeline = parse_launch(&launch_str)?;
let pipeline = pipeline
.dynamic_cast::<Pipeline>()
.map_err(Error::GstreamerElement)?;
Ok(pipeline)
}
+38 -3
View File
@@ -12,13 +12,14 @@
/// ```
///
use log::*;
use neolink_core::bc_protocol::BcCamera;
use neolink_core::{bc::xml::TalkConfig, bc_protocol::BcCamera};
use std::fs;
use validator::Validate;
mod cmdline;
mod config;
mod errors;
mod gst;
pub(crate) use cmdline::Opt;
use config::Config;
@@ -49,9 +50,43 @@ pub fn main(opt: Opt) -> Result<(), Error> {
info!("{}: Connected and logged in", camera_config.name);
let data = fs::read(&opt.adpcm_file)?;
let talk_ability = camera.talk_ability()?;
if talk_ability.duplex_list.is_empty()
|| talk_ability.audio_stream_mode_list.is_empty()
|| talk_ability.audio_config_list.is_empty()
{
return Err(Error::TalkUnsupported);
}
camera.talk(&data, opt.block_size, opt.sample_rate)?;
// Just copy that data from the first talk ability in the config have never seen more
// than one ability
let config = 0;
let talk_config = TalkConfig {
channel_id: camera_config.channel_id,
duplex: talk_ability.duplex_list[config].duplex.clone(),
audio_stream_mode: talk_ability.audio_stream_mode_list[config]
.audio_stream_mode
.clone(),
audio_config: talk_ability.audio_config_list[config].audio_config.clone(),
..Default::default()
};
let block_size = talk_config.audio_config.length_per_encoder / 2;
let sample_rate = talk_config.audio_config.sample_rate;
if block_size == 0 || sample_rate == 0 {
return Err(Error::TalkUnsupported);
}
let rx = gst::file_input(
&opt.media_path
.to_str()
.expect("File path not UTF8 complient"),
block_size,
sample_rate,
)?;
camera.talk_stream(rx, talk_config)?;
}
}