Trying new buffer push
This commit is contained in:
2 files changed
+71
-125
No files matched your search
+14
-11
@@ -491,21 +491,24 @@ impl NeoMediaFactoryImpl {
|
||||
let queue = make_queue("audqueue")?;
|
||||
// let queue2 = make_queue2("audqueue2")?;
|
||||
let parser = make_element("aacparse", "audparser")?;
|
||||
let decoder = make_element("decodebin", "auddecoder")?;
|
||||
let decoder = match make_element("faad", "auddecoder_faad") {
|
||||
Ok(ele) => Ok(ele),
|
||||
Err(_) => make_element("avdec_aac", "auddecoder_avdec_aac"),
|
||||
}?;
|
||||
let encoder = make_element("audioconvert", "audencoder")?;
|
||||
let payload = make_element("rtpL16pay", "pay1")?;
|
||||
|
||||
bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
|
||||
Element::link_many(&[&source, &queue, &parser, &decoder])?;
|
||||
Element::link_many(&[&encoder, &payload])?;
|
||||
decoder.connect_pad_added(move |_element, pad| {
|
||||
debug!("Linking encoder to decoder: {:?}", pad.caps());
|
||||
let sink_pad = encoder
|
||||
.static_pad("sink")
|
||||
.expect("Encoder is missing its pad");
|
||||
pad.link(&sink_pad)
|
||||
.expect("Failed to link AAC decoder to encoder");
|
||||
});
|
||||
Element::link_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
|
||||
// Element::link_many(&[])?;
|
||||
// decoder.connect_pad_added(move |_element, pad| {
|
||||
// debug!("Linking encoder to decoder: {:?}", pad.caps());
|
||||
// let sink_pad = encoder
|
||||
// .static_pad("sink")
|
||||
// .expect("Encoder is missing its pad");
|
||||
// pad.link(&sink_pad)
|
||||
// .expect("Failed to link AAC decoder to encoder");
|
||||
// });
|
||||
|
||||
let source = source
|
||||
.dynamic_cast::<AppSrc>()
|
||||
|
||||
+57
-114
@@ -453,6 +453,11 @@ pub(super) struct NeoMediaSender {
|
||||
use_smoothing: bool,
|
||||
}
|
||||
|
||||
enum Buffer {
|
||||
Video(gstreamer::Buffer),
|
||||
Audio(gstreamer::Buffer),
|
||||
}
|
||||
|
||||
impl NeoMediaSender {
|
||||
pub(super) fn new(buffer_size: usize, use_smoothing: bool) -> Self {
|
||||
let (tx, rx) = channel(30);
|
||||
@@ -615,15 +620,22 @@ impl NeoMediaSender {
|
||||
self.buffer.buf.clear();
|
||||
|
||||
let mut buffer_iter = buffer.buf.iter().cloned();
|
||||
// The prebuffer should starts from an Iframe
|
||||
let preprocess = buffer_iter
|
||||
.take_while_ref(|item| item.time < target_time)
|
||||
.take_while_ref(|item| {
|
||||
item.time < target_time
|
||||
|| !matches!(
|
||||
item.data.first().map(|arc| arc.as_ref()),
|
||||
Some(BcMedia::Iframe(_))
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut buffer = buffer_iter.collect::<Vec<_>>();
|
||||
self.start_time.reset_to(target_time as f64);
|
||||
|
||||
// Send preprocess now
|
||||
self.send_buffers(preprocess.as_slice(), true).await?;
|
||||
trace!("Preprocessed: {}", preprocess.len());
|
||||
debug!("Preprocessed: {}", preprocess.len());
|
||||
// Send these later
|
||||
for frame in buffer.drain(..) {
|
||||
self.buffer.buf.push_back(frame);
|
||||
@@ -717,17 +729,11 @@ impl NeoMediaSender {
|
||||
if medias.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
let mut vid_buffers: Vec<(FrameTime, Vec<u8>)> = vec![];
|
||||
let mut aud_buffers: Vec<(FrameTime, Vec<u8>)> = vec![];
|
||||
let mut buffers: Vec<Buffer> = vec![];
|
||||
|
||||
for media_sets in medias.iter() {
|
||||
tokio::task::yield_now().await;
|
||||
for media in media_sets.data.iter() {
|
||||
let buffer = match media.as_ref() {
|
||||
BcMedia::Iframe(_) | BcMedia::Pframe(_) => Some(&mut vid_buffers),
|
||||
BcMedia::Aac(_) | BcMedia::Adpcm(_) => Some(&mut aud_buffers),
|
||||
_ => None,
|
||||
};
|
||||
let data = match media.as_ref() {
|
||||
BcMedia::Iframe(data) => Some(&data.data),
|
||||
BcMedia::Pframe(data) => Some(&data.data),
|
||||
@@ -735,118 +741,55 @@ impl NeoMediaSender {
|
||||
BcMedia::Adpcm(data) => Some(&data.data),
|
||||
_ => None,
|
||||
};
|
||||
if let (Some(data), Some(buffer)) = (data, buffer) {
|
||||
let next_time = media_sets.time;
|
||||
if let Some(last) = buffer.last_mut() {
|
||||
let last_time = last.0;
|
||||
if next_time == last_time {
|
||||
last.1.extend(data.iter().copied());
|
||||
} else {
|
||||
buffer.push((next_time, data.clone()))
|
||||
if let Some(buf) = data {
|
||||
let time = media_sets.time;
|
||||
let runtime = match (self.get_runtime(), ignore_stamps) {
|
||||
(None, _) | (_, false) => self.buftime_to_runtime(time),
|
||||
(Some(runtime), true) => runtime,
|
||||
};
|
||||
trace!(
|
||||
" - Sending vid frame at time {} ({:?} Expect: {:?})",
|
||||
time,
|
||||
Duration::from_micros(runtime as u64),
|
||||
self.get_runtime().map(|i| Duration::from_micros(i as u64))
|
||||
);
|
||||
|
||||
let gst_buf = {
|
||||
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
|
||||
{
|
||||
let gst_buf_mut = gst_buf.get_mut().unwrap();
|
||||
|
||||
let time = ClockTime::from_useconds(runtime.try_into().unwrap());
|
||||
gst_buf_mut.set_dts(time);
|
||||
gst_buf_mut.set_pts(time);
|
||||
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
|
||||
gst_buf_data.copy_from_slice(buf.as_slice());
|
||||
}
|
||||
} else {
|
||||
buffer.push((next_time, data.clone()))
|
||||
gst_buf
|
||||
};
|
||||
match media.as_ref() {
|
||||
BcMedia::Iframe(_) | BcMedia::Pframe(_) => {
|
||||
buffers.push(Buffer::Video(gst_buf));
|
||||
}
|
||||
BcMedia::Aac(_) | BcMedia::Adpcm(_) => {
|
||||
buffers.push(Buffer::Audio(gst_buf));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
if !vid_buffers.is_empty() {
|
||||
// debug!("Sending video buffers: {}", vid_buffers.len());
|
||||
if let Some(appsrc) = self.vid.clone() {
|
||||
let buffers = {
|
||||
let mut buffers = gstreamer::BufferList::new_sized(vid_buffers.len());
|
||||
{
|
||||
let buffers_ref = buffers.get_mut().unwrap();
|
||||
for (time, buf) in vid_buffers.drain(..) {
|
||||
tokio::task::yield_now().await;
|
||||
let runtime = match (self.get_runtime(), ignore_stamps) {
|
||||
(None, _) | (_, false) => self.buftime_to_runtime(time),
|
||||
(Some(runtime), true) => runtime,
|
||||
};
|
||||
trace!(
|
||||
" - Sending vid frame at time {} ({:?} Expect: {:?})",
|
||||
time,
|
||||
Duration::from_micros(runtime as u64),
|
||||
self.get_runtime().map(|i| Duration::from_micros(i as u64))
|
||||
);
|
||||
|
||||
let gst_buf = {
|
||||
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
|
||||
{
|
||||
let gst_buf_mut = gst_buf.get_mut().unwrap();
|
||||
|
||||
let time =
|
||||
ClockTime::from_useconds(runtime.try_into().unwrap());
|
||||
gst_buf_mut.set_dts(time);
|
||||
gst_buf_mut.set_pts(time);
|
||||
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
|
||||
gst_buf_data.copy_from_slice(buf.as_slice());
|
||||
}
|
||||
gst_buf
|
||||
};
|
||||
buffers_ref.add(gst_buf);
|
||||
}
|
||||
}
|
||||
buffers
|
||||
};
|
||||
|
||||
let res = tokio::task::spawn_blocking(move || {
|
||||
// debug!(" - Pushing buffer: {}", buffers.len());
|
||||
appsrc
|
||||
.push_buffer_list(buffers.copy())
|
||||
.map(|_| ())
|
||||
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
|
||||
})
|
||||
.await;
|
||||
match &res {
|
||||
Err(e) => {
|
||||
debug!("Paniced on send buffer list: {:?}", e);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
debug!("Failed to send buffer list: {:?}", e);
|
||||
}
|
||||
Ok(Ok(_)) => {}
|
||||
};
|
||||
res??;
|
||||
}
|
||||
}
|
||||
if !aud_buffers.is_empty() {
|
||||
if let Some(appsrc) = self.aud.clone() {
|
||||
let buffers = {
|
||||
let mut buffers = gstreamer::BufferList::new_sized(aud_buffers.len());
|
||||
{
|
||||
let buffers_ref = buffers.get_mut().unwrap();
|
||||
for (time, buf) in aud_buffers.drain(..) {
|
||||
tokio::task::yield_now().await;
|
||||
let runtime = match (self.get_runtime(), ignore_stamps) {
|
||||
(None, _) | (_, false) => self.buftime_to_runtime(time),
|
||||
(Some(runtime), true) => runtime,
|
||||
};
|
||||
|
||||
let gst_buf = {
|
||||
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
|
||||
{
|
||||
let gst_buf_mut = gst_buf.get_mut().unwrap();
|
||||
|
||||
let time =
|
||||
ClockTime::from_useconds(runtime.try_into().unwrap());
|
||||
gst_buf_mut.set_dts(time);
|
||||
gst_buf_mut.set_pts(time);
|
||||
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
|
||||
gst_buf_data.copy_from_slice(buf.as_slice());
|
||||
}
|
||||
gst_buf
|
||||
};
|
||||
buffers_ref.add(gst_buf);
|
||||
}
|
||||
}
|
||||
buffers
|
||||
};
|
||||
|
||||
for buf in buffers.drain(..) {
|
||||
let buf_app = match (buf, self.vid.as_ref(), self.aud.as_ref()) {
|
||||
(Buffer::Video(buf), Some(app), _) => Some((buf, app.clone())),
|
||||
(Buffer::Audio(buf), _, Some(app)) => Some((buf, app.clone())),
|
||||
_ => None,
|
||||
};
|
||||
if let Some((buf, appsrc)) = buf_app {
|
||||
let res = tokio::task::spawn_blocking(move || {
|
||||
appsrc
|
||||
.push_buffer_list(buffers.copy())
|
||||
.push_buffer(buf.copy())
|
||||
.map(|_| ())
|
||||
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user