Trying new buffer push

This commit is contained in:
QuantumEntangledAndy committed 2023-07-17 09:20:13 +07:00
1 parent 5b3f845c83
commit 45e08e6d7d
2 files changed
+71 -125

No files matched your search

+14 -11
View File
@@ -491,21 +491,24 @@ impl NeoMediaFactoryImpl {
let queue = make_queue("audqueue")?;
// let queue2 = make_queue2("audqueue2")?;
let parser = make_element("aacparse", "audparser")?;
let decoder = make_element("decodebin", "auddecoder")?;
let decoder = match make_element("faad", "auddecoder_faad") {
Ok(ele) => Ok(ele),
Err(_) => make_element("avdec_aac", "auddecoder_avdec_aac"),
}?;
let encoder = make_element("audioconvert", "audencoder")?;
let payload = make_element("rtpL16pay", "pay1")?;
bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
Element::link_many(&[&source, &queue, &parser, &decoder])?;
Element::link_many(&[&encoder, &payload])?;
decoder.connect_pad_added(move |_element, pad| {
debug!("Linking encoder to decoder: {:?}", pad.caps());
let sink_pad = encoder
.static_pad("sink")
.expect("Encoder is missing its pad");
pad.link(&sink_pad)
.expect("Failed to link AAC decoder to encoder");
});
Element::link_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?;
// Element::link_many(&[])?;
// decoder.connect_pad_added(move |_element, pad| {
// debug!("Linking encoder to decoder: {:?}", pad.caps());
// let sink_pad = encoder
// .static_pad("sink")
// .expect("Encoder is missing its pad");
// pad.link(&sink_pad)
// .expect("Failed to link AAC decoder to encoder");
// });
let source = source
.dynamic_cast::<AppSrc>()
+57 -114
View File
@@ -453,6 +453,11 @@ pub(super) struct NeoMediaSender {
use_smoothing: bool,
}
enum Buffer {
Video(gstreamer::Buffer),
Audio(gstreamer::Buffer),
}
impl NeoMediaSender {
pub(super) fn new(buffer_size: usize, use_smoothing: bool) -> Self {
let (tx, rx) = channel(30);
@@ -615,15 +620,22 @@ impl NeoMediaSender {
self.buffer.buf.clear();
let mut buffer_iter = buffer.buf.iter().cloned();
// The prebuffer should starts from an Iframe
let preprocess = buffer_iter
.take_while_ref(|item| item.time < target_time)
.take_while_ref(|item| {
item.time < target_time
|| !matches!(
item.data.first().map(|arc| arc.as_ref()),
Some(BcMedia::Iframe(_))
)
})
.collect::<Vec<_>>();
let mut buffer = buffer_iter.collect::<Vec<_>>();
self.start_time.reset_to(target_time as f64);
// Send preprocess now
self.send_buffers(preprocess.as_slice(), true).await?;
trace!("Preprocessed: {}", preprocess.len());
debug!("Preprocessed: {}", preprocess.len());
// Send these later
for frame in buffer.drain(..) {
self.buffer.buf.push_back(frame);
@@ -717,17 +729,11 @@ impl NeoMediaSender {
if medias.is_empty() {
return Ok(());
}
tokio::task::yield_now().await;
let mut vid_buffers: Vec<(FrameTime, Vec<u8>)> = vec![];
let mut aud_buffers: Vec<(FrameTime, Vec<u8>)> = vec![];
let mut buffers: Vec<Buffer> = vec![];
for media_sets in medias.iter() {
tokio::task::yield_now().await;
for media in media_sets.data.iter() {
let buffer = match media.as_ref() {
BcMedia::Iframe(_) | BcMedia::Pframe(_) => Some(&mut vid_buffers),
BcMedia::Aac(_) | BcMedia::Adpcm(_) => Some(&mut aud_buffers),
_ => None,
};
let data = match media.as_ref() {
BcMedia::Iframe(data) => Some(&data.data),
BcMedia::Pframe(data) => Some(&data.data),
@@ -735,118 +741,55 @@ impl NeoMediaSender {
BcMedia::Adpcm(data) => Some(&data.data),
_ => None,
};
if let (Some(data), Some(buffer)) = (data, buffer) {
let next_time = media_sets.time;
if let Some(last) = buffer.last_mut() {
let last_time = last.0;
if next_time == last_time {
last.1.extend(data.iter().copied());
} else {
buffer.push((next_time, data.clone()))
if let Some(buf) = data {
let time = media_sets.time;
let runtime = match (self.get_runtime(), ignore_stamps) {
(None, _) | (_, false) => self.buftime_to_runtime(time),
(Some(runtime), true) => runtime,
};
trace!(
" - Sending vid frame at time {} ({:?} Expect: {:?})",
time,
Duration::from_micros(runtime as u64),
self.get_runtime().map(|i| Duration::from_micros(i as u64))
);
let gst_buf = {
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let time = ClockTime::from_useconds(runtime.try_into().unwrap());
gst_buf_mut.set_dts(time);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(buf.as_slice());
}
} else {
buffer.push((next_time, data.clone()))
gst_buf
};
match media.as_ref() {
BcMedia::Iframe(_) | BcMedia::Pframe(_) => {
buffers.push(Buffer::Video(gst_buf));
}
BcMedia::Aac(_) | BcMedia::Adpcm(_) => {
buffers.push(Buffer::Audio(gst_buf));
}
_ => {}
}
}
}
}
tokio::task::yield_now().await;
if !vid_buffers.is_empty() {
// debug!("Sending video buffers: {}", vid_buffers.len());
if let Some(appsrc) = self.vid.clone() {
let buffers = {
let mut buffers = gstreamer::BufferList::new_sized(vid_buffers.len());
{
let buffers_ref = buffers.get_mut().unwrap();
for (time, buf) in vid_buffers.drain(..) {
tokio::task::yield_now().await;
let runtime = match (self.get_runtime(), ignore_stamps) {
(None, _) | (_, false) => self.buftime_to_runtime(time),
(Some(runtime), true) => runtime,
};
trace!(
" - Sending vid frame at time {} ({:?} Expect: {:?})",
time,
Duration::from_micros(runtime as u64),
self.get_runtime().map(|i| Duration::from_micros(i as u64))
);
let gst_buf = {
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let time =
ClockTime::from_useconds(runtime.try_into().unwrap());
gst_buf_mut.set_dts(time);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(buf.as_slice());
}
gst_buf
};
buffers_ref.add(gst_buf);
}
}
buffers
};
let res = tokio::task::spawn_blocking(move || {
// debug!(" - Pushing buffer: {}", buffers.len());
appsrc
.push_buffer_list(buffers.copy())
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
})
.await;
match &res {
Err(e) => {
debug!("Paniced on send buffer list: {:?}", e);
}
Ok(Err(e)) => {
debug!("Failed to send buffer list: {:?}", e);
}
Ok(Ok(_)) => {}
};
res??;
}
}
if !aud_buffers.is_empty() {
if let Some(appsrc) = self.aud.clone() {
let buffers = {
let mut buffers = gstreamer::BufferList::new_sized(aud_buffers.len());
{
let buffers_ref = buffers.get_mut().unwrap();
for (time, buf) in aud_buffers.drain(..) {
tokio::task::yield_now().await;
let runtime = match (self.get_runtime(), ignore_stamps) {
(None, _) | (_, false) => self.buftime_to_runtime(time),
(Some(runtime), true) => runtime,
};
let gst_buf = {
let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap();
{
let gst_buf_mut = gst_buf.get_mut().unwrap();
let time =
ClockTime::from_useconds(runtime.try_into().unwrap());
gst_buf_mut.set_dts(time);
gst_buf_mut.set_pts(time);
let mut gst_buf_data = gst_buf_mut.map_writable().unwrap();
gst_buf_data.copy_from_slice(buf.as_slice());
}
gst_buf
};
buffers_ref.add(gst_buf);
}
}
buffers
};
for buf in buffers.drain(..) {
let buf_app = match (buf, self.vid.as_ref(), self.aud.as_ref()) {
(Buffer::Video(buf), Some(app), _) => Some((buf, app.clone())),
(Buffer::Audio(buf), _, Some(app)) => Some((buf, app.clone())),
_ => None,
};
if let Some((buf, appsrc)) = buf_app {
let res = tokio::task::spawn_blocking(move || {
appsrc
.push_buffer_list(buffers.copy())
.push_buffer(buf.copy())
.map(|_| ())
.map_err(|_| anyhow!("Could not push buffer to appsrc"))
})