From 45e08e6d7d61f369ae61242518162897eeea92c9 Mon Sep 17 00:00:00 2001 From: QuantumEntangledAndy Date: Mon, 17 Jul 2023 09:20:13 +0700 Subject: [PATCH] Trying new buffer push --- src/rtsp/gst/factory.rs | 25 +++--- src/rtsp/gst/sender.rs | 171 ++++++++++++++-------------------------- 2 files changed, 71 insertions(+), 125 deletions(-) diff --git a/src/rtsp/gst/factory.rs b/src/rtsp/gst/factory.rs index 71141c0..36b8aae 100644 --- a/src/rtsp/gst/factory.rs +++ b/src/rtsp/gst/factory.rs @@ -491,21 +491,24 @@ impl NeoMediaFactoryImpl { let queue = make_queue("audqueue")?; // let queue2 = make_queue2("audqueue2")?; let parser = make_element("aacparse", "audparser")?; - let decoder = make_element("decodebin", "auddecoder")?; + let decoder = match make_element("faad", "auddecoder_faad") { + Ok(ele) => Ok(ele), + Err(_) => make_element("avdec_aac", "auddecoder_avdec_aac"), + }?; let encoder = make_element("audioconvert", "audencoder")?; let payload = make_element("rtpL16pay", "pay1")?; bin.add_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; - Element::link_many(&[&source, &queue, &parser, &decoder])?; - Element::link_many(&[&encoder, &payload])?; - decoder.connect_pad_added(move |_element, pad| { - debug!("Linking encoder to decoder: {:?}", pad.caps()); - let sink_pad = encoder - .static_pad("sink") - .expect("Encoder is missing its pad"); - pad.link(&sink_pad) - .expect("Failed to link AAC decoder to encoder"); - }); + Element::link_many(&[&source, &queue, &parser, &decoder, &encoder, &payload])?; + // Element::link_many(&[])?; + // decoder.connect_pad_added(move |_element, pad| { + // debug!("Linking encoder to decoder: {:?}", pad.caps()); + // let sink_pad = encoder + // .static_pad("sink") + // .expect("Encoder is missing its pad"); + // pad.link(&sink_pad) + // .expect("Failed to link AAC decoder to encoder"); + // }); let source = source .dynamic_cast::() diff --git a/src/rtsp/gst/sender.rs b/src/rtsp/gst/sender.rs index 4844953..7bfae24 100644 --- a/src/rtsp/gst/sender.rs +++ b/src/rtsp/gst/sender.rs @@ -453,6 +453,11 @@ pub(super) struct NeoMediaSender { use_smoothing: bool, } +enum Buffer { + Video(gstreamer::Buffer), + Audio(gstreamer::Buffer), +} + impl NeoMediaSender { pub(super) fn new(buffer_size: usize, use_smoothing: bool) -> Self { let (tx, rx) = channel(30); @@ -615,15 +620,22 @@ impl NeoMediaSender { self.buffer.buf.clear(); let mut buffer_iter = buffer.buf.iter().cloned(); + // The prebuffer should starts from an Iframe let preprocess = buffer_iter - .take_while_ref(|item| item.time < target_time) + .take_while_ref(|item| { + item.time < target_time + || !matches!( + item.data.first().map(|arc| arc.as_ref()), + Some(BcMedia::Iframe(_)) + ) + }) .collect::>(); let mut buffer = buffer_iter.collect::>(); self.start_time.reset_to(target_time as f64); // Send preprocess now self.send_buffers(preprocess.as_slice(), true).await?; - trace!("Preprocessed: {}", preprocess.len()); + debug!("Preprocessed: {}", preprocess.len()); // Send these later for frame in buffer.drain(..) { self.buffer.buf.push_back(frame); @@ -717,17 +729,11 @@ impl NeoMediaSender { if medias.is_empty() { return Ok(()); } - tokio::task::yield_now().await; - let mut vid_buffers: Vec<(FrameTime, Vec)> = vec![]; - let mut aud_buffers: Vec<(FrameTime, Vec)> = vec![]; + let mut buffers: Vec = vec![]; + for media_sets in medias.iter() { tokio::task::yield_now().await; for media in media_sets.data.iter() { - let buffer = match media.as_ref() { - BcMedia::Iframe(_) | BcMedia::Pframe(_) => Some(&mut vid_buffers), - BcMedia::Aac(_) | BcMedia::Adpcm(_) => Some(&mut aud_buffers), - _ => None, - }; let data = match media.as_ref() { BcMedia::Iframe(data) => Some(&data.data), BcMedia::Pframe(data) => Some(&data.data), @@ -735,118 +741,55 @@ impl NeoMediaSender { BcMedia::Adpcm(data) => Some(&data.data), _ => None, }; - if let (Some(data), Some(buffer)) = (data, buffer) { - let next_time = media_sets.time; - if let Some(last) = buffer.last_mut() { - let last_time = last.0; - if next_time == last_time { - last.1.extend(data.iter().copied()); - } else { - buffer.push((next_time, data.clone())) + if let Some(buf) = data { + let time = media_sets.time; + let runtime = match (self.get_runtime(), ignore_stamps) { + (None, _) | (_, false) => self.buftime_to_runtime(time), + (Some(runtime), true) => runtime, + }; + trace!( + " - Sending vid frame at time {} ({:?} Expect: {:?})", + time, + Duration::from_micros(runtime as u64), + self.get_runtime().map(|i| Duration::from_micros(i as u64)) + ); + + let gst_buf = { + let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); + { + let gst_buf_mut = gst_buf.get_mut().unwrap(); + + let time = ClockTime::from_useconds(runtime.try_into().unwrap()); + gst_buf_mut.set_dts(time); + gst_buf_mut.set_pts(time); + let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); + gst_buf_data.copy_from_slice(buf.as_slice()); } - } else { - buffer.push((next_time, data.clone())) + gst_buf + }; + match media.as_ref() { + BcMedia::Iframe(_) | BcMedia::Pframe(_) => { + buffers.push(Buffer::Video(gst_buf)); + } + BcMedia::Aac(_) | BcMedia::Adpcm(_) => { + buffers.push(Buffer::Audio(gst_buf)); + } + _ => {} } } } } tokio::task::yield_now().await; - if !vid_buffers.is_empty() { - // debug!("Sending video buffers: {}", vid_buffers.len()); - if let Some(appsrc) = self.vid.clone() { - let buffers = { - let mut buffers = gstreamer::BufferList::new_sized(vid_buffers.len()); - { - let buffers_ref = buffers.get_mut().unwrap(); - for (time, buf) in vid_buffers.drain(..) { - tokio::task::yield_now().await; - let runtime = match (self.get_runtime(), ignore_stamps) { - (None, _) | (_, false) => self.buftime_to_runtime(time), - (Some(runtime), true) => runtime, - }; - trace!( - " - Sending vid frame at time {} ({:?} Expect: {:?})", - time, - Duration::from_micros(runtime as u64), - self.get_runtime().map(|i| Duration::from_micros(i as u64)) - ); - - let gst_buf = { - let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); - { - let gst_buf_mut = gst_buf.get_mut().unwrap(); - - let time = - ClockTime::from_useconds(runtime.try_into().unwrap()); - gst_buf_mut.set_dts(time); - gst_buf_mut.set_pts(time); - let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); - gst_buf_data.copy_from_slice(buf.as_slice()); - } - gst_buf - }; - buffers_ref.add(gst_buf); - } - } - buffers - }; - - let res = tokio::task::spawn_blocking(move || { - // debug!(" - Pushing buffer: {}", buffers.len()); - appsrc - .push_buffer_list(buffers.copy()) - .map(|_| ()) - .map_err(|_| anyhow!("Could not push buffer to appsrc")) - }) - .await; - match &res { - Err(e) => { - debug!("Paniced on send buffer list: {:?}", e); - } - Ok(Err(e)) => { - debug!("Failed to send buffer list: {:?}", e); - } - Ok(Ok(_)) => {} - }; - res??; - } - } - if !aud_buffers.is_empty() { - if let Some(appsrc) = self.aud.clone() { - let buffers = { - let mut buffers = gstreamer::BufferList::new_sized(aud_buffers.len()); - { - let buffers_ref = buffers.get_mut().unwrap(); - for (time, buf) in aud_buffers.drain(..) { - tokio::task::yield_now().await; - let runtime = match (self.get_runtime(), ignore_stamps) { - (None, _) | (_, false) => self.buftime_to_runtime(time), - (Some(runtime), true) => runtime, - }; - - let gst_buf = { - let mut gst_buf = gstreamer::Buffer::with_size(buf.len()).unwrap(); - { - let gst_buf_mut = gst_buf.get_mut().unwrap(); - - let time = - ClockTime::from_useconds(runtime.try_into().unwrap()); - gst_buf_mut.set_dts(time); - gst_buf_mut.set_pts(time); - let mut gst_buf_data = gst_buf_mut.map_writable().unwrap(); - gst_buf_data.copy_from_slice(buf.as_slice()); - } - gst_buf - }; - buffers_ref.add(gst_buf); - } - } - buffers - }; - + for buf in buffers.drain(..) { + let buf_app = match (buf, self.vid.as_ref(), self.aud.as_ref()) { + (Buffer::Video(buf), Some(app), _) => Some((buf, app.clone())), + (Buffer::Audio(buf), _, Some(app)) => Some((buf, app.clone())), + _ => None, + }; + if let Some((buf, appsrc)) = buf_app { let res = tokio::task::spawn_blocking(move || { appsrc - .push_buffer_list(buffers.copy()) + .push_buffer(buf.copy()) .map(|_| ()) .map_err(|_| anyhow!("Could not push buffer to appsrc")) })