Skip to content

perf(recording): drain macOS autorelease pools and reuse video frames in-place #3625

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
duongynhi000005-oss wants to merge 1 commit into from
+344 −233
Closed

perf(recording): drain macOS autorelease pools and reuse video frames in-place #3625

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 33 additions & 31 deletions crates/camera/src/macos.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,38 +24,40 @@ pub(super) fn list_cameras_impl() -> impl Iterator<Item = CameraInfo> {

impl CameraInfo {
pub(super) fn formats_impl(&self) -> Option<Vec<Format>> {
let device = find_device(self)?;

let mut ret = vec![];

for format in device.formats().iter() {
let desc = format.format_desc();
let width = desc.dims().width as u32;
let height = desc.dims().height as u32;
let pixel_format = fourcc_display(desc.media_sub_type());

for fr_range in format.video_supported_frame_rate_ranges().iter() {
// SAFETY: trust me bro it crashes on intel mac otherwise
let fr_range = unsafe {
&*(fr_range as *const av::capture::device::FrameRateRange)
.cast::<AVFrameRateRange>()
};

let min = unsafe { fr_range.minFrameDuration() };

ret.push(Format {
native: format.retained(),
info: FormatInfo {
width,
height,
frame_rate: min.timescale as f32 / min.value as f32,
},
pixel_format: Some(pixel_format.clone()),
})
objc::ar_pool(|| {
let device = find_device(self)?;

let mut ret = vec![];

for format in device.formats().iter() {
let desc = format.format_desc();
let width = desc.dims().width as u32;
let height = desc.dims().height as u32;
let pixel_format = fourcc_display(desc.media_sub_type());

for fr_range in format.video_supported_frame_rate_ranges().iter() {
// SAFETY: trust me bro it crashes on intel mac otherwise
let fr_range = unsafe {
&*(fr_range as *const av::capture::device::FrameRateRange)
.cast::<AVFrameRateRange>()
};

let min = unsafe { fr_range.minFrameDuration() };

ret.push(Format {
native: format.retained(),
info: FormatInfo {
width,
height,
frame_rate: min.timescale as f32 / min.value as f32,
},
pixel_format: Some(pixel_format.clone()),
})
}
}
}

Some(ret)
Some(ret)
})
}
}

Expand Down Expand Up @@ -156,7 +158,7 @@ pub(super) fn start_capturing_impl(
let input =
av::capture::DeviceInput::with_device(&device).map_err(AVFoundationError::Static)?;

let queue = dispatch::Queue::new();
let queue = dispatch::Queue::serial_with_ar_pool();
let mut missing_image_bufs: u64 = 0;
let delegate =
CallbackOutputDelegate::with(CallbackOutputDelegateInner::new(Box::new(move |data| {
Expand Down
13 changes: 13 additions & 0 deletions crates/enc-ffmpeg/src/mux/segmented_stream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,19 @@ impl SegmentedVideoEncoder {
Ok(())
}

pub fn queue_frame_reusable(
&mut self,
frame: &mut frame::Video,
timestamp: Duration,
) -> Result<(), QueueFrameError> {
self.encoder
.queue_frame_to_reusable(frame, timestamp, &mut self.mux)?;

self.mux.frame_queued();

Ok(())
}

pub fn finish(&mut self) -> Result<(), FinishError> {
let encoder = &mut self.encoder;
self.mux
Expand Down
49 changes: 49 additions & 0 deletions crates/enc-ffmpeg/src/video/h264.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -990,6 +990,40 @@ impl H264Encoder {
self.queue_frame_to(frame, timestamp, output)
}

pub fn queue_frame_to_reusable<S: PacketSink + ?Sized>(
&mut self,
frame: &mut frame::Video,
timestamp: Duration,
output: &mut S,
) -> Result<(), QueueFrameError> {
self.base.update_pts(frame, timestamp, &mut self.encoder);

let frame_to_send = if let Some(converter) = &mut self.converter {
let pts = frame.pts();
let converted = self.converted_frame_pool.as_mut().unwrap();
let ret = unsafe { ffmpeg::ffi::av_frame_make_writable(converted.as_mut_ptr()) };
if ret < 0 {
return Err(QueueFrameError::Encode(ffmpeg::Error::from(ret)));
}
super::convert_or_scale(frame, converted, converter)
.map_err(QueueFrameError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
} else {
frame as &frame::Video
};
let frame_to_send = self
.hw_upload
.frame(frame_to_send)
.map_err(QueueFrameError::Encode)?;

self.base
.send_frame(frame_to_send, output, &mut self.encoder)
.map_err(QueueFrameError::Encode)?;

Ok(())
}

pub fn queue_frame_to<S: PacketSink + ?Sized>(
&mut self,
mut frame: frame::Video,
Expand All @@ -1002,6 +1036,10 @@ impl H264Encoder {
let frame_to_send = if let Some(converter) = &mut self.converter {
let pts = frame.pts();
let converted = self.converted_frame_pool.as_mut().unwrap();
let ret = unsafe { ffmpeg::ffi::av_frame_make_writable(converted.as_mut_ptr()) };
if ret < 0 {
return Err(QueueFrameError::Encode(ffmpeg::Error::from(ret)));
}
super::convert_or_scale(&frame, converted, converter)
.map_err(QueueFrameError::Converter)?;
converted.set_pts(pts);
Expand Down Expand Up @@ -1032,9 +1070,20 @@ impl H264Encoder {

let frame_to_send = if let Some(converter) = &mut self.converter {
let pts = frame.pts();
if converted_frame.as_ref().is_some_and(|f| {
f.format() != self.output_format
|| f.width() != self.output_width
|| f.height() != self.output_height
}) {
*converted_frame = None;
}
let converted = converted_frame.get_or_insert_with(|| {
frame::Video::new(self.output_format, self.output_width, self.output_height)
});
let ret = unsafe { ffmpeg::ffi::av_frame_make_writable(converted.as_mut_ptr()) };
if ret < 0 {
return Err(QueueFrameError::Encode(ffmpeg::Error::from(ret)));
}
super::convert_or_scale(frame, converted, converter)
.map_err(QueueFrameError::Converter)?;
converted.set_pts(pts);
Expand Down
15 changes: 15 additions & 0 deletions crates/enc-ffmpeg/src/video/h264_packet.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ impl H264PacketEncoder {
let converted = self.converted_frame_pool.as_mut().expect(
"converted_frame_pool present whenever converter is Some (invariant preserved by from_opened)",
);
let ret = unsafe { ffmpeg::ffi::av_frame_make_writable(converted.as_mut_ptr()) };
if ret < 0 {
return Err(EncodePacketError::Encode(ffmpeg::Error::from(ret)));
}
super::convert_or_scale(&frame, converted, converter)
.map_err(EncodePacketError::Converter)?;
converted.set_pts(pts);
Expand Down Expand Up @@ -176,9 +180,20 @@ impl H264PacketEncoder {

let frame_to_send = if let Some(converter) = &mut self.converter {
let pts = frame.pts();
if converted_frame.as_ref().is_some_and(|f| {
f.format() != self.output_format
|| f.width() != self.output_width
|| f.height() != self.output_height
}) {
*converted_frame = None;
}
let converted = converted_frame.get_or_insert_with(|| {
frame::Video::new(self.output_format, self.output_width, self.output_height)
});
let ret = unsafe { ffmpeg::ffi::av_frame_make_writable(converted.as_mut_ptr()) };
if ret < 0 {
return Err(EncodePacketError::Encode(ffmpeg::Error::from(ret)));
}
super::convert_or_scale(frame, converted, converter)
.map_err(EncodePacketError::Converter)?;
converted.set_pts(pts);
Expand Down
24 changes: 12 additions & 12 deletions crates/recording/src/output_pipeline/macos.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ impl Muxer for AVFoundationMp4Muxer {
const MAX_RETRIES: u32 = 100;

loop {
let queue_result = {
let queue_result = cidre::objc::ar_pool(|| {
let mut encoder = match encoder_clone.lock() {
Ok(e) => e,
Err(_) => {
Expand All @@ -464,8 +464,8 @@ impl Muxer for AVFoundationMp4Muxer {
return Err(anyhow!(message));
}
};
encoder.queue_video_frame(sample_buf.clone(), timestamp)
};
Ok(encoder.queue_video_frame(sample_buf.clone(), timestamp))
})?;

match queue_result {
Ok(()) => break,
Expand Down Expand Up @@ -586,7 +586,7 @@ impl Muxer for AVFoundationMp4Muxer {
const MAX_RETRIES: u32 = 50;

loop {
let queue_result = {
let queue_result = cidre::objc::ar_pool(|| {
let mut encoder = match encoder_clone.lock() {
Ok(e) => e,
Err(_) => {
Expand All @@ -599,8 +599,8 @@ impl Muxer for AVFoundationMp4Muxer {
return Err(anyhow!(message));
}
};
encoder.queue_audio_frame(&frame, timestamp)
};
Ok(encoder.queue_audio_frame(&frame, timestamp))
})?;

match queue_result {
Ok(()) => break,
Expand Down Expand Up @@ -1072,7 +1072,7 @@ impl Muxer for AVFoundationCameraMuxer {
const MAX_RETRIES: u32 = 100;

loop {
let queue_result = {
let queue_result = cidre::objc::ar_pool(|| {
let mut encoder = match encoder_clone.lock() {
Ok(e) => e,
Err(_) => {
Expand All @@ -1082,8 +1082,8 @@ impl Muxer for AVFoundationCameraMuxer {
return Err(anyhow!(message));
}
};
encoder.queue_video_frame(sample_buf.clone(), timestamp)
};
Ok(encoder.queue_video_frame(sample_buf.clone(), timestamp))
})?;

match queue_result {
Ok(()) => break,
Expand Down Expand Up @@ -1203,7 +1203,7 @@ impl Muxer for AVFoundationCameraMuxer {
const MAX_RETRIES: u32 = 50;

loop {
let queue_result = {
let queue_result = cidre::objc::ar_pool(|| {
let mut encoder = match encoder_clone.lock() {
Ok(e) => e,
Err(_) => {
Expand All @@ -1212,8 +1212,8 @@ impl Muxer for AVFoundationCameraMuxer {
return Err(anyhow!(message));
}
};
encoder.queue_audio_frame(&frame, timestamp)
};
Ok(encoder.queue_audio_frame(&frame, timestamp))
})?;

match queue_result {
Ok(()) => break,
Expand Down
Loading