Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions crates/enc-ffmpeg/src/mux/segmented_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,10 @@ pub struct SegmentedVideoEncoderConfig {
/// [`SegmentedVideoEncoder::queue_hw_pixel_buffer`]) before falling back
/// to the software-frame path. Ignored on other platforms.
pub prefer_videotoolbox_hw_input: bool,
/// When set, pixel format conversion and scaling run outside the encoder
/// (for example via [`cap_frame_converter::AsyncConverterPool`]) so
/// readback and GPU conversion can overlap encoding.
pub prefer_external_conversion: bool,
}

impl Default for SegmentedVideoEncoderConfig {
Expand All @@ -300,6 +304,7 @@ impl Default for SegmentedVideoEncoderConfig {
bpp: H264EncoderBuilder::QUALITY_BPP,
output_size: None,
prefer_videotoolbox_hw_input: false,
prefer_external_conversion: false,
}
}
}
Expand Down Expand Up @@ -403,6 +408,10 @@ impl SegmentedVideoEncoder {
builder = builder.with_output_size(width, height)?;
}

if config.prefer_external_conversion {
builder = builder.with_external_conversion();
}

#[cfg(target_os = "macos")]
let hw_attempt = if config.prefer_videotoolbox_hw_input {
match builder.clone().build_videotoolbox_hw_input(&mut output) {
Expand Down Expand Up @@ -618,6 +627,35 @@ impl SegmentedVideoEncoder {
self.pending_fallback.take()
}

pub fn conversion_requirements(&self) -> crate::video::h264::ConversionRequirements {
self.encoder.conversion_requirements()
}

pub fn queue_preconverted_frame(
&mut self,
frame: frame::Video,
timestamp: Duration,
) -> Result<(), QueueFrameError> {
let mut sink = SinkProbe::new(&mut self.mux);
let result = self
.encoder
.queue_preconverted_frame_to(frame, timestamp, &mut sink);
let sink_failed = sink.failed;
match result {
Ok(()) => self.consecutive_codec_failures = 0,
Err(EncodeError::Encode(error)) if !sink_failed => {
if !self.recover_from_codec_failure(error) {
return Err(EncodeError::Encode(error).into());
}
}
Err(error) => return Err(error.into()),
}

self.mux.frame_queued();

Ok(())
}

pub fn finish(&mut self) -> Result<(), FinishError> {
// A hardware encoder that dies at the very end loses only the frames
// it still held, so its flush error alone does not fail the
Expand Down Expand Up @@ -2069,6 +2107,56 @@ mod tests {
assert!(manifest["total_duration"].as_f64().unwrap() > 4.5);
}

#[test]
fn preconverted_frames_fall_back_to_the_software_encoder() {
ffmpeg::init().unwrap();
let temp = tempfile::tempdir().unwrap();
let base_path = temp.path().to_path_buf();
let mut encoder = SegmentedVideoEncoder::init_libx264(
base_path.clone(),
test_video_info(),
SegmentedVideoEncoderConfig::default(),
)
.unwrap();
encoder.force_runtime_fallback(true);
encoder.encoder.inject_codec_failure_after(
30,
HARDWARE_FAILURE,
Some(H264Preset::Ultrafast),
);

let total_frames = 90u64;
let mut rejected = 0;
for i in 0..total_frames {
let requirements = encoder.conversion_requirements();
let mut frame = ffmpeg::frame::Video::new(
requirements.output_format,
requirements.output_width,
requirements.output_height,
);
for plane in 0..frame.planes() {
frame.data_mut(plane).fill(128);
}
if encoder
.queue_preconverted_frame(frame, Duration::from_micros(i * 33_333))
.is_err()
{
rejected += 1;
}
}

let fallback = encoder.take_encoder_fallback().expect("encoder replaced");
assert_eq!(fallback.replacement, "libx264");
assert_eq!(rejected, HARDWARE_FAILURES_BEFORE_FALLBACK - 1);
encoder.finish().unwrap();

let decoded = decode_every_packet(&recorded_track(&base_path));
assert_eq!(
decoded.frames,
total_frames as usize - HARDWARE_FAILURES_BEFORE_FALLBACK as usize
);
}

#[test]
fn hardware_encoder_failure_while_flushing_keeps_the_recording() {
ffmpeg::init().unwrap();
Expand Down
17 changes: 16 additions & 1 deletion crates/enc-ffmpeg/src/video/h264.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1043,6 +1043,7 @@ struct InjectedCodecFailure {
error: ffmpeg::Error,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ConversionRequirements {
pub input_format: format::Pixel,
pub input_width: u32,
Expand Down Expand Up @@ -1265,10 +1266,24 @@ impl H264Encoder {

pub fn queue_preconverted_frame(
&mut self,
mut frame: frame::Video,
frame: frame::Video,
timestamp: Duration,
output: &mut format::context::Output,
) -> Result<(), QueueFrameError> {
self.queue_preconverted_frame_to(frame, timestamp, output)
}

pub fn queue_preconverted_frame_to<S: PacketSink + ?Sized>(
&mut self,
mut frame: frame::Video,
timestamp: Duration,
output: &mut S,
) -> Result<(), QueueFrameError> {
#[cfg(test)]
if let Some(error) = self.injected_codec_failure() {
return Err(QueueFrameError::Encode(error));
}

trace!(
"Encoding pre-converted frame: format={:?}, size={}x{}, expected={:?} {}x{}",
frame.format(),
Expand Down
19 changes: 19 additions & 0 deletions crates/frame-converter/src/d3d11.rs
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,16 @@ impl D3D11Converter {
})?
};

if is_rgb_format(config.input_format) && !is_rgb_format(config.output_format) {
let output_color_space =
windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_COLOR_SPACE {
_bitfield: YCBCR_MATRIX_BT709,
};
unsafe {
video_context.VideoProcessorSetOutputColorSpace(&processor, &output_color_space);
}
}

let (input_texture, input_shared_handle) = create_shared_texture(
&device,
config.input_width,
Expand Down Expand Up @@ -873,6 +883,15 @@ impl FrameConverter for D3D11Converter {
}
}

// `D3D11_VIDEO_PROCESSOR_COLOR_SPACE.YCbCr_Matrix` (bit 2): the video processor
// converts RGB to YCbCr with BT.601 unless told otherwise, while every encoder
// stream is tagged BT.709.
const YCBCR_MATRIX_BT709: u32 = 1 << 2;

fn is_rgb_format(pixel: Pixel) -> bool {
matches!(pixel, Pixel::BGRA | Pixel::RGBA)
}

fn pixel_to_dxgi(pixel: Pixel) -> Result<DXGI_FORMAT, ConvertError> {
match pixel {
Pixel::NV12 => Ok(DXGI_FORMAT_NV12),
Expand Down
3 changes: 3 additions & 0 deletions crates/frame-converter/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ pub use pool::*;
mod frame_pool;
pub use frame_pool::*;

mod reorder;
pub use reorder::*;

#[cfg(target_os = "macos")]
mod videotoolbox;
#[cfg(target_os = "macos")]
Expand Down
Loading
Loading