From 612c861ef153cf0ee575d8dd1413b960e4e19dd6 Mon Sep 17 00:00:00 2001 From: Eduardo Ramos Date: Sat, 4 Jan 2025 06:31:00 +0100 Subject: [PATCH] Fix bug in MSFoundation backend where media formats are created not exactly as expected by the device and fail to open (#197) * Fix infinite loop when guid_to_frameformat fails * Fix bug in MSFoundation backend where media formats are created not exactly as expected by the device and fail to open. Also fix conversion of MSFoundation RGB24, it's RAWBGR instead of RAWRGB * Fix compilation issue --- nokhwa-bindings-linux/src/lib.rs | 3 + nokhwa-bindings-macos/src/lib.rs | 7 + nokhwa-bindings-windows/src/lib.rs | 223 ++++++++++++++++------------- nokhwa-core/src/pixel_format.rs | 55 ++++++- nokhwa-core/src/traits.rs | 2 +- nokhwa-core/src/types.rs | 59 ++++++++ 6 files changed, 239 insertions(+), 110 deletions(-) diff --git a/nokhwa-bindings-linux/src/lib.rs b/nokhwa-bindings-linux/src/lib.rs index 23647a6..2bd9c58 100644 --- a/nokhwa-bindings-linux/src/lib.rs +++ b/nokhwa-bindings-linux/src/lib.rs @@ -540,6 +540,7 @@ mod internal { FrameFormat::YUYV => FourCC::new(b"YUYV"), FrameFormat::GRAY => FourCC::new(b"GRAY"), FrameFormat::RAWRGB => FourCC::new(b"RGB3"), + FrameFormat::RAWBGR => FourCC::new(b"BGR3"), FrameFormat::NV12 => FourCC::new(b"NV12"), }; @@ -912,6 +913,7 @@ mod internal { "MJPG" => Some(FrameFormat::MJPEG), "GRAY" => Some(FrameFormat::GRAY), "RGB3" => Some(FrameFormat::RAWRGB), + "BGR3" => Some(FrameFormat::RAWBGR), "NV12" => Some(FrameFormat::NV12), _ => None, } @@ -923,6 +925,7 @@ mod internal { FrameFormat::YUYV => FourCC::new(b"YUYV"), FrameFormat::GRAY => FourCC::new(b"GRAY"), FrameFormat::RAWRGB => FourCC::new(b"RGB3"), + FrameFormat::RAWBGR => FourCC::new(b"BGR3"), FrameFormat::NV12 => FourCC::new(b"NV12"), } } diff --git a/nokhwa-bindings-macos/src/lib.rs b/nokhwa-bindings-macos/src/lib.rs index bd419bb..15b250b 100644 --- a/nokhwa-bindings-macos/src/lib.rs +++ b/nokhwa-bindings-macos/src/lib.rs @@ -2285,6 +2285,13 @@ mod internal { FrameFormat::GRAY => kCMPixelFormat_8IndexedGray_WhiteIsZero, FrameFormat::NV12 => kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, FrameFormat::RAWRGB => kCMPixelFormat_24RGB, + FrameFormat::RAWBGR => { + return Err(NokhwaError::SetPropertyError { + property: "setVideoSettings".to_string(), + value: "set frame format".to_string(), + error: "Unsupported frame format BGR".to_string(), + }); + } }; let obj = CFNumber::from(cmpixelfmt as i32); let obj = obj.as_CFTypeRef() as *mut Object; diff --git a/nokhwa-bindings-windows/src/lib.rs b/nokhwa-bindings-windows/src/lib.rs index 88682b5..d65aec0 100644 --- a/nokhwa-bindings-windows/src/lib.rs +++ b/nokhwa-bindings-windows/src/lib.rs @@ -48,7 +48,7 @@ pub mod wmf { }; use windows::Win32::Media::DirectShow::{CameraControl_Flags_Auto, CameraControl_Flags_Manual}; use windows::Win32::Media::MediaFoundation::{ - IMFMediaType, MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM, + MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM, }; use windows::{ core::{Interface, GUID, PWSTR}, @@ -65,14 +65,14 @@ pub mod wmf { KernelStreaming::GUID_NULL, MediaFoundation::{ IMFActivate, IMFAttributes, IMFMediaSource, IMFSample, IMFSourceReader, - MFCreateAttributes, MFCreateMediaType, MFCreateSourceReaderFromMediaSource, - MFEnumDeviceSources, MFMediaType_Video, MFShutdown, MFStartup, + MFCreateAttributes, MFCreateSourceReaderFromMediaSource, + MFEnumDeviceSources, MFShutdown, MFStartup, MFSTARTUP_NOSOCKET, MF_API_VERSION, MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, MF_MT_FRAME_RATE, MF_MT_FRAME_RATE_RANGE_MAX, MF_MT_FRAME_RATE_RANGE_MIN, MF_MT_FRAME_SIZE, - MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS, + MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS, }, }, System::Com::{CoInitializeEx, CoUninitialize, COINIT}, @@ -165,7 +165,7 @@ pub mod wmf { fn guid_to_frameformat(guid: GUID) -> Option { match guid { MF_VIDEO_FORMAT_NV12 => Some(FrameFormat::NV12), - MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWRGB), + MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWBGR), MF_VIDEO_FORMAT_GRAY => Some(FrameFormat::GRAY), MF_VIDEO_FORMAT_YUY2 => Some(FrameFormat::YUYV), MF_VIDEO_FORMAT_MJPEG => Some(FrameFormat::MJPEG), @@ -173,16 +173,6 @@ pub mod wmf { } } - fn frameformat_to_guid(frameformat: FrameFormat) -> GUID { - match frameformat { - FrameFormat::MJPEG => MF_VIDEO_FORMAT_MJPEG, - FrameFormat::YUYV => MF_VIDEO_FORMAT_YUY2, - FrameFormat::NV12 => MF_VIDEO_FORMAT_NV12, - FrameFormat::GRAY => MF_VIDEO_FORMAT_GRAY, - FrameFormat::RAWRGB => MF_VIDEO_FORMAT_RGB24, - } - } - pub fn initialize_mf() -> Result<(), NokhwaError> { if !(INITIALIZED.load(Ordering::SeqCst)) { if let Err(why) = unsafe { @@ -607,16 +597,6 @@ pub mod wmf { } framerates.push(numerator); }; - if let Ok(fraction_u64) = - unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) } - { - let mut numerator = (fraction_u64 >> 32) as u32; - let denominator = fraction_u64 as u32; - if denominator != 1 { - numerator = 0; - } - framerates.push(numerator); - }; if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } { let mut numerator = (fraction_u64 >> 32) as u32; let denominator = fraction_u64 as u32; @@ -1006,88 +986,125 @@ pub mod wmf { } pub fn set_format(&mut self, format: CameraFormat) -> Result<(), NokhwaError> { - // convert to media_type - let media_type: IMFMediaType = match unsafe { MFCreateMediaType() } { - Ok(mt) => mt, - Err(why) => { - return Err(NokhwaError::StructureError { - structure: "IMFMediaType".to_string(), - error: why.to_string(), - }) - } - }; + // We need to make sure to use all the original attributes of the IMFMediaType to avoid problems. + // Otherwise, constructing IMFMediaType from scratch can sometimes fail due to not exactly matching. + // Therefore, we search for the first media_type that matches and also works correctly. - // set relevant things - let resolution = (u64::from(format.resolution().width_x) << 32_u64) - + u64::from(format.resolution().height_y); - let fps = { - let frame_rate_u64 = 0_u64; - let mut bytes: [u8; 8] = frame_rate_u64.to_le_bytes(); - bytes[7] = format.frame_rate() as u8; - bytes[3] = 0x01; - u64::from_le_bytes(bytes) - }; - let fourcc = frameformat_to_guid(format.format()); - // setting to the new media_type - if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_MAJOR_TYPE".to_string(), - value: "MFMediaType_Video".to_string(), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_SUBTYPE, &fourcc) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_SUBTYPE".to_string(), - value: format!("{:?}", fourcc), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_SIZE, resolution) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_FRAME_SIZE".to_string(), - value: resolution.to_string(), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE, fps) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_FRAME_RATE".to_string(), - value: fps.to_string(), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN, fps) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_FRAME_RATE_RANGE_MIN".to_string(), - value: fps.to_string(), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX, fps) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_FRAME_RATE_RANGE_MAX".to_string(), - value: fps.to_string(), - error: why.to_string(), - }); - } + let mut last_error : Option = None; - if let Err(why) = unsafe { - self.source_reader.SetCurrentMediaType( - MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, - None, - &media_type, - ) + let mut index = 0; + while let Ok(media_type) = unsafe { + self.source_reader + .GetNativeMediaType(MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, index) } { - return Err(NokhwaError::SetPropertyError { - property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(), - value: format!("{media_type:?}"), - error: why.to_string(), - }); + index += 1; + let fourcc = match unsafe { media_type.GetGUID(&MF_MT_SUBTYPE) } { + Ok(fcc) => fcc, + Err(why) => { + return Err(NokhwaError::GetPropertyError { + property: "MF_MT_SUBTYPE".to_string(), + error: why.to_string(), + }) + } + }; + + let frame_fmt = match guid_to_frameformat(fourcc) { + Some(fcc) => fcc, + None => continue, + }; + + if frame_fmt != format.format() { + continue; + } + + let (width, height) = match unsafe { media_type.GetUINT64(&MF_MT_FRAME_SIZE) } { + Ok(res_u64) => { + let width = (res_u64 >> 32) as u32; + let height = res_u64 as u32; // the cast will truncate the upper bits + (width, height) + } + Err(why) => { + return Err(NokhwaError::GetPropertyError { + property: "MF_MT_FRAME_SIZE".to_string(), + error: why.to_string(), + }) + } + }; + + if (Resolution { width_x: width, height_y: height }) != format.resolution() { + continue; + } + + // MFRatio is represented as 2 u32s in memory. This means we can convert it to 2 + let framerate_list = { + let mut framerates = vec![0_u32; 3]; + if let Ok(fraction_u64) = + unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) } + { + let mut numerator = (fraction_u64 >> 32) as u32; + let denominator = fraction_u64 as u32; + if denominator != 1 { + numerator = 0; + } + framerates.push(numerator); + }; + if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } { + let mut numerator = (fraction_u64 >> 32) as u32; + let denominator = fraction_u64 as u32; + if denominator != 1 { + numerator = 0; + } + framerates.push(numerator); + }; + if let Ok(fraction_u64) = + unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN) } + { + let mut numerator = (fraction_u64 >> 32) as u32; + let denominator = fraction_u64 as u32; + if denominator != 1 { + numerator = 0; + } + framerates.push(numerator); + }; + framerates + }; + + for frame_rate in framerate_list { + if frame_rate == format.frame_rate() { + let result = unsafe { + self.source_reader.SetCurrentMediaType( + MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, + None, + &media_type, + ) + }; + + match result { + Ok(_) => { + self.device_format = format; + self.format_refreshed()?; + return Ok(()); + }, + Err(why) => { + last_error = Some(NokhwaError::SetPropertyError { + property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(), + value: format!("{media_type:?}"), + error: why.to_string(), + }); + } + } + } + } } - self.device_format = format; - self.format_refreshed()?; - Ok(()) + + if let Some(err) = last_error { + return Err(err); + } + + Err(NokhwaError::InitializeError { + backend: ApiBackend::MediaFoundation, + error: "Failed to fulfill requested format".to_string(), + }) } pub fn is_stream_open(&self) -> bool { diff --git a/nokhwa-core/src/pixel_format.rs b/nokhwa-core/src/pixel_format.rs index c197d36..0e8c43e 100644 --- a/nokhwa-core/src/pixel_format.rs +++ b/nokhwa-core/src/pixel_format.rs @@ -15,7 +15,7 @@ */ use crate::error::NokhwaError; use crate::types::{ - buf_mjpeg_to_rgb, buf_nv12_to_rgb, buf_yuyv422_to_rgb, color_frame_formats, frame_formats, + buf_bgr_to_rgb, buf_mjpeg_to_rgb, buf_nv12_to_rgb, buf_yuyv422_to_rgb, color_frame_formats, frame_formats, mjpeg_to_rgb, nv12_to_rgb, yuyv422_to_rgb, FrameFormat, Resolution, }; use image::{Luma, LumaA, Pixel, Rgb, Rgba}; @@ -76,6 +76,16 @@ impl FormatDecoder for RgbFormat { }) .collect()), FrameFormat::RAWRGB => Ok(data.to_vec()), + FrameFormat::RAWBGR => { + let mut rgb = vec![0u8; data.len()]; + data.chunks_exact(3).enumerate().for_each(|(idx, px)| { + let index = idx * 3; + rgb[index] = px[2]; + rgb[index + 1] = px[1]; + rgb[index + 2] = px[0]; + }); + Ok(rgb) + }, FrameFormat::NV12 => nv12_to_rgb(resolution, data, false), } } @@ -111,6 +121,7 @@ impl FormatDecoder for RgbFormat { dest.copy_from_slice(data); Ok(()) } + FrameFormat::RAWBGR => buf_bgr_to_rgb(resolution, data, dest), FrameFormat::NV12 => buf_nv12_to_rgb(resolution, data, dest, false), } } @@ -150,6 +161,10 @@ impl FormatDecoder for RgbAFormat { .chunks_exact(3) .flat_map(|x| [x[0], x[1], x[2], 255]) .collect()), + FrameFormat::RAWBGR => Ok(data + .chunks_exact(3) + .flat_map(|x| [x[2], x[1], x[0], 255]) + .collect()), FrameFormat::NV12 => nv12_to_rgb(resolution, data, true), } } @@ -193,6 +208,16 @@ impl FormatDecoder for RgbAFormat { }); Ok(()) } + FrameFormat::RAWBGR => { + data.chunks_exact(3).enumerate().for_each(|(idx, px)| { + let index = idx * 4; + dest[index] = px[2]; + dest[index + 1] = px[1]; + dest[index + 2] = px[0]; + dest[index + 3] = 255; + }); + Ok(()) + } FrameFormat::NV12 => buf_nv12_to_rgb(resolution, data, dest, true), } } @@ -253,6 +278,10 @@ impl FormatDecoder for LumaFormat { .chunks(3) .map(|px| ((i32::from(px[0]) + i32::from(px[1]) + i32::from(px[2])) / 3) as u8) .collect()), + FrameFormat::RAWBGR => Ok(data + .chunks(3) + .map(|px| ((i32::from(px[2]) + i32::from(px[1]) + i32::from(px[0])) / 3) as u8) + .collect()), } } @@ -268,11 +297,10 @@ impl FormatDecoder for LumaFormat { FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::NV12 => { Err(NokhwaError::ProcessFrameError { src: fcc, - destination: "Luma => RGB".to_string(), + destination: "RGB => Luma".to_string(), error: "Conversion Error".to_string(), }) } - FrameFormat::GRAY => { data.iter().zip(dest.iter_mut()).for_each(|(pxv, d)| { *d = *pxv; @@ -281,7 +309,12 @@ impl FormatDecoder for LumaFormat { } FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError { src: fcc, - destination: "RGB => RGB".to_string(), + destination: "RGB => Luma".to_string(), + error: "Conversion Error".to_string(), + }), + FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "BGR => Luma".to_string(), error: "Conversion Error".to_string(), }), } @@ -340,7 +373,12 @@ impl FormatDecoder for LumaAFormat { FrameFormat::GRAY => Ok(data.iter().flat_map(|x| [*x, 255]).collect()), FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError { src: fcc, - destination: "RGB => RGB".to_string(), + destination: "RGB => LumaA".to_string(), + error: "Conversion Error".to_string(), + }), + FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "BGR => LumaA".to_string(), error: "Conversion Error".to_string(), }), } @@ -393,7 +431,12 @@ impl FormatDecoder for LumaAFormat { } FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError { src: fcc, - destination: "RGB => RGB".to_string(), + destination: "RGB => LumaA".to_string(), + error: "Conversion Error".to_string(), + }), + FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "BGR => LumaA".to_string(), error: "Conversion Error".to_string(), }), } diff --git a/nokhwa-core/src/traits.rs b/nokhwa-core/src/traits.rs index 79d4db0..9de70ed 100644 --- a/nokhwa-core/src/traits.rs +++ b/nokhwa-core/src/traits.rs @@ -173,7 +173,7 @@ CaptureBackendTrait { let cfmt = self.camera_format(); let resolution = cfmt.resolution(); let pxwidth = match cfmt.format() { - FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::RAWRGB | FrameFormat::NV12 => 3, + FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::RAWRGB | FrameFormat::RAWBGR | FrameFormat::NV12 => 3, FrameFormat::GRAY => 1, }; if alpha { diff --git a/nokhwa-core/src/types.rs b/nokhwa-core/src/types.rs index fb39dac..975c9e4 100644 --- a/nokhwa-core/src/types.rs +++ b/nokhwa-core/src/types.rs @@ -299,6 +299,7 @@ pub enum FrameFormat { NV12, GRAY, RAWRGB, + RAWBGR, } impl Display for FrameFormat { @@ -316,6 +317,9 @@ impl Display for FrameFormat { FrameFormat::RAWRGB => { write!(f, "RAWRGB") } + FrameFormat::RAWBGR => { + write!(f, "RAWBGR") + } FrameFormat::NV12 => { write!(f, "NV12") } @@ -331,6 +335,7 @@ impl FromStr for FrameFormat { "YUYV" => Ok(FrameFormat::YUYV), "GRAY" => Ok(FrameFormat::GRAY), "RAWRGB" => Ok(FrameFormat::RAWRGB), + "RAWBGR" => Ok(FrameFormat::RAWBGR), "NV12" => Ok(FrameFormat::NV12), _ => Err(NokhwaError::StructureError { structure: "FrameFormat".to_string(), @@ -349,6 +354,7 @@ pub const fn frame_formats() -> &'static [FrameFormat] { FrameFormat::NV12, FrameFormat::GRAY, FrameFormat::RAWRGB, + FrameFormat::RAWBGR, ] } @@ -360,6 +366,7 @@ pub const fn color_frame_formats() -> &'static [FrameFormat] { FrameFormat::YUYV, FrameFormat::NV12, FrameFormat::RAWRGB, + FrameFormat::RAWBGR, ] } @@ -1812,3 +1819,55 @@ pub fn buf_nv12_to_rgb( Ok(()) } + +#[allow(clippy::similar_names)] +#[inline] +pub fn buf_bgr_to_rgb( + resolution: Resolution, + data: &[u8], + out: &mut [u8], +) -> Result<(), NokhwaError> { + let width = resolution.width(); + let height = resolution.height(); + + if width % 2 != 0 || height % 2 != 0 { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::RAWBGR, + destination: "RGB".to_string(), + error: "bad resolution".to_string(), + }); + } + + let input_size = (width * height * 3) as usize; // BGR is 3 bytes per pixel + let output_size = (width * height * 3) as usize; // RGB is 3 bytes per pixel + + if data.len() != input_size { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::RAWBGR, + destination: "RGB".to_string(), + error: "bad input buffer size".to_string(), + }); + } + + if out.len() != output_size { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::RAWBGR, + destination: "RGB".to_string(), + error: "bad output buffer size".to_string(), + }); + } + + for (idx, chunk) in data.chunks_exact(3).enumerate() { + // BGR Format: [Blue, Green, Red] + let b = chunk[0]; + let g = chunk[1]; + let r = chunk[2]; + + let out_idx = idx * 3; // 3 bytes per pixel in RGB + out[out_idx] = r; // Red + out[out_idx + 1] = g; // Green + out[out_idx + 2] = b; // Blue + } + + Ok(()) +}