GRIN/nokhwa
1
0
Fork 0
mirror of https://github.com/l1npengtul/nokhwa.git synced 2026-07-04 02:27:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix bug in MSFoundation backend where media formats are created not exactly as expected by the device and fail to open (#197)

Browse Source 
* Fix infinite loop when guid_to_frameformat fails

* Fix bug in MSFoundation backend where media formats are created not exactly as expected by the device and fail to open. Also fix conversion of MSFoundation RGB24, it's RAWBGR instead of RAWRGB

* Fix compilation issue
This commit is contained in:
Eduardo Ramos
2025-01-04 06:31:00 +01:00
committed by GitHub
parent 45c6a3aef3
commit 612c861ef1
6 changed files with 239 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nokhwa-bindings-linux/src/lib.rs
+3
View File
@@ -540,6 +540,7 @@ mod internal {
FrameFormat::YUYV => FourCC::new(b"YUYV"),
FrameFormat::GRAY => FourCC::new(b"GRAY"),
FrameFormat::RAWRGB => FourCC::new(b"RGB3"),
FrameFormat::RAWBGR => FourCC::new(b"BGR3"),
FrameFormat::NV12 => FourCC::new(b"NV12"),
};
@@ -912,6 +913,7 @@ mod internal {
"MJPG" => Some(FrameFormat::MJPEG),
"GRAY" => Some(FrameFormat::GRAY),
"RGB3" => Some(FrameFormat::RAWRGB),
"BGR3" => Some(FrameFormat::RAWBGR),
"NV12" => Some(FrameFormat::NV12),
_ => None,
}
@@ -923,6 +925,7 @@ mod internal {
FrameFormat::YUYV => FourCC::new(b"YUYV"),
FrameFormat::GRAY => FourCC::new(b"GRAY"),
FrameFormat::RAWRGB => FourCC::new(b"RGB3"),
FrameFormat::RAWBGR => FourCC::new(b"BGR3"),
FrameFormat::NV12 => FourCC::new(b"NV12"),
}
}
nokhwa-bindings-macos/src/lib.rs
+7
View File
@@ -2285,6 +2285,13 @@ mod internal {
FrameFormat::GRAY => kCMPixelFormat_8IndexedGray_WhiteIsZero,
FrameFormat::NV12 => kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange,
FrameFormat::RAWRGB => kCMPixelFormat_24RGB,
FrameFormat::RAWBGR => {
return Err(NokhwaError::SetPropertyError {
property: "setVideoSettings".to_string(),
value: "set frame format".to_string(),
error: "Unsupported frame format BGR".to_string(),
});
}
};
let obj = CFNumber::from(cmpixelfmt as i32);
let obj = obj.as_CFTypeRef() as *mut Object;
nokhwa-bindings-windows/src/lib.rs
+120 -103
View File
@@ -48,7 +48,7 @@ pub mod wmf {
};
use windows::Win32::Media::DirectShow::{CameraControl_Flags_Auto, CameraControl_Flags_Manual};
use windows::Win32::Media::MediaFoundation::{
IMFMediaType, MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM,
MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM,
};
use windows::{
core::{Interface, GUID, PWSTR},
@@ -65,14 +65,14 @@ pub mod wmf {
KernelStreaming::GUID_NULL,
MediaFoundation::{
IMFActivate, IMFAttributes, IMFMediaSource, IMFSample, IMFSourceReader,
MFCreateAttributes, MFCreateMediaType, MFCreateSourceReaderFromMediaSource,
MFEnumDeviceSources, MFMediaType_Video, MFShutdown, MFStartup,
MFCreateAttributes, MFCreateSourceReaderFromMediaSource,
MFEnumDeviceSources, MFShutdown, MFStartup,
MFSTARTUP_NOSOCKET, MF_API_VERSION, MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, MF_MT_FRAME_RATE,
MF_MT_FRAME_RATE_RANGE_MAX, MF_MT_FRAME_RATE_RANGE_MIN, MF_MT_FRAME_SIZE,
MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS,
MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS,
},
},
System::Com::{CoInitializeEx, CoUninitialize, COINIT},
@@ -165,7 +165,7 @@ pub mod wmf {
fn guid_to_frameformat(guid: GUID) -> Option<FrameFormat> {
match guid {
MF_VIDEO_FORMAT_NV12 => Some(FrameFormat::NV12),
MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWRGB),
MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWBGR),
MF_VIDEO_FORMAT_GRAY => Some(FrameFormat::GRAY),
MF_VIDEO_FORMAT_YUY2 => Some(FrameFormat::YUYV),
MF_VIDEO_FORMAT_MJPEG => Some(FrameFormat::MJPEG),
@@ -173,16 +173,6 @@ pub mod wmf {
}
}
fn frameformat_to_guid(frameformat: FrameFormat) -> GUID {
match frameformat {
FrameFormat::MJPEG => MF_VIDEO_FORMAT_MJPEG,
FrameFormat::YUYV => MF_VIDEO_FORMAT_YUY2,
FrameFormat::NV12 => MF_VIDEO_FORMAT_NV12,
FrameFormat::GRAY => MF_VIDEO_FORMAT_GRAY,
FrameFormat::RAWRGB => MF_VIDEO_FORMAT_RGB24,
}
}
pub fn initialize_mf() -> Result<(), NokhwaError> {
if !(INITIALIZED.load(Ordering::SeqCst)) {
if let Err(why) = unsafe {
@@ -607,16 +597,6 @@ pub mod wmf {
}
framerates.push(numerator);
};
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } {
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
@@ -1006,88 +986,125 @@ pub mod wmf {
}
pub fn set_format(&mut self, format: CameraFormat) -> Result<(), NokhwaError> {
// convert to media_type
let media_type: IMFMediaType = match unsafe { MFCreateMediaType() } {
Ok(mt) => mt,
Err(why) => {
return Err(NokhwaError::StructureError {
structure: "IMFMediaType".to_string(),
error: why.to_string(),
})
}
};
// We need to make sure to use all the original attributes of the IMFMediaType to avoid problems.
// Otherwise, constructing IMFMediaType from scratch can sometimes fail due to not exactly matching.
// Therefore, we search for the first media_type that matches and also works correctly.
// set relevant things
let resolution = (u64::from(format.resolution().width_x) << 32_u64)
+ u64::from(format.resolution().height_y);
let fps = {
let frame_rate_u64 = 0_u64;
let mut bytes: [u8; 8] = frame_rate_u64.to_le_bytes();
bytes[7] = format.frame_rate() as u8;
bytes[3] = 0x01;
u64::from_le_bytes(bytes)
};
let fourcc = frameformat_to_guid(format.format());
// setting to the new media_type
if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_MAJOR_TYPE".to_string(),
value: "MFMediaType_Video".to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_SUBTYPE, &fourcc) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_SUBTYPE".to_string(),
value: format!("{:?}", fourcc),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_SIZE, resolution) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_SIZE".to_string(),
value: resolution.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE_RANGE_MIN".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE_RANGE_MAX".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
}
let mut last_error : Option<NokhwaError> = None;
if let Err(why) = unsafe {
self.source_reader.SetCurrentMediaType(
MEDIA_FOUNDATION_FIRST_VIDEO_STREAM,
None,
&media_type,
)
let mut index = 0;
while let Ok(media_type) = unsafe {
self.source_reader
.GetNativeMediaType(MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, index)
} {
return Err(NokhwaError::SetPropertyError {
property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(),
value: format!("{media_type:?}"),
error: why.to_string(),
});
index += 1;
let fourcc = match unsafe { media_type.GetGUID(&MF_MT_SUBTYPE) } {
Ok(fcc) => fcc,
Err(why) => {
return Err(NokhwaError::GetPropertyError {
property: "MF_MT_SUBTYPE".to_string(),
error: why.to_string(),
})
}
};
let frame_fmt = match guid_to_frameformat(fourcc) {
Some(fcc) => fcc,
None => continue,
};
if frame_fmt != format.format() {
continue;
}
let (width, height) = match unsafe { media_type.GetUINT64(&MF_MT_FRAME_SIZE) } {
Ok(res_u64) => {
let width = (res_u64 >> 32) as u32;
let height = res_u64 as u32; // the cast will truncate the upper bits
(width, height)
}
Err(why) => {
return Err(NokhwaError::GetPropertyError {
property: "MF_MT_FRAME_SIZE".to_string(),
error: why.to_string(),
})
}
};
if (Resolution { width_x: width, height_y: height }) != format.resolution() {
continue;
}
// MFRatio is represented as 2 u32s in memory. This means we can convert it to 2
let framerate_list = {
let mut framerates = vec![0_u32; 3];
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } {
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
framerates
};
for frame_rate in framerate_list {
if frame_rate == format.frame_rate() {
let result = unsafe {
self.source_reader.SetCurrentMediaType(
MEDIA_FOUNDATION_FIRST_VIDEO_STREAM,
None,
&media_type,
)
};
match result {
Ok(_) => {
self.device_format = format;
self.format_refreshed()?;
return Ok(());
},
Err(why) => {
last_error = Some(NokhwaError::SetPropertyError {
property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(),
value: format!("{media_type:?}"),
error: why.to_string(),
});
}
}
}
}
}
self.device_format = format;
self.format_refreshed()?;
Ok(())
if let Some(err) = last_error {
return Err(err);
}
Err(NokhwaError::InitializeError {
backend: ApiBackend::MediaFoundation,
error: "Failed to fulfill requested format".to_string(),
})
}
pub fn is_stream_open(&self) -> bool {
nokhwa-core/src/pixel_format.rs
+49 -6
View File
@@ -15,7 +15,7 @@
*/
use crate::error::NokhwaError;
use crate::types::{
buf_mjpeg_to_rgb, buf_nv12_to_rgb, buf_yuyv422_to_rgb, color_frame_formats, frame_formats,
buf_bgr_to_rgb, buf_mjpeg_to_rgb, buf_nv12_to_rgb, buf_yuyv422_to_rgb, color_frame_formats, frame_formats,
mjpeg_to_rgb, nv12_to_rgb, yuyv422_to_rgb, FrameFormat, Resolution,
};
use image::{Luma, LumaA, Pixel, Rgb, Rgba};
@@ -76,6 +76,16 @@ impl FormatDecoder for RgbFormat {
})
.collect()),
FrameFormat::RAWRGB => Ok(data.to_vec()),
FrameFormat::RAWBGR => {
let mut rgb = vec![0u8; data.len()];
data.chunks_exact(3).enumerate().for_each(|(idx, px)| {
let index = idx * 3;
rgb[index] = px[2];
rgb[index + 1] = px[1];
rgb[index + 2] = px[0];
});
Ok(rgb)
},
FrameFormat::NV12 => nv12_to_rgb(resolution, data, false),
}
}
@@ -111,6 +121,7 @@ impl FormatDecoder for RgbFormat {
dest.copy_from_slice(data);
Ok(())
}
FrameFormat::RAWBGR => buf_bgr_to_rgb(resolution, data, dest),
FrameFormat::NV12 => buf_nv12_to_rgb(resolution, data, dest, false),
}
}
@@ -150,6 +161,10 @@ impl FormatDecoder for RgbAFormat {
.chunks_exact(3)
.flat_map(|x| [x[0], x[1], x[2], 255])
.collect()),
FrameFormat::RAWBGR => Ok(data
.chunks_exact(3)
.flat_map(|x| [x[2], x[1], x[0], 255])
.collect()),
FrameFormat::NV12 => nv12_to_rgb(resolution, data, true),
}
}
@@ -193,6 +208,16 @@ impl FormatDecoder for RgbAFormat {
});
Ok(())
}
FrameFormat::RAWBGR => {
data.chunks_exact(3).enumerate().for_each(|(idx, px)| {
let index = idx * 4;
dest[index] = px[2];
dest[index + 1] = px[1];
dest[index + 2] = px[0];
dest[index + 3] = 255;
});
Ok(())
}
FrameFormat::NV12 => buf_nv12_to_rgb(resolution, data, dest, true),
}
}
@@ -253,6 +278,10 @@ impl FormatDecoder for LumaFormat {
.chunks(3)
.map(|px| ((i32::from(px[0]) + i32::from(px[1]) + i32::from(px[2])) / 3) as u8)
.collect()),
FrameFormat::RAWBGR => Ok(data
.chunks(3)
.map(|px| ((i32::from(px[2]) + i32::from(px[1]) + i32::from(px[0])) / 3) as u8)
.collect()),
}
}
@@ -268,11 +297,10 @@ impl FormatDecoder for LumaFormat {
FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::NV12 => {
Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "Luma => RGB".to_string(),
destination: "RGB => Luma".to_string(),
error: "Conversion Error".to_string(),
})
}
FrameFormat::GRAY => {
data.iter().zip(dest.iter_mut()).for_each(|(pxv, d)| {
*d = *pxv;
@@ -281,7 +309,12 @@ impl FormatDecoder for LumaFormat {
}
FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "RGB => RGB".to_string(),
destination: "RGB => Luma".to_string(),
error: "Conversion Error".to_string(),
}),
FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "BGR => Luma".to_string(),
error: "Conversion Error".to_string(),
}),
}
@@ -340,7 +373,12 @@ impl FormatDecoder for LumaAFormat {
FrameFormat::GRAY => Ok(data.iter().flat_map(|x| [*x, 255]).collect()),
FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "RGB => RGB".to_string(),
destination: "RGB => LumaA".to_string(),
error: "Conversion Error".to_string(),
}),
FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "BGR => LumaA".to_string(),
error: "Conversion Error".to_string(),
}),
}
@@ -393,7 +431,12 @@ impl FormatDecoder for LumaAFormat {
}
FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "RGB => RGB".to_string(),
destination: "RGB => LumaA".to_string(),
error: "Conversion Error".to_string(),
}),
FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "BGR => LumaA".to_string(),
error: "Conversion Error".to_string(),
}),
}
nokhwa-core/src/traits.rs
+1 -1
View File
@@ -173,7 +173,7 @@ CaptureBackendTrait {
let cfmt = self.camera_format();
let resolution = cfmt.resolution();
let pxwidth = match cfmt.format() {
FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::RAWRGB | FrameFormat::NV12 => 3,
FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::RAWRGB | FrameFormat::RAWBGR | FrameFormat::NV12 => 3,
FrameFormat::GRAY => 1,
};
if alpha {
nokhwa-core/src/types.rs
+59
View File
@@ -299,6 +299,7 @@ pub enum FrameFormat {
NV12,
GRAY,
RAWRGB,
RAWBGR,
}
impl Display for FrameFormat {
@@ -316,6 +317,9 @@ impl Display for FrameFormat {
FrameFormat::RAWRGB => {
write!(f, "RAWRGB")
}
FrameFormat::RAWBGR => {
write!(f, "RAWBGR")
}
FrameFormat::NV12 => {
write!(f, "NV12")
}
@@ -331,6 +335,7 @@ impl FromStr for FrameFormat {
"YUYV" => Ok(FrameFormat::YUYV),
"GRAY" => Ok(FrameFormat::GRAY),
"RAWRGB" => Ok(FrameFormat::RAWRGB),
"RAWBGR" => Ok(FrameFormat::RAWBGR),
"NV12" => Ok(FrameFormat::NV12),
_ => Err(NokhwaError::StructureError {
structure: "FrameFormat".to_string(),
@@ -349,6 +354,7 @@ pub const fn frame_formats() -> &'static [FrameFormat] {
FrameFormat::NV12,
FrameFormat::GRAY,
FrameFormat::RAWRGB,
FrameFormat::RAWBGR,
]
}
@@ -360,6 +366,7 @@ pub const fn color_frame_formats() -> &'static [FrameFormat] {
FrameFormat::YUYV,
FrameFormat::NV12,
FrameFormat::RAWRGB,
FrameFormat::RAWBGR,
]
}
@@ -1812,3 +1819,55 @@ pub fn buf_nv12_to_rgb(
Ok(())
}
#[allow(clippy::similar_names)]
#[inline]
pub fn buf_bgr_to_rgb(
resolution: Resolution,
data: &[u8],
out: &mut [u8],
) -> Result<(), NokhwaError> {
let width = resolution.width();
let height = resolution.height();
if width % 2 != 0 || height % 2 != 0 {
return Err(NokhwaError::ProcessFrameError {
src: FrameFormat::RAWBGR,
destination: "RGB".to_string(),
error: "bad resolution".to_string(),
});
}
let input_size = (width * height * 3) as usize; // BGR is 3 bytes per pixel
let output_size = (width * height * 3) as usize; // RGB is 3 bytes per pixel
if data.len() != input_size {
return Err(NokhwaError::ProcessFrameError {
src: FrameFormat::RAWBGR,
destination: "RGB".to_string(),
error: "bad input buffer size".to_string(),
});
}
if out.len() != output_size {
return Err(NokhwaError::ProcessFrameError {
src: FrameFormat::RAWBGR,
destination: "RGB".to_string(),
error: "bad output buffer size".to_string(),
});
}
for (idx, chunk) in data.chunks_exact(3).enumerate() {
// BGR Format: [Blue, Green, Red]
let b = chunk[0];
let g = chunk[1];
let r = chunk[2];
let out_idx = idx * 3; // 3 bytes per pixel in RGB
out[out_idx] = r; // Red
out[out_idx + 1] = g; // Green
out[out_idx + 2] = b; // Blue
}
Ok(())
}