GRIN/nokhwa
1
0
Fork 0
mirror of https://github.com/l1npengtul/nokhwa.git synced 2026-07-04 02:27:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

add support for NV12 to fix #52, #51

Browse Source
This commit is contained in:
l1npengtul
2022-11-05 16:10:05 +09:00
parent 681a7507bb
commit 9da0c8fddf
5 changed files with 232 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nokhwa-core/src/buffer.rs
+7 -4
View File
@@ -21,8 +21,6 @@ use crate::{
};
use bytes::Bytes;
use image::ImageBuffer;
#[cfg(feature = "serialize")]
use serde::{Deserialize, Serialize};
/// A buffer returned by a camera to accomodate custom decoding.
/// Contains information of Resolution, the buffer's [`FrameFormat`], and the buffer.
@@ -74,7 +72,7 @@ impl Buffer {
pub fn decode_image<F: FormatDecoder>(
&self,
) -> Result<ImageBuffer<F::Output, Vec<u8>>, NokhwaError> {
let new_data = F::write_output(self.source_frame_format, &self.buffer)?;
let new_data = F::write_output(self.source_frame_format, self.resolution, &self.buffer)?;
let image =
ImageBuffer::from_raw(self.resolution.width_x, self.resolution.height_y, new_data)
.ok_or(NokhwaError::ProcessFrameError {
@@ -92,7 +90,12 @@ impl Buffer {
&self,
buffer: &mut [u8],
) -> Result<(), NokhwaError> {
F::write_output_buffer(self.source_frame_format, &self.buffer, buffer)
F::write_output_buffer(
self.source_frame_format,
self.resolution,
&self.buffer,
buffer,
)
}
/// Decodes a image with allocation using the provided [`FormatDecoder`] into a [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html).
nokhwa-core/src/pixel_format.rs
+81 -29
View File
@@ -15,7 +15,8 @@
*/
use crate::error::NokhwaError;
use crate::types::{
buf_mjpeg_to_rgb, buf_yuyv422_to_rgb, mjpeg_to_rgb, yuyv422_to_rgb, FrameFormat,
buf_mjpeg_to_rgb, buf_yuv_420_to_rgb, buf_yuyv422_to_rgb, mjpeg_to_rgb, yuv_420_to_rgb,
yuyv422_to_rgb, FrameFormat, Resolution,
};
use image::{Luma, LumaA, Pixel, Rgb, Rgba};
use std::fmt::Debug;
@@ -28,13 +29,18 @@ pub trait FormatDecoder: Clone + Sized + Send + Sync {
/// Allocates and returns a `Vec`
/// # Errors
/// If the data is malformed, or the source [`FrameFormat`] is incompatible, this will error.
fn write_output(fcc: FrameFormat, data: &[u8]) -> Result<Vec<u8>, NokhwaError>;
fn write_output(
fcc: FrameFormat,
resolution: Resolution,
data: &[u8],
) -> Result<Vec<u8>, NokhwaError>;
/// Writes to a user provided buffer.
/// # Errors
/// If the data is malformed, the source [`FrameFormat`] is incompatible, or the user-alloted buffer is not large enough, this will error.
fn write_output_buffer(
fcc: FrameFormat,
resolution: Resolution,
data: &[u8],
dest: &mut [u8],
) -> Result<(), NokhwaError>;
@@ -53,7 +59,11 @@ impl FormatDecoder for RgbFormat {
type Output = Rgb<u8>;
const FORMATS: &'static [FrameFormat] = &[FrameFormat::MJPEG, FrameFormat::YUYV];
fn write_output(fcc: FrameFormat, data: &[u8]) -> Result<Vec<u8>, NokhwaError> {
fn write_output(
fcc: FrameFormat,
resolution: Resolution,
data: &[u8],
) -> Result<Vec<u8>, NokhwaError> {
match fcc {
FrameFormat::MJPEG => mjpeg_to_rgb(data, false),
FrameFormat::YUYV => yuyv422_to_rgb(data, false),
@@ -65,11 +75,13 @@ impl FormatDecoder for RgbFormat {
})
.collect()),
FrameFormat::RAWRGB => Ok(data.to_vec()),
FrameFormat::NV12 => yuv_420_to_rgb(resolution, data, false),
}
}
fn write_output_buffer(
fcc: FrameFormat,
resolution: Resolution,
data: &[u8],
dest: &mut [u8],
) -> Result<(), NokhwaError> {
@@ -93,7 +105,11 @@ impl FormatDecoder for RgbFormat {
});
Ok(())
}
FrameFormat::RAWRGB => {dest.copy_from_slice(data); Ok(())}
FrameFormat::RAWRGB => {
dest.copy_from_slice(data);
Ok(())
}
FrameFormat::NV12 => buf_yuv_420_to_rgb(resolution, data, dest, false),
}
}
}
@@ -112,7 +128,11 @@ impl FormatDecoder for RgbAFormat {
const FORMATS: &'static [FrameFormat] = &[FrameFormat::MJPEG, FrameFormat::YUYV];
fn write_output(fcc: FrameFormat, data: &[u8]) -> Result<Vec<u8>, NokhwaError> {
fn write_output(
fcc: FrameFormat,
resolution: Resolution,
data: &[u8],
) -> Result<Vec<u8>, NokhwaError> {
match fcc {
FrameFormat::MJPEG => mjpeg_to_rgb(data, true),
FrameFormat::YUYV => yuyv422_to_rgb(data, true),
@@ -125,16 +145,16 @@ impl FormatDecoder for RgbAFormat {
.collect()),
FrameFormat::RAWRGB => Ok(data
.chunks_exact(3)
.flat_map(|x| {
[x[0], x[1], x[2], 255]
})
.collect()
),
.flat_map(|x| [x[0], x[1], x[2], 255])
.collect()),
FrameFormat::NV12 => yuv_420_to_rgb(resolution, data, true),
}
}
fn write_output_buffer(
fcc: FrameFormat,
resolution: Resolution,
data: &[u8],
dest: &mut [u8],
) -> Result<(), NokhwaError> {
@@ -160,19 +180,16 @@ impl FormatDecoder for RgbAFormat {
Ok(())
}
FrameFormat::RAWRGB => {
data
.chunks_exact(3)
.enumerate()
.for_each(|(idx, px)| {
data.chunks_exact(3).enumerate().for_each(|(idx, px)| {
let index = idx * 4;
dest[index] = px[0];
dest[index + 1] = px[1];
dest[index + 2] = px[2];
dest[index + 3] = 255;
});
Ok(())
}
FrameFormat::NV12 => buf_yuv_420_to_rgb(resolution, data, dest, true),
}
}
}
@@ -193,7 +210,11 @@ impl FormatDecoder for LumaFormat {
&[FrameFormat::MJPEG, FrameFormat::YUYV, FrameFormat::GRAY];
#[allow(clippy::cast_possible_truncation)]
fn write_output(fcc: FrameFormat, data: &[u8]) -> Result<Vec<u8>, NokhwaError> {
fn write_output(
fcc: FrameFormat,
resolution: Resolution,
data: &[u8],
) -> Result<Vec<u8>, NokhwaError> {
match fcc {
FrameFormat::MJPEG => Ok(mjpeg_to_rgb(data, false)?
.as_slice()
@@ -213,17 +234,26 @@ impl FormatDecoder for LumaFormat {
(avg / 3) as u8
})
.collect()),
FrameFormat::NV12 => Ok(yuv_420_to_rgb(resolution, data, false)?
.as_slice()
.chunks_exact(3)
.map(|x| {
let mut avg = 0;
x.iter().for_each(|v| avg += u16::from(*v));
(avg / 3) as u8
})
.collect()),
FrameFormat::GRAY => Ok(data.to_vec()),
FrameFormat::RAWRGB => {
Ok(data.chunks(3).map(|px| {
((px[0] as i32 + px[1] as i32 + px[2] as i32) / 3) as u8
}).collect())
},
FrameFormat::RAWRGB => Ok(data
.chunks(3)
.map(|px| ((px[0] as i32 + px[1] as i32 + px[2] as i32) / 3) as u8)
.collect()),
}
}
fn write_output_buffer(
fcc: FrameFormat,
_resolution: Resolution,
data: &[u8],
dest: &mut [u8],
) -> Result<(), NokhwaError> {
@@ -241,6 +271,11 @@ impl FormatDecoder for LumaFormat {
destination: "Luma => RGB".to_string(),
error: "Conversion Error".to_string(),
}),
FrameFormat::NV12 => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "Luma => RGB".to_string(),
error: "Conversion Error".to_string(),
}),
FrameFormat::GRAY => {
data.iter().zip(dest.iter_mut()).for_each(|(pxv, d)| {
*d = *pxv;
@@ -272,7 +307,11 @@ impl FormatDecoder for LumaAFormat {
&[FrameFormat::MJPEG, FrameFormat::YUYV, FrameFormat::GRAY];
#[allow(clippy::cast_possible_truncation)]
fn write_output(fcc: FrameFormat, data: &[u8]) -> Result<Vec<u8>, NokhwaError> {
fn write_output(
fcc: FrameFormat,
resolution: Resolution,
data: &[u8],
) -> Result<Vec<u8>, NokhwaError> {
match fcc {
FrameFormat::MJPEG => Ok(mjpeg_to_rgb(data, false)?
.as_slice()
@@ -292,6 +331,15 @@ impl FormatDecoder for LumaAFormat {
[(avg / 3) as u8, 255]
})
.collect()),
FrameFormat::NV12 => Ok(yuv_420_to_rgb(resolution, data, false)?
.as_slice()
.chunks_exact(3)
.flat_map(|x| {
let mut avg = 0;
x.iter().for_each(|v| avg += u16::from(*v));
[(avg / 3) as u8, 255]
})
.collect()),
FrameFormat::GRAY => Ok(data.iter().flat_map(|x| [*x, 255]).collect()),
FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError {
src: fcc,
@@ -304,6 +352,7 @@ impl FormatDecoder for LumaAFormat {
fn write_output_buffer(
fcc: FrameFormat,
data: &[u8],
_resolution: Resolution,
dest: &mut [u8],
) -> Result<(), NokhwaError> {
match fcc {
@@ -320,6 +369,11 @@ impl FormatDecoder for LumaAFormat {
destination: "YUYV => LumaA".to_string(),
error: "Conversion Error".to_string(),
}),
FrameFormat::NV12 => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "NV12 => LumaA".to_string(),
error: "Conversion Error".to_string(),
}),
FrameFormat::GRAY => {
if dest.len() != data.len() * 2 {
return Err(NokhwaError::ProcessFrameError {
@@ -339,13 +393,11 @@ impl FormatDecoder for LumaAFormat {
});
Ok(())
}
FrameFormat::RAWRGB => {
Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "RGB => RGB".to_string(),
error: "Conversion Error".to_string(),
})
},
FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError {
src: fcc,
destination: "RGB => RGB".to_string(),
error: "Conversion Error".to_string(),
}),
}
}
}
nokhwa-core/src/types.rs
+134 -11
View File
@@ -68,6 +68,11 @@ impl RequestedFormat<'_> {
wanted_decoder: decoder,
}
}
/// Gets the [`RequestedFormatType`]
pub fn requested_format_type(&self) -> RequestedFormatType {
self.requested_format
}
/// Fulfill the requested using a list of all available formats.
///
@@ -276,13 +281,16 @@ impl TryFrom<CameraIndex> for usize {
/// Describes a frame format (i.e. how the bytes themselves are encoded). Often called `FourCC`.
/// - YUYV is a mathematical color space. You can read more [here.](https://en.wikipedia.org/wiki/YCbCr)
/// - NV12 is same as above. Note that a partial compression (e.g. [16, 235] may be coerced to [0, 255].
/// - MJPEG is a motion-jpeg compressed frame, it allows for high frame rates.
/// - GRAY is a grayscale image format, usually for specialized cameras such as IR Cameras.
/// - RAWRGB is a Raw RGB888 format.
#[derive(Copy, Clone, Debug, Hash, Ord, PartialOrd, Eq, PartialEq)]
#[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))]
pub enum FrameFormat {
MJPEG,
YUYV,
NV12,
GRAY,
RAWRGB,
}
@@ -301,14 +309,23 @@ impl Display for FrameFormat {
}
FrameFormat::RAWRGB => {
write!(f, "RAWRGB")
},
}
FrameFormat::NV12 => {
write!(f, "NV12")
}
}
}
}
#[must_use]
pub const fn frame_formats() -> &'static [FrameFormat] {
&[FrameFormat::MJPEG, FrameFormat::YUYV, FrameFormat::GRAY, FrameFormat::RAWRGB]
&[
FrameFormat::MJPEG,
FrameFormat::YUYV,
FrameFormat::NV12,
FrameFormat::GRAY,
FrameFormat::RAWRGB,
]
}
/// Describes a Resolution.
@@ -343,6 +360,7 @@ impl Resolution {
/// This is exported as `get_Width`.
#[must_use]
#[cfg_attr(feature = "output-wasm", wasm_bindgen(getter = Width))]
#[inline]
pub fn width(self) -> u32 {
self.width_x
}
@@ -352,6 +370,7 @@ impl Resolution {
/// This is exported as `get_Height`.
#[must_use]
#[cfg_attr(feature = "output-wasm", wasm_bindgen(getter = Height))]
#[inline]
pub fn height(self) -> u32 {
self.height_y
}
@@ -359,6 +378,7 @@ impl Resolution {
/// Get the x (width) of Resolution
#[must_use]
#[cfg_attr(feature = "output-wasm", wasm_bindgen(skip))]
#[inline]
pub fn x(self) -> u32 {
self.width_x
}
@@ -366,6 +386,7 @@ impl Resolution {
/// Get the y (height) of Resolution
#[must_use]
#[cfg_attr(feature = "output-wasm", wasm_bindgen(skip))]
#[inline]
pub fn y(self) -> u32 {
self.height_y
}
@@ -1544,15 +1565,6 @@ pub fn buf_mjpeg_to_rgb(_data: &[u8], _dest: &mut [u8], _rgba: bool) -> Result<(
/// # Errors
/// This may error when the data stream size is not divisible by 4, a i32 -> u8 conversion fails, or it fails to read from a certain index.
pub fn yuyv422_to_rgb(data: &[u8], rgba: bool) -> Result<Vec<u8>, NokhwaError> {
if data.len() % 4 != 0 {
return Err(NokhwaError::ProcessFrameError {
src: FrameFormat::YUYV,
destination: "RGB888".to_string(),
error: "Assertion failure, the YUV stream isn't 4:2:2! (wrong number of bytes)"
.to_string(),
});
}
let pixel_size = if rgba { 4 } else { 3 };
// yuyv yields 2 3-byte pixels per yuyv chunk
let rgb_buf_size = (data.len() / 4) * (2 * pixel_size);
@@ -1672,3 +1684,114 @@ pub fn yuyv444_to_rgba(y: i32, u: i32, v: i32) -> [u8; 4] {
let [r, g, b] = yuyv444_to_rgb(y, u, v);
[r, g, b, 255]
}
/// Converts a YUYV 4:2:0 datastream to a RGB888 Stream. [For further reading](https://en.wikipedia.org/wiki/YUV#Converting_between_Y%E2%80%B2UV_and_RGB)
/// # Errors
/// This may error when the data stream size is wrong.
pub fn yuv_420_to_rgb(
resolution: Resolution,
data: &[u8],
rgba: bool,
) -> Result<Vec<u8>, NokhwaError> {
let pxsize = if rgba { 4 } else { 3 };
let mut dest = vec![0; (pxsize * resolution.width() * resolution.height()) as usize];
buf_yuv_420_to_rgb(resolution, data, &mut dest, rgba)?;
Ok(dest)
}
// this depresses me
// like, everytime i open this codebase all the life is sucked out of me
// i hate it
/// Converts a YUYV 4:2:0 datastream to a RGB888 Stream and outputs it into a destination buffer. [For further reading](https://en.wikipedia.org/wiki/YUV#Converting_between_Y%E2%80%B2UV_and_RGB)
/// # Errors
/// This may error when the data stream size is wrong.
pub fn buf_yuv_420_to_rgb(
resolution: Resolution,
data: &[u8],
out: &mut [u8],
rgba: bool,
) -> Result<(), NokhwaError> {
if resolution.x() % 2 != 0 || resolution.y() % 2 != 0 {
return Err(NokhwaError::ProcessFrameError {
src: FrameFormat::NV12,
destination: "RGB888".to_string(),
error: "Resolution must be even!".to_string(),
});
}
let x_lobcorp = resolution.y() * resolution.x();
let u_values = x_lobcorp / 4;
let v_values = x_lobcorp / 4;
if (x_lobcorp + u_values + v_values) as usize != data.len() {
return Err(NokhwaError::ProcessFrameError {
src: FrameFormat::NV12,
destination: "RGB888".to_string(),
error: "Ran out of data!".to_string(),
});
}
let px_b_size = if rgba { 4 } else { 3 };
if (x_lobcorp * px_b_size) as usize > out.len() {
return Err(NokhwaError::ProcessFrameError {
src: FrameFormat::NV12,
destination: "RGB888".to_string(),
error: "Output buffer too small!".to_string(),
});
};
let u_values_start = x_lobcorp;
let v_values_start = x_lobcorp + u_values;
let half_height = resolution.y() / 2;
let half_width = resolution.x() / 2;
for h in 0..half_height {
for _ in 0..half_width {
let r0_idx = ((h * 2) + (y * resolution.width())) as usize;
let r1_idx = ((h * 2) + ((y + 1) * resolution.width())) as usize;
let y0 = data[r0_idx] as i32;
let y1 = data[r0_idx + 1] as i32;
let y2 = data[r1_idx] as i32;
let y3 = data[r1_idx + 1] as i32;
let cell_number = (y * resolution.width()) + h;
let u0 = data[(u_values_start + cell_number) as usize] as i32;
let v0 = data[(v_values_start + cell_number) as usize] as i32;
if rgba {
let rgb0 = yuyv444_to_rgba(y0, u0, v0);
let rgb1 = yuyv444_to_rgba(y1, u0, v0);
let rgb2 = yuyv444_to_rgba(y2, u0, v0);
let rgb3 = yuyv444_to_rgba(y3, u0, v0);
for (i, (px0, px1)) in rgb0.into_iter().zip(rgb1.into_iter()).enumerate() {
out[r0_idx + i] = px0;
out[r0_idx + i + 4] = px1;
}
for (i, (px0, px1)) in rgb2.into_iter().zip(rgb3.into_iter()).enumerate() {
out[r1_idx + i] = px0;
out[r1_idx + i + 4] = px1;
}
} else {
let rgb0 = yuyv444_to_rgb(y0, u0, v0);
let rgb1 = yuyv444_to_rgb(y1, u0, v0);
let rgb2 = yuyv444_to_rgb(y2, u0, v0);
let rgb3 = yuyv444_to_rgb(y3, u0, v0);
for (i, (px0, px1)) in rgb0.into_iter().zip(rgb1.into_iter()).enumerate() {
out[r0_idx + i] = px0;
out[r0_idx + i + 3] = px1;
}
for (i, (px0, px1)) in rgb2.into_iter().zip(rgb3.into_iter()).enumerate() {
out[r1_idx + i] = px0;
out[r1_idx + i + 3] = px1;
}
}
}
}
Ok(())
}
src/backends/capture/opencv_backend.rs
+7 -1
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
use nokhwa_core::types::RequestedFormatType;
use nokhwa_core::{
buffer::Buffer,
error::NokhwaError,
@@ -114,7 +115,12 @@ impl OpenCvCaptureDevice {
NokhwaError::OpenDeviceError(format!("Failed to open {}", index), why.to_string())
})?;
let camera_format = cam_fmt.fulfill(&[CameraFormat::default()]).unwrap();
let camera_format =
if let RequestedFormatType::Exact(exact) = cam_fmt.requested_format_type() {
exact
} else {
return Err(NokhwaError::UnsupportedOperationError(ApiBackend::OpenCv));
};
set_properties(&mut video_capture, camera_format)?;
src/backends/capture/v4l2_backend.rs
+3
View File
@@ -421,6 +421,7 @@ impl<'a> CaptureBackendTrait for V4LCaptureDevice<'a> {
FrameFormat::YUYV => FourCC::new(b"YUYV"),
FrameFormat::GRAY => FourCC::new(b"GRAY"),
FrameFormat::RAWRGB => FourCC::new(b"RGB3"),
FrameFormat::NV12 => FourCC::new(b"NV12"),
};
let format = Format::new(new_fmt.width(), new_fmt.height(), v4l_fcc);
@@ -772,6 +773,7 @@ fn fourcc_to_frameformat(fourcc: FourCC) -> Option<FrameFormat> {
"MJPG" => Some(FrameFormat::MJPEG),
"GRAY" => Some(FrameFormat::GRAY),
"RGB3" => Some(FrameFormat::RAWRGB),
"NV12" => Some(FrameFormat::NV12)
_ => None,
}
}
@@ -782,5 +784,6 @@ fn frameformat_to_fourcc(fourcc: FrameFormat) -> FourCC {
FrameFormat::YUYV => FourCC::new(b"YUYV"),
FrameFormat::GRAY => FourCC::new(b"GRAY"),
FrameFormat::RAWRGB => FourCC::new(b"RGB3"),
FrameFormat::NV12 => FourCC::new(b"NV12"),
}
}