diff --git a/nokhwa-core/src/buffer.rs b/nokhwa-core/src/buffer.rs index cf48bcf..4f2e6b4 100644 --- a/nokhwa-core/src/buffer.rs +++ b/nokhwa-core/src/buffer.rs @@ -21,8 +21,6 @@ use crate::{ }; use bytes::Bytes; use image::ImageBuffer; -#[cfg(feature = "serialize")] -use serde::{Deserialize, Serialize}; /// A buffer returned by a camera to accomodate custom decoding. /// Contains information of Resolution, the buffer's [`FrameFormat`], and the buffer. @@ -74,7 +72,7 @@ impl Buffer { pub fn decode_image( &self, ) -> Result>, NokhwaError> { - let new_data = F::write_output(self.source_frame_format, &self.buffer)?; + let new_data = F::write_output(self.source_frame_format, self.resolution, &self.buffer)?; let image = ImageBuffer::from_raw(self.resolution.width_x, self.resolution.height_y, new_data) .ok_or(NokhwaError::ProcessFrameError { @@ -92,7 +90,12 @@ impl Buffer { &self, buffer: &mut [u8], ) -> Result<(), NokhwaError> { - F::write_output_buffer(self.source_frame_format, &self.buffer, buffer) + F::write_output_buffer( + self.source_frame_format, + self.resolution, + &self.buffer, + buffer, + ) } /// Decodes a image with allocation using the provided [`FormatDecoder`] into a [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). diff --git a/nokhwa-core/src/pixel_format.rs b/nokhwa-core/src/pixel_format.rs index 42277c9..d558d1a 100644 --- a/nokhwa-core/src/pixel_format.rs +++ b/nokhwa-core/src/pixel_format.rs @@ -15,7 +15,8 @@ */ use crate::error::NokhwaError; use crate::types::{ - buf_mjpeg_to_rgb, buf_yuyv422_to_rgb, mjpeg_to_rgb, yuyv422_to_rgb, FrameFormat, + buf_mjpeg_to_rgb, buf_yuv_420_to_rgb, buf_yuyv422_to_rgb, mjpeg_to_rgb, yuv_420_to_rgb, + yuyv422_to_rgb, FrameFormat, Resolution, }; use image::{Luma, LumaA, Pixel, Rgb, Rgba}; use std::fmt::Debug; @@ -28,13 +29,18 @@ pub trait FormatDecoder: Clone + Sized + Send + Sync { /// Allocates and returns a `Vec` /// # Errors /// If the data is malformed, or the source [`FrameFormat`] is incompatible, this will error. - fn write_output(fcc: FrameFormat, data: &[u8]) -> Result, NokhwaError>; + fn write_output( + fcc: FrameFormat, + resolution: Resolution, + data: &[u8], + ) -> Result, NokhwaError>; /// Writes to a user provided buffer. /// # Errors /// If the data is malformed, the source [`FrameFormat`] is incompatible, or the user-alloted buffer is not large enough, this will error. fn write_output_buffer( fcc: FrameFormat, + resolution: Resolution, data: &[u8], dest: &mut [u8], ) -> Result<(), NokhwaError>; @@ -53,7 +59,11 @@ impl FormatDecoder for RgbFormat { type Output = Rgb; const FORMATS: &'static [FrameFormat] = &[FrameFormat::MJPEG, FrameFormat::YUYV]; - fn write_output(fcc: FrameFormat, data: &[u8]) -> Result, NokhwaError> { + fn write_output( + fcc: FrameFormat, + resolution: Resolution, + data: &[u8], + ) -> Result, NokhwaError> { match fcc { FrameFormat::MJPEG => mjpeg_to_rgb(data, false), FrameFormat::YUYV => yuyv422_to_rgb(data, false), @@ -65,11 +75,13 @@ impl FormatDecoder for RgbFormat { }) .collect()), FrameFormat::RAWRGB => Ok(data.to_vec()), + FrameFormat::NV12 => yuv_420_to_rgb(resolution, data, false), } } fn write_output_buffer( fcc: FrameFormat, + resolution: Resolution, data: &[u8], dest: &mut [u8], ) -> Result<(), NokhwaError> { @@ -93,7 +105,11 @@ impl FormatDecoder for RgbFormat { }); Ok(()) } - FrameFormat::RAWRGB => {dest.copy_from_slice(data); Ok(())} + FrameFormat::RAWRGB => { + dest.copy_from_slice(data); + Ok(()) + } + FrameFormat::NV12 => buf_yuv_420_to_rgb(resolution, data, dest, false), } } } @@ -112,7 +128,11 @@ impl FormatDecoder for RgbAFormat { const FORMATS: &'static [FrameFormat] = &[FrameFormat::MJPEG, FrameFormat::YUYV]; - fn write_output(fcc: FrameFormat, data: &[u8]) -> Result, NokhwaError> { + fn write_output( + fcc: FrameFormat, + resolution: Resolution, + data: &[u8], + ) -> Result, NokhwaError> { match fcc { FrameFormat::MJPEG => mjpeg_to_rgb(data, true), FrameFormat::YUYV => yuyv422_to_rgb(data, true), @@ -125,16 +145,16 @@ impl FormatDecoder for RgbAFormat { .collect()), FrameFormat::RAWRGB => Ok(data .chunks_exact(3) - .flat_map(|x| { - [x[0], x[1], x[2], 255] - }) - .collect() - ), + .flat_map(|x| [x[0], x[1], x[2], 255]) + .collect()), + FrameFormat::NV12 => yuv_420_to_rgb(resolution, data, true), } } fn write_output_buffer( fcc: FrameFormat, + resolution: Resolution, + data: &[u8], dest: &mut [u8], ) -> Result<(), NokhwaError> { @@ -160,19 +180,16 @@ impl FormatDecoder for RgbAFormat { Ok(()) } FrameFormat::RAWRGB => { - data - .chunks_exact(3) - .enumerate() - .for_each(|(idx, px)| { + data.chunks_exact(3).enumerate().for_each(|(idx, px)| { let index = idx * 4; dest[index] = px[0]; dest[index + 1] = px[1]; dest[index + 2] = px[2]; dest[index + 3] = 255; - }); Ok(()) } + FrameFormat::NV12 => buf_yuv_420_to_rgb(resolution, data, dest, true), } } } @@ -193,7 +210,11 @@ impl FormatDecoder for LumaFormat { &[FrameFormat::MJPEG, FrameFormat::YUYV, FrameFormat::GRAY]; #[allow(clippy::cast_possible_truncation)] - fn write_output(fcc: FrameFormat, data: &[u8]) -> Result, NokhwaError> { + fn write_output( + fcc: FrameFormat, + resolution: Resolution, + data: &[u8], + ) -> Result, NokhwaError> { match fcc { FrameFormat::MJPEG => Ok(mjpeg_to_rgb(data, false)? .as_slice() @@ -213,17 +234,26 @@ impl FormatDecoder for LumaFormat { (avg / 3) as u8 }) .collect()), + FrameFormat::NV12 => Ok(yuv_420_to_rgb(resolution, data, false)? + .as_slice() + .chunks_exact(3) + .map(|x| { + let mut avg = 0; + x.iter().for_each(|v| avg += u16::from(*v)); + (avg / 3) as u8 + }) + .collect()), FrameFormat::GRAY => Ok(data.to_vec()), - FrameFormat::RAWRGB => { - Ok(data.chunks(3).map(|px| { - ((px[0] as i32 + px[1] as i32 + px[2] as i32) / 3) as u8 - }).collect()) - }, + FrameFormat::RAWRGB => Ok(data + .chunks(3) + .map(|px| ((px[0] as i32 + px[1] as i32 + px[2] as i32) / 3) as u8) + .collect()), } } fn write_output_buffer( fcc: FrameFormat, + _resolution: Resolution, data: &[u8], dest: &mut [u8], ) -> Result<(), NokhwaError> { @@ -241,6 +271,11 @@ impl FormatDecoder for LumaFormat { destination: "Luma => RGB".to_string(), error: "Conversion Error".to_string(), }), + FrameFormat::NV12 => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "Luma => RGB".to_string(), + error: "Conversion Error".to_string(), + }), FrameFormat::GRAY => { data.iter().zip(dest.iter_mut()).for_each(|(pxv, d)| { *d = *pxv; @@ -272,7 +307,11 @@ impl FormatDecoder for LumaAFormat { &[FrameFormat::MJPEG, FrameFormat::YUYV, FrameFormat::GRAY]; #[allow(clippy::cast_possible_truncation)] - fn write_output(fcc: FrameFormat, data: &[u8]) -> Result, NokhwaError> { + fn write_output( + fcc: FrameFormat, + resolution: Resolution, + data: &[u8], + ) -> Result, NokhwaError> { match fcc { FrameFormat::MJPEG => Ok(mjpeg_to_rgb(data, false)? .as_slice() @@ -292,6 +331,15 @@ impl FormatDecoder for LumaAFormat { [(avg / 3) as u8, 255] }) .collect()), + FrameFormat::NV12 => Ok(yuv_420_to_rgb(resolution, data, false)? + .as_slice() + .chunks_exact(3) + .flat_map(|x| { + let mut avg = 0; + x.iter().for_each(|v| avg += u16::from(*v)); + [(avg / 3) as u8, 255] + }) + .collect()), FrameFormat::GRAY => Ok(data.iter().flat_map(|x| [*x, 255]).collect()), FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError { src: fcc, @@ -304,6 +352,7 @@ impl FormatDecoder for LumaAFormat { fn write_output_buffer( fcc: FrameFormat, data: &[u8], + _resolution: Resolution, dest: &mut [u8], ) -> Result<(), NokhwaError> { match fcc { @@ -320,6 +369,11 @@ impl FormatDecoder for LumaAFormat { destination: "YUYV => LumaA".to_string(), error: "Conversion Error".to_string(), }), + FrameFormat::NV12 => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "NV12 => LumaA".to_string(), + error: "Conversion Error".to_string(), + }), FrameFormat::GRAY => { if dest.len() != data.len() * 2 { return Err(NokhwaError::ProcessFrameError { @@ -339,13 +393,11 @@ impl FormatDecoder for LumaAFormat { }); Ok(()) } - FrameFormat::RAWRGB => { - Err(NokhwaError::ProcessFrameError { - src: fcc, - destination: "RGB => RGB".to_string(), - error: "Conversion Error".to_string(), - }) - }, + FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "RGB => RGB".to_string(), + error: "Conversion Error".to_string(), + }), } } } diff --git a/nokhwa-core/src/types.rs b/nokhwa-core/src/types.rs index 7cbad53..ea0df7c 100644 --- a/nokhwa-core/src/types.rs +++ b/nokhwa-core/src/types.rs @@ -68,6 +68,11 @@ impl RequestedFormat<'_> { wanted_decoder: decoder, } } + + /// Gets the [`RequestedFormatType`] + pub fn requested_format_type(&self) -> RequestedFormatType { + self.requested_format + } /// Fulfill the requested using a list of all available formats. /// @@ -276,13 +281,16 @@ impl TryFrom for usize { /// Describes a frame format (i.e. how the bytes themselves are encoded). Often called `FourCC`. /// - YUYV is a mathematical color space. You can read more [here.](https://en.wikipedia.org/wiki/YCbCr) +/// - NV12 is same as above. Note that a partial compression (e.g. [16, 235] may be coerced to [0, 255]. /// - MJPEG is a motion-jpeg compressed frame, it allows for high frame rates. /// - GRAY is a grayscale image format, usually for specialized cameras such as IR Cameras. +/// - RAWRGB is a Raw RGB888 format. #[derive(Copy, Clone, Debug, Hash, Ord, PartialOrd, Eq, PartialEq)] #[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))] pub enum FrameFormat { MJPEG, YUYV, + NV12, GRAY, RAWRGB, } @@ -301,14 +309,23 @@ impl Display for FrameFormat { } FrameFormat::RAWRGB => { write!(f, "RAWRGB") - }, + } + FrameFormat::NV12 => { + write!(f, "NV12") + } } } } #[must_use] pub const fn frame_formats() -> &'static [FrameFormat] { - &[FrameFormat::MJPEG, FrameFormat::YUYV, FrameFormat::GRAY, FrameFormat::RAWRGB] + &[ + FrameFormat::MJPEG, + FrameFormat::YUYV, + FrameFormat::NV12, + FrameFormat::GRAY, + FrameFormat::RAWRGB, + ] } /// Describes a Resolution. @@ -343,6 +360,7 @@ impl Resolution { /// This is exported as `get_Width`. #[must_use] #[cfg_attr(feature = "output-wasm", wasm_bindgen(getter = Width))] + #[inline] pub fn width(self) -> u32 { self.width_x } @@ -352,6 +370,7 @@ impl Resolution { /// This is exported as `get_Height`. #[must_use] #[cfg_attr(feature = "output-wasm", wasm_bindgen(getter = Height))] + #[inline] pub fn height(self) -> u32 { self.height_y } @@ -359,6 +378,7 @@ impl Resolution { /// Get the x (width) of Resolution #[must_use] #[cfg_attr(feature = "output-wasm", wasm_bindgen(skip))] + #[inline] pub fn x(self) -> u32 { self.width_x } @@ -366,6 +386,7 @@ impl Resolution { /// Get the y (height) of Resolution #[must_use] #[cfg_attr(feature = "output-wasm", wasm_bindgen(skip))] + #[inline] pub fn y(self) -> u32 { self.height_y } @@ -1544,15 +1565,6 @@ pub fn buf_mjpeg_to_rgb(_data: &[u8], _dest: &mut [u8], _rgba: bool) -> Result<( /// # Errors /// This may error when the data stream size is not divisible by 4, a i32 -> u8 conversion fails, or it fails to read from a certain index. pub fn yuyv422_to_rgb(data: &[u8], rgba: bool) -> Result, NokhwaError> { - if data.len() % 4 != 0 { - return Err(NokhwaError::ProcessFrameError { - src: FrameFormat::YUYV, - destination: "RGB888".to_string(), - error: "Assertion failure, the YUV stream isn't 4:2:2! (wrong number of bytes)" - .to_string(), - }); - } - let pixel_size = if rgba { 4 } else { 3 }; // yuyv yields 2 3-byte pixels per yuyv chunk let rgb_buf_size = (data.len() / 4) * (2 * pixel_size); @@ -1672,3 +1684,114 @@ pub fn yuyv444_to_rgba(y: i32, u: i32, v: i32) -> [u8; 4] { let [r, g, b] = yuyv444_to_rgb(y, u, v); [r, g, b, 255] } + +/// Converts a YUYV 4:2:0 datastream to a RGB888 Stream. [For further reading](https://en.wikipedia.org/wiki/YUV#Converting_between_Y%E2%80%B2UV_and_RGB) +/// # Errors +/// This may error when the data stream size is wrong. +pub fn yuv_420_to_rgb( + resolution: Resolution, + data: &[u8], + rgba: bool, +) -> Result, NokhwaError> { + let pxsize = if rgba { 4 } else { 3 }; + let mut dest = vec![0; (pxsize * resolution.width() * resolution.height()) as usize]; + buf_yuv_420_to_rgb(resolution, data, &mut dest, rgba)?; + Ok(dest) +} + +// this depresses me +// like, everytime i open this codebase all the life is sucked out of me +// i hate it +/// Converts a YUYV 4:2:0 datastream to a RGB888 Stream and outputs it into a destination buffer. [For further reading](https://en.wikipedia.org/wiki/YUV#Converting_between_Y%E2%80%B2UV_and_RGB) +/// # Errors +/// This may error when the data stream size is wrong. +pub fn buf_yuv_420_to_rgb( + resolution: Resolution, + data: &[u8], + out: &mut [u8], + rgba: bool, +) -> Result<(), NokhwaError> { + if resolution.x() % 2 != 0 || resolution.y() % 2 != 0 { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::NV12, + destination: "RGB888".to_string(), + error: "Resolution must be even!".to_string(), + }); + } + + let x_lobcorp = resolution.y() * resolution.x(); + let u_values = x_lobcorp / 4; + let v_values = x_lobcorp / 4; + + if (x_lobcorp + u_values + v_values) as usize != data.len() { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::NV12, + destination: "RGB888".to_string(), + error: "Ran out of data!".to_string(), + }); + } + + let px_b_size = if rgba { 4 } else { 3 }; + + if (x_lobcorp * px_b_size) as usize > out.len() { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::NV12, + destination: "RGB888".to_string(), + error: "Output buffer too small!".to_string(), + }); + }; + + let u_values_start = x_lobcorp; + let v_values_start = x_lobcorp + u_values; + + let half_height = resolution.y() / 2; + let half_width = resolution.x() / 2; + + for h in 0..half_height { + for _ in 0..half_width { + let r0_idx = ((h * 2) + (y * resolution.width())) as usize; + let r1_idx = ((h * 2) + ((y + 1) * resolution.width())) as usize; + + let y0 = data[r0_idx] as i32; + let y1 = data[r0_idx + 1] as i32; + let y2 = data[r1_idx] as i32; + let y3 = data[r1_idx + 1] as i32; + + let cell_number = (y * resolution.width()) + h; + let u0 = data[(u_values_start + cell_number) as usize] as i32; + let v0 = data[(v_values_start + cell_number) as usize] as i32; + + if rgba { + let rgb0 = yuyv444_to_rgba(y0, u0, v0); + let rgb1 = yuyv444_to_rgba(y1, u0, v0); + let rgb2 = yuyv444_to_rgba(y2, u0, v0); + let rgb3 = yuyv444_to_rgba(y3, u0, v0); + + for (i, (px0, px1)) in rgb0.into_iter().zip(rgb1.into_iter()).enumerate() { + out[r0_idx + i] = px0; + out[r0_idx + i + 4] = px1; + } + for (i, (px0, px1)) in rgb2.into_iter().zip(rgb3.into_iter()).enumerate() { + out[r1_idx + i] = px0; + out[r1_idx + i + 4] = px1; + } + } else { + let rgb0 = yuyv444_to_rgb(y0, u0, v0); + let rgb1 = yuyv444_to_rgb(y1, u0, v0); + let rgb2 = yuyv444_to_rgb(y2, u0, v0); + let rgb3 = yuyv444_to_rgb(y3, u0, v0); + + for (i, (px0, px1)) in rgb0.into_iter().zip(rgb1.into_iter()).enumerate() { + out[r0_idx + i] = px0; + out[r0_idx + i + 3] = px1; + } + for (i, (px0, px1)) in rgb2.into_iter().zip(rgb3.into_iter()).enumerate() { + out[r1_idx + i] = px0; + out[r1_idx + i + 3] = px1; + } + } + } + } + + Ok(()) +} diff --git a/src/backends/capture/opencv_backend.rs b/src/backends/capture/opencv_backend.rs index f273a19..f476b61 100644 --- a/src/backends/capture/opencv_backend.rs +++ b/src/backends/capture/opencv_backend.rs @@ -14,6 +14,7 @@ * limitations under the License. */ +use nokhwa_core::types::RequestedFormatType; use nokhwa_core::{ buffer::Buffer, error::NokhwaError, @@ -114,7 +115,12 @@ impl OpenCvCaptureDevice { NokhwaError::OpenDeviceError(format!("Failed to open {}", index), why.to_string()) })?; - let camera_format = cam_fmt.fulfill(&[CameraFormat::default()]).unwrap(); + let camera_format = + if let RequestedFormatType::Exact(exact) = cam_fmt.requested_format_type() { + exact + } else { + return Err(NokhwaError::UnsupportedOperationError(ApiBackend::OpenCv)); + }; set_properties(&mut video_capture, camera_format)?; diff --git a/src/backends/capture/v4l2_backend.rs b/src/backends/capture/v4l2_backend.rs index 82788f6..1159a37 100644 --- a/src/backends/capture/v4l2_backend.rs +++ b/src/backends/capture/v4l2_backend.rs @@ -421,6 +421,7 @@ impl<'a> CaptureBackendTrait for V4LCaptureDevice<'a> { FrameFormat::YUYV => FourCC::new(b"YUYV"), FrameFormat::GRAY => FourCC::new(b"GRAY"), FrameFormat::RAWRGB => FourCC::new(b"RGB3"), + FrameFormat::NV12 => FourCC::new(b"NV12"), }; let format = Format::new(new_fmt.width(), new_fmt.height(), v4l_fcc); @@ -772,6 +773,7 @@ fn fourcc_to_frameformat(fourcc: FourCC) -> Option { "MJPG" => Some(FrameFormat::MJPEG), "GRAY" => Some(FrameFormat::GRAY), "RGB3" => Some(FrameFormat::RAWRGB), + "NV12" => Some(FrameFormat::NV12) _ => None, } } @@ -782,5 +784,6 @@ fn frameformat_to_fourcc(fourcc: FrameFormat) -> FourCC { FrameFormat::YUYV => FourCC::new(b"YUYV"), FrameFormat::GRAY => FourCC::new(b"GRAY"), FrameFormat::RAWRGB => FourCC::new(b"RGB3"), + FrameFormat::NV12 => FourCC::new(b"NV12"), } }