diff --git a/nokhwa-core/src/buffer.rs b/nokhwa-core/src/buffer.rs index b75d475..89a85b9 100644 --- a/nokhwa-core/src/buffer.rs +++ b/nokhwa-core/src/buffer.rs @@ -63,88 +63,153 @@ impl Buffer { pub fn source_frame_format(&self) -> SourceFrameFormat { self.source_frame_format } - - // /// Decodes a image with allocation using the provided [`FormatDecoder`]. - // /// # Errors - // /// Will error when the decoding fails. - // #[inline] - // pub fn decode_image( - // &self, - // ) -> Result>, NokhwaError> { - // let new_data = F::write_output(self.source_frame_format, self.resolution, &self.buffer)?; - // let image = - // ImageBuffer::from_raw(self.resolution.width_x, self.resolution.height_y, new_data) - // .ok_or(NokhwaError::ProcessFrameError { - // src: self.source_frame_format, - // destination: stringify!(F).to_string(), - // error: "Failed to create buffer".to_string(), - // })?; - // Ok(image) - // } - // - // /// Decodes a image with allocation using the provided [`FormatDecoder`] into a `buffer`. - // /// # Errors - // /// Will error when the decoding fails, or the provided buffer is too small. - // #[inline] - // pub fn decode_image_to_buffer( - // &self, - // buffer: &mut [u8], - // ) -> Result<(), NokhwaError> { - // F::write_output_buffer( - // self.source_frame_format, - // self.resolution, - // &self.buffer, - // buffer, - // ) - // } - // /// Decodes a image with allocation using the provided [`FormatDecoder`] into a [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). - // /// - // /// Note that this does a clone when creating the buffer, to decouple the lifetime of the internal data to the temporary Buffer. If you want to avoid this, please see [`decode_opencv_mat`](Self::decode_opencv_mat). - // /// # Errors - // /// Will error when the decoding fails, or `OpenCV` failed to create/copy the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). - // /// # Safety - // /// This function uses `unsafe` in order to create the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). Please see [`Mat::new_rows_cols_with_data`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html#method.new_rows_cols_with_data) for more. - // /// - // /// Most notably, the `data` **must** stay in scope for the duration of the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html) or bad, ***bad*** things happen. - // #[cfg(feature = "opencv-mat")] - // #[cfg_attr(feature = "docs-features", doc(cfg(feature = "opencv-mat")))] - // #[allow(clippy::cast_possible_wrap)] - // pub fn decode_opencv_mat( - // &mut self, - // ) -> Result { - // use image::Pixel; - // use opencv::core::{Mat, Mat_AUTO_STEP, CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4}; - // - // let array_type = match F::Output::CHANNEL_COUNT { - // 1 => CV_8UC1, - // 2 => CV_8UC2, - // 3 => CV_8UC3, - // 4 => CV_8UC4, - // _ => { - // return Err(NokhwaError::ProcessFrameError { - // src: FrameFormat::RAWRGB, - // destination: "OpenCV Mat".to_string(), - // error: "Invalid Decoder FormatDecoder Channel Count".to_string(), - // }) - // } - // }; - // - // unsafe { - // // TODO: Look into removing this unnecessary copy. - // let mat1 = Mat::new_rows_cols_with_data( - // self.resolution.height_y as i32, - // self.resolution.width_x as i32, - // array_type, - // self.buffer.as_ref().as_ptr().cast_mut().cast(), - // Mat_AUTO_STEP, - // ) - // .map_err(|why| NokhwaError::ProcessFrameError { - // src: FrameFormat::RAWRGB, - // destination: "OpenCV Mat".to_string(), - // error: why.to_string(), - // })?; - // - // Ok(mat1) - // } - // } +} + +#[cfg(feature = "opencv-mat")] +impl Buffer { + /// Decodes a image with allocation using the provided [`FormatDecoder`]. + /// # Errors + /// Will error when the decoding fails. + #[inline] + pub fn decode_image( + &self, + ) -> Result>, NokhwaError> { + let new_data = F::write_output(self.source_frame_format, self.resolution, &self.buffer)?; + let image = + ImageBuffer::from_raw(self.resolution.width_x, self.resolution.height_y, new_data) + .ok_or(NokhwaError::ProcessFrameError { + src: self.source_frame_format, + destination: stringify!(F).to_string(), + error: "Failed to create buffer".to_string(), + })?; + Ok(image) + } + + /// Decodes a image with allocation using the provided [`FormatDecoder`] into a `buffer`. + /// # Errors + /// Will error when the decoding fails, or the provided buffer is too small. + #[inline] + pub fn decode_image_to_buffer( + &self, + buffer: &mut [u8], + ) -> Result<(), NokhwaError> { + F::write_output_buffer( + self.source_frame_format, + self.resolution, + &self.buffer, + buffer, + ) + } + /// Decodes a image with allocation using the provided [`FormatDecoder`] into a [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). + /// + /// Note that this does a clone when creating the buffer, to decouple the lifetime of the internal data to the temporary Buffer. If you want to avoid this, please see [`decode_opencv_mat`](Self::decode_opencv_mat). + /// # Errors + /// Will error when the decoding fails, or `OpenCV` failed to create/copy the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). + /// # Safety + /// This function uses `unsafe` in order to create the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). Please see [`Mat::new_rows_cols_with_data`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html#method.new_rows_cols_with_data) for more. + /// + /// Most notably, the `data` **must** stay in scope for the duration of the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html) or bad, ***bad*** things happen. + #[cfg(feature = "opencv-mat")] + #[cfg_attr(feature = "docs-features", doc(cfg(feature = "opencv-mat")))] + #[allow(clippy::cast_possible_wrap)] + pub fn decode_opencv_mat( + &mut self, + ) -> Result { + use image::Pixel; + use opencv::core::{Mat, Mat_AUTO_STEP, CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4}; + + let array_type = match F::Output::CHANNEL_COUNT { + 1 => CV_8UC1, + 2 => CV_8UC2, + 3 => CV_8UC3, + 4 => CV_8UC4, + _ => { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::RAWRGB, + destination: "OpenCV Mat".to_string(), + error: "Invalid Decoder FormatDecoder Channel Count".to_string(), + }) + } + }; + + unsafe { + // TODO: Look into removing this unnecessary copy. + let mat1 = Mat::new_rows_cols_with_data( + self.resolution.height_y as i32, + self.resolution.width_x as i32, + array_type, + self.buffer.as_ref().as_ptr().cast_mut().cast(), + Mat_AUTO_STEP, + ) + .map_err(|why| NokhwaError::ProcessFrameError { + src: FrameFormat::RAWRGB, + destination: "OpenCV Mat".to_string(), + error: why.to_string(), + })?; + + Ok(mat1) + } + } +} + +#[cfg(feature = "wgpu-types")] +impl Buffer { + #[cfg_attr(feature = "docs-features", doc(cfg(feature = "wgpu-types")))] + /// Directly copies a frame to a Wgpu texture. This will automatically convert the frame into a RGBA frame. + /// # Errors + /// If the frame cannot be captured or the resolution is 0 on any axis, this will error. + fn frame_texture<'a>( + &mut self, + device: &WgpuDevice, + queue: &WgpuQueue, + label: Option<&'a str>, + ) -> Result { + use crate::pixel_format::RgbAFormat; + use std::num::NonZeroU32; + let frame = self.frame()?.decode_image::()?; + + let texture_size = Extent3d { + width: frame.width(), + height: frame.height(), + depth_or_array_layers: 1, + }; + + let texture = device.create_texture(&TextureDescriptor { + label, + size: texture_size, + mip_level_count: 1, + sample_count: 1, + dimension: TextureDimension::D2, + format: TextureFormat::Rgba8UnormSrgb, + usage: TextureUsages::TEXTURE_BINDING | TextureUsages::COPY_DST, + }); + + let width_nonzero = match NonZeroU32::try_from(4 * frame.width()) { + Ok(w) => Some(w), + Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())), + }; + + let height_nonzero = match NonZeroU32::try_from(frame.height()) { + Ok(h) => Some(h), + Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())), + }; + + queue.write_texture( + ImageCopyTexture { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: TextureAspect::All, + }, + &frame, + ImageDataLayout { + offset: 0, + bytes_per_row: width_nonzero, + rows_per_image: height_nonzero, + }, + texture_size, + ); + + Ok(texture) + } } diff --git a/nokhwa-core/src/format_filter.rs b/nokhwa-core/src/format_filter.rs index 8be64d4..b2408d0 100644 --- a/nokhwa-core/src/format_filter.rs +++ b/nokhwa-core/src/format_filter.rs @@ -29,12 +29,15 @@ pub enum RequestedFormatType { None, } +// TODO: Format Filter Builder to provide more interactive API for fulfillment of formats. + /// How you get your [`FrameFormat`] from the #[derive(Clone, Debug)] pub struct FormatFilter { filter_pref: RequestedFormatType, fcc_primary: BTreeSet, fcc_platform: BTreeMap>, + } impl FormatFilter { @@ -128,7 +131,7 @@ impl Default for FormatFilter { } } -fn format_fulfill( +pub fn format_fulfill( sources: impl AsRef<[CameraFormat]>, filter: FormatFilter, ) -> Option { diff --git a/nokhwa-core/src/traits.rs b/nokhwa-core/src/traits.rs index 3344dd0..c9758ca 100644 --- a/nokhwa-core/src/traits.rs +++ b/nokhwa-core/src/traits.rs @@ -251,16 +251,16 @@ where #[cfg_attr(feature = "async", async_trait::async_trait)] pub trait AsyncCaptureTrait: CaptureTrait { /// Initialize the camera, preparing it for use, with a random format (usually the first one). - async fn init(&mut self) -> Result<(), NokhwaError>; + async fn init_async(&mut self) -> Result<(), NokhwaError>; /// Initialize the camera, preparing it for use, with a format that fits the supplied [`FormatFilter`]. - async fn init_with_format(&mut self, format: FormatFilter) + async fn init_with_format_async(&mut self, format: FormatFilter) -> Result; /// Forcefully refreshes the stored camera format, bringing it into sync with "reality" (current camera state) /// # Errors /// If the camera can not get its most recent [`CameraFormat`]. this will error. - async fn refresh_camera_format(&mut self) -> Result<(), NokhwaError>; + async fn refresh_camera_format_async(&mut self) -> Result<(), NokhwaError>; /// Will set the current [`CameraFormat`] /// This will reset the current stream if used while stream is opened. @@ -268,7 +268,7 @@ pub trait AsyncCaptureTrait: CaptureTrait { /// This will also update the cache. /// # Errors /// If you started the stream and the camera rejects the new camera format, this will return an error. - async fn set_camera_format(&mut self, new_fmt: CameraFormat) -> Result<(), NokhwaError>; + async fn set_camera_format_async(&mut self, new_fmt: CameraFormat) -> Result<(), NokhwaError>; /// Will set the current [`Resolution`] /// This will reset the current stream if used while stream is opened. @@ -276,7 +276,7 @@ pub trait AsyncCaptureTrait: CaptureTrait { /// This will also update the cache. /// # Errors /// If you started the stream and the camera rejects the new resolution, this will return an error. - async fn set_resolution(&mut self, new_res: Resolution) -> Result<(), NokhwaError>; + async fn set_resolution_async(&mut self, new_res: Resolution) -> Result<(), NokhwaError>; /// Will set the current framerate /// This will reset the current stream if used while stream is opened. @@ -284,7 +284,7 @@ pub trait AsyncCaptureTrait: CaptureTrait { /// This will also update the cache. /// # Errors /// If you started the stream and the camera rejects the new framerate, this will return an error. - async fn set_frame_rate(&mut self, new_fps: u32) -> Result<(), NokhwaError>; + async fn set_frame_rate_async(&mut self, new_fps: u32) -> Result<(), NokhwaError>; /// Will set the current [`FrameFormat`] /// This will reset the current stream if used while stream is opened. @@ -292,7 +292,7 @@ pub trait AsyncCaptureTrait: CaptureTrait { /// This will also update the cache. /// # Errors /// If you started the stream and the camera rejects the new frame format, this will return an error. - async fn set_frame_format( + async fn set_frame_format_async( &mut self, fourcc: SourceFrameFormat, ) -> Result<(), NokhwaError>; @@ -303,7 +303,7 @@ pub trait AsyncCaptureTrait: CaptureTrait { /// # Errors /// If the `control` is not supported, the value is invalid (less than min, greater than max, not in step), or there was an error setting the control, /// this will error. - async fn set_camera_control( + async fn set_camera_control_async( &mut self, id: KnownCameraControl, value: ControlValueSetter, @@ -312,24 +312,24 @@ pub trait AsyncCaptureTrait: CaptureTrait { /// Will open the camera stream with set parameters. This will be called internally if you try and call [`frame()`](CaptureTrait::frame()) before you call [`open_stream()`](CaptureTrait::open_stream()). /// # Errors /// If the specific backend fails to open the camera (e.g. already taken, busy, doesn't exist anymore) this will error. - async fn open_stream(&mut self) -> Result<(), NokhwaError>; + async fn open_stream_async(&mut self) -> Result<(), NokhwaError>; /// Will get a frame from the camera as a [`Buffer`]. Depending on the backend, if you have not called [`open_stream()`](CaptureTrait::open_stream()) before you called this, /// it will either return an error. /// # Errors /// If the backend fails to get the frame (e.g. already taken, busy, doesn't exist anymore), the decoding fails (e.g. MJPEG -> u8), or [`open_stream()`](CaptureTrait::open_stream()) has not been called yet, /// this will error. - async fn frame(&mut self) -> Result; + async fn frame_async(&mut self) -> Result; /// Will get a frame from the camera **without** any processing applied, meaning you will usually get a frame you need to decode yourself. /// # Errors /// If the backend fails to get the frame (e.g. already taken, busy, doesn't exist anymore), or [`open_stream()`](CaptureTrait::open_stream()) has not been called yet, this will error. - async fn frame_raw(&mut self) -> Result, NokhwaError>; + async fn frame_raw_async(&mut self) -> Result, NokhwaError>; /// Will drop the stream. /// # Errors /// Please check the `Quirks` section of each backend. - async fn stop_stream(&mut self) -> Result<(), NokhwaError>; + async fn stop_stream_async(&mut self) -> Result<(), NokhwaError>; } #[cfg(feature = "async")] @@ -342,4 +342,32 @@ where } } +pub trait OneShot: CaptureTrait { + fn one_shot(&mut self) -> Result { + if self.is_stream_open() { + self.frame() + } else { + self.open_stream()?; + let frame = self.frame()?; + self.stop_stream()?; + Ok(frame) + } + } +} + +#[cfg(feature = "async")] +#[cfg_attr(feature = "async", async_trait::async_trait)] +pub trait AsyncOneShot: AsyncCaptureTrait { + async fn one_shot(&mut self) -> Result { + if self.is_stream_open() { + self.frame_async().await + } else { + self.open_stream_async().await?; + let frame = self.frame_async().await?; + self.stop_stream_async().await?; + Ok(frame) + } + } +} + pub trait VirtualBackendTrait {} diff --git a/src/backends/capture/browser_camera.rs b/src/backends/capture/browser_camera.rs index c449d66..49ccfb2 100644 --- a/src/backends/capture/browser_camera.rs +++ b/src/backends/capture/browser_camera.rs @@ -12,6 +12,7 @@ use nokhwa_core::types::{ use wasm_bindgen_futures::JsFuture; use std::borrow::Cow; use std::collections::HashMap; +use std::future::Future; use wasm_bindgen::{JsCast, JsValue}; use web_sys::{ CanvasRenderingContext2d, Document, Element, MediaDevices, Navigator, OffscreenCanvas, Window, MediaStream, MediaStreamConstraints, HtmlCanvasElement, MediaDeviceInfo, MediaDeviceKind, @@ -205,6 +206,8 @@ enum CanvasType { HtmlCanvas(HtmlCanvasElement), } + + /// Quirks: /// - Regular [`CaptureTrait`] will block. Use [``] /// - [REQUIRES AN UP-TO-DATE BROWSER DUE TO USE OF OFFSCREEN CANVAS.](https://caniuse.com/?search=OffscreenCanvas) @@ -220,7 +223,9 @@ pub struct BrowserCamera { } impl BrowserCamera { - pub fn new(index: &CameraIndex) -> Result {} + pub fn new(index: &CameraIndex) -> Result { + wasm_rs_async_executor::single_threaded::block_on(Self::new_async(index)) + } pub async fn new_async(index: &CameraIndex) -> Result { let window = window()?; @@ -236,25 +241,88 @@ impl BrowserCamera { media_stream } Err(why) => { - return Err(NokhwaError::StructureError { - structure: "MediaDevicesGetUserMediaJsFuture".to_string(), - error: format!("{why:?}"), - }) + return Err(NokhwaError::OpenDeviceError( + "MediaDevicesGetUserMediaJsFuture".to_string(), format!("{why:?}"), + )) } } } Err(why) => { - return Err(NokhwaError::StructureError { - structure: "MediaDevicesGetUserMedia".to_string(), - error: format!("{why:?}"), - }) + return Err(NokhwaError::OpenDeviceError( + "MediaDevicesGetUserMedia".to_string(), format!("{why:?}"), + )) } }; + let media_info = match media_devices.enumerate_devices() { + Ok(i) => { + let future = JsFuture::from(promise); + match future.await { + Ok(devs) => { + let arr = Array::from(&devs); + match index { + CameraIndex::Index(i) => { + let dr = arr.get(i as u32); + if dr == JsValue::UNDEFINED { + return Err(NokhwaError::StructureError { structure: "MediaDeviceInfo".to_string(), error: "undefined".to_string() }) + } + + MediaDeviceInfo::from(dr) + } + CameraIndex::String(s) => { + match arr.iter().map(MediaDeviceInfo::from) + .filter(|mdi| { + mdi.device_id() == s + }).nth(0) { + Some(i) => i, + None => return Err(NokhwaError::StructureError { structure: "MediaDeviceInfo".to_string(), error: "no id".to_string() }) + + } + } + } + } + Err(why) => { + return Err(NokhwaError::StructureError { structure: "MediaDeviceInfo Enumerate Devices Promise".to_string(), error: format!("{why:?}") }) + } + } + } + Err(why) => { + return Err(NokhwaError::GetPropertyError { property: "MediaDeviceInfo".to_string(), error: format!("{why:?}") }) + }, + }; + + let info = CameraInfo { + human_name: media_info.label(), + description: "videoinfo".to_string(), + misc: media_info.device_id(), + index: index.clone(), + }; + + let controls = CustomControls { + min_aspect_ratio: None, + aspect_ratio: 0.00, + max_aspect_ratio: None, + facing_mode: JSCameraFacingMode::Any, + facing_mode_exact: false, + resize_mode: JSCameraResizeMode::None, + resize_mode_exact: false, + device_id: media_info.device_id(), + device_id_exact: true, + group_id: media_info.group_id(), + group_id_exact: true, + }; + Ok(BrowserCamera { index: index.clone(), info, format: CameraFormat::default(), init: false, controls, cavnas: None, context: None }) + } + + async fn measure_controls(&mut self) -> Result<(), NokhwaError> { } + + async fn measure_info(&mut self) -> Result<(), NokhwaError> { + todo!() + } } impl Backend for BrowserCamera {