GRIN/nokhwa
1
0
Fork 0
mirror of https://github.com/l1npengtul/nokhwa.git synced 2026-07-15 06:28:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

format filter todo, browser camera new, buffer stuffs (todo), new traits for oneshots

Browse Source
This commit is contained in:
l1npengtul
2023-03-16 19:09:41 +09:00
parent ad93e0a6c5
commit 3f570ee0e5
4 changed files with 270 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nokhwa-core/src/buffer.rs
+149 -84
View File
@@ -63,88 +63,153 @@ impl Buffer {
pub fn source_frame_format(&self) -> SourceFrameFormat {
self.source_frame_format
}
// /// Decodes a image with allocation using the provided [`FormatDecoder`].
// /// # Errors
// /// Will error when the decoding fails.
// #[inline]
// pub fn decode_image<F: FormatDecoder>(
// &self,
// ) -> Result<ImageBuffer<F::Output, Vec<u8>>, NokhwaError> {
// let new_data = F::write_output(self.source_frame_format, self.resolution, &self.buffer)?;
// let image =
// ImageBuffer::from_raw(self.resolution.width_x, self.resolution.height_y, new_data)
// .ok_or(NokhwaError::ProcessFrameError {
// src: self.source_frame_format,
// destination: stringify!(F).to_string(),
// error: "Failed to create buffer".to_string(),
// })?;
// Ok(image)
// }
//
// /// Decodes a image with allocation using the provided [`FormatDecoder`] into a `buffer`.
// /// # Errors
// /// Will error when the decoding fails, or the provided buffer is too small.
// #[inline]
// pub fn decode_image_to_buffer<F: FormatDecoder>(
// &self,
// buffer: &mut [u8],
// ) -> Result<(), NokhwaError> {
// F::write_output_buffer(
// self.source_frame_format,
// self.resolution,
// &self.buffer,
// buffer,
// )
// }
// /// Decodes a image with allocation using the provided [`FormatDecoder`] into a [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html).
// ///
// /// Note that this does a clone when creating the buffer, to decouple the lifetime of the internal data to the temporary Buffer. If you want to avoid this, please see [`decode_opencv_mat`](Self::decode_opencv_mat).
// /// # Errors
// /// Will error when the decoding fails, or `OpenCV` failed to create/copy the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html).
// /// # Safety
// /// This function uses `unsafe` in order to create the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). Please see [`Mat::new_rows_cols_with_data`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html#method.new_rows_cols_with_data) for more.
// ///
// /// Most notably, the `data` **must** stay in scope for the duration of the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html) or bad, ***bad*** things happen.
// #[cfg(feature = "opencv-mat")]
// #[cfg_attr(feature = "docs-features", doc(cfg(feature = "opencv-mat")))]
// #[allow(clippy::cast_possible_wrap)]
// pub fn decode_opencv_mat<F: FormatDecoder>(
// &mut self,
// ) -> Result<opencv::core::Mat, NokhwaError> {
// use image::Pixel;
// use opencv::core::{Mat, Mat_AUTO_STEP, CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4};
//
// let array_type = match F::Output::CHANNEL_COUNT {
// 1 => CV_8UC1,
// 2 => CV_8UC2,
// 3 => CV_8UC3,
// 4 => CV_8UC4,
// _ => {
// return Err(NokhwaError::ProcessFrameError {
// src: FrameFormat::RAWRGB,
// destination: "OpenCV Mat".to_string(),
// error: "Invalid Decoder FormatDecoder Channel Count".to_string(),
// })
// }
// };
//
// unsafe {
// // TODO: Look into removing this unnecessary copy.
// let mat1 = Mat::new_rows_cols_with_data(
// self.resolution.height_y as i32,
// self.resolution.width_x as i32,
// array_type,
// self.buffer.as_ref().as_ptr().cast_mut().cast(),
// Mat_AUTO_STEP,
// )
// .map_err(|why| NokhwaError::ProcessFrameError {
// src: FrameFormat::RAWRGB,
// destination: "OpenCV Mat".to_string(),
// error: why.to_string(),
// })?;
//
// Ok(mat1)
// }
// }
}
#[cfg(feature = "opencv-mat")]
impl Buffer {
/// Decodes a image with allocation using the provided [`FormatDecoder`].
/// # Errors
/// Will error when the decoding fails.
#[inline]
pub fn decode_image<F: FormatDecoder>(
&self,
) -> Result<ImageBuffer<F::Output, Vec<u8>>, NokhwaError> {
let new_data = F::write_output(self.source_frame_format, self.resolution, &self.buffer)?;
let image =
ImageBuffer::from_raw(self.resolution.width_x, self.resolution.height_y, new_data)
.ok_or(NokhwaError::ProcessFrameError {
src: self.source_frame_format,
destination: stringify!(F).to_string(),
error: "Failed to create buffer".to_string(),
})?;
Ok(image)
}
/// Decodes a image with allocation using the provided [`FormatDecoder`] into a `buffer`.
/// # Errors
/// Will error when the decoding fails, or the provided buffer is too small.
#[inline]
pub fn decode_image_to_buffer<F: FormatDecoder>(
&self,
buffer: &mut [u8],
) -> Result<(), NokhwaError> {
F::write_output_buffer(
self.source_frame_format,
self.resolution,
&self.buffer,
buffer,
)
}
/// Decodes a image with allocation using the provided [`FormatDecoder`] into a [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html).
///
/// Note that this does a clone when creating the buffer, to decouple the lifetime of the internal data to the temporary Buffer. If you want to avoid this, please see [`decode_opencv_mat`](Self::decode_opencv_mat).
/// # Errors
/// Will error when the decoding fails, or `OpenCV` failed to create/copy the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html).
/// # Safety
/// This function uses `unsafe` in order to create the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). Please see [`Mat::new_rows_cols_with_data`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html#method.new_rows_cols_with_data) for more.
///
/// Most notably, the `data` **must** stay in scope for the duration of the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html) or bad, ***bad*** things happen.
#[cfg(feature = "opencv-mat")]
#[cfg_attr(feature = "docs-features", doc(cfg(feature = "opencv-mat")))]
#[allow(clippy::cast_possible_wrap)]
pub fn decode_opencv_mat<F: FormatDecoder>(
&mut self,
) -> Result<opencv::core::Mat, NokhwaError> {
use image::Pixel;
use opencv::core::{Mat, Mat_AUTO_STEP, CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4};
let array_type = match F::Output::CHANNEL_COUNT {
1 => CV_8UC1,
2 => CV_8UC2,
3 => CV_8UC3,
4 => CV_8UC4,
_ => {
return Err(NokhwaError::ProcessFrameError {
src: FrameFormat::RAWRGB,
destination: "OpenCV Mat".to_string(),
error: "Invalid Decoder FormatDecoder Channel Count".to_string(),
})
}
};
unsafe {
// TODO: Look into removing this unnecessary copy.
let mat1 = Mat::new_rows_cols_with_data(
self.resolution.height_y as i32,
self.resolution.width_x as i32,
array_type,
self.buffer.as_ref().as_ptr().cast_mut().cast(),
Mat_AUTO_STEP,
)
.map_err(|why| NokhwaError::ProcessFrameError {
src: FrameFormat::RAWRGB,
destination: "OpenCV Mat".to_string(),
error: why.to_string(),
})?;
Ok(mat1)
}
}
}
#[cfg(feature = "wgpu-types")]
impl Buffer {
#[cfg_attr(feature = "docs-features", doc(cfg(feature = "wgpu-types")))]
/// Directly copies a frame to a Wgpu texture. This will automatically convert the frame into a RGBA frame.
/// # Errors
/// If the frame cannot be captured or the resolution is 0 on any axis, this will error.
fn frame_texture<'a>(
&mut self,
device: &WgpuDevice,
queue: &WgpuQueue,
label: Option<&'a str>,
) -> Result<WgpuTexture, NokhwaError> {
use crate::pixel_format::RgbAFormat;
use std::num::NonZeroU32;
let frame = self.frame()?.decode_image::<RgbAFormat>()?;
let texture_size = Extent3d {
width: frame.width(),
height: frame.height(),
depth_or_array_layers: 1,
};
let texture = device.create_texture(&TextureDescriptor {
label,
size: texture_size,
mip_level_count: 1,
sample_count: 1,
dimension: TextureDimension::D2,
format: TextureFormat::Rgba8UnormSrgb,
usage: TextureUsages::TEXTURE_BINDING | TextureUsages::COPY_DST,
});
let width_nonzero = match NonZeroU32::try_from(4 * frame.width()) {
Ok(w) => Some(w),
Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())),
};
let height_nonzero = match NonZeroU32::try_from(frame.height()) {
Ok(h) => Some(h),
Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())),
};
queue.write_texture(
ImageCopyTexture {
texture: &texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: TextureAspect::All,
},
&frame,
ImageDataLayout {
offset: 0,
bytes_per_row: width_nonzero,
rows_per_image: height_nonzero,
},
texture_size,
);
Ok(texture)
}
}
nokhwa-core/src/format_filter.rs
+4 -1
View File
@@ -29,12 +29,15 @@ pub enum RequestedFormatType {
None,
}
// TODO: Format Filter Builder to provide more interactive API for fulfillment of formats.
/// How you get your [`FrameFormat`] from the
#[derive(Clone, Debug)]
pub struct FormatFilter {
filter_pref: RequestedFormatType,
fcc_primary: BTreeSet<FrameFormat>,
fcc_platform: BTreeMap<ApiBackend, BTreeSet<u128>>,
}
impl FormatFilter {
@@ -128,7 +131,7 @@ impl Default for FormatFilter {
}
}
fn format_fulfill(
pub fn format_fulfill(
sources: impl AsRef<[CameraFormat]>,
filter: FormatFilter,
) -> Option<CameraFormat> {
nokhwa-core/src/traits.rs
+40 -12
View File
@@ -251,16 +251,16 @@ where
#[cfg_attr(feature = "async", async_trait::async_trait)]
pub trait AsyncCaptureTrait: CaptureTrait {
/// Initialize the camera, preparing it for use, with a random format (usually the first one).
async fn init(&mut self) -> Result<(), NokhwaError>;
async fn init_async(&mut self) -> Result<(), NokhwaError>;
/// Initialize the camera, preparing it for use, with a format that fits the supplied [`FormatFilter`].
async fn init_with_format(&mut self, format: FormatFilter)
async fn init_with_format_async(&mut self, format: FormatFilter)
-> Result<CameraFormat, NokhwaError>;
/// Forcefully refreshes the stored camera format, bringing it into sync with "reality" (current camera state)
/// # Errors
/// If the camera can not get its most recent [`CameraFormat`]. this will error.
async fn refresh_camera_format(&mut self) -> Result<(), NokhwaError>;
async fn refresh_camera_format_async(&mut self) -> Result<(), NokhwaError>;
/// Will set the current [`CameraFormat`]
/// This will reset the current stream if used while stream is opened.
@@ -268,7 +268,7 @@ pub trait AsyncCaptureTrait: CaptureTrait {
/// This will also update the cache.
/// # Errors
/// If you started the stream and the camera rejects the new camera format, this will return an error.
async fn set_camera_format(&mut self, new_fmt: CameraFormat) -> Result<(), NokhwaError>;
async fn set_camera_format_async(&mut self, new_fmt: CameraFormat) -> Result<(), NokhwaError>;
/// Will set the current [`Resolution`]
/// This will reset the current stream if used while stream is opened.
@@ -276,7 +276,7 @@ pub trait AsyncCaptureTrait: CaptureTrait {
/// This will also update the cache.
/// # Errors
/// If you started the stream and the camera rejects the new resolution, this will return an error.
async fn set_resolution(&mut self, new_res: Resolution) -> Result<(), NokhwaError>;
async fn set_resolution_async(&mut self, new_res: Resolution) -> Result<(), NokhwaError>;
/// Will set the current framerate
/// This will reset the current stream if used while stream is opened.
@@ -284,7 +284,7 @@ pub trait AsyncCaptureTrait: CaptureTrait {
/// This will also update the cache.
/// # Errors
/// If you started the stream and the camera rejects the new framerate, this will return an error.
async fn set_frame_rate(&mut self, new_fps: u32) -> Result<(), NokhwaError>;
async fn set_frame_rate_async(&mut self, new_fps: u32) -> Result<(), NokhwaError>;
/// Will set the current [`FrameFormat`]
/// This will reset the current stream if used while stream is opened.
@@ -292,7 +292,7 @@ pub trait AsyncCaptureTrait: CaptureTrait {
/// This will also update the cache.
/// # Errors
/// If you started the stream and the camera rejects the new frame format, this will return an error.
async fn set_frame_format(
async fn set_frame_format_async(
&mut self,
fourcc: SourceFrameFormat,
) -> Result<(), NokhwaError>;
@@ -303,7 +303,7 @@ pub trait AsyncCaptureTrait: CaptureTrait {
/// # Errors
/// If the `control` is not supported, the value is invalid (less than min, greater than max, not in step), or there was an error setting the control,
/// this will error.
async fn set_camera_control(
async fn set_camera_control_async(
&mut self,
id: KnownCameraControl,
value: ControlValueSetter,
@@ -312,24 +312,24 @@ pub trait AsyncCaptureTrait: CaptureTrait {
/// Will open the camera stream with set parameters. This will be called internally if you try and call [`frame()`](CaptureTrait::frame()) before you call [`open_stream()`](CaptureTrait::open_stream()).
/// # Errors
/// If the specific backend fails to open the camera (e.g. already taken, busy, doesn't exist anymore) this will error.
async fn open_stream(&mut self) -> Result<(), NokhwaError>;
async fn open_stream_async(&mut self) -> Result<(), NokhwaError>;
/// Will get a frame from the camera as a [`Buffer`]. Depending on the backend, if you have not called [`open_stream()`](CaptureTrait::open_stream()) before you called this,
/// it will either return an error.
/// # Errors
/// If the backend fails to get the frame (e.g. already taken, busy, doesn't exist anymore), the decoding fails (e.g. MJPEG -> u8), or [`open_stream()`](CaptureTrait::open_stream()) has not been called yet,
/// this will error.
async fn frame(&mut self) -> Result<Buffer, NokhwaError>;
async fn frame_async(&mut self) -> Result<Buffer, NokhwaError>;
/// Will get a frame from the camera **without** any processing applied, meaning you will usually get a frame you need to decode yourself.
/// # Errors
/// If the backend fails to get the frame (e.g. already taken, busy, doesn't exist anymore), or [`open_stream()`](CaptureTrait::open_stream()) has not been called yet, this will error.
async fn frame_raw(&mut self) -> Result<Cow<[u8]>, NokhwaError>;
async fn frame_raw_async(&mut self) -> Result<Cow<[u8]>, NokhwaError>;
/// Will drop the stream.
/// # Errors
/// Please check the `Quirks` section of each backend.
async fn stop_stream(&mut self) -> Result<(), NokhwaError>;
async fn stop_stream_async(&mut self) -> Result<(), NokhwaError>;
}
#[cfg(feature = "async")]
@@ -342,4 +342,32 @@ where
}
}
pub trait OneShot: CaptureTrait {
fn one_shot(&mut self) -> Result<Buffer, NokhwaError> {
if self.is_stream_open() {
self.frame()
} else {
self.open_stream()?;
let frame = self.frame()?;
self.stop_stream()?;
Ok(frame)
}
}
}
#[cfg(feature = "async")]
#[cfg_attr(feature = "async", async_trait::async_trait)]
pub trait AsyncOneShot: AsyncCaptureTrait {
async fn one_shot(&mut self) -> Result<Buffer, NokhwaError> {
if self.is_stream_open() {
self.frame_async().await
} else {
self.open_stream_async().await?;
let frame = self.frame_async().await?;
self.stop_stream_async().await?;
Ok(frame)
}
}
}
pub trait VirtualBackendTrait {}
src/backends/capture/browser_camera.rs
+77 -9
View File
@@ -12,6 +12,7 @@ use nokhwa_core::types::{
use wasm_bindgen_futures::JsFuture;
use std::borrow::Cow;
use std::collections::HashMap;
use std::future::Future;
use wasm_bindgen::{JsCast, JsValue};
use web_sys::{
CanvasRenderingContext2d, Document, Element, MediaDevices, Navigator, OffscreenCanvas, Window, MediaStream, MediaStreamConstraints, HtmlCanvasElement, MediaDeviceInfo, MediaDeviceKind,
@@ -205,6 +206,8 @@ enum CanvasType {
HtmlCanvas(HtmlCanvasElement),
}
/// Quirks:
/// - Regular [`CaptureTrait`] will block. Use [``]
/// - [REQUIRES AN UP-TO-DATE BROWSER DUE TO USE OF OFFSCREEN CANVAS.](https://caniuse.com/?search=OffscreenCanvas)
@@ -220,7 +223,9 @@ pub struct BrowserCamera {
}
impl BrowserCamera {
pub fn new(index: &CameraIndex) -> Result<BrowserCamera, NokhwaError> {}
pub fn new(index: &CameraIndex) -> Result<BrowserCamera, NokhwaError> {
wasm_rs_async_executor::single_threaded::block_on(Self::new_async(index))
}
pub async fn new_async(index: &CameraIndex) -> Result<BrowserCamera, NokhwaError> {
let window = window()?;
@@ -236,25 +241,88 @@ impl BrowserCamera {
media_stream
}
Err(why) => {
return Err(NokhwaError::StructureError {
structure: "MediaDevicesGetUserMediaJsFuture".to_string(),
error: format!("{why:?}"),
})
return Err(NokhwaError::OpenDeviceError(
"MediaDevicesGetUserMediaJsFuture".to_string(), format!("{why:?}"),
))
}
}
}
Err(why) => {
return Err(NokhwaError::StructureError {
structure: "MediaDevicesGetUserMedia".to_string(),
error: format!("{why:?}"),
})
return Err(NokhwaError::OpenDeviceError(
"MediaDevicesGetUserMedia".to_string(), format!("{why:?}"),
))
}
};
let media_info = match media_devices.enumerate_devices() {
Ok(i) => {
let future = JsFuture::from(promise);
match future.await {
Ok(devs) => {
let arr = Array::from(&devs);
match index {
CameraIndex::Index(i) => {
let dr = arr.get(i as u32);
if dr == JsValue::UNDEFINED {
return Err(NokhwaError::StructureError { structure: "MediaDeviceInfo".to_string(), error: "undefined".to_string() })
}
MediaDeviceInfo::from(dr)
}
CameraIndex::String(s) => {
match arr.iter().map(MediaDeviceInfo::from)
.filter(|mdi| {
mdi.device_id() == s
}).nth(0) {
Some(i) => i,
None => return Err(NokhwaError::StructureError { structure: "MediaDeviceInfo".to_string(), error: "no id".to_string() })
}
}
}
}
Err(why) => {
return Err(NokhwaError::StructureError { structure: "MediaDeviceInfo Enumerate Devices Promise".to_string(), error: format!("{why:?}") })
}
}
}
Err(why) => {
return Err(NokhwaError::GetPropertyError { property: "MediaDeviceInfo".to_string(), error: format!("{why:?}") })
},
};
let info = CameraInfo {
human_name: media_info.label(),
description: "videoinfo".to_string(),
misc: media_info.device_id(),
index: index.clone(),
};
let controls = CustomControls {
min_aspect_ratio: None,
aspect_ratio: 0.00,
max_aspect_ratio: None,
facing_mode: JSCameraFacingMode::Any,
facing_mode_exact: false,
resize_mode: JSCameraResizeMode::None,
resize_mode_exact: false,
device_id: media_info.device_id(),
device_id_exact: true,
group_id: media_info.group_id(),
group_id_exact: true,
};
Ok(BrowserCamera { index: index.clone(), info, format: CameraFormat::default(), init: false, controls, cavnas: None, context: None })
}
async fn measure_controls(&mut self) -> Result<(), NokhwaError> {
}
async fn measure_info(&mut self) -> Result<(), NokhwaError> {
todo!()
}
}
impl Backend for BrowserCamera {