GRIN/nokhwa
1
0
Fork 0
mirror of https://github.com/l1npengtul/nokhwa.git synced 2026-07-04 02:27:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

work for this week

Browse Source
This commit is contained in:
l1npengtul
2022-09-25 05:27:13 -07:00
parent 51e58e163d
commit 56dee21591
8 changed files with 150 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+1 -1
View File
@@ -30,7 +30,7 @@ input-opencv = ["opencv", "opencv/rgb", "rgb"]
input-ipcam = ["input-opencv"]
input-gst = ["gstreamer", "glib", "gstreamer-app", "gstreamer-video", "regex", "parking_lot"]
input-jscam = ["web-sys", "js-sys", "wasm-bindgen-futures", "wasm-bindgen", "wasm-rs-async-executor"]
output-wgpu = ["wgpu"]
output-wgpu = ["wgpu", "nokhwa-core/wgpu-types"]
output-wasm = ["input-jscam"]
output-threaded = []
small-wasm = []
nokhwa-bindings-macos/src/lib.rs
+70 -26
View File
@@ -190,20 +190,14 @@ use core_media_sys::{
};
use flume::{Receiver, Sender};
use nokhwa_core::{
types::{
Resolution,
ApiBackend,
CameraFormat,
CameraIndex,
CameraInfo,
FrameFormat
},
error::NokhwaError,
types::{ApiBackend, CameraFormat, CameraIndex, CameraInfo, FrameFormat, Resolution},
};
use objc::{
declare::ClassDecl,
runtime::{Class, Object, Protocol, Sel, BOOL, YES},
};
use std::collections::HashSet;
use std::{
borrow::Cow,
cmp::Ordering,
@@ -335,7 +329,7 @@ fn default_callback(_: bool) {}
pub type CompressionData<'a> = (Cow<'a, [u8]>, FrameFormat);
pub type DataPipe<'a> = (Sender<CompressionData<'a>>, Receiver<CompressionData<'a>>);
static CALLBACK_CLASS: &'static Class = {
static CALLBACK_CLASS: &Class = {
let mut decl = ClassDecl::new("MyCaptureCallback", class!(NSObject)).unwrap();
// frame stack
@@ -390,7 +384,10 @@ static CALLBACK_CLASS: &'static Class = {
let boxed_fn = unsafe {
// AAAAAAAAAAAAAAAAAAAAAAAAA
// https://c.tenor.com/0e_zWtFLOzQAAAAC/needy-streamer-overload-needy-girl-overdose.gif
std::mem::transmute::<*mut c_void, &mut dyn FnMut(Vec<u8>, FrameFormat) + Sized + Sync + 'static>(function_cvoid)
std::mem::transmute_copy::<
*mut c_void,
Box<dyn FnMut(Vec<u8>, FrameFormat) + Send + Sync + 'static>,
>(&function_cvoid)
};
boxed_fn(buffer_as_vec, fourcc);
}
@@ -431,7 +428,7 @@ static CALLBACK_CLASS: &'static Class = {
decl.register()
};
pub fn request_permission_with_callback(callback: Box<dyn FnMut(bool) + Send + Sync + 'static>) {
pub fn request_permission_with_callback(callback: Box<dyn Fn(bool) + Send + Sync + 'static>) {
let cls = class!(AVCaptureDevice);
let objc_fn_block = ConcreteBlock::new(callback);
let objc_fn_pass = objc_fn_block.copy();
@@ -449,12 +446,50 @@ pub fn current_authorization_status() -> AVAuthorizationStatus {
}
// fuck it, use deprecated APIs
pub fn query_avfoundation() -> Result<Vec<AVCaptureDeviceDescriptor>, AVFError> {
Ok(AVCaptureDevice::devices_with_type(AVMediaType::Video)
.into_iter()
.enumerate()
.map(|(idx, dev)| AVCaptureDeviceDescriptor::from_capture_device(dev, idx))
.collect::<Vec<AVCaptureDeviceDescriptor>>())
pub fn query_avfoundation() -> Result<Vec<CameraInfo>, NokhwaError> {
let front = AVCaptureDeviceDiscoverySession::new(
vec![
AVCaptureDeviceType::UltraWide,
AVCaptureDeviceType::WideAngle,
AVCaptureDeviceType::Telephoto,
AVCaptureDeviceType::TrueDepth,
AVCaptureDeviceType::ExternalUnknown,
],
AVMediaType::Video,
AVCaptureDevicePosition::Front,
)?
.devices();
let back = AVCaptureDeviceDiscoverySession::new(
vec![
AVCaptureDeviceType::UltraWide,
AVCaptureDeviceType::WideAngle,
AVCaptureDeviceType::Telephoto,
AVCaptureDeviceType::TrueDepth,
AVCaptureDeviceType::ExternalUnknown,
],
AVMediaType::Video,
AVCaptureDevicePosition::Back,
)?
.devices();
let unspecified = AVCaptureDeviceDiscoverySession::new(
vec![
AVCaptureDeviceType::UltraWide,
AVCaptureDeviceType::WideAngle,
AVCaptureDeviceType::Telephoto,
AVCaptureDeviceType::TrueDepth,
AVCaptureDeviceType::ExternalUnknown,
],
AVMediaType::Video,
AVCaptureDevicePosition::Unspecified,
)?
.devices();
let mut device_set = HashSet::with_capacity(front.len() + back.len() + unspecified.len());
device_set.extend(front);
device_set.extend(back);
device_set.extend(unspecified);
Ok(device_set.into_iter().collect())
}
#[derive(Copy, Clone, Debug, Hash, Ord, PartialOrd, Eq, PartialEq)]
@@ -598,18 +633,24 @@ pub struct AVCaptureVideoCallback {
}
impl AVCaptureVideoCallback {
pub fn new(callback: Box<dyn FnMut(Vec<u8>, FrameFormat) + Send + Sync + 'static>) -> Self {
pub fn new(
device_spec: &CStr,
callback: Box<dyn FnMut(Vec<u8>, FrameFormat) + Send + Sync + 'static>,
) -> Result<Self, NokhwaError> {
let cls = &CALLBACK_CLASS as &Class;
let delegate: *mut Object = unsafe { msg_send![cls, alloc] };
let delegate: *mut Object = unsafe { msg_send![delegate, init] };
let fnptr_pinned = Box::leak(callback);
unsafe { let _ = msg_send![delegate, setFnPtr:fnptr_pinned]; }
unsafe {
let _ = msg_send![delegate, setFnPtr: fnptr_pinned];
}
let queue = unsafe { dispatch_queue_create(avf_queue_str, NSObject(std::ptr::null_mut())) };
let queue =
unsafe { dispatch_queue_create(device_spec.as_ptr(), NSObject(std::ptr::null_mut())) };
AVCaptureVideoCallback { delegate, queue }
Ok(AVCaptureVideoCallback { delegate, queue })
}
pub fn data_len(&self) -> usize {
@@ -634,7 +675,6 @@ impl Drop for AVCaptureVideoCallback {
create_boilerplate_impl! {
[pub AVFrameRateRange],
[pub AVCaptureDeviceDiscoverySession],
[pub AVCaptureDevice],
[pub AVCaptureDeviceInput],
[pub AVCaptureSession]
}
@@ -778,14 +818,18 @@ impl Drop for AVCaptureDeviceDiscoverySession {
unsafe { msg_send![self.inner, autorelease] }
}
}
pub struct AVCaptureDevice {
inner: *mut Object,
device: CameraInfo,
}
impl AVCaptureDevice {
pub fn devices_with_type(video_type: AVMediaType) -> Vec<AVCaptureDevice> {
let cls = class!(AVCaptureDevice);
let devices: *mut Object = unsafe { msg_send![cls, devicesWithMediaType: video_type] };
ns_arr_to_vec(devices)
pub fn inner(&self) -> *mut Object {
self.inner
}
}
impl AVCaptureDevice {
pub fn new(index: CameraIndex) -> Result<Self, NokhwaError> {
match index {
CameraIndex::Index(index) => {
nokhwa-core/src/buffer.rs
+1 -2
View File
@@ -23,11 +23,10 @@ use bytes::Bytes;
use image::ImageBuffer;
#[cfg(feature = "serialize")]
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
/// A buffer returned by a camera to accomodate custom decoding.
/// Contains information of Resolution, the buffer's [`FrameFormat`], and the buffer.
#[derive(Clone, Debug, Hash, PartialOrd, PartialEq)]
#[derive(Clone, Debug, Hash, PartialOrd, PartialEq, Eq)]
#[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))]
pub struct Buffer {
resolution: Resolution,
nokhwa-core/src/types.rs
+17 -24
View File
@@ -29,30 +29,29 @@ impl RequestedFormat {
match self {
RequestedFormat::HighestResolution => {
let mut formats = all_formats.to_vec();
formats.sort_by(|a, b| a.resolution().cmp(&b.resolution()));
let resolution = formats.iter().last()?;
formats.sort_by_key(|a| a.resolution());
let resolution = *formats.iter().last()?;
let mut format_resolutions = formats
.into_iter()
.filter(|fmt| fmt.resolution() == resolution.resolution())
.collect::<Vec<CameraFormat>>();
format_resolutions.sort_by(|a, b| a.frame_rate().cmp(&b.frame_rate()));
format_resolutions.last().map(|x| *x)
format_resolutions.sort_by_key(|a| a.frame_rate());
format_resolutions.last().copied()
}
RequestedFormat::HighestFrameRate => {
let mut formats = all_formats.to_vec();
formats.sort_by(|a, b| a.frame_rate().cmp(&b.frame_rate()));
let frame_rate = formats.iter().last()?;
formats.sort_by_key(|a| a.frame_rate());
let frame_rate = *formats.iter().last()?;
let mut format_framerates = formats
.into_iter()
.filter(|fmt| fmt.frame_rate() == frame_rate.frame_rate())
.copied()
.collect::<Vec<CameraFormat>>();
format_framerates.sort_by(|a, b| a.resolution().cmp(&b.resolution()));
format_framerates.last().map(|x| *x)
format_framerates.sort_by_key(|a| a.resolution());
format_framerates.last().copied()
}
RequestedFormat::Exact(fmt) => *fmt,
RequestedFormat::Exact(fmt) => Some(*fmt),
RequestedFormat::Closest(c) => {
let mut same_fmt_formats = all_formats
let same_fmt_formats = all_formats
.iter()
.filter(|x| x.format() == c.format())
.copied()
@@ -63,11 +62,11 @@ impl RequestedFormat {
let res = x.resolution();
let x_diff = res.x() as i32 - c.resolution().x() as i32;
let y_diff = res.y() as i32 - c.resolution().y() as i32;
let dist_no_sqrt = (x_diff.abs()).pow(2) + (y_diff.abs()).pow(2) as u32;
let dist_no_sqrt = (x_diff.abs()).pow(2) + (y_diff.abs()).pow(2);
(dist_no_sqrt, res)
})
.collect::<Vec<(u32, Resolution)>>();
resolution_map.sort_by(|a, b| a.0.cmp(*b.0));
.collect::<Vec<(i32, Resolution)>>();
resolution_map.sort_by(|a, b| a.0.cmp(&b.0));
resolution_map.dedup_by(|a, b| a.0.eq(&b.0));
let resolution = resolution_map.first()?.1;
@@ -85,14 +84,14 @@ impl RequestedFormat {
.iter()
.map(|x| {
let abs = *x as i32 - c.frame_rate() as i32;
(abs.abs() as u32, *x)
(abs.unsigned_abs(), *x)
})
.collect::<Vec<(u32, u32)>>();
framerate_map.sort();
let frame_rate = framerate_map.first()?.1;
Some(CameraFormat::new(resolution, c.format(), frame_rate))
}
RequestedFormat::None => all_formats.first().map(|x| *x),
RequestedFormat::None => all_formats.first().copied(),
}
}
}
@@ -157,12 +156,6 @@ impl Default for CameraIndex {
}
}
impl AsRef<str> for CameraIndex {
fn as_ref(&self) -> &str {
self.to_string().as_str()
}
}
impl TryFrom<CameraIndex> for u32 {
type Error = NokhwaError;
@@ -296,7 +289,7 @@ impl Ord for Resolution {
/// This is a convenience struct that holds all information about the format of a webcam stream.
/// It consists of a [`Resolution`], [`FrameFormat`], and a frame rate(u8).
#[derive(Copy, Clone, Debug, Hash, PartialEq, PartialOrd)]
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
#[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))]
pub struct CameraFormat {
resolution: Resolution,
@@ -397,7 +390,7 @@ impl Display for CameraFormat {
/// Information about a Camera e.g. its name.
/// `description` amd `misc` may contain information that may differ from backend to backend. Refer to each backend for details.
/// `index` is a camera's index given to it by (usually) the OS usually in the order it is known to the system.
#[derive(Clone, Debug, Hash, PartialEq, PartialOrd)]
#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd)]
#[cfg_attr(feature = "output-wasm", wasm_bindgen)]
#[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))]
pub struct CameraInfo {
src/backends/capture/avfoundation.rs
+55 -50
View File
@@ -14,14 +14,14 @@
* limitations under the License.
*/
use image::{ImageBuffer, Rgb};
use image::{Frame, ImageBuffer, Rgb};
use nokhwa_bindings_macos::{
AVCaptureDevice, AVCaptureDeviceInput, AVCaptureSession, AVCaptureVideoCallback,
AVCaptureVideoDataOutput,
};
use nokhwa_core::buffer::Buffer;
use nokhwa_core::{
error::NokhwaError,
pixel_format::FormatDecoder,
traits::CaptureBackendTrait,
types::{
mjpeg_to_rgb, yuyv422_to_rgb, ApiBackend, CameraControl, CameraFormat, CameraIndex,
@@ -29,8 +29,13 @@ use nokhwa_core::{
Resolution,
},
};
use std::{borrow::Cow, collections::HashMap};
use v4l::frameinterval::FrameIntervalEnum;
use std::sync::LockResult;
use std::{
borrow::Cow,
collections::HashMap,
sync::{Arc, Mutex},
};
use std::ffi::CString;
/// The backend struct that interfaces with V4L2.
/// To see what this does, please see [`CaptureBackendTrait`].
@@ -48,7 +53,9 @@ pub struct AVFoundationCaptureDevice {
data_out: Option<AVCaptureVideoDataOutput>,
data_collect: Option<AVCaptureVideoCallback>,
info: CameraInfo,
buffername: CString,
format: CameraFormat,
framebuf: Arc<Mutex<(Vec<u8>, FrameFormat)>>,
}
impl AVFoundationCaptureDevice {
@@ -63,6 +70,7 @@ impl AVFoundationCaptureDevice {
let formats = device.supported_formats()?;
let camera_fmt = req_fmt.fulfill(&formats)?;
device.set_all(camera_fmt)?;
let device_descriptor = device.
Ok(AVFoundationCaptureDevice {
device,
@@ -70,8 +78,10 @@ impl AVFoundationCaptureDevice {
session: None,
data_out: None,
data_collect: None,
info: device_descriptor,
info: ,
buffername: ,
format: camera_format,
framebuf: Arc::new(Mutex::new((vec![], FrameFormat::MJPEG))),
})
}
@@ -132,7 +142,7 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice {
let mut res_list = HashMap::new();
for format in supported_cfmt {
match res_list.get_mut(&format.resolution()) {
Some(fpses) => fpses.push(format.frame_rate()),
Some(fpses) => Vec::push(fpses, format.frame_rate()),
None => {
vec![format.frame_rate()]
}
@@ -210,7 +220,17 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice {
let session = AVCaptureSession::new();
session.begin_configuration();
session.add_input(&input)?;
let callback = AVCaptureVideoCallback::new();
let mut frame_mutex = self.framebuf.clone();
let func_callback: Box<dyn FnMut(Vec<u8>, FrameFormat) + Send + Sync + 'static> =
Box::new(|data: Vec<u8>, frame_format: FrameFormat| {
if let Ok(lck) = frame_mutex.lock() {
*lck = (data, frame_format);
}
});
let videocallback = AVCaptureVideoCallback::new(func_callback)?;
let output = AVCaptureVideoDataOutput::new();
output.add_delegate(&callback)?;
session.add_output(&output)?;
@@ -219,7 +239,7 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice {
self.dev_input = Some(input);
self.session = Some(session);
self.data_collect = Some(callback);
self.data_collect = Some(videocallback);
self.data_out = Some(output);
Ok(())
}
@@ -238,57 +258,42 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice {
}
}
fn frame(&mut self) -> Result<ImageBuffer<Rgb<u8>, Vec<u8>>, NokhwaError> {
let cam_fmt = self.camera_format();
let conv = self.frame_raw()?.to_vec();
let image_buf =
match ImageBuffer::from_vec(cam_fmt.width(), cam_fmt.height(), conv) {
Some(buf) => {
let rgb_buf: ImageBuffer<Rgb<u8>, Vec<u8>> = buf;
rgb_buf
}
None => return Err(NokhwaError::ReadFrameError(
"ImageBuffer is not large enough! This is probably a bug, please report it!"
.to_string(),
)),
};
Ok(image_buf)
fn frame(&mut self) -> Result<Buffer, NokhwaError> {
self.refresh_camera_format()?;
let cfmt = self.camera_format();
let buffer = Buffer::new(cfmt.resolution(), &self.frame_raw()?, cfmt.format());
Ok(buffer)
}
fn frame_raw(&mut self) -> Result<Cow<[u8]>, NokhwaError> {
match &self.session {
Some(session) => {
if !session.is_running() {
return Err(NokhwaError::ReadFrameError(
"Stream Not Started".to_string(),
));
}
if session.is_interrupted() {
return Err(NokhwaError::ReadFrameError(
"Stream Interrupted".to_string(),
));
let mut framebuffer_empty = match self.framebuf.lock() {
Ok(f) => {
if f.0.is_empty() {
true
}
false
}
None => {
return Err(NokhwaError::ReadFrameError(
"Stream Not Started".to_string(),
))
Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())),
};
loop {
if framebuffer_empty {
match self.framebuf.lock() {
Ok(f) => framebuffer_empty = f.0.is_empty(),
Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())),
}
} else {
break;
}
}
match &self.data_collect {
Some(collector) => {
let data = collector.frame_to_slice()?;
let data = match data.1 {
AVFourCC::YUV2 => Cow::from(yuyv422_to_rgb(data.0.borrow(), false)),
AVFourCC::MJPEG => Cow::from(mjpeg_to_rgb(data.0.borrow(), false)),
AVFourCC::GRAY8 => {}
};
Ok(data)
match self.framebuf.lock() {
Ok(f) => {
let mut new_frame = vec![];
std::mem::swap(&mut new_frame, &mut f.0);
Ok(Cow::from(new_frame))
}
None => Err(NokhwaError::ReadFrameError(
"Stream Not Started".to_string(),
)),
Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())),
}
}
src/camera.rs
+1 -2
View File
@@ -24,7 +24,6 @@ use nokhwa_core::{
FrameFormat, KnownCameraControl, RequestedFormat, Resolution,
},
};
use std::fmt::format;
use std::{borrow::Cow, collections::HashMap};
#[cfg(feature = "output-wgpu")]
use wgpu::{Device as WgpuDevice, Queue as WgpuQueue, Texture as WgpuTexture};
@@ -156,7 +155,7 @@ impl Camera {
request: RequestedFormat,
) -> Result<CameraFormat, NokhwaError> {
let new_format = request
.fufill(self.device.compatible_camera_formats()?.as_slice())
.fulfill(self.device.compatible_camera_formats()?.as_slice())
.ok_or(NokhwaError::GetPropertyError {
property: "Compatible Camera Format by request".to_string(),
error: "Failed to fufill".to_string(),
src/init.rs
+4 -4
View File
@@ -18,16 +18,16 @@
feature = "input-avfoundation",
any(target_os = "macos", target_os = "ios")
)))]
fn init_avfoundation(callback: impl FnMut(bool) + Send + 'static) {
callback(true);
fn init_avfoundation(callback: impl Fn(bool) + Send + 'static) {
callback.call(true);
}
#[cfg(all(
feature = "input-avfoundation",
any(target_os = "macos", target_os = "ios")
))]
fn init_avfoundation(callback: impl FnMut(bool) + Send + 'static) {
use nokhwa_bindings_macos::avfoundation::request_permission_with_callback;
fn init_avfoundation(callback: impl Fn(bool) + Send + 'static) {
use nokhwa_bindings_macos::request_permission_with_callback;
request_permission_with_callback(callback);
}
src/query.rs
+1 -1
View File
@@ -16,7 +16,7 @@
use nokhwa_core::{
error::NokhwaError,
types::{ApiBackend, CameraIndex, CameraInfo},
types::{ApiBackend, CameraInfo},
};
// TODO: Update as this goes