diff --git a/Cargo.toml b/Cargo.toml index 0d9c2c9..428a9a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ input-opencv = ["opencv", "opencv/rgb", "rgb"] input-ipcam = ["input-opencv"] input-gst = ["gstreamer", "glib", "gstreamer-app", "gstreamer-video", "regex", "parking_lot"] input-jscam = ["web-sys", "js-sys", "wasm-bindgen-futures", "wasm-bindgen", "wasm-rs-async-executor"] -output-wgpu = ["wgpu"] +output-wgpu = ["wgpu", "nokhwa-core/wgpu-types"] output-wasm = ["input-jscam"] output-threaded = [] small-wasm = [] diff --git a/nokhwa-bindings-macos/src/lib.rs b/nokhwa-bindings-macos/src/lib.rs index 9d26052..97cadb7 100644 --- a/nokhwa-bindings-macos/src/lib.rs +++ b/nokhwa-bindings-macos/src/lib.rs @@ -190,20 +190,14 @@ use core_media_sys::{ }; use flume::{Receiver, Sender}; use nokhwa_core::{ - types::{ - Resolution, - ApiBackend, - CameraFormat, - CameraIndex, - CameraInfo, - FrameFormat - }, error::NokhwaError, + types::{ApiBackend, CameraFormat, CameraIndex, CameraInfo, FrameFormat, Resolution}, }; use objc::{ declare::ClassDecl, runtime::{Class, Object, Protocol, Sel, BOOL, YES}, }; +use std::collections::HashSet; use std::{ borrow::Cow, cmp::Ordering, @@ -335,7 +329,7 @@ fn default_callback(_: bool) {} pub type CompressionData<'a> = (Cow<'a, [u8]>, FrameFormat); pub type DataPipe<'a> = (Sender>, Receiver>); -static CALLBACK_CLASS: &'static Class = { +static CALLBACK_CLASS: &Class = { let mut decl = ClassDecl::new("MyCaptureCallback", class!(NSObject)).unwrap(); // frame stack @@ -390,7 +384,10 @@ static CALLBACK_CLASS: &'static Class = { let boxed_fn = unsafe { // AAAAAAAAAAAAAAAAAAAAAAAAA // https://c.tenor.com/0e_zWtFLOzQAAAAC/needy-streamer-overload-needy-girl-overdose.gif - std::mem::transmute::<*mut c_void, &mut dyn FnMut(Vec, FrameFormat) + Sized + Sync + 'static>(function_cvoid) + std::mem::transmute_copy::< + *mut c_void, + Box, FrameFormat) + Send + Sync + 'static>, + >(&function_cvoid) }; boxed_fn(buffer_as_vec, fourcc); } @@ -431,7 +428,7 @@ static CALLBACK_CLASS: &'static Class = { decl.register() }; -pub fn request_permission_with_callback(callback: Box) { +pub fn request_permission_with_callback(callback: Box) { let cls = class!(AVCaptureDevice); let objc_fn_block = ConcreteBlock::new(callback); let objc_fn_pass = objc_fn_block.copy(); @@ -449,12 +446,50 @@ pub fn current_authorization_status() -> AVAuthorizationStatus { } // fuck it, use deprecated APIs -pub fn query_avfoundation() -> Result, AVFError> { - Ok(AVCaptureDevice::devices_with_type(AVMediaType::Video) - .into_iter() - .enumerate() - .map(|(idx, dev)| AVCaptureDeviceDescriptor::from_capture_device(dev, idx)) - .collect::>()) +pub fn query_avfoundation() -> Result, NokhwaError> { + let front = AVCaptureDeviceDiscoverySession::new( + vec![ + AVCaptureDeviceType::UltraWide, + AVCaptureDeviceType::WideAngle, + AVCaptureDeviceType::Telephoto, + AVCaptureDeviceType::TrueDepth, + AVCaptureDeviceType::ExternalUnknown, + ], + AVMediaType::Video, + AVCaptureDevicePosition::Front, + )? + .devices(); + let back = AVCaptureDeviceDiscoverySession::new( + vec![ + AVCaptureDeviceType::UltraWide, + AVCaptureDeviceType::WideAngle, + AVCaptureDeviceType::Telephoto, + AVCaptureDeviceType::TrueDepth, + AVCaptureDeviceType::ExternalUnknown, + ], + AVMediaType::Video, + AVCaptureDevicePosition::Back, + )? + .devices(); + let unspecified = AVCaptureDeviceDiscoverySession::new( + vec![ + AVCaptureDeviceType::UltraWide, + AVCaptureDeviceType::WideAngle, + AVCaptureDeviceType::Telephoto, + AVCaptureDeviceType::TrueDepth, + AVCaptureDeviceType::ExternalUnknown, + ], + AVMediaType::Video, + AVCaptureDevicePosition::Unspecified, + )? + .devices(); + + let mut device_set = HashSet::with_capacity(front.len() + back.len() + unspecified.len()); + device_set.extend(front); + device_set.extend(back); + device_set.extend(unspecified); + + Ok(device_set.into_iter().collect()) } #[derive(Copy, Clone, Debug, Hash, Ord, PartialOrd, Eq, PartialEq)] @@ -598,18 +633,24 @@ pub struct AVCaptureVideoCallback { } impl AVCaptureVideoCallback { - pub fn new(callback: Box, FrameFormat) + Send + Sync + 'static>) -> Self { + pub fn new( + device_spec: &CStr, + callback: Box, FrameFormat) + Send + Sync + 'static>, + ) -> Result { let cls = &CALLBACK_CLASS as &Class; let delegate: *mut Object = unsafe { msg_send![cls, alloc] }; let delegate: *mut Object = unsafe { msg_send![delegate, init] }; let fnptr_pinned = Box::leak(callback); - unsafe { let _ = msg_send![delegate, setFnPtr:fnptr_pinned]; } + unsafe { + let _ = msg_send![delegate, setFnPtr: fnptr_pinned]; + } - let queue = unsafe { dispatch_queue_create(avf_queue_str, NSObject(std::ptr::null_mut())) }; + let queue = + unsafe { dispatch_queue_create(device_spec.as_ptr(), NSObject(std::ptr::null_mut())) }; - AVCaptureVideoCallback { delegate, queue } + Ok(AVCaptureVideoCallback { delegate, queue }) } pub fn data_len(&self) -> usize { @@ -634,7 +675,6 @@ impl Drop for AVCaptureVideoCallback { create_boilerplate_impl! { [pub AVFrameRateRange], [pub AVCaptureDeviceDiscoverySession], - [pub AVCaptureDevice], [pub AVCaptureDeviceInput], [pub AVCaptureSession] } @@ -778,14 +818,18 @@ impl Drop for AVCaptureDeviceDiscoverySession { unsafe { msg_send![self.inner, autorelease] } } } +pub struct AVCaptureDevice { + inner: *mut Object, + device: CameraInfo, +} impl AVCaptureDevice { - pub fn devices_with_type(video_type: AVMediaType) -> Vec { - let cls = class!(AVCaptureDevice); - let devices: *mut Object = unsafe { msg_send![cls, devicesWithMediaType: video_type] }; - ns_arr_to_vec(devices) + pub fn inner(&self) -> *mut Object { + self.inner } +} +impl AVCaptureDevice { pub fn new(index: CameraIndex) -> Result { match index { CameraIndex::Index(index) => { diff --git a/nokhwa-core/src/buffer.rs b/nokhwa-core/src/buffer.rs index e5a44f2..6422bfb 100644 --- a/nokhwa-core/src/buffer.rs +++ b/nokhwa-core/src/buffer.rs @@ -23,11 +23,10 @@ use bytes::Bytes; use image::ImageBuffer; #[cfg(feature = "serialize")] use serde::{Deserialize, Serialize}; -use std::borrow::Cow; /// A buffer returned by a camera to accomodate custom decoding. /// Contains information of Resolution, the buffer's [`FrameFormat`], and the buffer. -#[derive(Clone, Debug, Hash, PartialOrd, PartialEq)] +#[derive(Clone, Debug, Hash, PartialOrd, PartialEq, Eq)] #[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))] pub struct Buffer { resolution: Resolution, diff --git a/nokhwa-core/src/types.rs b/nokhwa-core/src/types.rs index c5bd65c..368cd6c 100644 --- a/nokhwa-core/src/types.rs +++ b/nokhwa-core/src/types.rs @@ -29,30 +29,29 @@ impl RequestedFormat { match self { RequestedFormat::HighestResolution => { let mut formats = all_formats.to_vec(); - formats.sort_by(|a, b| a.resolution().cmp(&b.resolution())); - let resolution = formats.iter().last()?; + formats.sort_by_key(|a| a.resolution()); + let resolution = *formats.iter().last()?; let mut format_resolutions = formats .into_iter() .filter(|fmt| fmt.resolution() == resolution.resolution()) .collect::>(); - format_resolutions.sort_by(|a, b| a.frame_rate().cmp(&b.frame_rate())); - format_resolutions.last().map(|x| *x) + format_resolutions.sort_by_key(|a| a.frame_rate()); + format_resolutions.last().copied() } RequestedFormat::HighestFrameRate => { let mut formats = all_formats.to_vec(); - formats.sort_by(|a, b| a.frame_rate().cmp(&b.frame_rate())); - let frame_rate = formats.iter().last()?; + formats.sort_by_key(|a| a.frame_rate()); + let frame_rate = *formats.iter().last()?; let mut format_framerates = formats .into_iter() .filter(|fmt| fmt.frame_rate() == frame_rate.frame_rate()) - .copied() .collect::>(); - format_framerates.sort_by(|a, b| a.resolution().cmp(&b.resolution())); - format_framerates.last().map(|x| *x) + format_framerates.sort_by_key(|a| a.resolution()); + format_framerates.last().copied() } - RequestedFormat::Exact(fmt) => *fmt, + RequestedFormat::Exact(fmt) => Some(*fmt), RequestedFormat::Closest(c) => { - let mut same_fmt_formats = all_formats + let same_fmt_formats = all_formats .iter() .filter(|x| x.format() == c.format()) .copied() @@ -63,11 +62,11 @@ impl RequestedFormat { let res = x.resolution(); let x_diff = res.x() as i32 - c.resolution().x() as i32; let y_diff = res.y() as i32 - c.resolution().y() as i32; - let dist_no_sqrt = (x_diff.abs()).pow(2) + (y_diff.abs()).pow(2) as u32; + let dist_no_sqrt = (x_diff.abs()).pow(2) + (y_diff.abs()).pow(2); (dist_no_sqrt, res) }) - .collect::>(); - resolution_map.sort_by(|a, b| a.0.cmp(*b.0)); + .collect::>(); + resolution_map.sort_by(|a, b| a.0.cmp(&b.0)); resolution_map.dedup_by(|a, b| a.0.eq(&b.0)); let resolution = resolution_map.first()?.1; @@ -85,14 +84,14 @@ impl RequestedFormat { .iter() .map(|x| { let abs = *x as i32 - c.frame_rate() as i32; - (abs.abs() as u32, *x) + (abs.unsigned_abs(), *x) }) .collect::>(); framerate_map.sort(); let frame_rate = framerate_map.first()?.1; Some(CameraFormat::new(resolution, c.format(), frame_rate)) } - RequestedFormat::None => all_formats.first().map(|x| *x), + RequestedFormat::None => all_formats.first().copied(), } } } @@ -157,12 +156,6 @@ impl Default for CameraIndex { } } -impl AsRef for CameraIndex { - fn as_ref(&self) -> &str { - self.to_string().as_str() - } -} - impl TryFrom for u32 { type Error = NokhwaError; @@ -296,7 +289,7 @@ impl Ord for Resolution { /// This is a convenience struct that holds all information about the format of a webcam stream. /// It consists of a [`Resolution`], [`FrameFormat`], and a frame rate(u8). -#[derive(Copy, Clone, Debug, Hash, PartialEq, PartialOrd)] +#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))] pub struct CameraFormat { resolution: Resolution, @@ -397,7 +390,7 @@ impl Display for CameraFormat { /// Information about a Camera e.g. its name. /// `description` amd `misc` may contain information that may differ from backend to backend. Refer to each backend for details. /// `index` is a camera's index given to it by (usually) the OS usually in the order it is known to the system. -#[derive(Clone, Debug, Hash, PartialEq, PartialOrd)] +#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd)] #[cfg_attr(feature = "output-wasm", wasm_bindgen)] #[cfg_attr(feature = "serialize", derive(Serialize, Deserialize))] pub struct CameraInfo { diff --git a/src/backends/capture/avfoundation.rs b/src/backends/capture/avfoundation.rs index ad2025c..e45a6a3 100644 --- a/src/backends/capture/avfoundation.rs +++ b/src/backends/capture/avfoundation.rs @@ -14,14 +14,14 @@ * limitations under the License. */ -use image::{ImageBuffer, Rgb}; +use image::{Frame, ImageBuffer, Rgb}; use nokhwa_bindings_macos::{ AVCaptureDevice, AVCaptureDeviceInput, AVCaptureSession, AVCaptureVideoCallback, AVCaptureVideoDataOutput, }; +use nokhwa_core::buffer::Buffer; use nokhwa_core::{ error::NokhwaError, - pixel_format::FormatDecoder, traits::CaptureBackendTrait, types::{ mjpeg_to_rgb, yuyv422_to_rgb, ApiBackend, CameraControl, CameraFormat, CameraIndex, @@ -29,8 +29,13 @@ use nokhwa_core::{ Resolution, }, }; -use std::{borrow::Cow, collections::HashMap}; -use v4l::frameinterval::FrameIntervalEnum; +use std::sync::LockResult; +use std::{ + borrow::Cow, + collections::HashMap, + sync::{Arc, Mutex}, +}; +use std::ffi::CString; /// The backend struct that interfaces with V4L2. /// To see what this does, please see [`CaptureBackendTrait`]. @@ -48,7 +53,9 @@ pub struct AVFoundationCaptureDevice { data_out: Option, data_collect: Option, info: CameraInfo, + buffername: CString, format: CameraFormat, + framebuf: Arc, FrameFormat)>>, } impl AVFoundationCaptureDevice { @@ -63,6 +70,7 @@ impl AVFoundationCaptureDevice { let formats = device.supported_formats()?; let camera_fmt = req_fmt.fulfill(&formats)?; device.set_all(camera_fmt)?; + let device_descriptor = device. Ok(AVFoundationCaptureDevice { device, @@ -70,8 +78,10 @@ impl AVFoundationCaptureDevice { session: None, data_out: None, data_collect: None, - info: device_descriptor, + info: , + buffername: , format: camera_format, + framebuf: Arc::new(Mutex::new((vec![], FrameFormat::MJPEG))), }) } @@ -132,7 +142,7 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice { let mut res_list = HashMap::new(); for format in supported_cfmt { match res_list.get_mut(&format.resolution()) { - Some(fpses) => fpses.push(format.frame_rate()), + Some(fpses) => Vec::push(fpses, format.frame_rate()), None => { vec![format.frame_rate()] } @@ -210,7 +220,17 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice { let session = AVCaptureSession::new(); session.begin_configuration(); session.add_input(&input)?; - let callback = AVCaptureVideoCallback::new(); + + let mut frame_mutex = self.framebuf.clone(); + + let func_callback: Box, FrameFormat) + Send + Sync + 'static> = + Box::new(|data: Vec, frame_format: FrameFormat| { + if let Ok(lck) = frame_mutex.lock() { + *lck = (data, frame_format); + } + }); + + let videocallback = AVCaptureVideoCallback::new(func_callback)?; let output = AVCaptureVideoDataOutput::new(); output.add_delegate(&callback)?; session.add_output(&output)?; @@ -219,7 +239,7 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice { self.dev_input = Some(input); self.session = Some(session); - self.data_collect = Some(callback); + self.data_collect = Some(videocallback); self.data_out = Some(output); Ok(()) } @@ -238,57 +258,42 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice { } } - fn frame(&mut self) -> Result, Vec>, NokhwaError> { - let cam_fmt = self.camera_format(); - let conv = self.frame_raw()?.to_vec(); - let image_buf = - match ImageBuffer::from_vec(cam_fmt.width(), cam_fmt.height(), conv) { - Some(buf) => { - let rgb_buf: ImageBuffer, Vec> = buf; - rgb_buf - } - None => return Err(NokhwaError::ReadFrameError( - "ImageBuffer is not large enough! This is probably a bug, please report it!" - .to_string(), - )), - }; - Ok(image_buf) + fn frame(&mut self) -> Result { + self.refresh_camera_format()?; + let cfmt = self.camera_format(); + let buffer = Buffer::new(cfmt.resolution(), &self.frame_raw()?, cfmt.format()); + Ok(buffer) } fn frame_raw(&mut self) -> Result, NokhwaError> { - match &self.session { - Some(session) => { - if !session.is_running() { - return Err(NokhwaError::ReadFrameError( - "Stream Not Started".to_string(), - )); - } - if session.is_interrupted() { - return Err(NokhwaError::ReadFrameError( - "Stream Interrupted".to_string(), - )); + let mut framebuffer_empty = match self.framebuf.lock() { + Ok(f) => { + if f.0.is_empty() { + true } + false } - None => { - return Err(NokhwaError::ReadFrameError( - "Stream Not Started".to_string(), - )) + Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())), + }; + + loop { + if framebuffer_empty { + match self.framebuf.lock() { + Ok(f) => framebuffer_empty = f.0.is_empty(), + Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())), + } + } else { + break; } } - match &self.data_collect { - Some(collector) => { - let data = collector.frame_to_slice()?; - let data = match data.1 { - AVFourCC::YUV2 => Cow::from(yuyv422_to_rgb(data.0.borrow(), false)), - AVFourCC::MJPEG => Cow::from(mjpeg_to_rgb(data.0.borrow(), false)), - AVFourCC::GRAY8 => {} - }; - Ok(data) + match self.framebuf.lock() { + Ok(f) => { + let mut new_frame = vec![]; + std::mem::swap(&mut new_frame, &mut f.0); + Ok(Cow::from(new_frame)) } - None => Err(NokhwaError::ReadFrameError( - "Stream Not Started".to_string(), - )), + Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())), } } diff --git a/src/camera.rs b/src/camera.rs index f20ae35..ddd0406 100644 --- a/src/camera.rs +++ b/src/camera.rs @@ -24,7 +24,6 @@ use nokhwa_core::{ FrameFormat, KnownCameraControl, RequestedFormat, Resolution, }, }; -use std::fmt::format; use std::{borrow::Cow, collections::HashMap}; #[cfg(feature = "output-wgpu")] use wgpu::{Device as WgpuDevice, Queue as WgpuQueue, Texture as WgpuTexture}; @@ -156,7 +155,7 @@ impl Camera { request: RequestedFormat, ) -> Result { let new_format = request - .fufill(self.device.compatible_camera_formats()?.as_slice()) + .fulfill(self.device.compatible_camera_formats()?.as_slice()) .ok_or(NokhwaError::GetPropertyError { property: "Compatible Camera Format by request".to_string(), error: "Failed to fufill".to_string(), diff --git a/src/init.rs b/src/init.rs index e6686f0..5be7a51 100644 --- a/src/init.rs +++ b/src/init.rs @@ -18,16 +18,16 @@ feature = "input-avfoundation", any(target_os = "macos", target_os = "ios") )))] -fn init_avfoundation(callback: impl FnMut(bool) + Send + 'static) { - callback(true); +fn init_avfoundation(callback: impl Fn(bool) + Send + 'static) { + callback.call(true); } #[cfg(all( feature = "input-avfoundation", any(target_os = "macos", target_os = "ios") ))] -fn init_avfoundation(callback: impl FnMut(bool) + Send + 'static) { - use nokhwa_bindings_macos::avfoundation::request_permission_with_callback; +fn init_avfoundation(callback: impl Fn(bool) + Send + 'static) { + use nokhwa_bindings_macos::request_permission_with_callback; request_permission_with_callback(callback); } diff --git a/src/query.rs b/src/query.rs index 31febfa..8744cd6 100644 --- a/src/query.rs +++ b/src/query.rs @@ -16,7 +16,7 @@ use nokhwa_core::{ error::NokhwaError, - types::{ApiBackend, CameraIndex, CameraInfo}, + types::{ApiBackend, CameraInfo}, }; // TODO: Update as this goes