Add support for sensor capture timestamp (#234)

* adding sensor capture timestamp

* fix v4l timestamp

* dont set timestamp if not available

* return None if v4l timestamp is 0

* windows use sensor ts

nokhwa-bindings-linux/Cargo.toml

@@ -21,3 +21,4 @@ path    = "../nokhwa-core"
 
 [target.'cfg(target_os="linux")'.dependencies]
 v4l = { version = "0.14", features = [ "v4l2-sys" ] }
+libc = "0.2"

nokhwa-bindings-linux/src/lib.rs

@@ -877,12 +877,23 @@ mod internal {
 
         fn frame(&mut self) -> Result<Buffer, NokhwaError> {
             let cam_fmt = self.camera_format;
-            let raw_frame = self.frame_raw()?;
-            Ok(Buffer::new(
-                cam_fmt.resolution(),
-                &raw_frame,
-                cam_fmt.format(),
-            ))
+            match &mut self.stream_handle {
+                Some(sh) => match sh.next() {
+                    Ok((data, meta)) => {
+                        let wall_ts = monotonic_to_wallclock(meta.timestamp);
+                        Ok(Buffer::with_timestamp(
+                            cam_fmt.resolution(),
+                            data,
+                            cam_fmt.format(),
+                            wall_ts,
+                        ))
+                    }
+                    Err(why) => Err(NokhwaError::ReadFrameError(why.to_string())),
+                },
+                None => Err(NokhwaError::ReadFrameError(
+                    "Stream Not Started".to_string(),
+                )),
+            }
         }
 
         fn frame_raw(&mut self) -> Result<Cow<'_, [u8]>, NokhwaError> {
@@ -927,6 +938,36 @@ mod internal {
             FrameFormat::NV12 => FourCC::new(b"NV12"),
         }
     }
+
+    /// Convert a V4L2 CLOCK_MONOTONIC timestamp to a wallclock Duration since UNIX_EPOCH.
+    fn monotonic_to_wallclock(ts: v4l::Timestamp) -> Option<std::time::Duration> {
+        let frame_mono = std::time::Duration::from(ts);
+        if frame_mono.is_zero() {
+            return None;
+        }
+
+        let mut mono_now = libc::timespec {
+            tv_sec: 0,
+            tv_nsec: 0,
+        };
+        let mut wall_now = libc::timespec {
+            tv_sec: 0,
+            tv_nsec: 0,
+        };
+        // SAFETY: passing valid pointers to kernel clock_gettime
+        unsafe {
+            libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut mono_now);
+            libc::clock_gettime(libc::CLOCK_REALTIME, &mut wall_now);
+        }
+        let mono_now =
+            std::time::Duration::new(mono_now.tv_sec as u64, mono_now.tv_nsec as u32);
+        let wall_now =
+            std::time::Duration::new(wall_now.tv_sec as u64, wall_now.tv_nsec as u32);
+
+        // frame_age = how long ago the frame was captured (monotonic delta)
+        let frame_age = mono_now.checked_sub(frame_mono)?;
+        wall_now.checked_sub(frame_age)
+    }
 }
 
 #[cfg(not(target_os = "linux"))]

nokhwa-bindings-macos/src/lib.rs

@@ -97,6 +97,8 @@ mod internal {
 
             pub fn CMSampleBufferGetDataBuffer(sbuf: CMSampleBufferRef) -> CMBlockBufferRef;
 
+            pub fn CMSampleBufferGetPresentationTimeStamp(sbuf: CMSampleBufferRef) -> CMTime;
+
             pub fn dispatch_queue_create(
                 label: *const std::os::raw::c_char,
                 attr: NSObject,
@@ -248,11 +250,37 @@ mod internal {
         error::Error,
         ffi::{c_float, c_void, CStr},
         sync::Arc,
+        time::Duration,
     };
 
     const UTF8_ENCODING: usize = 4;
     type CGFloat = c_float;
 
+    extern "C" {
+        fn mach_absolute_time() -> u64;
+    }
+
+    #[repr(C)]
+    struct MachTimebaseInfo {
+        numer: u32,
+        denom: u32,
+    }
+
+    extern "C" {
+        fn mach_timebase_info(info: *mut MachTimebaseInfo) -> i32;
+    }
+
+    fn mach_absolute_time_nanos() -> u64 {
+        static TIMEBASE: once_cell::sync::Lazy<(u32, u32)> = once_cell::sync::Lazy::new(|| {
+            let mut info = MachTimebaseInfo { numer: 0, denom: 0 };
+            unsafe { mach_timebase_info(&mut info) };
+            (info.numer, info.denom)
+        });
+        let ticks = unsafe { mach_absolute_time() };
+        let (numer, denom) = *TIMEBASE;
+        ticks.wrapping_mul(numer as u64) / (denom as u64)
+    }
+
     macro_rules! create_boilerplate_impl {
         {
             $( [$class_vis:vis $class_name:ident : $( {$field_vis:vis $field_name:ident : $field_type:ty} ),*] ),+
@@ -378,7 +406,7 @@ mod internal {
         }
     }
 
-    pub type CompressionData<'a> = (Cow<'a, [u8]>, FrameFormat);
+    pub type CompressionData<'a> = (Cow<'a, [u8]>, FrameFormat, Option<Duration>);
     pub type DataPipe<'a> = (Sender<CompressionData<'a>>, Receiver<CompressionData<'a>>);
 
     static CALLBACK_CLASS: Lazy<&'static Class> = Lazy::new(|| {
@@ -427,15 +455,45 @@ mod internal {
                 };
 
                 unsafe { CVPixelBufferUnlockBaseAddress(image_buffer, 0) };
+
+                // CMSampleBufferGetPresentationTimeStamp returns the sensor
+                // capture instant on a monotonic clock (mach_absolute_time
+                // timebase).  Convert to Unix wallclock:
+                //   wall = SystemTime::now() - (mach_now - pts)
+                let capture_ts = {
+                    let pts = unsafe {
+                        core_media::CMSampleBufferGetPresentationTimeStamp(
+                            didOutputSampleBuffer,
+                        )
+                    };
+                    if pts.timescale > 0 {
+                        let pts_nanos = (pts.value as u128)
+                            .saturating_mul(1_000_000_000)
+                            / (pts.timescale as u128);
+                        let mono_now_nanos = mach_absolute_time_nanos() as u128;
+                        let wall_now = std::time::SystemTime::now();
+
+                        let age = Duration::from_nanos(
+                            mono_now_nanos.saturating_sub(pts_nanos) as u64,
+                        );
+                        wall_now
+                            .duration_since(std::time::UNIX_EPOCH)
+                            .ok()
+                            .and_then(|wall_dur| wall_dur.checked_sub(age))
+                    } else {
+                        None
+                    }
+                };
+
                 // oooooh scarey unsafe
                 // AAAAAAAAAAAAAAAAAAAAAAAAA
                 // https://c.tenor.com/0e_zWtFLOzQAAAAC/needy-streamer-overload-needy-girl-overdose.gif
                 let bufferlck_cv: *const c_void = unsafe { msg_send![this, bufferPtr] };
                 let buffer_sndr = unsafe {
-                    let ptr = bufferlck_cv.cast::<Sender<(Vec<u8>, FrameFormat)>>();
+                    let ptr = bufferlck_cv.cast::<Sender<(Vec<u8>, FrameFormat, Option<Duration>)>>();
                     Arc::from_raw(ptr)
                 };
-                if let Err(_) = buffer_sndr.send((buffer_as_vec, FrameFormat::GRAY)) {
+                if let Err(_) = buffer_sndr.send((buffer_as_vec, FrameFormat::GRAY, capture_ts)) {
                     // FIXME: dont, what the fuck???
                     return;
                 }
@@ -681,7 +739,7 @@ mod internal {
     impl AVCaptureVideoCallback {
         pub fn new(
             device_spec: &CStr,
-            buffer: &Arc<Sender<(Vec<u8>, FrameFormat)>>,
+            buffer: &Arc<Sender<(Vec<u8>, FrameFormat, Option<Duration>)>>,
         ) -> Result<Self, NokhwaError> {
             let cls = &CALLBACK_CLASS as &Class;
             let delegate: *mut Object = unsafe { msg_send![cls, alloc] };

nokhwa-bindings-windows/src/lib.rs

@@ -45,6 +45,7 @@ pub mod wmf {
             atomic::{AtomicBool, AtomicUsize, Ordering},
             Arc,
         },
+        time::Duration,
     };
     use windows::Win32::Media::DirectShow::{CameraControl_Flags_Auto, CameraControl_Flags_Manual};
     use windows::Win32::Media::MediaFoundation::{
@@ -410,6 +411,10 @@ pub mod wmf {
         device_specifier: CameraInfo,
         device_format: CameraFormat,
         source_reader: IMFSourceReader,
+        /// Wallclock instant captured when the stream was started.
+        /// MF sample timestamps are relative to stream start, so
+        /// `stream_epoch + sample_time` gives us an absolute wallclock.
+        stream_epoch: Option<Duration>,
     }
 
     impl MediaFoundationDevice {
@@ -494,6 +499,7 @@ pub mod wmf {
                         device_specifier: device_descriptor,
                         device_format: CameraFormat::default(),
                         source_reader,
+                        stream_epoch: None,
                     })
                 }
                 CameraIndex::String(s) => {
@@ -1125,11 +1131,14 @@ pub mod wmf {
                 return Err(NokhwaError::OpenStreamError(why.to_string()));
             }
 
+            self.stream_epoch = std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .ok();
             self.is_open.set(true);
             Ok(())
         }
 
-        pub fn raw_bytes(&mut self) -> Result<Cow<'_, [u8]>, NokhwaError> {
+        pub fn raw_bytes(&mut self) -> Result<(Cow<'_, [u8]>, Option<Duration>), NokhwaError> {
             let mut imf_sample: Option<IMFSample> = match unsafe { MFCreateSample() } {
                 Ok(sample) => Some(sample),
                 Err(why) => {
@@ -1137,6 +1146,7 @@ pub mod wmf {
                 }
             };
             let mut stream_flags = 0;
+            let mut sample_time_100ns: i64 = 0;
             {
                 loop {
                     if let Err(why) = unsafe {
@@ -1145,7 +1155,7 @@ pub mod wmf {
                             0,
                             None,
                             Some(&mut stream_flags),
-                            None,
+                            Some(&mut sample_time_100ns),
                             Some(&mut imf_sample),
                         )
                     } {
@@ -1166,6 +1176,15 @@ pub mod wmf {
                 }
             };
 
+            // Calculate absolute capture timestamp.
+            let capture_ts = if sample_time_100ns > 0 {
+                let sample_offset = Duration::from_nanos(sample_time_100ns as u64 * 100);
+                self.stream_epoch
+                    .and_then(|epoch| epoch.checked_add(sample_offset))
+            } else {
+                None
+            };
+
             let buffer = match unsafe { imf_sample.ConvertToContiguousBuffer() } {
                 Ok(buf) => buf,
                 Err(why) => return Err(NokhwaError::ReadFrameError(why.to_string())),
@@ -1200,10 +1219,11 @@ pub mod wmf {
                 ) as &[u8]);
             }
 
-            Ok(Cow::from(data_slice))
+            Ok((Cow::from(data_slice), capture_ts))
         }
 
         pub fn stop_stream(&mut self) {
+            self.stream_epoch = None;
             self.is_open.set(false);
         }
     }
@@ -1242,7 +1262,7 @@ pub mod wmf {
         CameraControl, CameraFormat, CameraIndex, CameraInfo, ControlValueSetter,
         KnownCameraControl,
     };
-    use std::borrow::Cow;
+    use std::{borrow::Cow, time::Duration};
 
     pub fn initialize_mf() -> Result<(), NokhwaError> {
         Err(NokhwaError::NotImplementedError(
@@ -1333,7 +1353,7 @@ pub mod wmf {
             ))
         }
 
-        pub fn raw_bytes(&mut self) -> Result<Cow<'_, [u8]>, NokhwaError> {
+        pub fn raw_bytes(&mut self) -> Result<(Cow<'_, [u8]>, Option<Duration>), NokhwaError> {
             Err(NokhwaError::NotImplementedError(
                 "Only on Windows".to_string(),
             ))

nokhwa-core/src/buffer.rs

@@ -20,6 +20,7 @@ use crate::{
 };
 use bytes::Bytes;
 use image::ImageBuffer;
+use std::time::Duration;
 #[cfg(feature = "opencv-mat")]
 use opencv::{boxed_ref::BoxedRef, core::Mat};
 
@@ -32,6 +33,7 @@ pub struct Buffer {
     resolution: Resolution,
     buffer: Bytes,
     source_frame_format: FrameFormat,
+    capture_timestamp: Option<Duration>,
 }
 
 impl Buffer {
@@ -43,9 +45,33 @@ impl Buffer {
             resolution: res,
             buffer: Bytes::copy_from_slice(buf),
             source_frame_format,
+            capture_timestamp: None,
         }
     }
 
+    /// Creates a new buffer with a [`&[u8]`] and a backend-provided capture timestamp.
+    #[must_use]
+    #[inline]
+    pub fn with_timestamp(
+        res: Resolution,
+        buf: &[u8],
+        source_frame_format: FrameFormat,
+        capture_timestamp: Option<Duration>,
+    ) -> Self {
+        Self {
+            resolution: res,
+            buffer: Bytes::copy_from_slice(buf),
+            source_frame_format,
+            capture_timestamp,
+        }
+    }
+
+    /// Get the backend-provided capture timestamp, if available.
+    #[must_use]
+    pub fn capture_timestamp(&self) -> Option<Duration> {
+        self.capture_timestamp
+    }
+
     /// Get the [`Resolution`] of this buffer.
     #[must_use]
     pub fn resolution(&self) -> Resolution {

src/backends/capture/avfoundation.rs

@@ -32,7 +32,7 @@ use nokhwa_core::{
 #[cfg(target_os = "macos")]
 use nokhwa_core::{pixel_format::RgbFormat, types::RequestedFormatType};
 #[cfg(target_os = "macos")]
-use std::{ffi::CString, sync::Arc};
+use std::{ffi::CString, sync::Arc, time::Duration};
 
 use std::{borrow::Cow, collections::HashMap};
 
@@ -55,8 +55,8 @@ pub struct AVFoundationCaptureDevice {
     info: CameraInfo,
     buffer_name: CString,
     format: CameraFormat,
-    frame_buffer_receiver: Arc<Receiver<(Vec<u8>, FrameFormat)>>,
-    fbufsnd: Arc<Sender<(Vec<u8>, FrameFormat)>>,
+    frame_buffer_receiver: Arc<Receiver<(Vec<u8>, FrameFormat, Option<Duration>)>>,
+    fbufsnd: Arc<Sender<(Vec<u8>, FrameFormat, Option<Duration>)>>,
 }
 
 #[cfg(target_os = "macos")]
@@ -282,8 +282,11 @@ impl CaptureBackendTrait for AVFoundationCaptureDevice {
     fn frame(&mut self) -> Result<Buffer, NokhwaError> {
         self.refresh_camera_format()?;
         let cfmt = self.camera_format();
-        let b = self.frame_raw()?;
-        let buffer = Buffer::new(cfmt.resolution(), b.as_ref(), cfmt.format());
+        let (bytes, _fmt, capture_ts) = self
+            .frame_buffer_receiver
+            .recv()
+            .map_err(|why| NokhwaError::ReadFrameError(why.to_string()))?;
+        let buffer = Buffer::with_timestamp(cfmt.resolution(), &bytes, cfmt.format(), capture_ts);
         let _ = self.frame_buffer_receiver.drain();
         Ok(buffer)
     }

src/backends/capture/msmf_backend.rs

@@ -244,15 +244,18 @@ impl CaptureBackendTrait for MediaFoundationCaptureDevice {
     fn frame(&mut self) -> Result<Buffer, NokhwaError> {
         self.refresh_camera_format()?;
         let self_ctrl = self.camera_format();
-        Ok(Buffer::new(
+        let (bytes, capture_ts) = self.inner.raw_bytes()?;
+        Ok(Buffer::with_timestamp(
             self_ctrl.resolution(),
-            &self.inner.raw_bytes()?,
+            &bytes,
             self_ctrl.format(),
+            capture_ts,
         ))
     }
 
     fn frame_raw(&mut self) -> Result<Cow<'_, [u8]>, NokhwaError> {
-        self.inner.raw_bytes()
+        let (bytes, _capture_ts) = self.inner.raw_bytes()?;
+        Ok(bytes)
     }
 
     fn stop_stream(&mut self) -> Result<(), NokhwaError> {