preliminary browsercamera commit

l1npengtul · Aug 18, 2024 · 13633ae · 13633ae
1 parent 1433938
commit 13633ae
Show file tree

Hide file tree

Showing 21 changed files with 1,409 additions and 4,470 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -30,7 +30,7 @@ input-native = ["input-avfoundation", "input-v4l", "input-msmf"]
 # Re-enable it once soundness has been proven + mozjpeg is updated to 0.9.x
 # input-uvc = ["uvc", "uvc/vendor", "usb_enumeration", "lazy_static"]
 input-opencv = ["opencv", "opencv/rgb", "rgb", "nokhwa-core/opencv-mat"]
-input-jscam = [ "wasm-bindgen-futures", "wasm-rs-async-executor", "output-async", "js-sys", "web-sys"]
+input-jscam = [ "wasm-bindgen-futures", "wasm-rs-async-executor", "output-async", "js-sys", "web-sys", "serde-wasm-bindgen", "serde"]
 output-wgpu = ["wgpu", "nokhwa-core/wgpu-types"]
 #output-wasm = ["input-jscam"]
 output-threaded = []
@@ -73,11 +73,11 @@ version = "0.2"
 optional = true
 
 [dependencies.wgpu]
-version = "0.19"
+version = "0.20"
 optional = true
 
 [dependencies.opencv]
-version = "0.89"
+version = "0.92"
 default-features = false
 features = ["videoio"]
 optional = true
@@ -143,6 +143,10 @@ optional = true
 version = "0.9"
 optional = true
 
+[dependencies.serde-wasm-bindgen]
+version = "0.6"
+optional = true
+
 [dependencies.async-trait]
 version = "0.1"
 optional = true

diff --git a/nokhwa-bindings-linux/src/lib.rs b/nokhwa-bindings-linux/src/lib.rs
@@ -178,11 +178,11 @@ mod internal {
                         }
                     };
 
-                    self.camera_format = CameraFormat::new(
+                    self.camera_format = Some(CameraFormat::new(
                         Resolution::new(format.width, format.height),
                         frame_format: ,
                         fps,
-                    );
+                    ));
                     Ok(())
                 }
                 Err(why) => Err(NokhwaError::GetPropertyError {
@@ -198,10 +198,6 @@ mod internal {
             todo!()
         }
 
-        fn init_with_format(&mut self, format: FormatFilter) -> Result<CameraFormat, NokhwaError> {
-            todo!()
-        }
-
         fn backend(&self) -> ApiBackend {
             ApiBackend::Video4Linux
         }
@@ -596,8 +592,8 @@ mod internal {
 
     fn fourcc_to_frameformat(fourcc: FourCC) -> Option<FrameFormat> {
         match fourcc.str().ok()? {
-            "YUYV" => Some(FrameFormat::Yuv422),
-            "UYVY" => Some(FrameFormat::Uyv422),
+            "YUYV" => Some(FrameFormat::Yuy2_422),
+            "UYVY" => Some(FrameFormat::Uyvy_422),
             "YV12" => Some(FrameFormat::Yv12),
             "MJPG" => Some(FrameFormat::MJpeg),
             "GRAY" => Some(FrameFormat::Luma8),

diff --git a/nokhwa-bindings-macos/Cargo.toml b/nokhwa-bindings-macos/Cargo.toml
@@ -21,5 +21,7 @@ core-video-sys = "0.1"
 cocoa-foundation = "0.1"
 objc = { version = "0.2", features = ["exception"] }
 block = "0.1"
-flume = "0.10"
+flume = "0.11.0"
 once_cell = "1.16"
+
+av-foundation = "0.3.0"
diff --git a/nokhwa-core/Cargo.toml b/nokhwa-core/Cargo.toml
@@ -16,6 +16,7 @@ serialize = ["serde"]
 wgpu-types = ["wgpu"]
 opencv-mat = ["opencv"]
 docs-features = ["serialize", "wgpu-types"]
+conversions = ["dcv-color-primitives", "yuvutils-rs", "mozjpeg"]
 async = ["async-trait"]
 test-fail-warnings = []
 
@@ -35,11 +36,11 @@ features = ["derive"]
 optional = true
 
 [dependencies.wgpu]
-version = "0.19"
+version = "22"
 optional = true
 
 [dependencies.opencv]
-version = "0.89.0"
+version = "0.92"
 default-features = false
 optional = true
 
@@ -51,5 +52,13 @@ optional = true
 version = "0.1"
 optional = true
 
+[dependencies.dcv-color-primitives]
+version = "0.6"
+optional = true
+
+[dependencies.yuvutils-rs]
+version = "0.3"
+optional = true
+
 [package.metadata.docs.rs]
 features = ["docs-features"]
diff --git a/nokhwa-core/src/buffer.rs b/nokhwa-core/src/buffer.rs
@@ -14,8 +14,9 @@
  * limitations under the License.
  */
 
-use crate::{ types::Resolution};
+use crate::types::Resolution;
 use bytes::Bytes;
+use crate::frame_format::FrameFormat;
 
 /// A buffer returned by a camera to accommodate custom decoding.
 /// Contains information of Resolution, the buffer's [`FrameFormat`], and the buffer.
@@ -64,154 +65,3 @@ impl Buffer {
         self.source_frame_format
     }
 }
-
-#[cfg(feature = "opencv-mat")]
-use crate::error::NokhwaError;
-#[cfg(feature = "opencv-mat")]
-use image::ImageBuffer;
-
-#[cfg(feature = "opencv-mat")]
-impl Buffer {
-
-    /// Decodes a image with allocation using the provided [`FormatDecoder`].
-    /// # Errors
-    /// Will error when the decoding fails.
-    #[inline]
-    pub fn decode_image<F: FormatDecoder>(
-        &self,
-    ) -> Result<ImageBuffer<F::Output, Vec<u8>>, NokhwaError> {
-        let new_data = F::write_output(self.source_frame_format, self.resolution, &self.buffer)?;
-        let image =
-            ImageBuffer::from_raw(self.resolution.width_x, self.resolution.height_y, new_data)
-                .ok_or(NokhwaError::ProcessFrameError {
-                    src: self.source_frame_format,
-                    destination: stringify!(F).to_string(),
-                    error: "Failed to create buffer".to_string(),
-                })?;
-        Ok(image)
-    }
-
-    /// Decodes a image with allocation using the provided [`FormatDecoder`] into a `buffer`.
-    /// # Errors
-    /// Will error when the decoding fails, or the provided buffer is too small.
-    #[inline]
-    pub fn decode_image_to_buffer<F: FormatDecoder>(
-        &self,
-        buffer: &mut [u8],
-    ) -> Result<(), NokhwaError> {
-        F::write_output_buffer(
-            self.source_frame_format,
-            self.resolution,
-            &self.buffer,
-            buffer,
-        )
-    }
-    /// Decodes a image with allocation using the provided [`FormatDecoder`] into a [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html).
-    ///
-    /// Note that this does a clone when creating the buffer, to decouple the lifetime of the internal data to the temporary Buffer. If you want to avoid this, please see [`decode_opencv_mat`](Self::decode_opencv_mat).
-    /// # Errors
-    /// Will error when the decoding fails, or `OpenCV` failed to create/copy the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html).
-    /// # Safety
-    /// This function uses `unsafe` in order to create the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html). Please see [`Mat::new_rows_cols_with_data`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html#method.new_rows_cols_with_data) for more.
-    ///
-    /// Most notably, the `data` **must** stay in scope for the duration of the [`Mat`](https://docs.rs/opencv/latest/opencv/core/struct.Mat.html) or bad, ***bad*** things happen.
-    #[cfg(feature = "opencv-mat")]
-    #[cfg_attr(feature = "docs-features", doc(cfg(feature = "opencv-mat")))]
-    #[allow(clippy::cast_possible_wrap)]
-    pub fn decode_opencv_mat<F: FormatDecoder>(
-        &mut self,
-    ) -> Result<opencv::core::Mat, NokhwaError> {
-        use image::Pixel;
-        use opencv::core::{Mat, Mat_AUTO_STEP, CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4};
-
-        let array_type = match F::Output::CHANNEL_COUNT {
-            1 => CV_8UC1,
-            2 => CV_8UC2,
-            3 => CV_8UC3,
-            4 => CV_8UC4,
-            _ => {
-                return Err(NokhwaError::ProcessFrameError {
-                    src: FrameFormat::RAWRGB,
-                    destination: "OpenCV Mat".to_string(),
-                    error: "Invalid Decoder FormatDecoder Channel Count".to_string(),
-                })
-            }
-        };
-
-        unsafe {
-            // TODO: Look into removing this unnecessary copy.
-            let mat1 = Mat::new_rows_cols_with_data(
-                self.resolution.height_y as i32,
-                self.resolution.width_x as i32,
-                array_type,
-                self.buffer.as_ref().as_ptr().cast_mut().cast(),
-                Mat_AUTO_STEP,
-            )
-            .map_err(|why| NokhwaError::ProcessFrameError {
-                src: FrameFormat::Rgb8,
-                destination: "OpenCV Mat".to_string(),
-                error: why.to_string(),
-            })?;
-
-            Ok(mat1)
-        }
-    }
-}
-
-#[cfg(feature = "wgpu-types")]
-use wgpu::{Extent3d, TextureDescriptor, TextureDimension, TextureFormat, TextureUsages, ImageCopyTexture, TextureAspect, ImageDataLayout};
-use crate::frame_format::FrameFormat;
-
-#[cfg(feature = "wgpu-types")]
-impl Buffer {
-    #[cfg_attr(feature = "docs-features", doc(cfg(feature = "wgpu-types")))]
-    /// Directly copies a frame to a Wgpu texture. This will automatically convert the frame into a RGBA frame.
-    /// # Errors
-    /// If the frame cannot be captured or the resolution is 0 on any axis, this will error.
-    fn frame_texture<'a>(
-        &mut self,
-        device: &wgpu::Device,
-        queue: &wgpu::Queue,
-        label: Option<&'a str>,
-    ) -> Result<wgpu::Texture, NokhwaError> {
-        let frame = self.frame()?.decode_image::<RgbAFormat>()?;
-
-        let texture_size = Extent3d {
-            width: frame.width(),
-            height: frame.height(),
-            depth_or_array_layers: 1,
-        };
-
-        let texture = device.create_texture(&TextureDescriptor {
-            label,
-            size: texture_size,
-            mip_level_count: 1,
-            sample_count: 1,
-            dimension: TextureDimension::D2,
-            format: TextureFormat::Rgba8UnormSrgb,
-            usage: TextureUsages::TEXTURE_BINDING | TextureUsages::COPY_DST,
-            view_formats: &[TextureFormat::Rgba8UnormSrgb],
-        });
-
-        let width_nonzero = 4 * frame.width();
-        let height_nonzero = frame.height();
-
-        queue.write_texture(
-            ImageCopyTexture {
-                texture: &texture,
-                mip_level: 0,
-                origin: wgpu::Origin3d::ZERO,
-                aspect: TextureAspect::All,
-            },
-            &frame,
-            ImageDataLayout {
-                offset: 0,
-                bytes_per_row: width_nonzero,
-                rows_per_image: height_nonzero,
-            },
-            texture_size,
-        );
-
-        Ok(texture)
-    }
-}
diff --git a/nokhwa-core/src/decoder.rs b/nokhwa-core/src/decoder.rs