[camera_android_camerax] Add support for NV21 image format

packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java

@@ -831,6 +831,7 @@ void unlockAutoFocus() {
             dartMessenger.error(flutterResult, errorCode, errorMessage, null));
   }
 
+  // TODO(camsim99): double check that this nv21 just for video recording
   public void startVideoRecording(@Nullable EventChannel imageStreamChannel) {
     prepareRecording();
 

packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java

@@ -211,7 +211,7 @@ public List<Map<String, Object>> parsePlanesForYuvOrJpeg(@NonNull Image image) {
    * @return parsed map describing the image planes to be sent to dart.
    */
   @NonNull
-  public List<Map<String, Object>> parsePlanesForNv21(@NonNull Image image) {
+  public List<Map<String, Object>> yuv420ThreePlanesToNV21(@NonNull Image image) {
     List<Map<String, Object>> planes = new ArrayList<>();
 
     // We will convert the YUV data to NV21 which is a single-plane image

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/PlaneProxyUtils.java

@@ -0,0 +1,157 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// Note: the code in this file is taken directly from the official Google MLKit example:
+// https://github.com/googlesamples/mlkit
+
+package io.flutter.plugins.camerax;
+
+import android.media.Image;
+import androidx.annotation.NonNull;
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+// TODO(camsim99): make sure license stuff is handled
+public class PlaneProxyUtils {
+  /**
+   * Converts YUV_420_888 to NV21 bytebuffer.
+   *
+   * <p>The NV21 format consists of a single byte array containing the Y, U and V values. For an
+   * image of size S, the first S positions of the array contain all the Y values. The remaining
+   * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both
+   * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain
+   * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU
+   *
+   * <p>YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled
+   * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and
+   * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into
+   * the first part of the NV21 array. The U and V planes may already have the representation in the
+   * NV21 format. This happens if the planes share the same buffer, the V buffer is one position
+   * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy
+   * them to the NV21 array.
+   *
+   * <p>https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java
+   */
+  @NonNull
+  public static ByteBuffer yuv420ThreePlanesToNV21(
+      @NonNull List<PlaneProxy> yuv420888planes, int width, int height) {
+    int imageSize = width * height;
+    byte[] out = new byte[imageSize + 2 * (imageSize / 4)];
+
+    if (areUVPlanesNV21(yuv420888planes, width, height)) {
+      // Copy the Y values.
+      yuv420888planes.get(0).getBuffer().get(out, 0, imageSize);
+
+      ByteBuffer uBuffer = yuv420888planes.get(1).getBuffer();
+      ByteBuffer vBuffer = yuv420888planes.get(2).getBuffer();
+      // Get the first V value from the V buffer, since the U buffer does not contain it.
+      vBuffer.get(out, imageSize, 1);
+      // Copy the first U value and the remaining VU values from the U buffer.
+      uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1);
+    } else {
+      // Fallback to copying the UV values one by one, which is slower but also works.
+      // Unpack Y.
+      unpackPlane(yuv420888planes.get(0), width, height, out, 0, 1);
+      // Unpack U.
+      unpackPlane(yuv420888planes.get(1), width, height, out, imageSize + 1, 2);
+      // Unpack V.
+      unpackPlane(yuv420888planes.get(2), width, height, out, imageSize, 2);
+    }
+
+    return ByteBuffer.wrap(out);
+  }
+
+  /**
+   * Copyright 2020 Google LLC. All rights reserved.
+   *
+   * <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+   * except in compliance with the License. You may obtain a copy of the License at
+   *
+   * <p>http://www.apache.org/licenses/LICENSE-2.0
+   *
+   * <p>Unless required by applicable law or agreed to in writing, software distributed under the
+   * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+   * either express or implied. See the License for the specific language governing permissions and
+   * limitations under the License.
+   *
+   * <p>Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format.
+   *
+   * <p>https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java
+   */
+  private static boolean areUVPlanesNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+    int imageSize = width * height;
+
+    ByteBuffer uBuffer = planes.get(1).getBuffer();
+    ByteBuffer vBuffer = planes.get(2).getBuffer();
+
+    // Backup buffer properties.
+    int vBufferPosition = vBuffer.position();
+    int uBufferLimit = uBuffer.limit();
+
+    // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value.
+    vBuffer.position(vBufferPosition + 1);
+    // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value.
+    uBuffer.limit(uBufferLimit - 1);
+
+    // Check that the buffers are equal and have the expected number of elements.
+    boolean areNV21 =
+        (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0);
+
+    // Restore buffers to their initial state.
+    vBuffer.position(vBufferPosition);
+    uBuffer.limit(uBufferLimit);
+
+    return areNV21;
+  }
+
+  /**
+   * Copyright 2020 Google LLC. All rights reserved.
+   *
+   * <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+   * except in compliance with the License. You may obtain a copy of the License at
+   *
+   * <p>http://www.apache.org/licenses/LICENSE-2.0
+   *
+   * <p>Unless required by applicable law or agreed to in writing, software distributed under the
+   * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+   * either express or implied. See the License for the specific language governing permissions and
+   * limitations under the License.
+   *
+   * <p>Unpack an image plane into a byte array.
+   *
+   * <p>The input plane data will be copied in 'out', starting at 'offset' and every pixel will be
+   * spaced by 'pixelStride'. Note that there is no row padding on the output.
+   *
+   * <p>https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java
+   */
+  private static void unpackPlane(
+      @NonNull PlaneProxy plane, int width, int height, byte[] out, int offset, int pixelStride)
+      throws IllegalStateException {
+    ByteBuffer buffer = plane.getBuffer();
+    buffer.rewind();
+
+    // Compute the size of the current plane.
+    // We assume that it has the aspect ratio as the original image.
+    int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride();
+    if (numRow == 0) {
+      return;
+    }
+    int scaleFactor = height / numRow;
+    int numCol = width / scaleFactor;
+
+    // Extract the data in the output buffer.
+    int outputPos = offset;
+    int rowStart = 0;
+    for (int row = 0; row < numRow; row++) {
+      int inputPos = rowStart;
+      for (int col = 0; col < numCol; col++) {
+        out[outputPos] = buffer.get(inputPos);
+        outputPos += pixelStride;
+        inputPos += plane.getPixelStride();
+      }
+      rowStart += plane.getRowStride();
+    }
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/PlaneProxyUtilsProxyApi.java

@@ -0,0 +1,30 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import androidx.annotation.NonNull;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/**
+ * ProxyApi implementation for {@link PlaneProxyUtils}. This class may handle instantiating native object
+ * instances that are attached to a Dart instance or handle method calls on the associated native
+ * class or an instance of that class.
+ */
+class PlaneProxyUtilsProxyApi extends PigeonApiPlaneProxyUtils {
+  PlaneProxyUtilsProxyApi(@NonNull ProxyApiRegistrar pigeonRegistrar) {
+    super(pigeonRegistrar);
+  }
+
+  // List<? extends PlaneProxy> can be considered the same as List<PlaneProxy>.
+  @SuppressWarnings("unchecked")
+  @NonNull
+  @Override
+  public byte[] getNv21Plane(@NonNull List<? extends PlaneProxy> planeProxyList, long imageWidth, long imageHeight) {
+    ByteBuffer nv21Bytes = PlaneProxyUtils.yuv420ThreePlanesToNV21((List<PlaneProxy>) planeProxyList, (int) imageWidth, (int) imageHeight);
+    return nv21Bytes.array();
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ProxyApiRegistrar.java

@@ -425,4 +425,10 @@ public PigeonApiMeteringPointFactory getPigeonApiMeteringPointFactory() {
   public CameraPermissionsErrorProxyApi getPigeonApiCameraPermissionsError() {
     return new CameraPermissionsErrorProxyApi(this);
   }
+
+  @NonNull
+  @Override
+  public PlaneProxyUtilsProxyApi getPigeonApiPlaneProxyUtils() {
+    return new PlaneProxyUtilsProxyApi(this);
+  }
 }

packages/camera/camera_android_camerax/example/lib/main.dart

@@ -12,6 +12,7 @@ import 'package:flutter/scheduler.dart';
 import 'package:video_player/video_player.dart';
 
 import 'camera_controller.dart';
+import 'camera_image.dart';
 import 'camera_preview.dart';
 
 /// Camera example home widget.
@@ -495,9 +496,12 @@ class _CameraExampleHomeState extends State<CameraExampleHome>
                   TextButton(
                     style: styleAuto,
                     onPressed:
-                        controller != null
-                            ? () => onSetFocusModeButtonPressed(FocusMode.auto)
-                            : null,
+                        // controller != null
+                        //     ? () => onSetFocusModeButtonPressed(FocusMode.auto)
+                        //     : null,
+                        () => controller!.startImageStream((CameraImage image) {
+                          print('image available!');
+                        }),
                     onLongPress: () {
                       if (controller != null) {
                         CameraPlatform.instance.setFocusPoint(
@@ -668,7 +672,7 @@ class _CameraExampleHomeState extends State<CameraExampleHome>
         audioBitrate: 32000,
         enableAudio: enableAudio,
       ),
-      imageFormatGroup: ImageFormatGroup.jpeg,
+      imageFormatGroup: ImageFormatGroup.nv21,
     );
 
     controller = cameraController;

packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart

@@ -7,6 +7,7 @@ import 'dart:math' show Point;
 
 import 'package:async/async.dart';
 import 'package:camera_platform_interface/camera_platform_interface.dart';
+import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart'
     show DeviceOrientation, PlatformException;
 import 'package:flutter/widgets.dart' show Texture, Widget, visibleForTesting;
@@ -266,6 +267,8 @@ class AndroidCameraCameraX extends CameraPlatform {
   @visibleForTesting
   late bool enableRecordingAudio;
 
+  late ImageFormatGroup _imageFormatForImageStreaming;
+
   /// Returns list of all available cameras and their descriptions.
   @override
   Future<List<CameraDescription>> availableCameras() async {
@@ -456,8 +459,12 @@ class AndroidCameraCameraX extends CameraPlatform {
   @override
   Future<void> initializeCamera(
     int cameraId, {
+    // TODO(camsim99): look into nvv21 for images as well
     ImageFormatGroup imageFormatGroup = ImageFormatGroup.unknown,
   }) async {
+    // Save imageFormatGroup to configure image streaming.
+    _imageFormatForImageStreaming = imageFormatGroup;
+
     // Configure CameraInitializedEvent to send as representation of a
     // configured camera:
     // Retrieve preview resolution.
@@ -1265,14 +1272,38 @@ class AndroidCameraCameraX extends CameraPlatform {
     Future<void> analyze(ImageProxy imageProxy) async {
       final List<PlaneProxy> planes = await imageProxy.getPlanes();
       final List<CameraImagePlane> cameraImagePlanes = <CameraImagePlane>[];
-      for (final PlaneProxy plane in planes) {
+
+      // CameraX records videos with the YUV420 format by default, but we convert
+      // to NV21 if requested when initializeCamera is called. JPEG requires no
+      // further processing.
+      final bool convertToNv21 =
+          _imageFormatForImageStreaming == ImageFormatGroup.nv21;
+      final int imageWidth = imageProxy.width;
+      final int imageHeight = imageProxy.height;
+
+      if (convertToNv21) {
+        final Uint8List nv21PlaneBytes = await PlaneProxyUtils.getNv21Plane(
+          planes,
+          imageWidth,
+          imageHeight,
+        );
         cameraImagePlanes.add(
           CameraImagePlane(
-            bytes: plane.buffer,
-            bytesPerRow: plane.rowStride,
-            bytesPerPixel: plane.pixelStride,
+            bytes: nv21PlaneBytes,
+            bytesPerRow: imageWidth,
+            bytesPerPixel: 1,
           ),
         );
+      } else {
+        for (final PlaneProxy plane in planes) {
+          cameraImagePlanes.add(
+            CameraImagePlane(
+              bytes: plane.buffer,
+              bytesPerRow: plane.rowStride,
+              bytesPerPixel: plane.pixelStride,
+            ),
+          );
+        }
       }
 
       final int format = imageProxy.format;
@@ -1284,8 +1315,8 @@ class AndroidCameraCameraX extends CameraPlatform {
       final CameraImageData cameraImageData = CameraImageData(
         format: cameraImageFormat,
         planes: cameraImagePlanes,
-        height: imageProxy.height,
-        width: imageProxy.width,
+        height: imageHeight,
+        width: imageWidth,
       );
 
       weakThis.target!.cameraImageDataStreamController!.add(cameraImageData);